package net.osmand.binary; import gnu.trove.iterator.TLongIterator; import gnu.trove.list.array.TIntArrayList; import gnu.trove.list.array.TLongArrayList; import gnu.trove.map.hash.TIntObjectHashMap; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.RandomAccessFile; import java.text.MessageFormat; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; import net.osmand.ResultMatcher; import net.osmand.binary.BinaryMapAddressReaderAdapter.AddressRegion; import net.osmand.binary.BinaryMapAddressReaderAdapter.CitiesBlock; import net.osmand.binary.BinaryMapIndexReader.MapIndex; import net.osmand.binary.BinaryMapIndexReader.MapRoot; import net.osmand.binary.BinaryMapIndexReader.SearchFilter; import net.osmand.binary.BinaryMapIndexReader.SearchPoiTypeFilter; import net.osmand.binary.BinaryMapIndexReader.SearchRequest; import net.osmand.binary.BinaryMapIndexReader.TagValuePair; import net.osmand.binary.BinaryMapPoiReaderAdapter.PoiRegion; import net.osmand.binary.BinaryMapPoiReaderAdapter.PoiSubType; import net.osmand.binary.BinaryMapRouteReaderAdapter.RouteRegion; import net.osmand.binary.BinaryMapTransportReaderAdapter.TransportIndex; import net.osmand.data.Amenity; import net.osmand.data.AmenityType; import net.osmand.data.Building; import net.osmand.data.City; import net.osmand.data.MapObject; import net.osmand.data.Street; import net.osmand.util.MapUtils; import com.google.protobuf.CodedOutputStream; import com.google.protobuf.WireFormat; public class BinaryInspector { public static final int BUFFER_SIZE = 1 << 20; private VerboseInfo vInfo; public static void main(String[] args) throws IOException { BinaryInspector in = new BinaryInspector(); // test cases show info if(args.length == 1 && "test".equals(args[0])) { in.inspector(new String[]{ //"-vpoi", // "-vmap", "-vmapobjects", // "-vaddress", "-vcities", "-vstreets", "-vstreetgroups","-vbuildings", //"-zoom=16", //"-bbox=4,55,7,50", //"/home/victor/projects/osmand/osm-gen/Map.obf" // "/home/victor/projects/osmand/osm-gen/Russia_bashkiria_asia_2.obf" }); } else { in.inspector(args); } } private void printToFile(String s) throws IOException { if(vInfo.osmOut != null) { vInfo.osmOut.write(s.getBytes()); } else { System.out.println(s); } } private void println(String s) { if(vInfo != null && vInfo.osm && vInfo.osmOut == null) { // ignore } else { System.out.println(s); } } private void print(String s) { if(vInfo != null && vInfo.osm && vInfo.osmOut == null) { // ignore } else { System.out.print(s); } } protected static class VerboseInfo { boolean vaddress; boolean vcities; boolean vstreetgroups; boolean vstreets; boolean vbuildings; boolean vintersections; boolean vtransport; boolean vpoi; boolean vmap; boolean vmapObjects; boolean osm; FileOutputStream osmOut = null; double lattop = 85; double latbottom = -85; double lonleft = -180; double lonright = 180; int zoom = -1; public boolean isVaddress() { return vaddress; } public int getZoom() { return zoom; } public boolean isVmap() { return vmap; } public boolean isVpoi() { return vpoi; } public boolean isVtransport() { return vtransport; } public VerboseInfo(String[] params) throws FileNotFoundException { for(int i=0;i= o.getLocation().getLatitude() && latbottom <= o.getLocation().getLatitude() && lonleft <= o.getLocation().getLongitude() && lonright >= o.getLocation().getLongitude(); } public void close() throws IOException { if(osmOut != null) { osmOut.close();; osmOut = null; } } } public void inspector(String[] args) throws IOException { if(args == null || args.length == 0){ printUsage(null); return; } String f = args[0]; if (f.charAt(0) == '-') { // command if (f.equals("-c") || f.equals("-combine")) { if (args.length < 4) { printUsage("Too few parameters to extract (require minimum 4)"); } else { Map parts = new HashMap(); for (int i = 2; i < args.length; i++) { File file = new File(args[i]); if (!file.exists()) { System.err.println("File to extract from doesn't exist " + args[i]); return; } parts.put(file, null); if (i < args.length - 1) { if (args[i + 1].startsWith("-") || args[i + 1].startsWith("+")) { parts.put(file, args[i + 1]); i++; } } } List extracted = combineParts(new File(args[1]), parts); if (extracted != null) { println("\n" + extracted.size() + " parts were successfully extracted to " + args[1]); } } } else if (f.startsWith("-v")) { if (args.length < 2) { printUsage("Missing file parameter"); } else { vInfo = new VerboseInfo(args); printFileInformation(args[args.length - 1]); vInfo.close(); } } else { printUsage("Unknown command : " + f); } } else { vInfo = null; printFileInformation(f); } } public static final void writeInt(CodedOutputStream ous, int v) throws IOException { ous.writeRawByte((v >>> 24) & 0xFF); ous.writeRawByte((v >>> 16) & 0xFF); ous.writeRawByte((v >>> 8) & 0xFF); ous.writeRawByte(v & 0xFF); //written += 4; } @SuppressWarnings("unchecked") public static List combineParts(File fileToExtract, Map partsToExtractFrom) throws IOException { BinaryMapIndexReader[] indexes = new BinaryMapIndexReader[partsToExtractFrom.size()]; RandomAccessFile[] rafs = new RandomAccessFile[partsToExtractFrom.size()]; LinkedHashSet[] partsSet = new LinkedHashSet[partsToExtractFrom.size()]; int c = 0; Set addressNames = new LinkedHashSet(); int version = -1; // Go through all files and validate conistency for(File f : partsToExtractFrom.keySet()){ if(f.getAbsolutePath().equals(fileToExtract.getAbsolutePath())){ System.err.println("Error : Input file is equal to output file " + f.getAbsolutePath()); return null; } rafs[c] = new RandomAccessFile(f.getAbsolutePath(), "r"); indexes[c] = new BinaryMapIndexReader(rafs[c]); partsSet[c] = new LinkedHashSet(); if(version == -1){ version = indexes[c].getVersion(); } else { if(indexes[c].getVersion() != version){ System.err.println("Error : Different input files has different input versions " + indexes[c].getVersion() + " != " + version); return null; } } LinkedHashSet temp = new LinkedHashSet(); String pattern = partsToExtractFrom.get(f); boolean minus = true; for (int i = 0; i < indexes[c].getIndexes().size(); i++) { partsSet[c].add(i + 1f); BinaryIndexPart part = indexes[c].getIndexes().get(i); if(part instanceof MapIndex){ List roots = ((MapIndex) part).getRoots(); int rsize = roots.size(); for(int j=0; j p = partsSet[c].iterator(); while (p.hasNext()) { Float part = p.next(); if (minus) { if (temp.contains(part)) { p.remove(); } } else { if (!temp.contains(part)) { p.remove(); } } } c++; } // write files FileOutputStream fout = new FileOutputStream(fileToExtract); CodedOutputStream ous = CodedOutputStream.newInstance(fout, BUFFER_SIZE); List list = new ArrayList(); byte[] BUFFER_TO_READ = new byte[BUFFER_SIZE]; ous.writeInt32(OsmandOdb.OsmAndStructure.VERSION_FIELD_NUMBER, version); ous.writeInt64(OsmandOdb.OsmAndStructure.DATECREATED_FIELD_NUMBER, System.currentTimeMillis()); for (int k = 0; k < indexes.length; k++) { LinkedHashSet partSet = partsSet[k]; BinaryMapIndexReader index = indexes[k]; RandomAccessFile raf = rafs[k]; for (int i = 0; i < index.getIndexes().size(); i++) { if (!partSet.contains(Float.valueOf(i + 1f))) { continue; } list.add(i + 1f); BinaryIndexPart part = index.getIndexes().get(i); String map; if (part instanceof MapIndex) { ous.writeTag(OsmandOdb.OsmAndStructure.MAPINDEX_FIELD_NUMBER, WireFormat.WIRETYPE_FIXED32_LENGTH_DELIMITED); map = "Map"; } else if (part instanceof AddressRegion) { ous.writeTag(OsmandOdb.OsmAndStructure.ADDRESSINDEX_FIELD_NUMBER, WireFormat.WIRETYPE_FIXED32_LENGTH_DELIMITED); map = "Address"; if (addressNames.contains(part.getName())) { System.err.println("Error : going to merge 2 addresses with same names. Skip " + part.getName()); continue; } addressNames.add(part.getName()); } else if (part instanceof TransportIndex) { ous.writeTag(OsmandOdb.OsmAndStructure.TRANSPORTINDEX_FIELD_NUMBER, WireFormat.WIRETYPE_FIXED32_LENGTH_DELIMITED); map = "Transport"; } else if (part instanceof PoiRegion) { ous.writeTag(OsmandOdb.OsmAndStructure.POIINDEX_FIELD_NUMBER, WireFormat.WIRETYPE_FIXED32_LENGTH_DELIMITED); map = "POI"; } else if (part instanceof RouteRegion) { ous.writeTag(OsmandOdb.OsmAndStructure.ROUTINGINDEX_FIELD_NUMBER, WireFormat.WIRETYPE_FIXED32_LENGTH_DELIMITED); map = "Routing"; } else { throw new UnsupportedOperationException(); } writeInt(ous, part.getLength()); copyBinaryPart(ous, BUFFER_TO_READ, raf, part.getFilePointer(), part.getLength()); System.out.println(MessageFormat.format("{2} part {0} is extracted {1} bytes", new Object[]{part.getName(), part.getLength(), map})); } } ous.writeInt32(OsmandOdb.OsmAndStructure.VERSIONCONFIRM_FIELD_NUMBER, version); ous.flush(); fout.close(); return list; } private static void copyBinaryPart(CodedOutputStream ous, byte[] BUFFER, RandomAccessFile raf, long fp, int length) throws IOException { raf.seek(fp); int toRead = length; while (toRead > 0) { int read = raf.read(BUFFER); if (read == -1) { throw new IllegalArgumentException("Unexpected end of file"); } if (toRead < read) { read = toRead; } ous.writeRawBytes(BUFFER, 0, read); toRead -= read; } } protected String formatBounds(int left, int right, int top, int bottom){ double l = MapUtils.get31LongitudeX(left); double r = MapUtils.get31LongitudeX(right); double t = MapUtils.get31LatitudeY(top); double b = MapUtils.get31LatitudeY(bottom); return formatLatBounds(l, r, t, b); } protected String formatLatBounds(double l, double r, double t, double b){ MessageFormat format = new MessageFormat("(left top - right bottom) : {0,number,#.####}, {1,number,#.####} NE - {2,number,#.####}, {3,number,#.####} NE", new Locale("EN", "US")); return format.format(new Object[]{l, t, r, b}); } public void printFileInformation(String fileName) throws IOException { File file = new File(fileName); if(!file.exists()){ println("Binary OsmAnd index " + fileName + " was not found."); return; } printFileInformation(file); } public void printFileInformation(File file) throws IOException { RandomAccessFile r = new RandomAccessFile(file.getAbsolutePath(), "r"); printFileInformation(r, file.getName()); } public void printFileInformation(RandomAccessFile r, String filename) throws IOException { try { BinaryMapIndexReader index = new BinaryMapIndexReader(r); int i = 1; println("Binary index " + filename + " version = " + index.getVersion() +" edition = " + new Date(index.getDateCreated())); for(BinaryIndexPart p : index.getIndexes()){ String partname = ""; if(p instanceof MapIndex ){ partname = "Map"; } else if(p instanceof TransportIndex){ partname = "Transport"; } else if(p instanceof RouteRegion){ partname = "Routing"; } else if(p instanceof PoiRegion){ partname = "Poi"; } else if(p instanceof AddressRegion){ partname = "Address"; } String name = p.getName() == null ? "" : p.getName(); println(MessageFormat.format("{0} {1} data {3} - {2,number,#} bytes", new Object[]{i, partname, p.getLength(), name})); if(p instanceof TransportIndex){ TransportIndex ti = ((TransportIndex) p); int sh = (31 - BinaryMapIndexReader.TRANSPORT_STOP_ZOOM); println("\tBounds " + formatBounds(ti.getLeft() << sh, ti.getRight() << sh, ti.getTop() << sh, ti.getBottom() << sh)); } else if(p instanceof RouteRegion){ RouteRegion ri = ((RouteRegion) p); println("\tBounds " + formatLatBounds(ri.getLeftLongitude(), ri.getRightLongitude(), ri.getTopLatitude(), ri.getBottomLatitude())); } else if(p instanceof MapIndex){ MapIndex m = ((MapIndex) p); int j = 1; for(MapRoot mi : m.getRoots()){ println(MessageFormat.format("\t{4}.{5} Map level minZoom = {0}, maxZoom = {1}, size = {2,number,#} bytes \n\t\tBounds {3}", new Object[] { mi.getMinZoom(), mi.getMaxZoom(), mi.getLength(), formatBounds(mi.getLeft(), mi.getRight(), mi.getTop(), mi.getBottom()), i, j++})); } if((vInfo != null && vInfo.isVmap())){ printMapDetailInfo(index, m); } } else if(p instanceof PoiRegion && (vInfo != null && vInfo.isVpoi())){ printPOIDetailInfo(vInfo, index, (PoiRegion) p); } else if (p instanceof AddressRegion) { List cities = ((AddressRegion) p).cities; for (CitiesBlock c : cities) { println("\t" + i + "." + c.type + " Address part size=" + c.length + " bytes"); } if (vInfo != null && vInfo.isVaddress()) { printAddressDetailedInfo(vInfo, index, (AddressRegion) p); } } i++; } } catch (IOException e) { System.err.println("File doesn't have valid structure : " + filename + " " + e.getMessage() ); throw e; } } private void printAddressDetailedInfo(VerboseInfo verbose, BinaryMapIndexReader index, AddressRegion region) throws IOException { String[] cityType_String = new String[] { "Cities/Towns section", "Villages section", "Postcodes section", }; int[] cityType = new int[] { BinaryMapAddressReaderAdapter.CITY_TOWN_TYPE, BinaryMapAddressReaderAdapter.VILLAGES_TYPE, BinaryMapAddressReaderAdapter.POSTCODES_TYPE }; for (int j = 0; j < cityType.length; j++) { int type = cityType[j]; final List cities = index.getCities(region, null, type); print(MessageFormat.format("\t{0}, {1,number,#} group(s)", new Object[]{cityType_String[j], cities.size()})); if (BinaryMapAddressReaderAdapter.CITY_TOWN_TYPE == type) { if (!verbose.vstreetgroups && !verbose.vcities) { println(""); continue; } } else if (!verbose.vstreetgroups) { println(""); continue; } println(":"); for (City c : cities) { int size = index.preloadStreets(c, null); List streets = new ArrayList(c.getStreets()); print(MessageFormat.format("\t\t''{0}'' [{1,number,#}], {2,number,#} street(s) size {3,number,#} bytes", new Object[]{c.getEnName(), c.getId(), streets.size(), size})); if(!verbose.vstreets) { println(""); continue; } println(":"); if (!verbose.contains(c)) continue; for (Street t : streets) { if (!verbose.contains(t)) continue; index.preloadBuildings(t, null); final List buildings = t.getBuildings(); final List intersections = t.getIntersectedStreets(); println(MessageFormat.format("\t\t\t''{0}'' [{1,number,#}], {2,number,#} building(s), {3,number,#} intersections(s)", new Object[]{t.getEnName(), t.getId(), buildings.size(), intersections.size()})); if (buildings != null && !buildings.isEmpty() && verbose.vbuildings) { println("\t\t\t\tBuildings:"); for (Building b : buildings) { println(MessageFormat.format("\t\t\t\t{0} [{1,number,#}]", new Object[]{b.getName(true), b.getId()})); } } if (intersections != null && !intersections.isEmpty() && verbose.vintersections) { print("\t\t\t\tIntersects with:"); for (Street s : intersections) { println("\t\t\t\t\t" + s.getEnName()); } } } } } } private static class DamnCounter { int value; } private void printMapDetailInfo(BinaryMapIndexReader index, MapIndex mapIndex) throws IOException { final StringBuilder b = new StringBuilder(); final DamnCounter mapObjectsCounter = new DamnCounter(); if(vInfo.osm){ printToFile("\n" + "\n"); } SearchRequest req = BinaryMapIndexReader.buildSearchRequest( MapUtils.get31TileNumberX(vInfo.lonleft), MapUtils.get31TileNumberX(vInfo.lonright), MapUtils.get31TileNumberY(vInfo.lattop), MapUtils.get31TileNumberY(vInfo.latbottom), vInfo.getZoom(), new SearchFilter() { @Override public boolean accept(TIntArrayList types, MapIndex index) { return true; } }, new ResultMatcher() { @Override public boolean publish(BinaryMapDataObject obj) { mapObjectsCounter.value++; if(vInfo.vmapObjects) { b.setLength(0); if(vInfo.osm) { printOsmMapDetails(obj, b); try { printToFile(b.toString()); } catch (IOException e) { throw new RuntimeException(e); } } else { printMapDetails(obj, b); println(b.toString()); } } return false; } @Override public boolean isCancelled() { return false; } }); index.searchMapIndex(req, mapIndex); if(vInfo.osm){ printToFile("\n"); } println("\tTotal map objects: " + mapObjectsCounter.value); } private static void printMapDetails(BinaryMapDataObject obj, StringBuilder b) { boolean multipolygon = obj.getPolygonInnerCoordinates() != null && obj.getPolygonInnerCoordinates().length > 0; if(multipolygon ) { b.append("Multipolygon"); } else { b.append(obj.area? "Area" : (obj.getPointsLength() > 1? "Way" : "Point")); } int[] types = obj.getTypes(); b.append(" types ["); for(int j = 0; j 0) { b.append(", "); } TagValuePair pair = obj.getMapIndex().decodeType(types[j]); if(pair == null) { System.err.println("Type " + types[j] + "was not found"); continue; // throw new NullPointerException("Type " + obj.getAdditionalTypes()[j] + "was not found"); } b.append(pair.toSimpleString()+" ("+types[j]+")"); } b.append("]"); if(obj.getAdditionalTypes() != null && obj.getAdditionalTypes().length > 0){ b.append(" add_types ["); for(int j = 0; j 0) { b.append(", "); } TagValuePair pair = obj.getMapIndex().decodeType(obj.getAdditionalTypes()[j]); if(pair == null) { System.err.println("Type " + obj.getAdditionalTypes()[j] + "was not found"); continue; // throw new NullPointerException("Type " + obj.getAdditionalTypes()[j] + "was not found"); } b.append(pair.toSimpleString()+"("+obj.getAdditionalTypes()[j]+")"); } b.append("]"); } TIntObjectHashMap names = obj.getObjectNames(); if(names != null && !names.isEmpty()) { b.append(" Names ["); int[] keys = names.keys(); for(int j = 0; j 0) { b.append(", "); } TagValuePair pair = obj.getMapIndex().decodeType(keys[j]); if(pair == null) { throw new NullPointerException("Type " + keys[j] + "was not found"); } b.append(pair.toSimpleString()+"("+keys[j]+")"); b.append(" - ").append(names.get(keys[j])); } b.append("]"); } b.append(" id ").append((obj.getId() >> 1)); b.append(" lat/lon : "); for(int i=0; i 0; boolean point = obj.getPointsLength() == 1; StringBuilder tags = new StringBuilder(); int[] types = obj.getTypes(); for(int j = 0; j\n"); } if(obj.getAdditionalTypes() != null && obj.getAdditionalTypes().length > 0){ for(int j = 0; j\n"); } } TIntObjectHashMap names = obj.getObjectNames(); if(names != null && !names.isEmpty()) { int[] keys = names.keys(); for(int j = 0; j\n"); } } tags.append("\t\n"); if(point) { float lon= (float) MapUtils.get31LongitudeX(obj.getPoint31XTile(0)); float lat = (float) MapUtils.get31LatitudeY(obj.getPoint31YTile(0)); b.append("\n" ); b.append(tags); b.append("\n"); } else { TLongArrayList innerIds = new TLongArrayList(); TLongArrayList ids = new TLongArrayList(); for(int i=0; i\n" ); ids.add(id); } long outerId = printWay(ids, b, multipolygon ? null : tags); if (multipolygon) { int[][] polygonInnerCoordinates = obj.getPolygonInnerCoordinates(); for (int j = 0; j < polygonInnerCoordinates.length; j++) { ids.clear();; for (int i = 0; i < polygonInnerCoordinates[j].length; i += 2) { float lon = (float) MapUtils.get31LongitudeX(polygonInnerCoordinates[j][i]); float lat = (float) MapUtils.get31LatitudeY(polygonInnerCoordinates[j][i + 1]); int id = OSM_ID++; b.append("\n"); ids.add(id); } innerIds.add(printWay(ids, b, null)); } int id = OSM_ID++; b.append("\n" ); b.append(tags); b.append("\t\n" ); TLongIterator it = innerIds.iterator(); while(it.hasNext()) { long ref = it.next(); b.append("\n" ); } b.append("\n" ); } } } private long printWay(TLongArrayList ids, StringBuilder b , StringBuilder tags){ int id = OSM_ID++; b.append("\n" ); if(tags != null) { b.append(tags); } TLongIterator it = ids.iterator(); while(it.hasNext()) { long ref = it.next(); b.append("\t\n" ); } b.append("\n" ); return id; } private void printPOIDetailInfo(VerboseInfo verbose, BinaryMapIndexReader index, PoiRegion p) throws IOException { SearchRequest req = BinaryMapIndexReader.buildSearchPoiRequest( MapUtils.get31TileNumberX(verbose.lonleft), MapUtils.get31TileNumberX(verbose.lonright), MapUtils.get31TileNumberY(verbose.lattop), MapUtils.get31TileNumberY(verbose.latbottom), verbose.getZoom(), new SearchPoiTypeFilter() { @Override public boolean accept(AmenityType type, String subcategory) { return true; } }, new ResultMatcher() { @Override public boolean publish(Amenity object) { println(object.getType().toString() + " : " + object.getSubType() + " " + object.getName() + " " + object.getLocation() + " id=" + (object.getId() >> 1) + " " + object.getAdditionalInfo()); return false; } @Override public boolean isCancelled() { return false; } }); index.initCategories(p); println("\tRegion: " + p.name); println("\t\tBounds " + formatLatBounds(p.getLeftLongitude(), p.getRightLongitude(), p.getTopLatitude(), p.getBottomLatitude())); println("\t\tCategories:"); for(int i =0; i< p.categories.size(); i++) { println("\t\t\t" + p.categories.get(i)); for(int j = 0; j < p.subcategories.get(i).size(); j++) println("\t\t\t\t" + p.subcategories.get(i).get(j)); } println("\t\tSubtypes:"); for(int i =0; i< p.subTypes.size(); i++) { PoiSubType st = p.subTypes.get(i); println("\t\t\t" + st.name + " " + (st.text ? "text":(" encoded " + st.possibleValues.size()))); } req.poiTypeFilter = null;//for test only index.searchPoi(p, req); } public void printUsage(String warning) { if(warning != null){ println(warning); } println("Inspector is console utility for working with binary indexes of OsmAnd."); println("It allows print info about file, extract parts and merge indexes."); println("\nUsage for print info : inspector [-vaddress] [-vstreetgroups] [-vstreets] [-vbuildings] [-vintersections] [-vmap] [-vmapobjects] [-osm] [-vpoi] [-vtransport] [-zoom=Zoom] [-bbox=LeftLon,TopLat,RightLon,BottomLan] [file]"); println(" Prints information about [file] binary index of OsmAnd."); println(" -v.. more verbouse output (like all cities and their streets or all map objects with tags/values and coordinates)"); println("\nUsage for combining indexes : inspector -c file_to_create (file_from_extract ((+|-)parts_to_extract)? )*"); println("\tCreate new file of extracted parts from input file. [parts_to_extract] could be parts to include or exclude."); println(" Example : inspector -c output_file input_file +1,2,3\n\tExtracts 1, 2, 3 parts (could be find in print info)"); println(" Example : inspector -c output_file input_file -2,3\n\tExtracts all parts excluding 2, 3"); println(" Example : inspector -c output_file input_file1 input_file2 input_file3\n\tSimply combine 3 files"); println(" Example : inspector -c output_file input_file1 input_file2 -4\n\tCombine all parts of 1st file and all parts excluding 4th part of 2nd file"); } }