diff --git a/OsmAnd-java/src/net/osmand/binary/BinaryInspector.java b/OsmAnd-java/src/net/osmand/binary/BinaryInspector.java index 583a182730..50e5c8fa67 100644 --- a/OsmAnd-java/src/net/osmand/binary/BinaryInspector.java +++ b/OsmAnd-java/src/net/osmand/binary/BinaryInspector.java @@ -12,11 +12,15 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.RandomAccessFile; +import java.text.DecimalFormat; import java.text.MessageFormat; import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; @@ -27,6 +31,7 @@ import net.osmand.ResultMatcher; import net.osmand.binary.BinaryMapAddressReaderAdapter.AddressRegion; import net.osmand.binary.BinaryMapAddressReaderAdapter.CitiesBlock; import net.osmand.binary.BinaryMapIndexReader.MapIndex; +import net.osmand.binary.BinaryMapIndexReader.MapObjectStat; import net.osmand.binary.BinaryMapIndexReader.MapRoot; import net.osmand.binary.BinaryMapIndexReader.SearchFilter; import net.osmand.binary.BinaryMapIndexReader.SearchPoiTypeFilter; @@ -61,13 +66,14 @@ public class BinaryInspector { if(args.length == 1 && "test".equals(args[0])) { in.inspector(new String[]{ // "-vpoi", -// "-vmap", "-vmapobjects", + "-vmap",// "-vmapobjects", // "-vrouting", // "-vaddress", "-vcities","-vstreetgroups", // "-vstreets", "-vbuildings", "-vintersections", // "-zoom=16", -// "-bbox=1.74,51.17,1.75,51.16", -// "/Users/victorshcherb/osmand/osm-gen/World_seamarks_2.obf" +// "-bbox=1.74,51.17,1.75,51.16", + "-vstats", + "/Users/victorshcherb/osmand/maps/World_basemap_2.obf" }); } else { in.inspector(args); @@ -111,6 +117,7 @@ public class BinaryInspector { boolean vmap; boolean vrouting; boolean vmapObjects; + boolean vstats; boolean osm; FileOutputStream osmOut = null; double lattop = 85; @@ -143,6 +150,10 @@ public class BinaryInspector { return vtransport; } + public boolean isVStats() { + return vstats; + } + public VerboseInfo(String[] params) throws FileNotFoundException { for(int i=0;i types = new LinkedHashMap(); + private SearchRequest req; + + public void processKey(String simpleString, MapObjectStat st, TIntObjectHashMap objectNames, + int coordinates, boolean names ) { + TIntObjectIterator it = objectNames.iterator(); + int nameLen = 0; + while(it.hasNext()) { + it.advance(); + nameLen ++; + nameLen += it.value().length(); + } + if(!types.containsKey(simpleString)) { + MapStatKey stt = new MapStatKey(); + stt.key = simpleString; + types.put(simpleString, stt); + } + MapStatKey key = types.get(simpleString); + if (names) { + key.namesLength += nameLen; + } else { + key.statCoordinates += st.lastObjectCoordinates; + key.statCoordinatesCount += coordinates; + key.statObjectSize += st.lastObjectSize; + key.count++; + } + } + + + public void process(BinaryMapDataObject obj) { + MapObjectStat st = req.stat; + int cnt = 0; + boolean names = st.lastObjectCoordinates == 0; + this.lastStringNamesSize += st.lastStringNamesSize; + this.lastObjectIdSize += st.lastObjectIdSize; + this.lastObjectHeaderInfo += st.lastObjectHeaderInfo; + this.lastObjectAdditionalTypes += st.lastObjectAdditionalTypes; + this.lastObjectTypes += st.lastObjectTypes; + this.lastObjectCoordinates += st.lastObjectCoordinates; + if (!names) { + cnt = obj.getPointsLength(); + this.lastObjectIdSize += st.lastObjectSize; + if (obj.getPolygonInnerCoordinates() != null) { + for (int[] i : obj.getPolygonInnerCoordinates()) { + cnt += i.length; + } + } + this.lastObjectCoordinatesCount += cnt; + } + for (int i = 0; i < obj.getTypes().length; i++) { + int tp = obj.getTypes()[i]; + TagValuePair pair = obj.mapIndex.decodeType(tp); + processKey(pair.toSimpleString(), st, obj.getObjectNames(), cnt, names); + } + st.clearObjectStats(); + st.lastObjectSize = 0; + + } + + public void print() { + MapObjectStat st = req.stat; + println("MAP BLOCK INFO:"); + long b = 0; + b += out("Header", st.lastBlockHeaderInfo); + b += out("String table", st.lastBlockStringTableSize); + b += out("Map Objects", st.lastObjectSize); + out("TOTAL", b); + println("\nMAP OBJECTS INFO:"); + b = 0; + b += out("Header", lastObjectHeaderInfo); + b += out("Coordinates", lastObjectCoordinates); + b += out("Coordinates Count(pair)", lastObjectCoordinatesCount); + b += out("Types", lastObjectTypes); + b += out("Additonal Types", lastObjectAdditionalTypes); + b += out("Ids", lastObjectIdSize); + b += out("String names", lastStringNamesSize); + out("TOTAL", b); + + println("\n\nOBJECT BY TYPE STATS: "); + ArrayList stats = new ArrayList(types.values()); + Collections.sort(stats, new Comparator() { + + @Override + public int compare(MapStatKey o1, MapStatKey o2) { + return -Long.compare(o1.statObjectSize, o2.statObjectSize); + } + }); + + for(MapStatKey s : stats) { + println(s.key + " (" + s.count + ") \t " + s.statObjectSize + " bytes \t coord="+ + s.statCoordinatesCount+ + " (" +s.statCoordinates +" bytes) " + + " names "+s.namesLength + " bytes"); + } + + } + + private long out(String s, long i) { + while (s.length() < 25) { + s += " "; + } + DecimalFormat df = new DecimalFormat("0,000,000,000"); + println(s+": " + df.format(i)); + return i; + } + + + public void setReq(SearchRequest req) { + this.req = req; + } + + } + private void printMapDetailInfo(BinaryMapIndexReader index, MapIndex mapIndex) throws IOException { final StringBuilder b = new StringBuilder(); final DamnCounter mapObjectsCounter = new DamnCounter(); + final MapStats mapObjectStats = new MapStats(); if(vInfo.osm){ printToFile("\n" + "\n"); } - SearchRequest req = BinaryMapIndexReader.buildSearchRequest( + if(vInfo.isVStats()) { + BinaryMapIndexReader.READ_STATS = true; + } + final SearchRequest req = BinaryMapIndexReader.buildSearchRequest( MapUtils.get31TileNumberX(vInfo.lonleft), MapUtils.get31TileNumberX(vInfo.lonright), MapUtils.get31TileNumberY(vInfo.lattop), @@ -636,10 +787,11 @@ public class BinaryInspector { @Override public boolean publish(BinaryMapDataObject obj) { mapObjectsCounter.value++; - if(vInfo.vmapObjects) - { + if(vInfo.isVStats()) { + mapObjectStats.process(obj); + } else if (vInfo.vmapObjects) { b.setLength(0); - if(vInfo.osm) { + if (vInfo.osm) { printOsmMapDetails(obj, b); try { printToFile(b.toString()); @@ -658,13 +810,21 @@ public class BinaryInspector { return false; } }); + if(vInfo.vstats) { + mapObjectStats.setReq(req); + } index.searchMapIndex(req, mapIndex); if(vInfo.osm){ printToFile("\n"); + } + if(vInfo.vstats) { + mapObjectStats.print(); } println("\tTotal map objects: " + mapObjectsCounter.value); } + + private static void printMapDetails(BinaryMapDataObject obj, StringBuilder b) { boolean multipolygon = obj.getPolygonInnerCoordinates() != null && obj.getPolygonInnerCoordinates().length > 0; if(multipolygon ) { diff --git a/OsmAnd-java/src/net/osmand/binary/BinaryMapIndexReader.java b/OsmAnd-java/src/net/osmand/binary/BinaryMapIndexReader.java index f6e5b11fa7..441d92e8d1 100644 --- a/OsmAnd-java/src/net/osmand/binary/BinaryMapIndexReader.java +++ b/OsmAnd-java/src/net/osmand/binary/BinaryMapIndexReader.java @@ -1,7 +1,6 @@ package net.osmand.binary; -import gnu.trove.iterator.TIntIterator; import gnu.trove.list.array.TIntArrayList; import gnu.trove.map.TIntObjectMap; import gnu.trove.map.hash.TIntObjectHashMap; @@ -16,7 +15,6 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.RandomAccessFile; import java.io.Reader; -import java.io.StringReader; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -41,8 +39,6 @@ import net.osmand.ResultMatcher; import net.osmand.StringMatcher; import net.osmand.binary.BinaryMapAddressReaderAdapter.AddressRegion; import net.osmand.binary.BinaryMapAddressReaderAdapter.CitiesBlock; -import net.osmand.binary.BinaryMapIndexReader.SearchPoiTypeFilter; -import net.osmand.binary.BinaryMapIndexReader.SearchRequest; import net.osmand.binary.BinaryMapPoiReaderAdapter.PoiRegion; import net.osmand.binary.BinaryMapRouteReaderAdapter.RouteRegion; import net.osmand.binary.BinaryMapRouteReaderAdapter.RouteSubregion; @@ -73,6 +69,7 @@ import org.xml.sax.InputSource; import org.xml.sax.SAXException; import com.google.protobuf.CodedInputStream; +import com.google.protobuf.CodedOutputStream; import com.google.protobuf.WireFormat; public class BinaryMapIndexReader { @@ -80,6 +77,7 @@ public class BinaryMapIndexReader { public final static int TRANSPORT_STOP_ZOOM = 24; public static final int SHIFT_COORDINATES = 5; private final static Log log = PlatformUtil.getLog(BinaryMapIndexReader.class); + public static boolean READ_STATS = false; private final RandomAccessFile raf; /*private*/ int version; @@ -938,13 +936,23 @@ public class BinaryMapIndexReader { return; case MapDataBlock.BASEID_FIELD_NUMBER: baseId = codedIS.readUInt64(); + if(READ_STATS) { + req.stat.addBlockHeader(MapDataBlock.BASEID_FIELD_NUMBER, 0); + } break; case MapDataBlock.DATAOBJECTS_FIELD_NUMBER: int length = codedIS.readRawVarint32(); int oldLimit = codedIS.pushLimit(length); + if(READ_STATS) { + req.stat.lastObjectSize += length; + req.stat.addBlockHeader(MapDataBlock.DATAOBJECTS_FIELD_NUMBER, length); + } BinaryMapDataObject mapObject = readMapDataObject(tree, req, root); if (mapObject != null) { mapObject.setId(mapObject.getId() + baseId); + if (READ_STATS) { + req.publish(mapObject); + } if (tempResults == null) { tempResults = new ArrayList(); } @@ -955,6 +963,10 @@ public class BinaryMapIndexReader { case MapDataBlock.STRINGTABLE_FIELD_NUMBER: length = codedIS.readRawVarint32(); oldLimit = codedIS.pushLimit(length); + if(READ_STATS) { + req.stat.addBlockHeader(MapDataBlock.STRINGTABLE_FIELD_NUMBER, length); + req.stat.lastBlockStringTableSize += length; + } if (tempResults != null) { List stringTable = readStringTable(); for (int i = 0; i < tempResults.size(); i++) { @@ -1060,6 +1072,11 @@ public class BinaryMapIndexReader { } req.cacheCoordinates.clear(); int size = codedIS.readRawVarint32(); + if(READ_STATS) { + req.stat.lastObjectCoordinates += size; + req.stat.addTagHeader(OsmandOdb.MapData.COORDINATES_FIELD_NUMBER, + size); + } int old = codedIS.pushLimit(size); int px = tree.left & MASK_TO_READ; int py = tree.top & MASK_TO_READ; @@ -1123,6 +1140,11 @@ public class BinaryMapIndexReader { px = tree.left & MASK_TO_READ; py = tree.top & MASK_TO_READ; size = codedIS.readRawVarint32(); + if(READ_STATS) { + req.stat.lastObjectCoordinates += size; + req.stat.addTagHeader(OsmandOdb.MapData.POLYGONINNERCOORDINATES_FIELD_NUMBER, + size); + } old = codedIS.pushLimit(size); while (codedIS.getBytesUntilLimit() > 0) { int x = (codedIS.readSInt32() << SHIFT_COORDINATES) + px; @@ -1138,15 +1160,25 @@ public class BinaryMapIndexReader { additionalTypes = new TIntArrayList(); int sizeL = codedIS.readRawVarint32(); old = codedIS.pushLimit(sizeL); + if(READ_STATS) { + req.stat.lastObjectAdditionalTypes += sizeL; + req.stat.addTagHeader(OsmandOdb.MapData.ADDITIONALTYPES_FIELD_NUMBER, + sizeL); + } while (codedIS.getBytesUntilLimit() > 0) { additionalTypes.add(codedIS.readRawVarint32()); } codedIS.popLimit(old); + break; case OsmandOdb.MapData.TYPES_FIELD_NUMBER: req.cacheTypes.clear(); sizeL = codedIS.readRawVarint32(); old = codedIS.pushLimit(sizeL); + if(READ_STATS) { + req.stat.addTagHeader(OsmandOdb.MapData.TYPES_FIELD_NUMBER, sizeL); + req.stat.lastObjectTypes += sizeL; + } while (codedIS.getBytesUntilLimit() > 0) { req.cacheTypes.add(codedIS.readRawVarint32()); } @@ -1163,6 +1195,10 @@ public class BinaryMapIndexReader { break; case OsmandOdb.MapData.ID_FIELD_NUMBER: id = codedIS.readSInt64(); + if(READ_STATS) { + req.stat.addTagHeader(OsmandOdb.MapData.ID_FIELD_NUMBER, 0); + req.stat.lastObjectIdSize += CodedOutputStream.computeSInt64SizeNoTag(id); + } break; case OsmandOdb.MapData.STRINGNAMES_FIELD_NUMBER: stringNames = new TIntObjectHashMap(); @@ -1176,6 +1212,10 @@ public class BinaryMapIndexReader { stringOrder.add(stag); } codedIS.popLimit(old); + if(READ_STATS) { + req.stat.addTagHeader(OsmandOdb.MapData.STRINGNAMES_FIELD_NUMBER, sizeL); + req.stat.lastStringNamesSize += sizeL; + } break; default: skipUnknownField(t); @@ -1500,6 +1540,40 @@ public class BinaryMapIndexReader { } + public static class MapObjectStat { + public int lastStringNamesSize; + public int lastObjectIdSize; + public int lastObjectHeaderInfo; + public int lastObjectAdditionalTypes; + public int lastObjectTypes; + public int lastObjectCoordinates; + + public int lastObjectSize ; + public int lastBlockStringTableSize; + public int lastBlockHeaderInfo; + + public void addBlockHeader(int typesFieldNumber, int sizeL) { + lastBlockHeaderInfo += + CodedOutputStream.computeTagSize(typesFieldNumber) + + CodedOutputStream.computeRawVarint32Size(sizeL); + } + + public void addTagHeader(int typesFieldNumber, int sizeL) { + lastObjectHeaderInfo += + CodedOutputStream.computeTagSize(typesFieldNumber) + + CodedOutputStream.computeRawVarint32Size(sizeL); + } + + public void clearObjectStats() { + lastStringNamesSize = 0; + lastObjectIdSize = 0; + lastObjectHeaderInfo = 0; + lastObjectAdditionalTypes = 0; + lastObjectTypes = 0; + lastObjectCoordinates = 0; + } + } + public static class SearchRequest { public final static int ZOOM_TO_SEARCH_POI = 16; private List searchResults = new ArrayList(); @@ -1539,6 +1613,8 @@ public class BinaryMapIndexReader { TIntArrayList cacheCoordinates = new TIntArrayList(); TIntArrayList cacheTypes = new TIntArrayList(); + MapObjectStat stat = new MapObjectStat(); + // TRACE INFO int numberOfVisitedObjects = 0;