Android-builders-club/OsmAnd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Update binary inspector to print map details size

Browse source
This commit is contained in:
Victor Shcherb 2015-03-09 22:47:38 +01:00
parent 9678bfc8cc
commit c397a428a1
2 changed files with 229 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

159
OsmAnd-java/src/net/osmand/binary/BinaryInspector.java
View file

@ -12,11 +12,15 @@ import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.text.DecimalFormat;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
@ -27,6 +31,7 @@ import net.osmand.ResultMatcher;
import net.osmand.binary.BinaryMapAddressReaderAdapter.AddressRegion;
import net.osmand.binary.BinaryMapAddressReaderAdapter.CitiesBlock;
import net.osmand.binary.BinaryMapIndexReader.MapIndex;
import net.osmand.binary.BinaryMapIndexReader.MapObjectStat;
import net.osmand.binary.BinaryMapIndexReader.MapRoot;
import net.osmand.binary.BinaryMapIndexReader.SearchFilter;
import net.osmand.binary.BinaryMapIndexReader.SearchPoiTypeFilter;
@ -61,13 +66,14 @@ public class BinaryInspector {
if(args.length == 1 && "test".equals(args[0])) {
in.inspector(new String[]{
// "-vpoi",
// "-vmap", "-vmapobjects",
"-vmap",// "-vmapobjects",
// "-vrouting",
// "-vaddress", "-vcities","-vstreetgroups",
// "-vstreets", "-vbuildings", "-vintersections",
// "-zoom=16",
// "-bbox=1.74,51.17,1.75,51.16",
// "/Users/victorshcherb/osmand/osm-gen/World_seamarks_2.obf"
"-vstats",
"/Users/victorshcherb/osmand/maps/World_basemap_2.obf"
});
} else {
in.inspector(args);
@ -111,6 +117,7 @@ public class BinaryInspector {
boolean vmap;
boolean vrouting;
boolean vmapObjects;
boolean vstats;
boolean osm;
FileOutputStream osmOut = null;
double lattop = 85;
@ -143,6 +150,10 @@ public class BinaryInspector {
return vtransport;
}
public boolean isVStats() {
return vstats;
}
public VerboseInfo(String[] params) throws FileNotFoundException {
for(int i=0;i<params.length;i++){
if(params[i].equals("-vaddress")){
@ -159,6 +170,8 @@ public class BinaryInspector {
vintersections = true;
} else if(params[i].equals("-vmap")){
vmap = true;
} else if(params[i].equals("-vstats")){
vstats = true;
} else if(params[i].equals("-vrouting")){
vrouting = true;
} else if(params[i].equals("-vmapobjects")){
@ -609,18 +622,139 @@ public class BinaryInspector {
}
}
}
private static class DamnCounter
{
private static class DamnCounter {
int value;
}
private static class MapStatKey {
String key = "";
long statCoordinates;
long statObjectSize;
int count;
int namesLength;
}
private class MapStats {
public int lastStringNamesSize;
public int lastObjectIdSize;
public int lastObjectHeaderInfo;
public int lastObjectAdditionalTypes;
public int lastObjectTypes;
public int lastObjectCoordinates;
public int lastObjectCoordinatesCount;
private Map<String, MapStatKey> types = new LinkedHashMap<String, BinaryInspector.MapStatKey>();
private SearchRequest<BinaryMapDataObject> req;
public void processKey(String simpleString, MapObjectStat st, TIntObjectHashMap<String> objectNames) {
TIntObjectIterator<String> it = objectNames.iterator();
int nameLen = 0;
while(it.hasNext()) {
it.advance();
nameLen ++;
nameLen += it.value().length();
}
if(!types.containsKey(simpleString)) {
MapStatKey stt = new MapStatKey();
stt.key = simpleString;
types.put(simpleString, stt);
}
MapStatKey key = types.get(simpleString);
key.statCoordinates += st.lastObjectCoordinates;
key.statObjectSize += st.lastObjectSize;
key.count++;
key.namesLength += nameLen;
}
public void process(BinaryMapDataObject obj) {
MapObjectStat st = req.stat;
this.lastStringNamesSize += st.lastStringNamesSize;
this.lastObjectIdSize += st.lastObjectIdSize;
this.lastObjectHeaderInfo += st.lastObjectHeaderInfo;
this.lastObjectAdditionalTypes += st.lastObjectAdditionalTypes;
this.lastObjectTypes += st.lastObjectTypes;
this.lastObjectCoordinates += st.lastObjectCoordinates;
this.lastObjectCoordinatesCount += obj.getPointsLength();
if(obj.getPolygonInnerCoordinates() != null) {
for(int[] i : obj.getPolygonInnerCoordinates()) {
this.lastObjectCoordinatesCount += i.length;
}
}
for(int i = 0; i < obj.getTypes().length; i++) {
int tp = obj.getTypes()[i];
TagValuePair pair = obj.mapIndex.decodeType(tp);
processKey(pair.toSimpleString(), st, obj.getObjectNames());
}
st.clearObjectStats();
}
public void print() {
MapObjectStat st = req.stat;
println("MAP BLOCK INFO:");
long b = 0;
b += out("Header", st.lastBlockHeaderInfo);
b += out("String table", st.lastBlockStringTableSize);
b += out("Map Objects", st.lastObjectSize);
out("TOTAL", b);
println("\nMAP OBJECTS INFO:");
b = 0;
b += out("Header", lastObjectHeaderInfo);
b += out("Coordinates", lastObjectCoordinates);
b += out("Coordinates Count", lastObjectCoordinatesCount);
b += out("Types", lastObjectTypes);
b += out("Additonal Types", lastObjectAdditionalTypes);
b += out("Ids", lastObjectIdSize);
b += out("String names", lastStringNamesSize);
out("TOTAL", b);
println("\n\nOBJECT BY TYPE STATS: ");
ArrayList<MapStatKey> stats = new ArrayList<MapStatKey>(types.values());
Collections.sort(stats, new Comparator<MapStatKey>() {
@Override
public int compare(MapStatKey o1, MapStatKey o2) {
return -Long.compare(o1.statObjectSize, o2.statObjectSize);
}
});
for(MapStatKey s : stats) {
println(s.key + " cnt=" + s.count + " size=" + s.statObjectSize + " coordsize="+s.statCoordinates+
" namelen="+s.namesLength);
}
}
private long out(String s, long i) {
while (s.length() < 20) {
s += " ";
}
DecimalFormat df = new DecimalFormat("0,000,000,000");
println(s+": " + df.format(i));
return i;
}
public void setReq(SearchRequest<BinaryMapDataObject> req) {
this.req = req;
}
}
private void printMapDetailInfo(BinaryMapIndexReader index, MapIndex mapIndex) throws IOException {
final StringBuilder b = new StringBuilder();
final DamnCounter mapObjectsCounter = new DamnCounter();
final MapStats mapObjectStats = new MapStats();
if(vInfo.osm){
printToFile("<?xml version='1.0' encoding='UTF-8'?>\n" +
"<osm version='0.5'>\n");
}
SearchRequest<BinaryMapDataObject> req = BinaryMapIndexReader.buildSearchRequest(
if(vInfo.isVStats()) {
BinaryMapIndexReader.READ_STATS = true;
}
final SearchRequest<BinaryMapDataObject> req = BinaryMapIndexReader.buildSearchRequest(
MapUtils.get31TileNumberX(vInfo.lonleft),
MapUtils.get31TileNumberX(vInfo.lonright),
MapUtils.get31TileNumberY(vInfo.lattop),
@ -636,10 +770,11 @@ public class BinaryInspector {
@Override
public boolean publish(BinaryMapDataObject obj) {
mapObjectsCounter.value++;
if(vInfo.vmapObjects)
{
if(vInfo.isVStats()) {
mapObjectStats.process(obj);
} else if (vInfo.vmapObjects) {
b.setLength(0);
if(vInfo.osm) {
if (vInfo.osm) {
printOsmMapDetails(obj, b);
try {
printToFile(b.toString());
@ -658,13 +793,21 @@ public class BinaryInspector {
return false;
}
});
if(vInfo.vstats) {
mapObjectStats.setReq(req);
}
index.searchMapIndex(req, mapIndex);
if(vInfo.osm){
printToFile("</osm >\n");
}
if(vInfo.vstats) {
mapObjectStats.print();
}
println("\tTotal map objects: " + mapObjectsCounter.value);
}
private static void printMapDetails(BinaryMapDataObject obj, StringBuilder b) {
boolean multipolygon = obj.getPolygonInnerCoordinates() != null && obj.getPolygonInnerCoordinates().length > 0;
if(multipolygon ) {

81
OsmAnd-java/src/net/osmand/binary/BinaryMapIndexReader.java
View file

@ -1,7 +1,6 @@
package net.osmand.binary;
import gnu.trove.iterator.TIntIterator;
import gnu.trove.list.array.TIntArrayList;
import gnu.trove.map.TIntObjectMap;
import gnu.trove.map.hash.TIntObjectHashMap;
@ -16,7 +15,6 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.RandomAccessFile;
import java.io.Reader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@ -41,8 +39,6 @@ import net.osmand.ResultMatcher;
import net.osmand.StringMatcher;
import net.osmand.binary.BinaryMapAddressReaderAdapter.AddressRegion;
import net.osmand.binary.BinaryMapAddressReaderAdapter.CitiesBlock;
import net.osmand.binary.BinaryMapIndexReader.SearchPoiTypeFilter;
import net.osmand.binary.BinaryMapIndexReader.SearchRequest;
import net.osmand.binary.BinaryMapPoiReaderAdapter.PoiRegion;
import net.osmand.binary.BinaryMapRouteReaderAdapter.RouteRegion;
import net.osmand.binary.BinaryMapRouteReaderAdapter.RouteSubregion;
@ -73,6 +69,7 @@ import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import com.google.protobuf.CodedInputStream;
import com.google.protobuf.CodedOutputStream;
import com.google.protobuf.WireFormat;
public class BinaryMapIndexReader {
@ -80,6 +77,7 @@ public class BinaryMapIndexReader {
public final static int TRANSPORT_STOP_ZOOM = 24;
public static final int SHIFT_COORDINATES = 5;
private final static Log log = PlatformUtil.getLog(BinaryMapIndexReader.class);
public static boolean READ_STATS = false;
private final RandomAccessFile raf;
/*private*/ int version;
@ -938,10 +936,17 @@ public class BinaryMapIndexReader {
return;
case MapDataBlock.BASEID_FIELD_NUMBER:
baseId = codedIS.readUInt64();
if(READ_STATS) {
req.stat.addBlockHeader(MapDataBlock.BASEID_FIELD_NUMBER, 0);
}
break;
case MapDataBlock.DATAOBJECTS_FIELD_NUMBER:
int length = codedIS.readRawVarint32();
int oldLimit = codedIS.pushLimit(length);
if(READ_STATS) {
req.stat.lastObjectSize += length;
req.stat.addBlockHeader(MapDataBlock.DATAOBJECTS_FIELD_NUMBER, length);
}
BinaryMapDataObject mapObject = readMapDataObject(tree, req, root);
if (mapObject != null) {
mapObject.setId(mapObject.getId() + baseId);
@ -955,6 +960,10 @@ public class BinaryMapIndexReader {
case MapDataBlock.STRINGTABLE_FIELD_NUMBER:
length = codedIS.readRawVarint32();
oldLimit = codedIS.pushLimit(length);
if(READ_STATS) {
req.stat.addBlockHeader(MapDataBlock.STRINGTABLE_FIELD_NUMBER, length);
req.stat.lastBlockStringTableSize += length;
}
if (tempResults != null) {
List<String> stringTable = readStringTable();
for (int i = 0; i < tempResults.size(); i++) {
@ -1060,6 +1069,11 @@ public class BinaryMapIndexReader {
}
req.cacheCoordinates.clear();
int size = codedIS.readRawVarint32();
if(READ_STATS) {
req.stat.lastObjectCoordinates += size;
req.stat.addTagHeader(OsmandOdb.MapData.COORDINATES_FIELD_NUMBER,
size);
}
int old = codedIS.pushLimit(size);
int px = tree.left & MASK_TO_READ;
int py = tree.top & MASK_TO_READ;
@ -1123,6 +1137,11 @@ public class BinaryMapIndexReader {
px = tree.left & MASK_TO_READ;
py = tree.top & MASK_TO_READ;
size = codedIS.readRawVarint32();
if(READ_STATS) {
req.stat.lastObjectCoordinates += size;
req.stat.addTagHeader(OsmandOdb.MapData.POLYGONINNERCOORDINATES_FIELD_NUMBER,
size);
}
old = codedIS.pushLimit(size);
while (codedIS.getBytesUntilLimit() > 0) {
int x = (codedIS.readSInt32() << SHIFT_COORDINATES) + px;
@ -1138,15 +1157,25 @@ public class BinaryMapIndexReader {
additionalTypes = new TIntArrayList();
int sizeL = codedIS.readRawVarint32();
old = codedIS.pushLimit(sizeL);
if(READ_STATS) {
req.stat.lastObjectAdditionalTypes += sizeL;
req.stat.addTagHeader(OsmandOdb.MapData.ADDITIONALTYPES_FIELD_NUMBER,
sizeL);
}
while (codedIS.getBytesUntilLimit() > 0) {
additionalTypes.add(codedIS.readRawVarint32());
}
codedIS.popLimit(old);
break;
case OsmandOdb.MapData.TYPES_FIELD_NUMBER:
req.cacheTypes.clear();
sizeL = codedIS.readRawVarint32();
old = codedIS.pushLimit(sizeL);
if(READ_STATS) {
req.stat.addTagHeader(OsmandOdb.MapData.TYPES_FIELD_NUMBER, sizeL);
req.stat.lastObjectTypes += sizeL;
}
while (codedIS.getBytesUntilLimit() > 0) {
req.cacheTypes.add(codedIS.readRawVarint32());
}
@ -1163,6 +1192,10 @@ public class BinaryMapIndexReader {
break;
case OsmandOdb.MapData.ID_FIELD_NUMBER:
id = codedIS.readSInt64();
if(READ_STATS) {
req.stat.addTagHeader(OsmandOdb.MapData.ID_FIELD_NUMBER, 0);
req.stat.lastObjectIdSize += CodedOutputStream.computeSInt64SizeNoTag(id);
}
break;
case OsmandOdb.MapData.STRINGNAMES_FIELD_NUMBER:
stringNames = new TIntObjectHashMap<String>();
@ -1176,6 +1209,10 @@ public class BinaryMapIndexReader {
stringOrder.add(stag);
}
codedIS.popLimit(old);
if(READ_STATS) {
req.stat.addTagHeader(OsmandOdb.MapData.STRINGNAMES_FIELD_NUMBER, sizeL);
req.stat.lastStringNamesSize += sizeL;
}
break;
default:
skipUnknownField(t);
@ -1500,6 +1537,40 @@ public class BinaryMapIndexReader {
}
public static class MapObjectStat {
public int lastStringNamesSize;
public int lastObjectIdSize;
public int lastObjectHeaderInfo;
public int lastObjectAdditionalTypes;
public int lastObjectTypes;
public int lastObjectCoordinates;
public int lastObjectSize ;
public int lastBlockStringTableSize;
public int lastBlockHeaderInfo;
public void addBlockHeader(int typesFieldNumber, int sizeL) {
lastBlockHeaderInfo +=
CodedOutputStream.computeTagSize(typesFieldNumber) +
CodedOutputStream.computeRawVarint32Size(sizeL);
}
public void addTagHeader(int typesFieldNumber, int sizeL) {
lastObjectHeaderInfo +=
CodedOutputStream.computeTagSize(typesFieldNumber) +
CodedOutputStream.computeRawVarint32Size(sizeL);
}
public void clearObjectStats() {
lastStringNamesSize = 0;
lastObjectIdSize = 0;
lastObjectHeaderInfo = 0;
lastObjectAdditionalTypes = 0;
lastObjectTypes = 0;
lastObjectCoordinates = 0;
}
}
public static class SearchRequest<T> {
public final static int ZOOM_TO_SEARCH_POI = 16;
private List<T> searchResults = new ArrayList<T>();
@ -1539,6 +1610,8 @@ public class BinaryMapIndexReader {
TIntArrayList cacheCoordinates = new TIntArrayList();
TIntArrayList cacheTypes = new TIntArrayList();
MapObjectStat stat = new MapObjectStat();
// TRACE INFO
int numberOfVisitedObjects = 0;