From b7553405f96c3c94b53dc6788b4ae7291ecee385 Mon Sep 17 00:00:00 2001 From: Victor Shcherb Date: Wed, 15 Aug 2012 00:25:41 +0200 Subject: [PATCH] Speed up loading --- .../osmand/binary/BinaryMapIndexReader.java | 47 +++++++++------- .../binary/BinaryMapPoiReaderAdapter.java | 54 ++++++++++++++----- Osmand-kernel/osmand/src/binaryRead.cpp | 54 ++++++++++++++----- Osmand-kernel/osmand/src/java_wrap.cpp | 2 - 4 files changed, 112 insertions(+), 45 deletions(-) diff --git a/DataExtractionOSM/src/net/osmand/binary/BinaryMapIndexReader.java b/DataExtractionOSM/src/net/osmand/binary/BinaryMapIndexReader.java index 9a367037ec..8259583a16 100644 --- a/DataExtractionOSM/src/net/osmand/binary/BinaryMapIndexReader.java +++ b/DataExtractionOSM/src/net/osmand/binary/BinaryMapIndexReader.java @@ -145,9 +145,7 @@ public class BinaryMapIndexReader { mapIndex.length = readInt(); mapIndex.filePointer = codedIS.getTotalBytesRead(); int oldLimit = codedIS.pushLimit(mapIndex.length); - // FIXME -// codedIS.skipRawBytes(codedIS.getBytesUntilLimit()); - readMapIndex(mapIndex); + readMapIndex(mapIndex, false); basemap = basemap || mapIndex.isBaseMap(); codedIS.popLimit(oldLimit); codedIS.seek(mapIndex.filePointer + mapIndex.length); @@ -209,7 +207,7 @@ public class BinaryMapIndexReader { oldLimit = codedIS.pushLimit(poiInd.length); // FIXME // codedIS.skipRawBytes(codedIS.getBytesUntilLimit()); - poiAdapter.readPoiIndex(poiInd); + poiAdapter.readPoiIndex(poiInd, false); codedIS.popLimit(oldLimit); poiIndexes.add(poiInd); indexes.add(poiInd); @@ -574,8 +572,9 @@ public class BinaryMapIndexReader { * Map public methods */ - private void readMapIndex(MapIndex index) throws IOException { + private void readMapIndex(MapIndex index, boolean onlyInitEncodingRules) throws IOException { int defaultId = 1; + int oldLimit ; while(true){ int t = codedIS.readTag(); int tag = WireFormat.getTagFieldNumber(t); @@ -583,29 +582,34 @@ public class BinaryMapIndexReader { switch (tag) { case 0: // encoding rules are required! - if(index.encodingRules.isEmpty()){ - throw new IllegalStateException("Encoding rules are not defined for the map index"); + if (onlyInitEncodingRules) { + index.finishInitializingTags(); } - index.finishInitializingTags(); return; case OsmandOdb.OsmAndMapIndex.NAME_FIELD_NUMBER : index.setName(codedIS.readString()); break; case OsmandOdb.OsmAndMapIndex.RULES_FIELD_NUMBER : - int len = codedIS.readInt32(); - int oldLimit = codedIS.pushLimit(len); - readMapEncodingRule(index, defaultId++); - codedIS.popLimit(oldLimit); + if (onlyInitEncodingRules) { + int len = codedIS.readInt32(); + oldLimit = codedIS.pushLimit(len); + readMapEncodingRule(index, defaultId++); + codedIS.popLimit(oldLimit); + } else { + skipUnknownField(t); + } break; case OsmandOdb.OsmAndMapIndex.LEVELS_FIELD_NUMBER : int length = readInt(); int filePointer = codedIS.getTotalBytesRead(); - oldLimit = codedIS.pushLimit(length); - MapRoot mapRoot = readMapLevel(new MapRoot()); - mapRoot.length = length; - mapRoot.filePointer = filePointer; - index.getRoots().add(mapRoot); - codedIS.popLimit(oldLimit); + if (!onlyInitEncodingRules) { + oldLimit = codedIS.pushLimit(length); + MapRoot mapRoot = readMapLevel(new MapRoot()); + mapRoot.length = length; + mapRoot.filePointer = filePointer; + index.getRoots().add(mapRoot); + codedIS.popLimit(oldLimit); + } codedIS.seek(filePointer + length); break; default: @@ -749,6 +753,13 @@ public class BinaryMapIndexReader { if (index.right < req.left || index.left > req.right || index.top > req.bottom || index.bottom < req.top) { continue; } + // lazy initializing rules + if(mapIndex.encodingRules.isEmpty()) { + codedIS.seek(mapIndex.filePointer); + int oldLimit = codedIS.pushLimit(mapIndex.length); + readMapIndex(mapIndex, true); + codedIS.popLimit(oldLimit); + } // lazy initializing trees if(index.trees == null){ index.trees = new ArrayList(); diff --git a/DataExtractionOSM/src/net/osmand/binary/BinaryMapPoiReaderAdapter.java b/DataExtractionOSM/src/net/osmand/binary/BinaryMapPoiReaderAdapter.java index 708b24a247..c308e17df0 100644 --- a/DataExtractionOSM/src/net/osmand/binary/BinaryMapPoiReaderAdapter.java +++ b/DataExtractionOSM/src/net/osmand/binary/BinaryMapPoiReaderAdapter.java @@ -79,7 +79,7 @@ public class BinaryMapPoiReaderAdapter { return map.readInt(); } - protected void readPoiBoundariesIndex(PoiRegion region) throws IOException { + private void readPoiBoundariesIndex(PoiRegion region) throws IOException { while(true){ int t = codedIS.readTag(); int tag = WireFormat.getTagFieldNumber(t); @@ -106,7 +106,9 @@ public class BinaryMapPoiReaderAdapter { } - protected void readPoiIndex(PoiRegion region) throws IOException { + protected void readPoiIndex(PoiRegion region, boolean readCategories) throws IOException { + int length; + int oldLimit; while(true){ int t = codedIS.readTag(); int tag = WireFormat.getTagFieldNumber(t); @@ -116,19 +118,21 @@ public class BinaryMapPoiReaderAdapter { case OsmandOdb.OsmAndPoiIndex.NAME_FIELD_NUMBER : region.name = codedIS.readString(); break; - case OsmandOdb.OsmAndPoiIndex.BOUNDARIES_FIELD_NUMBER: { - int length = codedIS.readRawVarint32(); - int oldLimit = codedIS.pushLimit(length); + case OsmandOdb.OsmAndPoiIndex.BOUNDARIES_FIELD_NUMBER: + length = codedIS.readRawVarint32(); + oldLimit = codedIS.pushLimit(length); readPoiBoundariesIndex(region); codedIS.popLimit(oldLimit); - } break; - case OsmandOdb.OsmAndPoiIndex.CATEGORIESTABLE_FIELD_NUMBER : { - int length = codedIS.readRawVarint32(); - int oldLimit = codedIS.pushLimit(length); + case OsmandOdb.OsmAndPoiIndex.CATEGORIESTABLE_FIELD_NUMBER : + if(!readCategories){ + codedIS.skipRawBytes(codedIS.getBytesUntilLimit()); + return; + } + length = codedIS.readRawVarint32(); + oldLimit = codedIS.pushLimit(length); readCategory(region); codedIS.popLimit(oldLimit); - } break; case OsmandOdb.OsmAndPoiIndex.BOXES_FIELD_NUMBER : codedIS.skipRawBytes(codedIS.getBytesUntilLimit()); return; @@ -162,6 +166,10 @@ public class BinaryMapPoiReaderAdapter { } } + protected void initCategories(PoiRegion region){ + + } + protected void searchPoiByName( PoiRegion region, SearchRequest req) throws IOException { TIntLongHashMap offsets = new TIntLongHashMap(); Collator instance = Collator.getInstance(); @@ -170,6 +178,7 @@ public class BinaryMapPoiReaderAdapter { StringMatcherMode.CHECK_STARTS_FROM_SPACE); long time = System.currentTimeMillis(); int indexOffset = codedIS.getTotalBytesRead(); + boolean readCategories = region.categories.isEmpty(); while(true){ if(req.isCancelled()){ return; @@ -186,6 +195,15 @@ public class BinaryMapPoiReaderAdapter { offsets = readPoiNameIndex(instance, req.nameQuery, req); codedIS.popLimit(oldLimit); break; + case OsmandOdb.OsmAndPoiIndex.CATEGORIESTABLE_FIELD_NUMBER : + if(!readCategories){ + skipUnknownField(t); + } else { + length = codedIS.readRawVarint32(); + oldLimit = codedIS.pushLimit(length); + readCategory(region); + codedIS.popLimit(oldLimit); + } case OsmandOdb.OsmAndPoiIndex.POIDATA_FIELD_NUMBER : // also offsets can be randomly skipped by limit Integer[] offKeys = new Integer[offsets.size()]; @@ -349,6 +367,9 @@ public class BinaryMapPoiReaderAdapter { skipTiles = new TLongHashSet(); zoomToSkip = req.zoom + ZOOM_TO_SKIP_FILTER; } + int length ; + int oldLimit ; + boolean readCategories = region.categories.isEmpty(); TIntLongHashMap offsetsMap = new TIntLongHashMap(); while(true){ if(req.isCancelled()){ @@ -359,9 +380,18 @@ public class BinaryMapPoiReaderAdapter { switch (tag) { case 0: return; + case OsmandOdb.OsmAndPoiIndex.CATEGORIESTABLE_FIELD_NUMBER : + if(!readCategories){ + skipUnknownField(t); + } else { + length = codedIS.readRawVarint32(); + oldLimit = codedIS.pushLimit(length); + readCategory(region); + codedIS.popLimit(oldLimit); + } case OsmandOdb.OsmAndPoiIndex.BOXES_FIELD_NUMBER : - int length = readInt(); - int oldLimit = codedIS.pushLimit(length); + length = readInt(); + oldLimit = codedIS.pushLimit(length); readBoxField(left31, right31, top31, bottom31, 0, 0, 0, offsetsMap, skipTiles, req, region); codedIS.popLimit(oldLimit); break; diff --git a/Osmand-kernel/osmand/src/binaryRead.cpp b/Osmand-kernel/osmand/src/binaryRead.cpp index 41e3b62530..a751b8818d 100644 --- a/Osmand-kernel/osmand/src/binaryRead.cpp +++ b/Osmand-kernel/osmand/src/binaryRead.cpp @@ -97,7 +97,7 @@ bool readMapTreeBounds(CodedInputStream* input, MapTreeBounds* tree, MapRoot* ro return true; } -bool readMapLevel(CodedInputStream* input, MapRoot* root) { +bool readMapLevel(CodedInputStream* input, MapRoot* root, bool initSubtrees) { int tag; int si; while ((tag = input->ReadTag()) != 0) { @@ -131,6 +131,10 @@ bool readMapLevel(CodedInputStream* input, MapRoot* root) { break; } case OsmAndMapIndex_MapRootLevel::kBoxesFieldNumber: { + if(!initSubtrees){ + input->Skip(input->BytesUntilLimit()); + return true; + } MapTreeBounds bounds; readInt(input, &bounds.length); bounds.filePointer = input->getTotalBytesRead(); @@ -345,7 +349,7 @@ bool readRoutingIndex(CodedInputStream* input, RoutingIndex* routingIndex) { return true; } -bool readMapIndex(CodedInputStream* input, MapIndex* mapIndex) { +bool readMapIndex(CodedInputStream* input, MapIndex* mapIndex, bool onlyInitEncodingRules) { uint32_t tag; uint32_t defaultId = 1; while ((tag = input->ReadTag()) != 0) { @@ -355,22 +359,29 @@ bool readMapIndex(CodedInputStream* input, MapIndex* mapIndex) { break; } case OsmAndMapIndex::kRulesFieldNumber: { - int len; - WireFormatLite::ReadPrimitive(input, &len); - int oldLimit = input->PushLimit(len); - readMapEncodingRule(input, mapIndex, defaultId++); - input->PopLimit(oldLimit); + if(onlyInitEncodingRules) { + int len; + WireFormatLite::ReadPrimitive(input, &len); + int oldLimit = input->PushLimit(len); + readMapEncodingRule(input, mapIndex, defaultId++); + input->PopLimit(oldLimit); + } else { + skipUnknownFields(input, tag); + } break; } case OsmAndMapIndex::kLevelsFieldNumber: { MapRoot mapLevel; readInt(input, &mapLevel.length); mapLevel.filePointer = input->getTotalBytesRead(); - int oldLimit = input->PushLimit(mapLevel.length); - readMapLevel(input, &mapLevel); - input->PopLimit(oldLimit); + if (!onlyInitEncodingRules) { + int oldLimit = input->PushLimit(mapLevel.length); + readMapLevel(input, &mapLevel, false); + input->PopLimit(oldLimit); + mapIndex->levels.push_back(mapLevel); + } input->Seek(mapLevel.filePointer + mapLevel.length); - mapIndex->levels.push_back(mapLevel); + break; } default: { @@ -384,7 +395,9 @@ bool readMapIndex(CodedInputStream* input, MapIndex* mapIndex) { } } } - mapIndex->finishInitializingTags(); + if(onlyInitEncodingRules) { + mapIndex->finishInitializingTags(); + } return true; } @@ -410,7 +423,7 @@ bool initMapStructure(CodedInputStream* input, BinaryMapFile* file) { readInt(input, &mapIndex.length); mapIndex.filePointer = input->getTotalBytesRead(); int oldLimit = input->PushLimit(mapIndex.length); - readMapIndex(input, &mapIndex); + readMapIndex(input, &mapIndex, false); input->PopLimit(oldLimit); input->Seek(mapIndex.filePointer + mapIndex.length); file->mapIndexes.push_back(mapIndex); @@ -830,6 +843,20 @@ bool readMapDataBlocks(CodedInputStream* input, SearchQuery* req, MapTreeBounds* bool sortTreeBounds (const MapTreeBounds& i,const MapTreeBounds& j) { return (i.mapDataBlockdecodingRules.size() == 0) { + input->Seek(ind->filePointer); + int oldLimit = input->PushLimit(ind->length); + readMapIndex(input, ind, true); + input->PopLimit(oldLimit); + } + // lazy initializing subtrees + if (root->bounds.size() == 0) { + input->Seek(root->filePointer); + int oldLimit = input->PushLimit(root->length); + readMapLevel(input, root, true); + input->PopLimit(oldLimit); + } // search for (std::vector::iterator i = root->bounds.begin(); i != root->bounds.end(); i++) { @@ -839,6 +866,7 @@ void searchMapData(CodedInputStream* input, MapRoot* root, MapIndex* ind, Search if (i->right < req->left || i->left > req->right || i->top > req->bottom || i->bottom < req->top) { continue; } + std::vector foundSubtrees; input->Seek(i->filePointer); int oldLimit = input->PushLimit(i->length); diff --git a/Osmand-kernel/osmand/src/java_wrap.cpp b/Osmand-kernel/osmand/src/java_wrap.cpp index f645f2eb54..55dd04b4c6 100644 --- a/Osmand-kernel/osmand/src/java_wrap.cpp +++ b/Osmand-kernel/osmand/src/java_wrap.cpp @@ -61,8 +61,6 @@ extern "C" JNIEXPORT jboolean JNICALL Java_net_osmand_NativeLibrary_initBinaryMa BinaryMapFile* fl = initBinaryMapFile(inputName); if(fl == NULL) { osmand_log_print(LOG_WARN, "File %s was not initialized", inputName.c_str()); - } else { - osmand_log_print(LOG_INFO, "File %s is initialized.", fl->inputName.c_str()); } return fl != NULL; }