From cd15a3b4a4251a3eef73f7e73ce9facc71a964b8 Mon Sep 17 00:00:00 2001 From: Victor Shcherb Date: Wed, 13 Jun 2012 19:54:50 +0200 Subject: [PATCH] C++ reader --- .../binary/BinaryMapRouteReaderAdapter.java | 17 +- Osmand-kernel/osmand/src/binaryRead.cpp | 430 ++++++++++++++++++ Osmand-kernel/osmand/src/binaryRead.h | 46 ++ Osmand-kernel/osmand/src/osmand_main.cpp | 2 + 4 files changed, 487 insertions(+), 8 deletions(-) diff --git a/DataExtractionOSM/src/net/osmand/binary/BinaryMapRouteReaderAdapter.java b/DataExtractionOSM/src/net/osmand/binary/BinaryMapRouteReaderAdapter.java index a23ff31aa8..973f3e7574 100644 --- a/DataExtractionOSM/src/net/osmand/binary/BinaryMapRouteReaderAdapter.java +++ b/DataExtractionOSM/src/net/osmand/binary/BinaryMapRouteReaderAdapter.java @@ -124,7 +124,15 @@ public class BinaryMapRouteReaderAdapter { private static final int RESTRICTION_MASK = 7; public static class RouteDataObject { - public final RouteRegion region; + public final RouteRegion region; + // all these arrays supposed to be immutable! + // These feilds accessible from C++ + public int[] types ; + public int[] pointsX ; + public int[] pointsY ; + public long[] restrictions ; + public int[][] pointTypes ; + public long id; public RouteDataObject(RouteRegion region) { this.region = region; @@ -140,13 +148,6 @@ public class BinaryMapRouteReaderAdapter { this.id = copy.id; } - public int[] types ; - public int[] pointsX ; - public int[] pointsY ; - public long[] restrictions ; - public int[][] pointTypes ; - public long id; - public long getId() { return id; } diff --git a/Osmand-kernel/osmand/src/binaryRead.cpp b/Osmand-kernel/osmand/src/binaryRead.cpp index 3e7d6c111a..ba1e271eb3 100644 --- a/Osmand-kernel/osmand/src/binaryRead.cpp +++ b/Osmand-kernel/osmand/src/binaryRead.cpp @@ -161,6 +161,41 @@ bool readMapLevel(CodedInputStream* input, MapRoot* root) { return true; } +bool readRouteEncodingRule(CodedInputStream* input, RoutingIndex* index, uint32 id) { + int tag; + std::string tagS; + std::string value; + uint32 type = 0; + while ((tag = input->ReadTag()) != 0) { + switch (WireFormatLite::GetTagFieldNumber(tag)) { + case OsmAndRoutingIndex_RouteEncodingRule::kValueFieldNumber: { + DO_((WireFormatLite::ReadString(input, &value))); + break; + } + case OsmAndRoutingIndex_RouteEncodingRule::kTagFieldNumber: { + DO_((WireFormatLite::ReadString(input, &tagS))); + break; + } + case OsmAndRoutingIndex_RouteEncodingRule::kIdFieldNumber: { + DO_((WireFormatLite::ReadPrimitive(input, &id))); + break; + } + default: { + if (WireFormatLite::GetTagWireType(tag) == WireFormatLite::WIRETYPE_END_GROUP) { + return true; + } + if (!skipUnknownFields(input, tag)) { + return false; + } + break; + } + } + } + // Special case for check to not replace primary with primary_link + index->initRouteEncodingRule(id, tagS, value); + return true; +} + bool readMapEncodingRule(CodedInputStream* input, MapIndex* index, uint32 id) { int tag; std::string tagS; @@ -200,6 +235,114 @@ bool readMapEncodingRule(CodedInputStream* input, MapIndex* index, uint32 id) { return true; } + +bool readRouteTree(CodedInputStream* input, RouteSubregion* thisTree, RouteSubregion* parentTree, int depth, bool readCoordinates) { + bool readChildren = depth != 0; + uint32 tag; + int i; + while ((tag = input->ReadTag()) != 0) { + switch (WireFormatLite::GetTagFieldNumber(tag)) { + + case OsmAndRoutingIndex_RouteDataBox::kLeftFieldNumber: { + WireFormatLite::ReadPrimitive(input, &i); + if (readCoordinates) { + thisTree->left = i + (parentTree != NULL ? parentTree->left : 0); + } + break; + } + case OsmAndRoutingIndex_RouteDataBox::kRightFieldNumber: { + WireFormatLite::ReadPrimitive(input, &i); + if (readCoordinates) { + thisTree->right = i + (parentTree != NULL ? parentTree->right : 0); + } + break; + } + case OsmAndRoutingIndex_RouteDataBox::kTopFieldNumber: { + WireFormatLite::ReadPrimitive(input, &i); + if (readCoordinates) { + thisTree->top = i + (parentTree != NULL ? parentTree->top : 0); + } + break; + } + case OsmAndRoutingIndex_RouteDataBox::kBottomFieldNumber: { + WireFormatLite::ReadPrimitive(input, &i); + if (readCoordinates) { + thisTree->bottom = i + (parentTree != NULL ? parentTree->bottom : 0); + } + break; + } + case OsmAndRoutingIndex_RouteDataBox::kShiftToDataFieldNumber: { + readInt(input, &thisTree->mapDataBlock); + break; + } + case OsmAndRoutingIndex_RouteDataBox::kBoxesFieldNumber: { + RouteSubregion subregion; + readInt(input, &subregion.length); + subregion.filePointer = input->getTotalBytesRead(); + int oldLimit = input->PushLimit(subregion.length); + readRouteTree(input, &subregion, thisTree, depth - 1, true); + input->PopLimit(oldLimit); + input->Seek(subregion.filePointer + subregion.length); + thisTree->subregions.push_back(subregion); + break; + } + default: { + if (WireFormatLite::GetTagWireType(tag) == WireFormatLite::WIRETYPE_END_GROUP) { + return true; + } + if (!skipUnknownFields(input, tag)) { + return false; + } + break; + } + } + + } + return true; +} + +bool readRoutingIndex(CodedInputStream* input, RoutingIndex* routingIndex) { + uint32 defaultId = 1; + uint32 tag; + while ((tag = input->ReadTag()) != 0) { + switch (WireFormatLite::GetTagFieldNumber(tag)) { + case OsmAndRoutingIndex::kNameFieldNumber: { + DO_((WireFormatLite::ReadString(input, &routingIndex->name))); + break; + } + case OsmAndRoutingIndex::kRulesFieldNumber: { + int len; + WireFormatLite::ReadPrimitive(input, &len); + int oldLimit = input->PushLimit(len); + readRouteEncodingRule(input, routingIndex, defaultId++); + input->PopLimit(oldLimit); + break; + } + case OsmAndRoutingIndex::kRootBoxesFieldNumber: { + RouteSubregion subregion; + readInt(input, &subregion.length); + subregion.filePointer = input->getTotalBytesRead(); + int oldLimit = input->PushLimit(subregion.length); + readRouteTree(input, &subregion, NULL, 0, true); + input->PopLimit(oldLimit); + input->Seek(subregion.filePointer + subregion.length); + routingIndex->subregions.push_back(subregion); + break; + } + default: { + if (WireFormatLite::GetTagWireType(tag) == WireFormatLite::WIRETYPE_END_GROUP) { + return true; + } + if (!skipUnknownFields(input, tag)) { + return false; + } + break; + } + } + } + return true; +} + bool readMapIndex(CodedInputStream* input, MapIndex* mapIndex) { uint32 tag; uint32 defaultId = 1; @@ -273,6 +416,18 @@ bool initMapStructure(CodedInputStream* input, BinaryMapFile* file) { file->basemap = file->basemap || mapIndex.name.find("basemap") != string::npos; break; } + case OsmAndStructure::kRoutingIndexFieldNumber: { + RoutingIndex routingIndex; + readInt(input, &routingIndex.length); + routingIndex.filePointer = input->getTotalBytesRead(); + int oldLimit = input->PushLimit(routingIndex.length); + readRoutingIndex(input, &routingIndex); + input->PopLimit(oldLimit); + input->Seek(routingIndex.filePointer + routingIndex.length); + file->routingIndexes.push_back(routingIndex); + file->indexes.push_back(&file->routingIndexes.back()); + break; + } case OsmAndStructure::kVersionConfirmFieldNumber: { DO_((WireFormatLite::ReadPrimitive(input, &versionConfirm))); break; @@ -326,6 +481,7 @@ bool readStringTable(CodedInputStream* input, std::vector& list) { } static const int SHIFT_COORDINATES = 5; +static const int ROUTE_SHIFT_COORDINATES = 4; static const int MASK_TO_READ = ~((1 << SHIFT_COORDINATES) - 1); bool acceptTypes(SearchQuery* req, std::vector& types, MapIndex* root) { @@ -839,6 +995,280 @@ ResultPublisher* searchObjectsForRendering(SearchQuery* q, bool skipDuplicates, return q->publisher; } +void searchRouteRegion(CodedInputStream* input, SearchQuery* q, RoutingIndex* ind, std::vector& subregions, + std::vector& toLoad) { + for (std::vector::iterator subreg = subregions.begin(); + subreg != subregions.end(); subreg++) { + if (subreg->right >= q->left && q->right >= subreg->left && subreg->bottom >= q->top + && q->bottom >= subreg->top) { + if(subreg->subregions.empty()){ + bool contains = subreg->right <= q->right && q->left <= subreg->left && subreg->top <= q->top + && subreg->bottom >= q->bottom; + input->Seek(subreg->filePointer); + uint32 old = input -> PushLimit(subreg->length); + readRouteTree(input, &(*subreg), NULL, contains? -1 : 1, false); + input->PopLimit(old); + } + searchRouteRegion(input, q, ind, subreg->subregions, toLoad); + if(subreg->mapDataBlock != 0) { + toLoad.push_back(*subreg); + } + } + } +} + +bool readRouteDataObject(CodedInputStream* input, uint32 left, uint32 top, RouteDataObject* obj) { + int tag; + while ((tag = input->ReadTag()) != 0) { + switch (WireFormatLite::GetTagFieldNumber(tag)) { + case RouteData::kTypesFieldNumber: { + uint32 length; + DO_((WireFormatLite::ReadPrimitive(input, &length))); + int oldLimit = input->PushLimit(length); + uint32 t; + while (input->BytesUntilLimit() > 0) { + DO_((WireFormatLite::ReadPrimitive(input, &t))); + obj->types.push_back(t); + + } + input->PopLimit(oldLimit); + break; + } + case RouteData::kRouteIdFieldNumber: { + DO_((WireFormatLite::ReadPrimitive(input, &obj->id))); + break; + } + case RouteData::kPointsFieldNumber: { + uint32 length; + DO_((WireFormatLite::ReadPrimitive(input, &length))); + int oldLimit = input->PushLimit(length); + uint32 t; + int s; + int px = left >> ROUTE_SHIFT_COORDINATES; + int py = top >> ROUTE_SHIFT_COORDINATES; + while (input->BytesUntilLimit() > 0) { + DO_((WireFormatLite::ReadPrimitive(input, &s))); + uint32 x = s + px; + DO_((WireFormatLite::ReadPrimitive(input, &s))); + uint32 y = s + py; + + obj->pointsX.push_back(x << ROUTE_SHIFT_COORDINATES); + obj->pointsY.push_back(y << ROUTE_SHIFT_COORDINATES); + px = x; + py = y; + } + input->PopLimit(oldLimit); + break; + } + case RouteData::kPointTypesFieldNumber: { + uint32 length; + DO_((WireFormatLite::ReadPrimitive(input, &length))); + int oldLimit = input->PushLimit(length); + while (input->BytesUntilLimit() > 0) { + uint32 pointInd; + uint32 lens; + uint32 t; + DO_((WireFormatLite::ReadPrimitive(input, &pointInd))); + DO_((WireFormatLite::ReadPrimitive(input, &lens))); + int oldLimits = input->PushLimit(lens); + + if (obj->pointTypes.size() <= pointInd) { + obj->pointTypes.resize(pointInd, std::vector()); + } + while (input->BytesUntilLimit() > 0) { + DO_((WireFormatLite::ReadPrimitive(input, &t))); + obj->pointTypes[pointInd].push_back(t); + } + input->PopLimit(oldLimits); + } + input->PopLimit(oldLimit); + break; + } + + + default: { + if (WireFormatLite::GetTagWireType(tag) == WireFormatLite::WIRETYPE_END_GROUP) { + return true; + } + if (!skipUnknownFields(input, tag)) { + return false; + } + break; + } + } + } + return true; +} + +const static int RESTRICTION_SHIFT = 3; +const static int RESTRICTION_MASK = 7; +bool readRouteTreeData(CodedInputStream* input, RouteSubregion* s, std::vector& dataObjects) { + int tag; + std::vector idTables; + HMAP::hash_map > restrictions; + while ((tag = input->ReadTag()) != 0) { + switch (WireFormatLite::GetTagFieldNumber(tag)) { + // required uint32 version = 1; + case OsmAndRoutingIndex_RouteDataBlock::kDataObjectsFieldNumber: { + uint32 length; + DO_((WireFormatLite::ReadPrimitive(input, &length))); + int oldLimit = input->PushLimit(length); + RouteDataObject* obj = new RouteDataObject; + readRouteDataObject(input, s->left, s->top, obj); + if(dataObjects.size() <= obj->id ) { + dataObjects.resize((uint32) obj->id + 1, NULL); + } + dataObjects[obj->id] = obj; + input->PopLimit(oldLimit); + break; + } + case OsmAndRoutingIndex_RouteDataBlock::kStringTableFieldNumber: { + uint32 length; + DO_((WireFormatLite::ReadPrimitive(input, &length))); + int oldLimit = input->PushLimit(length); + // std::vector stringTable; + // readStringTable(input, stringTable); + input->Skip(input->BytesUntilLimit()); + input->PopLimit(oldLimit); + break; + } + case OsmAndRoutingIndex_RouteDataBlock::kRestrictionsFieldNumber: { + uint32 length; + DO_((WireFormatLite::ReadPrimitive(input, &length))); + int oldLimit = input->PushLimit(length); + uint64 from = 0; + uint64 to = 0; + uint64 type = 0; + int ts; + while ((ts = input->ReadTag()) != 0) { + switch (WireFormatLite::GetTagFieldNumber(ts)) { + case RestrictionData::kFromFieldNumber: { + DO_((WireFormatLite::ReadPrimitive(input, &from))); + break; + } + case RestrictionData::kToFieldNumber: { + DO_((WireFormatLite::ReadPrimitive(input, &to))); + break; + } + case RestrictionData::kTypeFieldNumber: { + DO_((WireFormatLite::ReadPrimitive(input, &type))); + break; + } + default: { + if (WireFormatLite::GetTagWireType(ts) == WireFormatLite::WIRETYPE_END_GROUP) { + return true; + } + if (!skipUnknownFields(input, ts)) { + return false; + } + break; + } + } + } + restrictions[from].push_back((to << RESTRICTION_SHIFT) + type); + input->PopLimit(oldLimit); + break; + } + case OsmAndRoutingIndex_RouteDataBlock::kIdTableFieldNumber: { + uint32 length; + DO_((WireFormatLite::ReadPrimitive(input, &length))); + int oldLimit = input->PushLimit(length); + int64 routeId = 0; + int ts; + while ((ts = input->ReadTag()) != 0) { + switch (WireFormatLite::GetTagFieldNumber(ts)) { + case IdTable::kRouteIdFieldNumber: { + int64 val; + DO_((WireFormatLite::ReadPrimitive(input, &val))); + routeId += val; + idTables.push_back(routeId); + break; + } + default: { + if (WireFormatLite::GetTagWireType(ts) == WireFormatLite::WIRETYPE_END_GROUP) { + return true; + } + if (!skipUnknownFields(input, ts)) { + return false; + } + break; + } + } + } + input->PopLimit(oldLimit); + break; + } + default: { + if (WireFormatLite::GetTagWireType(tag) == WireFormatLite::WIRETYPE_END_GROUP) { + return true; + } + if (!skipUnknownFields(input, tag)) { + return false; + } + break; + } + } + } + HMAP::hash_map >::iterator itRestrictions = restrictions.begin(); + for (; itRestrictions != restrictions.end(); itRestrictions++) { + RouteDataObject* fromr = dataObjects[itRestrictions->first]; + if (fromr != NULL) { + fromr->restrictions = itRestrictions->second; + for (int i = 0; i < fromr->restrictions.size(); i++) { + uint32 to = fromr->restrictions[i] >> RESTRICTION_SHIFT; + uint64 valto = (idTables[to] << RESTRICTION_SHIFT) | ((long) fromr->restrictions[i] & RESTRICTION_MASK); + fromr->restrictions[i] = valto; + } + } + } + std::vector::iterator dobj = dataObjects.begin(); + for (; dobj != dataObjects.end(); dobj++) { + if (*dobj != NULL) { + if ((*dobj)->id < idTables.size()) { + (*dobj)->id = idTables[(*dobj)->id]; + } + } + } + + return true; + +} + +bool sortRouteRegions (const RouteSubregion& i,const RouteSubregion& j) { return (i.mapDataBlock::iterator i = openFiles.begin(); + HMAP::hash_set ids; + int count = 0; + + bool basemapExists = false; + for (; i != openFiles.end() && !q->publisher->isCancelled(); i++) { + BinaryMapFile* file = i->second; + fseek(file->f, 0, 0); + FileInputStream input(fileno(file->f)); + input.SetCloseOnDelete(false); + CodedInputStream cis(&input); + cis.SetTotalBytesLimit(INT_MAX, INT_MAX >> 2); + for (std::vector::iterator routingIndex = file->routingIndexes.begin(); + routingIndex != file->routingIndexes.end(); routingIndex++) { + if (q->publisher->isCancelled()) { + break; + } + std::vector toLoad; + searchRouteRegion(&cis, q, &(*routingIndex), routingIndex->subregions, toLoad); + sort(toLoad.begin(), toLoad.end(), sortRouteRegions); + std::vector list; + for (std::vector::iterator subreg = toLoad.begin(); subreg != toLoad.end(); subreg++) { + cis.Seek(subreg->filePointer + subreg->mapDataBlock); + uint32 old = cis.PushLimit(subreg->length); + readRouteTreeData(&cis, &(*subreg), list); + cis.PopLimit(old); + } + } + + } +} + bool closeBinaryMapFile(std::string inputName) { std::map::iterator iterator; if ((iterator = openFiles.find(inputName)) != openFiles.end()) { diff --git a/Osmand-kernel/osmand/src/binaryRead.h b/Osmand-kernel/osmand/src/binaryRead.h index 0573ec12d2..7f84677d9c 100644 --- a/Osmand-kernel/osmand/src/binaryRead.h +++ b/Osmand-kernel/osmand/src/binaryRead.h @@ -35,6 +35,18 @@ struct MapTreeBounds { } }; +struct RouteSubregion { + uint32 length; + uint32 filePointer; + uint32 mapDataBlock; + uint32 left; + uint32 right; + uint32 top; + uint32 bottom; + std::vector subregions; +}; + + struct MapRoot: MapTreeBounds { int minZoom ; int maxZoom ; @@ -58,6 +70,30 @@ struct BinaryPartIndex { BinaryPartIndex(PART_INDEXES tp) : type(tp) {} }; +struct RoutingIndex : BinaryPartIndex { + HMAP::hash_map decodingRules; + std::vector subregions; + RoutingIndex() : BinaryPartIndex(ROUTING_INDEX) { + } + + void initRouteEncodingRule(uint32 id, std::string tag, std::string val) { + tag_value pair = tag_value(tag, val); + // DEFINE hash + //encodingRules[pair] = id; + decodingRules[id] = pair; + } +}; + +struct RouteDataObject { + RoutingIndex* region; + std::vector types ; + std::vector pointsX ; + std::vector pointsY ; + std::vector restrictions ; + std::vector > pointTypes; + int64 id; +}; + struct MapIndex : BinaryPartIndex { @@ -129,6 +165,7 @@ struct BinaryMapFile { uint32 version; uint64 dateCreated; std::vector mapIndexes; + std::vector routingIndexes; std::vector indexes; FILE* f; bool basemap; @@ -178,6 +215,7 @@ struct SearchQuery { int numberOfAcceptedObjects; int numberOfReadSubtrees; int numberOfAcceptedSubtrees; + std::vector routeObjects; SearchQuery(int l, int r, int t, int b, RenderingRuleSearchRequest* req, ResultPublisher* publisher) : req(req), left(l), right(r), top(t), bottom(b),publisher(publisher) { @@ -185,12 +223,20 @@ struct SearchQuery { numberOfAcceptedSubtrees = numberOfReadSubtrees = 0; ocean = land = false; } + SearchQuery(int l, int r, int t, int b, std::vector& result) : + req(req), left(l), right(r), top(t), bottom(b), routeObjects(result) { + } bool publish(MapDataObject* obj) { return publisher->publish(obj); } + bool publishRouteObject(RouteDataObject& obj) { + routeObjects.push_back(obj); + return true; + } }; +void searchRouteRegion(SearchQuery* q); ResultPublisher* searchObjectsForRendering(SearchQuery* q, bool skipDuplicates, std::string msgNothingFound); diff --git a/Osmand-kernel/osmand/src/osmand_main.cpp b/Osmand-kernel/osmand/src/osmand_main.cpp index b33807432d..35d94d159d 100644 --- a/Osmand-kernel/osmand/src/osmand_main.cpp +++ b/Osmand-kernel/osmand/src/osmand_main.cpp @@ -142,6 +142,8 @@ void printFileInformation(const char* fileName, VerboseInfo* verbose) { partname = "Map"; } else if (it->type == TRANSPORT_INDEX) { partname = "Transport"; + } else if (it->type == ROUTING_INDEX) { + partname = "Routing"; } else if (it->type == POI_INDEX) { partname = "Poi"; } else if (it->type == ADDRESS_INDEX) {