From a32daf7a6ed4110740d9ad00cb8ffd23c58bf239 Mon Sep 17 00:00:00 2001 From: Victor Shcherb Date: Sat, 6 Nov 2010 16:34:40 +0000 Subject: [PATCH] fix transport reader/writer git-svn-id: https://osmand.googlecode.com/svn/trunk@637 e29c36b1-1cfa-d876-8d93-3434fc2bb7b8 --- .../google/protobuf/CodedInputStreamRAF.java | 16 -- .../osmand/binary/BinaryMapIndexReader.java | 216 +++++++++++++++++- .../osmand/binary/BinaryMapIndexWriter.java | 4 +- .../osmand/data/index/DataIndexWriter.java | 2 +- .../osmand/data/preparation/IndexCreator.java | 4 +- DataExtractionOSM/src/osmand_odb.proto | 3 +- 6 files changed, 213 insertions(+), 32 deletions(-) diff --git a/DataExtractionOSM/src/com/google/protobuf/CodedInputStreamRAF.java b/DataExtractionOSM/src/com/google/protobuf/CodedInputStreamRAF.java index 8b7dc641a0..bd507b0a61 100644 --- a/DataExtractionOSM/src/com/google/protobuf/CodedInputStreamRAF.java +++ b/DataExtractionOSM/src/com/google/protobuf/CodedInputStreamRAF.java @@ -541,22 +541,6 @@ public final class CodedInputStreamRAF { return oldLimit; } - public int pushAbsoluteLimit(int byteLimit) throws InvalidProtocolBufferException { - if (byteLimit < 0) { - throw InvalidProtocolBufferException.negativeSize(); - } - // that's absolute limit as parameter - // byteLimit += totalBytesRetired + bufferPos; - final int oldLimit = currentLimit; - if (byteLimit > oldLimit) { - throw InvalidProtocolBufferException.truncatedMessage(); - } - currentLimit = byteLimit; - - recomputeBufferSizeAfterLimit(); - - return oldLimit; - } private void recomputeBufferSizeAfterLimit() { bufferSize += bufferSizeAfterLimit; diff --git a/DataExtractionOSM/src/net/osmand/binary/BinaryMapIndexReader.java b/DataExtractionOSM/src/net/osmand/binary/BinaryMapIndexReader.java index 0686cfd4c9..45bbef1852 100644 --- a/DataExtractionOSM/src/net/osmand/binary/BinaryMapIndexReader.java +++ b/DataExtractionOSM/src/net/osmand/binary/BinaryMapIndexReader.java @@ -17,6 +17,7 @@ import net.osmand.data.Building; import net.osmand.data.City; import net.osmand.data.PostCode; import net.osmand.data.Street; +import net.osmand.data.TransportRoute; import net.osmand.data.TransportStop; import net.osmand.data.City.CityType; import net.osmand.osm.LatLon; @@ -134,7 +135,7 @@ public class BinaryMapIndexReader { IndexStringTable st = new IndexStringTable(); st.length = codedIS.readRawVarint32(); st.fileOffset = codedIS.getTotalBytesRead(); - readStringTable(st, 0, 30, true); + readStringTable(st, 0, 20, true); ind.stringTable = st; codedIS.seek(st.length + st.fileOffset); break; @@ -147,13 +148,13 @@ public class BinaryMapIndexReader { // if cache false put into window private int readStringTable(IndexStringTable st, int startOffset, int length, boolean cache) throws IOException { - int old = codedIS.pushAbsoluteLimit(st.fileOffset + st.length); + int toSkip = seekUsingOffsets(st, startOffset); if(!cache){ st.window.clear(); st.windowOffset = startOffset; } - + int old = codedIS.pushLimit(st.fileOffset + st.length - codedIS.getTotalBytesRead()); while (length > 0) { int t = codedIS.readTag(); int tag = WireFormat.getTagFieldNumber(t); @@ -210,7 +211,7 @@ public class BinaryMapIndexReader { shift = st.offsets.get(a - 1); } codedIS.seek(st.fileOffset + shift); - return index - a; + return index - a * IndexStringTable.SIZE_OFFSET_ARRAY; } private void initStringOffsets(IndexStringTable st, int index) throws IOException { @@ -321,12 +322,14 @@ public class BinaryMapIndexReader { init |= 8; break; case OsmandOdb.TransportStopsTree.LEAFS_FIELD_NUMBER : + int stopOffset = codedIS.getTotalBytesRead(); int length = codedIS.readRawVarint32(); int oldLimit = codedIS.pushLimit(length); if(lastIndexResult == -1){ lastIndexResult = req.searchResults.size(); } - TransportStop transportStop = readTransportStop(cleft, cright, ctop, cbottom, req); + req.numberOfVisitedObjects++; + TransportStop transportStop = readTransportStop(stopOffset, cleft, cright, ctop, cbottom, req); if(transportStop != null){ req.searchResults.add(transportStop); @@ -361,9 +364,149 @@ public class BinaryMapIndexReader { } } + public net.osmand.data.TransportRoute getTransportRoute(int filePointer) throws IOException { + TransportIndex ind = null; + for(TransportIndex i : transportIndexes){ + if(i.fileOffset <= filePointer && (filePointer - i.fileOffset) < i.length){ + ind = i; + break; + } + } + if(ind == null){ + return null; + } + codedIS.seek(filePointer); + int routeLength = codedIS.readRawVarint32(); + int old = codedIS.pushLimit(routeLength); + net.osmand.data.TransportRoute dataObject = new net.osmand.data.TransportRoute(); + boolean end = false; + int name = -1; + int nameEn = -1; + int operator = -1; + long rid = 0; + int rx = 0; + int ry = 0; + long did = 0; + int dx = 0; + int dy = 0; + while(!end){ + int t = codedIS.readTag(); + int tag = WireFormat.getTagFieldNumber(t); + switch (tag) { + case 0: + end = true; + break; + case OsmandOdb.TransportRoute.DISTANCE_FIELD_NUMBER : + dataObject.setDistance(codedIS.readUInt32()); + break; + case OsmandOdb.TransportRoute.ID_FIELD_NUMBER : + dataObject.setId(codedIS.readUInt64()); + break; + case OsmandOdb.TransportRoute.REF_FIELD_NUMBER : + dataObject.setRef(codedIS.readString()); + break; + case OsmandOdb.TransportRoute.NAME_EN_FIELD_NUMBER : + nameEn = codedIS.readUInt32(); + break; + case OsmandOdb.TransportRoute.NAME_FIELD_NUMBER : + name = codedIS.readUInt32(); + break; + case OsmandOdb.TransportRoute.OPERATOR_FIELD_NUMBER: + operator = codedIS.readUInt32(); + break; + case OsmandOdb.TransportRoute.REVERSESTOPS_FIELD_NUMBER: + int length = codedIS.readRawVarint32(); + int olds = codedIS.pushLimit(length); + TransportStop stop = readTransportRouteStop(dx, dy, did); + dataObject.getBackwardStops().add(stop); + did = stop.getId(); + dx = (int) MapUtils.getTileNumberX(TRANSPORT_STOP_ZOOM, stop.getLocation().getLongitude()); + dy = (int) MapUtils.getTileNumberY(TRANSPORT_STOP_ZOOM, stop.getLocation().getLatitude()); + codedIS.popLimit(olds); + break; + case OsmandOdb.TransportRoute.DIRECTSTOPS_FIELD_NUMBER: + length = codedIS.readRawVarint32(); + olds = codedIS.pushLimit(length); + stop = readTransportRouteStop(rx, ry, rid); + dataObject.getForwardStops().add(stop); + rid = stop.getId(); + rx = (int) MapUtils.getTileNumberX(TRANSPORT_STOP_ZOOM, stop.getLocation().getLongitude()); + ry = (int) MapUtils.getTileNumberY(TRANSPORT_STOP_ZOOM, stop.getLocation().getLatitude()); + codedIS.popLimit(olds); + break; + default: + skipUnknownField(t); + break; + } + } + codedIS.popLimit(old); + if(name != -1){ + dataObject.setName(getStringFromStringTable(ind.stringTable, name)); + } + if(nameEn != -1){ + dataObject.setEnName(getStringFromStringTable(ind.stringTable, nameEn)); + } else { + dataObject.setEnName(Junidecode.unidecode(dataObject.getName())); + } + + if(operator != -1){ + dataObject.setOperator(getStringFromStringTable(ind.stringTable, operator)); + } + for(int i=0; i< 2; i++){ + List stops = i == 0 ? dataObject.getForwardStops() : dataObject.getBackwardStops(); + for(TransportStop s : stops){ + if(s.getName().length() > 0){ + s.setName(getStringFromStringTable(ind.stringTable, s.getName().charAt(0))); + } + if(s.getEnName().length() > 0){ + s.setEnName(getStringFromStringTable(ind.stringTable, s.getEnName().charAt(0))); + } else { + s.setEnName(Junidecode.unidecode(s.getName())); + } + + } + } + + + return dataObject; + } - private TransportStop readTransportStop(int cleft, int cright, int ctop, int cbottom, SearchRequest req) throws IOException { - int shift = codedIS.getTotalBytesRead(); + private TransportStop readTransportRouteStop(int dx, int dy, long did) throws IOException { + TransportStop dataObject = new TransportStop(); + boolean end = false; + while(!end){ + int t = codedIS.readTag(); + int tag = WireFormat.getTagFieldNumber(t); + switch (tag) { + case 0: + end = true; + break; + case OsmandOdb.TransportRouteStop.NAME_EN_FIELD_NUMBER : + dataObject.setEnName(""+((char) codedIS.readUInt32())); //$NON-NLS-1$ + break; + case OsmandOdb.TransportRouteStop.NAME_FIELD_NUMBER : + dataObject.setName(""+((char) codedIS.readUInt32())); //$NON-NLS-1$ + break; + case OsmandOdb.TransportRouteStop.ID_FIELD_NUMBER : + did += codedIS.readSInt64(); + break; + case OsmandOdb.TransportRouteStop.DX_FIELD_NUMBER : + dx += codedIS.readSInt32(); + break; + case OsmandOdb.TransportRouteStop.DY_FIELD_NUMBER : + dy += codedIS.readSInt32(); + break; + default: + skipUnknownField(t); + break; + } + } + dataObject.setId(did); + dataObject.setLocation(MapUtils.getLatitudeFromTile(TRANSPORT_STOP_ZOOM, dy), MapUtils.getLongitudeFromTile(TRANSPORT_STOP_ZOOM, dx)); + return dataObject; + } + + private TransportStop readTransportStop(int shift, int cleft, int cright, int ctop, int cbottom, SearchRequest req) throws IOException { int tag = WireFormat.getTagFieldNumber(codedIS.readTag()); if(OsmandOdb.TransportStop.DX_FIELD_NUMBER != tag) { throw new IllegalArgumentException(); @@ -375,7 +518,7 @@ public class BinaryMapIndexReader { throw new IllegalArgumentException(); } int y = codedIS.readSInt32() + ctop; - if(cright < x || cleft > x || ctop > y || cbottom < y){ + if(req.right < x || req.left > x || req.top > y || req.bottom < y){ codedIS.skipRawBytes(codedIS.getBytesUntilLimit()); return null; } @@ -400,7 +543,8 @@ public class BinaryMapIndexReader { dataObject.setName(""+((char) codedIS.readUInt32())); //$NON-NLS-1$ break; case OsmandOdb.TransportStop.NAME_FIELD_NUMBER : - dataObject.setName(""+((char) codedIS.readUInt32())); //$NON-NLS-1$ + int i = codedIS.readUInt32(); + dataObject.setName(""+((char) i)); //$NON-NLS-1$ break; case OsmandOdb.TransportStop.ID_FIELD_NUMBER : dataObject.setId(codedIS.readSInt64()); @@ -581,6 +725,34 @@ public class BinaryMapIndexReader { } + public List searchTransportIndex(SearchRequest req) throws IOException { + for (TransportIndex index : transportIndexes) { + if (index.stopsFileLength == 0 || index.right < req.left || index.left > req.right || index.top > req.bottom + || index.bottom < req.top) { + continue; + } + codedIS.seek(index.stopsFileOffset); + int oldLimit = codedIS.pushLimit(index.stopsFileLength); + int offset = req.searchResults.size(); + searchTransportTreeBounds(0, 0, 0, 0, req); + codedIS.popLimit(oldLimit); + for (int i = offset; i < req.searchResults.size(); i++) { + TransportStop st = req.searchResults.get(i); + if (st.getName().length() != 0) { + st.setName(getStringFromStringTable(index.stringTable, st.getName().charAt(0))); + } + if (st.getEnName().length() != 0) { + st.setEnName(getStringFromStringTable(index.stringTable, st.getEnName().charAt(0))); + } else { + st.setEnName(Junidecode.unidecode(st.getName())); + } + } + } + log.info("Search is done. Visit " + req.numberOfVisitedObjects + " objects. Read " + req.numberOfAcceptedObjects + " objects."); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + log.info("Read " + req.numberOfReadSubtrees + " subtrees. Go through " + req.numberOfAcceptedSubtrees + " subtrees."); //$NON-NLS-1$//$NON-NLS-2$//$NON-NLS-3$ + return req.getSearchResults(); + } + public List getRegionNames(){ List names = new ArrayList(); for(AddressRegion r : addressIndexes){ @@ -1397,6 +1569,16 @@ public class BinaryMapIndexReader { return request; } + public static SearchRequest buildSearchTransportRequest(int sleft, int sright, int stop, int sbottom, int zoom){ + SearchRequest request = new SearchRequest(); + request.left = sleft >> (31 - TRANSPORT_STOP_ZOOM); + request.right = sright >> (31 - TRANSPORT_STOP_ZOOM); + request.top = stop >> (31 - TRANSPORT_STOP_ZOOM); + request.bottom = sbottom >> (31 - TRANSPORT_STOP_ZOOM); + request.zoom = zoom; + return request; + } + public void close() throws IOException{ if(codedIS != null){ raf.close(); @@ -1498,7 +1680,7 @@ public class BinaryMapIndexReader { } private static class IndexStringTable { - private static final int SIZE_OFFSET_ARRAY = 4; + private static final int SIZE_OFFSET_ARRAY = 100; private static final int WINDOW_SIZE = 25; int fileOffset = 0; int length = 0; @@ -1532,6 +1714,8 @@ public class BinaryMapIndexReader { System.out.println("VERSION " + reader.getVersion()); long time = System.currentTimeMillis(); + + // test search // int sleft = MapUtils.get31TileNumberX(27.596); // int sright = MapUtils.get31TileNumberX(27.599); @@ -1569,10 +1753,20 @@ public class BinaryMapIndexReader { for(TransportIndex i : reader.transportIndexes){ System.out.println(i.left + " " + i.right + " " + i.top + " " + i.bottom); System.out.println(i.stringTable.cacheOfStrings); - System.out.println(reader.getStringFromStringTable(i.stringTable, 58)); System.out.println(i.stringTable.offsets); System.out.println(i.stringTable.window); } + int sleft = MapUtils.get31TileNumberX(27.573); + int sright = MapUtils.get31TileNumberX(27.581); + int stop = MapUtils.get31TileNumberY(53.912); + int sbottom = MapUtils.get31TileNumberY(53.908); + for(TransportStop s : reader.searchTransportIndex(buildSearchTransportRequest(sleft, sright, stop, sbottom, 15))){ + System.out.println(s.getName()); + for (int i : s.getReferencesToRoutes()) { + TransportRoute route = reader.getTransportRoute(i); + System.out.println(" " + route.getRef() + " " + route.getName() + " " + route.getDistance() + " " + route.getAvgBothDistance()); + } + } System.out.println("MEMORY " + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())); System.out.println("Time " + (System.currentTimeMillis() - time)); diff --git a/DataExtractionOSM/src/net/osmand/binary/BinaryMapIndexWriter.java b/DataExtractionOSM/src/net/osmand/binary/BinaryMapIndexWriter.java index 9f19d54afd..4ac07c468b 100644 --- a/DataExtractionOSM/src/net/osmand/binary/BinaryMapIndexWriter.java +++ b/DataExtractionOSM/src/net/osmand/binary/BinaryMapIndexWriter.java @@ -662,8 +662,10 @@ public class BinaryMapIndexWriter { stackBaseIds.pop(); stackBaseIds.push(id); } + codedOutStream.writeTag(OsmandOdb.TransportStopsTree.LEAFS_FIELD_NUMBER, WireFormat.FieldType.MESSAGE.getWireType()); codedOutStream.flush(); long fp = raf.getFilePointer(); + OsmandOdb.TransportStop.Builder ts = OsmandOdb.TransportStop.newBuilder(); ts.setName(registerString(stringTable, name)); if(nameEn != null){ @@ -676,7 +678,7 @@ public class BinaryMapIndexWriter { ts.addRoutes((int)(fp - i)); } - codedOutStream.writeMessage(OsmandOdb.TransportStopsTree.LEAFS_FIELD_NUMBER, ts.build()); + codedOutStream.writeMessageNoTag(ts.build()); } diff --git a/DataExtractionOSM/src/net/osmand/data/index/DataIndexWriter.java b/DataExtractionOSM/src/net/osmand/data/index/DataIndexWriter.java index 5a38f4a39e..6e3551b14a 100644 --- a/DataExtractionOSM/src/net/osmand/data/index/DataIndexWriter.java +++ b/DataExtractionOSM/src/net/osmand/data/index/DataIndexWriter.java @@ -146,7 +146,7 @@ public class DataIndexWriter { prepStops.setString(IndexTransportStop.NAME.ordinal() + 1, s.getName()); prepStops.setString(IndexTransportStop.NAME_EN.ordinal() + 1, s.getEnName()); int x = (int) MapUtils.getTileNumberX(24, s.getLocation().getLongitude()); - int y = (int) MapUtils.getTileNumberX(24, s.getLocation().getLatitude()); + int y = (int) MapUtils.getTileNumberY(24, s.getLocation().getLatitude()); addBatch(count, prepStops); try { transportStopsTree.insert(new LeafElement(new Rect(x, y, x, y), s.getId())); diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java b/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java index c40b9057ff..5d0b8af7a7 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java @@ -1734,8 +1734,8 @@ public class IndexCreator { selectTransportRouteStop.setLong(1, id); ResultSet rs = selectTransportStop.executeQuery(); if (rs.next()) { - int x24 = (int) MapUtils.getLongitudeFromTile(24, rs.getDouble(2)); - int y24 = (int) MapUtils.getLongitudeFromTile(24, rs.getDouble(3)); + int x24 = (int) MapUtils.getTileNumberX(24, rs.getDouble(3)); + int y24 = (int) MapUtils.getTileNumberY(24, rs.getDouble(2)); String name = rs.getString(4); String nameEn = rs.getString(5); if(nameEn != null && nameEn.equals(Junidecode.unidecode(name))){ diff --git a/DataExtractionOSM/src/osmand_odb.proto b/DataExtractionOSM/src/osmand_odb.proto index 2a8fc4b58d..0f6e02cb88 100644 --- a/DataExtractionOSM/src/osmand_odb.proto +++ b/DataExtractionOSM/src/osmand_odb.proto @@ -204,7 +204,8 @@ message TransportStop { optional uint32 name_en = 7; // index in message table repeated uint32 routes = 16; // -shift to transport route containing that stop - // TransportStop.Message.start - routes[i] = TransportRoute.Message.start - sizeof(TransportRoute.Message.length) + // TransportStop.Message.start - sizeof(TransportStop.Message.length) - routes[i] + // = TransportRoute.Message.start - sizeof(TransportRoute.Message.length) // message is started when body is started }