Android-builders-club/OsmAnd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Improve Transport reader algorith (make it much faster?)

Browse source
This commit is contained in:
Victor Shcherb 2019-07-31 00:56:54 +02:00
parent c89fda16f4
commit 39845c3601
2 changed files with 63 additions and 73 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
OsmAnd-java/src/main/java/net/osmand/binary/BinaryMapIndexReader.java
View file

@ -504,9 +504,9 @@ public class BinaryMapIndexReader {
result.put(filePointer, transportRoute); result.put(filePointer, transportRoute);
finishInit.add(transportRoute); finishInit.add(transportRoute);
} }
transportAdapter.initializeStringTable(ind, stringTable); TIntObjectHashMap<String> indexedStringTable = transportAdapter.initializeStringTable(ind, stringTable);
for(TransportRoute transportRoute : finishInit ) { for(TransportRoute transportRoute : finishInit ) {
transportAdapter.initializeNames(false, transportRoute, stringTable); transportAdapter.initializeNames(false, transportRoute, indexedStringTable);
} }
} }
@ -556,27 +556,29 @@ public class BinaryMapIndexReader {
} }
return false; return false;
} }
public List<TransportStop> searchTransportIndex(TransportIndex index, SearchRequest<TransportStop> req) throws IOException {
if (index.stopsFileLength == 0 || index.right < req.left || index.left > req.right || index.top > req.bottom
|| index.bottom < req.top) {
return req.getSearchResults();
}
codedIS.seek(index.stopsFileOffset);
int oldLimit = codedIS.pushLimit(index.stopsFileLength);
int offset = req.searchResults.size();
TIntObjectHashMap<String> stringTable = new TIntObjectHashMap<String>();
transportAdapter.searchTransportTreeBounds(0, 0, 0, 0, req, stringTable);
codedIS.popLimit(oldLimit);
TIntObjectHashMap<String> indexedStringTable = transportAdapter.initializeStringTable(index, stringTable);
for (int i = offset; i < req.searchResults.size(); i++) {
TransportStop st = req.searchResults.get(i);
transportAdapter.initializeNames(indexedStringTable, st);
}
return req.getSearchResults();
}
public List<TransportStop> searchTransportIndex(SearchRequest<TransportStop> req) throws IOException { public List<TransportStop> searchTransportIndex(SearchRequest<TransportStop> req) throws IOException {
for (TransportIndex index : transportIndexes) { for (TransportIndex index : transportIndexes) {
if (index.stopsFileLength == 0 || index.right < req.left || index.left > req.right || index.top > req.bottom searchTransportIndex(index, req);
|| index.bottom < req.top) {
continue;
}
if (req.stringTable != null) {
req.stringTable.clear();
}
codedIS.seek(index.stopsFileOffset);
int oldLimit = codedIS.pushLimit(index.stopsFileLength);
int offset = req.searchResults.size();
transportAdapter.searchTransportTreeBounds(0, 0, 0, 0, req);
codedIS.popLimit(oldLimit);
if (req.stringTable != null) {
transportAdapter.initializeStringTable(index, req.stringTable);
for (int i = offset; i < req.searchResults.size(); i++) {
TransportStop st = req.searchResults.get(i);
transportAdapter.initializeNames(req.stringTable, st);
}
}
} }
if (req.numberOfVisitedObjects > 0) { if (req.numberOfVisitedObjects > 0) {
log.debug("Search is done. Visit " + req.numberOfVisitedObjects + " objects. Read " + req.numberOfAcceptedObjects + " objects."); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ log.debug("Search is done. Visit " + req.numberOfVisitedObjects + " objects. Read " + req.numberOfAcceptedObjects + " objects."); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
@ -1522,7 +1524,6 @@ public class BinaryMapIndexReader {
if (stops != null) { if (stops != null) {
request.searchResults = stops; request.searchResults = stops;
} }
request.stringTable = new TIntObjectHashMap<String>();
request.left = sleft >> (31 - TRANSPORT_STOP_ZOOM); request.left = sleft >> (31 - TRANSPORT_STOP_ZOOM);
request.right = sright >> (31 - TRANSPORT_STOP_ZOOM); request.right = sright >> (31 - TRANSPORT_STOP_ZOOM);
request.top = stop >> (31 - TRANSPORT_STOP_ZOOM); request.top = stop >> (31 - TRANSPORT_STOP_ZOOM);
@ -1620,9 +1621,6 @@ public class BinaryMapIndexReader {
SearchPoiTypeFilter poiTypeFilter = null; SearchPoiTypeFilter poiTypeFilter = null;
// internal read information
TIntObjectHashMap<String> stringTable = null;
// cache information // cache information
TIntArrayList cacheCoordinates = new TIntArrayList(); TIntArrayList cacheCoordinates = new TIntArrayList();
TIntArrayList cacheTypes = new TIntArrayList(); TIntArrayList cacheTypes = new TIntArrayList();
@ -1746,9 +1744,6 @@ public class BinaryMapIndexReader {
searchResults = new ArrayList<T>(); searchResults = new ArrayList<T>();
cacheCoordinates.clear(); cacheCoordinates.clear();
cacheTypes.clear(); cacheTypes.clear();
if(stringTable != null) {
stringTable.clear();
}
land = false; land = false;
ocean = false; ocean = false;
numberOfVisitedObjects = 0; numberOfVisitedObjects = 0;

85
OsmAnd-java/src/main/java/net/osmand/binary/BinaryMapTransportReaderAdapter.java
View file

@ -1,7 +1,6 @@
package net.osmand.binary; package net.osmand.binary;
import java.io.IOException; import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.Map; import java.util.Map;
@ -9,7 +8,6 @@ import java.util.Map;
import com.google.protobuf.CodedInputStream; import com.google.protobuf.CodedInputStream;
import com.google.protobuf.WireFormat; import com.google.protobuf.WireFormat;
import gnu.trove.list.array.TIntArrayList;
import gnu.trove.map.hash.TIntObjectHashMap; import gnu.trove.map.hash.TIntObjectHashMap;
import net.osmand.binary.BinaryMapIndexReader.SearchRequest; import net.osmand.binary.BinaryMapIndexReader.SearchRequest;
import net.osmand.data.TransportSchedule; import net.osmand.data.TransportSchedule;
@ -76,10 +74,7 @@ public class BinaryMapTransportReaderAdapter {
protected static class IndexStringTable { protected static class IndexStringTable {
int fileOffset = 0; int fileOffset = 0;
int length = 0; int length = 0;
TIntObjectHashMap<String> stringTable = null;
// offset from start for each SIZE_OFFSET_ARRAY elements
// (SIZE_OFFSET_ARRAY + 1) offset = offsets[0] + skipOneString()
TIntArrayList offsets = new TIntArrayList();
} }
@ -147,7 +142,7 @@ public class BinaryMapTransportReaderAdapter {
} }
protected void searchTransportTreeBounds(int pleft, int pright, int ptop, int pbottom, protected void searchTransportTreeBounds(int pleft, int pright, int ptop, int pbottom,
SearchRequest<TransportStop> req) throws IOException { SearchRequest<TransportStop> req, TIntObjectHashMap<String> stringTable) throws IOException {
int init = 0; int init = 0;
int lastIndexResult = -1; int lastIndexResult = -1;
int cright = 0; int cright = 0;
@ -197,7 +192,7 @@ public class BinaryMapTransportReaderAdapter {
lastIndexResult = req.getSearchResults().size(); lastIndexResult = req.getSearchResults().size();
} }
req.numberOfVisitedObjects++; req.numberOfVisitedObjects++;
TransportStop transportStop = readTransportStop(stopOffset, cleft, cright, ctop, cbottom, req); TransportStop transportStop = readTransportStop(stopOffset, cleft, cright, ctop, cbottom, req, stringTable);
if(transportStop != null){ if(transportStop != null){
req.publish(transportStop); req.publish(transportStop);
} }
@ -209,7 +204,7 @@ public class BinaryMapTransportReaderAdapter {
int filePointer = codedIS.getTotalBytesRead(); int filePointer = codedIS.getTotalBytesRead();
if (req.limit == -1 || req.limit >= req.getSearchResults().size()) { if (req.limit == -1 || req.limit >= req.getSearchResults().size()) {
oldLimit = codedIS.pushLimit(length); oldLimit = codedIS.pushLimit(length);
searchTransportTreeBounds(cleft, cright, ctop, cbottom, req); searchTransportTreeBounds(cleft, cright, ctop, cbottom, req, stringTable);
codedIS.popLimit(oldLimit); codedIS.popLimit(oldLimit);
} }
codedIS.seek(filePointer + length); codedIS.seek(filePointer + length);
@ -392,35 +387,33 @@ public class BinaryMapTransportReaderAdapter {
} }
protected void initializeStringTable(TransportIndex ind, TIntObjectHashMap<String> stringTable) throws IOException { protected TIntObjectHashMap<String> initializeStringTable(TransportIndex ind,
int[] values = stringTable.keys(); TIntObjectHashMap<String> requested) throws IOException {
Arrays.sort(values); if (ind.stringTable.stringTable == null) {
codedIS.seek(ind.stringTable.fileOffset); ind.stringTable.stringTable = new TIntObjectHashMap<>();
int oldLimit = codedIS.pushLimit(ind.stringTable.length); codedIS.seek(ind.stringTable.fileOffset);
int current = 0; int oldLimit = codedIS.pushLimit(ind.stringTable.length);
int i = 0; int current = 0;
while (i < values.length && codedIS.getBytesUntilLimit() > 0) { int i = 0;
int t = codedIS.readTag(); while (codedIS.getBytesUntilLimit() > 0) {
int tag = WireFormat.getTagFieldNumber(t); int t = codedIS.readTag();
switch (tag) { int tag = WireFormat.getTagFieldNumber(t);
case 0: switch (tag) {
break; case 0:
case OsmandOdb.StringTable.S_FIELD_NUMBER: break;
if (current == values[i]) { case OsmandOdb.StringTable.S_FIELD_NUMBER:
String value = codedIS.readString(); String value = codedIS.readString();
stringTable.put(values[i], value); ind.stringTable.stringTable.put(current, value);
i++; current++;
} else { break;
default:
skipUnknownField(t); skipUnknownField(t);
break;
} }
current ++;
break;
default:
skipUnknownField(t);
break;
} }
codedIS.popLimit(oldLimit);
} }
codedIS.popLimit(oldLimit); return ind.stringTable.stringTable;
} }
protected void initializeNames(boolean onlyDescription, net.osmand.data.TransportRoute dataObject, protected void initializeNames(boolean onlyDescription, net.osmand.data.TransportRoute dataObject,
@ -513,7 +506,8 @@ public class BinaryMapTransportReaderAdapter {
return dataObject; return dataObject;
} }
private TransportStop readTransportStop(int shift, int cleft, int cright, int ctop, int cbottom, SearchRequest<TransportStop> req) throws IOException { private TransportStop readTransportStop(int shift, int cleft, int cright, int ctop, int cbottom,
SearchRequest<TransportStop> req, TIntObjectHashMap<String> stringTable) throws IOException {
int tag = WireFormat.getTagFieldNumber(codedIS.readTag()); int tag = WireFormat.getTagFieldNumber(codedIS.readTag());
if(OsmandOdb.TransportStop.DX_FIELD_NUMBER != tag) { if(OsmandOdb.TransportStop.DX_FIELD_NUMBER != tag) {
throw new IllegalArgumentException(); throw new IllegalArgumentException();
@ -560,26 +554,26 @@ public class BinaryMapTransportReaderAdapter {
req.cacheIdsB.add(codedIS.readUInt64()); req.cacheIdsB.add(codedIS.readUInt64());
break; break;
case OsmandOdb.TransportStop.NAME_EN_FIELD_NUMBER : case OsmandOdb.TransportStop.NAME_EN_FIELD_NUMBER :
if (req.stringTable != null) { if (stringTable != null) {
dataObject.setEnName(regStr(req.stringTable)); //$NON-NLS-1$ dataObject.setEnName(regStr(stringTable)); //$NON-NLS-1$
} else { } else {
skipUnknownField(t); skipUnknownField(t);
} }
break; break;
case OsmandOdb.TransportStop.NAME_FIELD_NUMBER : case OsmandOdb.TransportStop.NAME_FIELD_NUMBER :
if (req.stringTable != null) { if (stringTable != null) {
dataObject.setName(regStr(req.stringTable)); //$NON-NLS-1$ dataObject.setName(regStr(stringTable)); //$NON-NLS-1$
} else { } else {
skipUnknownField(t); skipUnknownField(t);
} }
break; break;
case OsmandOdb.TransportStop.ADDITIONALNAMEPAIRS_FIELD_NUMBER : case OsmandOdb.TransportStop.ADDITIONALNAMEPAIRS_FIELD_NUMBER :
if (req.stringTable != null) { if (stringTable != null) {
int sizeL = codedIS.readRawVarint32(); int sizeL = codedIS.readRawVarint32();
int oldRef = codedIS.pushLimit(sizeL); int oldRef = codedIS.pushLimit(sizeL);
while (codedIS.getBytesUntilLimit() > 0) { while (codedIS.getBytesUntilLimit() > 0) {
dataObject.setName(regStr(req.stringTable,codedIS.readRawVarint32()), dataObject.setName(regStr(stringTable,codedIS.readRawVarint32()),
regStr(req.stringTable,codedIS.readRawVarint32())); regStr(stringTable,codedIS.readRawVarint32()));
} }
codedIS.popLimit(oldRef); codedIS.popLimit(oldRef);
} else { } else {
@ -593,7 +587,7 @@ public class BinaryMapTransportReaderAdapter {
int length = codedIS.readRawVarint32(); int length = codedIS.readRawVarint32();
int oldLimit = codedIS.pushLimit(length); int oldLimit = codedIS.pushLimit(length);
TransportStopExit transportStopExit = readTransportStopExit(cleft, ctop, req); TransportStopExit transportStopExit = readTransportStopExit(cleft, ctop, req, stringTable);
dataObject.addExit(transportStopExit); dataObject.addExit(transportStopExit);
codedIS.popLimit(oldLimit); codedIS.popLimit(oldLimit);
break; break;
@ -604,7 +598,8 @@ public class BinaryMapTransportReaderAdapter {
} }
} }
private TransportStopExit readTransportStopExit(int cleft, int ctop, SearchRequest<TransportStop> req) throws IOException { private TransportStopExit readTransportStopExit(int cleft, int ctop, SearchRequest<TransportStop> req,
TIntObjectHashMap<String> stringTable) throws IOException {
TransportStopExit dataObject = new TransportStopExit(); TransportStopExit dataObject = new TransportStopExit();
int x = 0; int x = 0;
@ -624,8 +619,8 @@ public class BinaryMapTransportReaderAdapter {
} }
return dataObject; return dataObject;
case OsmandOdb.TransportStopExit.REF_FIELD_NUMBER: case OsmandOdb.TransportStopExit.REF_FIELD_NUMBER:
if (req.stringTable != null) { if (stringTable != null) {
dataObject.setRef(regStr(req.stringTable)); dataObject.setRef(regStr(stringTable));
} else { } else {
skipUnknownField(t); skipUnknownField(t);
} }