Android-builders-club/OsmAnd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

fix transport reader/writer

Browse source 
git-svn-id: https://osmand.googlecode.com/svn/trunk@637 e29c36b1-1cfa-d876-8d93-3434fc2bb7b8
This commit is contained in:
Victor Shcherb 2010-11-06 16:34:40 +00:00
parent 4dd4efe6ba
commit a32daf7a6e
6 changed files with 213 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
DataExtractionOSM/src/com/google/protobuf/CodedInputStreamRAF.java
View file

@ -541,22 +541,6 @@ public final class CodedInputStreamRAF {
return oldLimit; return oldLimit;
} }
public int pushAbsoluteLimit(int byteLimit) throws InvalidProtocolBufferException {
if (byteLimit < 0) {
throw InvalidProtocolBufferException.negativeSize();
}
// that's absolute limit as parameter
// byteLimit += totalBytesRetired + bufferPos;
final int oldLimit = currentLimit;
if (byteLimit > oldLimit) {
throw InvalidProtocolBufferException.truncatedMessage();
}
currentLimit = byteLimit;
recomputeBufferSizeAfterLimit();
return oldLimit;
}
private void recomputeBufferSizeAfterLimit() { private void recomputeBufferSizeAfterLimit() {
bufferSize += bufferSizeAfterLimit; bufferSize += bufferSizeAfterLimit;

216
DataExtractionOSM/src/net/osmand/binary/BinaryMapIndexReader.java
View file

@ -17,6 +17,7 @@ import net.osmand.data.Building;
import net.osmand.data.City; import net.osmand.data.City;
import net.osmand.data.PostCode; import net.osmand.data.PostCode;
import net.osmand.data.Street; import net.osmand.data.Street;
import net.osmand.data.TransportRoute;
import net.osmand.data.TransportStop; import net.osmand.data.TransportStop;
import net.osmand.data.City.CityType; import net.osmand.data.City.CityType;
import net.osmand.osm.LatLon; import net.osmand.osm.LatLon;
@ -134,7 +135,7 @@ public class BinaryMapIndexReader {
IndexStringTable st = new IndexStringTable(); IndexStringTable st = new IndexStringTable();
st.length = codedIS.readRawVarint32(); st.length = codedIS.readRawVarint32();
st.fileOffset = codedIS.getTotalBytesRead(); st.fileOffset = codedIS.getTotalBytesRead();
readStringTable(st, 0, 30, true); readStringTable(st, 0, 20, true);
ind.stringTable = st; ind.stringTable = st;
codedIS.seek(st.length + st.fileOffset); codedIS.seek(st.length + st.fileOffset);
break; break;
@ -147,13 +148,13 @@ public class BinaryMapIndexReader {
// if cache false put into window // if cache false put into window
private int readStringTable(IndexStringTable st, int startOffset, int length, boolean cache) throws IOException { private int readStringTable(IndexStringTable st, int startOffset, int length, boolean cache) throws IOException {
int old = codedIS.pushAbsoluteLimit(st.fileOffset + st.length);
int toSkip = seekUsingOffsets(st, startOffset); int toSkip = seekUsingOffsets(st, startOffset);
if(!cache){ if(!cache){
st.window.clear(); st.window.clear();
st.windowOffset = startOffset; st.windowOffset = startOffset;
} }
int old = codedIS.pushLimit(st.fileOffset + st.length - codedIS.getTotalBytesRead());
while (length > 0) { while (length > 0) {
int t = codedIS.readTag(); int t = codedIS.readTag();
int tag = WireFormat.getTagFieldNumber(t); int tag = WireFormat.getTagFieldNumber(t);
@ -210,7 +211,7 @@ public class BinaryMapIndexReader {
shift = st.offsets.get(a - 1); shift = st.offsets.get(a - 1);
} }
codedIS.seek(st.fileOffset + shift); codedIS.seek(st.fileOffset + shift);
return index - a; return index - a * IndexStringTable.SIZE_OFFSET_ARRAY;
} }
private void initStringOffsets(IndexStringTable st, int index) throws IOException { private void initStringOffsets(IndexStringTable st, int index) throws IOException {
@ -321,12 +322,14 @@ public class BinaryMapIndexReader {
init |= 8; init |= 8;
break; break;
case OsmandOdb.TransportStopsTree.LEAFS_FIELD_NUMBER : case OsmandOdb.TransportStopsTree.LEAFS_FIELD_NUMBER :
int stopOffset = codedIS.getTotalBytesRead();
int length = codedIS.readRawVarint32(); int length = codedIS.readRawVarint32();
int oldLimit = codedIS.pushLimit(length); int oldLimit = codedIS.pushLimit(length);
if(lastIndexResult == -1){ if(lastIndexResult == -1){
lastIndexResult = req.searchResults.size(); lastIndexResult = req.searchResults.size();
} }
TransportStop transportStop = readTransportStop(cleft, cright, ctop, cbottom, req); req.numberOfVisitedObjects++;
TransportStop transportStop = readTransportStop(stopOffset, cleft, cright, ctop, cbottom, req);
if(transportStop != null){ if(transportStop != null){
req.searchResults.add(transportStop); req.searchResults.add(transportStop);
@ -361,9 +364,149 @@ public class BinaryMapIndexReader {
} }
} }
public net.osmand.data.TransportRoute getTransportRoute(int filePointer) throws IOException {
TransportIndex ind = null;
for(TransportIndex i : transportIndexes){
if(i.fileOffset <= filePointer && (filePointer - i.fileOffset) < i.length){
ind = i;
break;
}
}
if(ind == null){
return null;
}
codedIS.seek(filePointer);
int routeLength = codedIS.readRawVarint32();
int old = codedIS.pushLimit(routeLength);
net.osmand.data.TransportRoute dataObject = new net.osmand.data.TransportRoute();
boolean end = false;
int name = -1;
int nameEn = -1;
int operator = -1;
long rid = 0;
int rx = 0;
int ry = 0;
long did = 0;
int dx = 0;
int dy = 0;
while(!end){
int t = codedIS.readTag();
int tag = WireFormat.getTagFieldNumber(t);
switch (tag) {
case 0:
end = true;
break;
case OsmandOdb.TransportRoute.DISTANCE_FIELD_NUMBER :
dataObject.setDistance(codedIS.readUInt32());
break;
case OsmandOdb.TransportRoute.ID_FIELD_NUMBER :
dataObject.setId(codedIS.readUInt64());
break;
case OsmandOdb.TransportRoute.REF_FIELD_NUMBER :
dataObject.setRef(codedIS.readString());
break;
case OsmandOdb.TransportRoute.NAME_EN_FIELD_NUMBER :
nameEn = codedIS.readUInt32();
break;
case OsmandOdb.TransportRoute.NAME_FIELD_NUMBER :
name = codedIS.readUInt32();
break;
case OsmandOdb.TransportRoute.OPERATOR_FIELD_NUMBER:
operator = codedIS.readUInt32();
break;
case OsmandOdb.TransportRoute.REVERSESTOPS_FIELD_NUMBER:
int length = codedIS.readRawVarint32();
int olds = codedIS.pushLimit(length);
TransportStop stop = readTransportRouteStop(dx, dy, did);
dataObject.getBackwardStops().add(stop);
did = stop.getId();
dx = (int) MapUtils.getTileNumberX(TRANSPORT_STOP_ZOOM, stop.getLocation().getLongitude());
dy = (int) MapUtils.getTileNumberY(TRANSPORT_STOP_ZOOM, stop.getLocation().getLatitude());
codedIS.popLimit(olds);
break;
case OsmandOdb.TransportRoute.DIRECTSTOPS_FIELD_NUMBER:
length = codedIS.readRawVarint32();
olds = codedIS.pushLimit(length);
stop = readTransportRouteStop(rx, ry, rid);
dataObject.getForwardStops().add(stop);
rid = stop.getId();
rx = (int) MapUtils.getTileNumberX(TRANSPORT_STOP_ZOOM, stop.getLocation().getLongitude());
ry = (int) MapUtils.getTileNumberY(TRANSPORT_STOP_ZOOM, stop.getLocation().getLatitude());
codedIS.popLimit(olds);
break;
default:
skipUnknownField(t);
break;
}
}
codedIS.popLimit(old);
if(name != -1){
dataObject.setName(getStringFromStringTable(ind.stringTable, name));
}
if(nameEn != -1){
dataObject.setEnName(getStringFromStringTable(ind.stringTable, nameEn));
} else {
dataObject.setEnName(Junidecode.unidecode(dataObject.getName()));
}
private TransportStop readTransportStop(int cleft, int cright, int ctop, int cbottom, SearchRequest<TransportStop> req) throws IOException { if(operator != -1){
int shift = codedIS.getTotalBytesRead(); dataObject.setOperator(getStringFromStringTable(ind.stringTable, operator));
}
for(int i=0; i< 2; i++){
List<TransportStop> stops = i == 0 ? dataObject.getForwardStops() : dataObject.getBackwardStops();
for(TransportStop s : stops){
if(s.getName().length() > 0){
s.setName(getStringFromStringTable(ind.stringTable, s.getName().charAt(0)));
}
if(s.getEnName().length() > 0){
s.setEnName(getStringFromStringTable(ind.stringTable, s.getEnName().charAt(0)));
} else {
s.setEnName(Junidecode.unidecode(s.getName()));
}
}
}
return dataObject;
}
private TransportStop readTransportRouteStop(int dx, int dy, long did) throws IOException {
TransportStop dataObject = new TransportStop();
boolean end = false;
while(!end){
int t = codedIS.readTag();
int tag = WireFormat.getTagFieldNumber(t);
switch (tag) {
case 0:
end = true;
break;
case OsmandOdb.TransportRouteStop.NAME_EN_FIELD_NUMBER :
dataObject.setEnName(""+((char) codedIS.readUInt32())); //$NON-NLS-1$
break;
case OsmandOdb.TransportRouteStop.NAME_FIELD_NUMBER :
dataObject.setName(""+((char) codedIS.readUInt32())); //$NON-NLS-1$
break;
case OsmandOdb.TransportRouteStop.ID_FIELD_NUMBER :
did += codedIS.readSInt64();
break;
case OsmandOdb.TransportRouteStop.DX_FIELD_NUMBER :
dx += codedIS.readSInt32();
break;
case OsmandOdb.TransportRouteStop.DY_FIELD_NUMBER :
dy += codedIS.readSInt32();
break;
default:
skipUnknownField(t);
break;
}
}
dataObject.setId(did);
dataObject.setLocation(MapUtils.getLatitudeFromTile(TRANSPORT_STOP_ZOOM, dy), MapUtils.getLongitudeFromTile(TRANSPORT_STOP_ZOOM, dx));
return dataObject;
}
private TransportStop readTransportStop(int shift, int cleft, int cright, int ctop, int cbottom, SearchRequest<TransportStop> req) throws IOException {
int tag = WireFormat.getTagFieldNumber(codedIS.readTag()); int tag = WireFormat.getTagFieldNumber(codedIS.readTag());
if(OsmandOdb.TransportStop.DX_FIELD_NUMBER != tag) { if(OsmandOdb.TransportStop.DX_FIELD_NUMBER != tag) {
throw new IllegalArgumentException(); throw new IllegalArgumentException();
@ -375,7 +518,7 @@ public class BinaryMapIndexReader {
throw new IllegalArgumentException(); throw new IllegalArgumentException();
} }
int y = codedIS.readSInt32() + ctop; int y = codedIS.readSInt32() + ctop;
if(cright < x || cleft > x || ctop > y || cbottom < y){ if(req.right < x || req.left > x || req.top > y || req.bottom < y){
codedIS.skipRawBytes(codedIS.getBytesUntilLimit()); codedIS.skipRawBytes(codedIS.getBytesUntilLimit());
return null; return null;
} }
@ -400,7 +543,8 @@ public class BinaryMapIndexReader {
dataObject.setName(""+((char) codedIS.readUInt32())); //$NON-NLS-1$ dataObject.setName(""+((char) codedIS.readUInt32())); //$NON-NLS-1$
break; break;
case OsmandOdb.TransportStop.NAME_FIELD_NUMBER : case OsmandOdb.TransportStop.NAME_FIELD_NUMBER :
dataObject.setName(""+((char) codedIS.readUInt32())); //$NON-NLS-1$ int i = codedIS.readUInt32();
dataObject.setName(""+((char) i)); //$NON-NLS-1$
break; break;
case OsmandOdb.TransportStop.ID_FIELD_NUMBER : case OsmandOdb.TransportStop.ID_FIELD_NUMBER :
dataObject.setId(codedIS.readSInt64()); dataObject.setId(codedIS.readSInt64());
@ -581,6 +725,34 @@ public class BinaryMapIndexReader {
} }
public List<TransportStop> searchTransportIndex(SearchRequest<TransportStop> req) throws IOException {
for (TransportIndex index : transportIndexes) {
if (index.stopsFileLength == 0 || index.right < req.left || index.left > req.right || index.top > req.bottom
|| index.bottom < req.top) {
continue;
}
codedIS.seek(index.stopsFileOffset);
int oldLimit = codedIS.pushLimit(index.stopsFileLength);
int offset = req.searchResults.size();
searchTransportTreeBounds(0, 0, 0, 0, req);
codedIS.popLimit(oldLimit);
for (int i = offset; i < req.searchResults.size(); i++) {
TransportStop st = req.searchResults.get(i);
if (st.getName().length() != 0) {
st.setName(getStringFromStringTable(index.stringTable, st.getName().charAt(0)));
}
if (st.getEnName().length() != 0) {
st.setEnName(getStringFromStringTable(index.stringTable, st.getEnName().charAt(0)));
} else {
st.setEnName(Junidecode.unidecode(st.getName()));
}
}
}
log.info("Search is done. Visit " + req.numberOfVisitedObjects + " objects. Read " + req.numberOfAcceptedObjects + " objects."); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
log.info("Read " + req.numberOfReadSubtrees + " subtrees. Go through " + req.numberOfAcceptedSubtrees + " subtrees."); //$NON-NLS-1$//$NON-NLS-2$//$NON-NLS-3$
return req.getSearchResults();
}
public List<String> getRegionNames(){ public List<String> getRegionNames(){
List<String> names = new ArrayList<String>(); List<String> names = new ArrayList<String>();
for(AddressRegion r : addressIndexes){ for(AddressRegion r : addressIndexes){
@ -1397,6 +1569,16 @@ public class BinaryMapIndexReader {
return request; return request;
} }
public static SearchRequest<TransportStop> buildSearchTransportRequest(int sleft, int sright, int stop, int sbottom, int zoom){
SearchRequest<TransportStop> request = new SearchRequest<TransportStop>();
request.left = sleft >> (31 - TRANSPORT_STOP_ZOOM);
request.right = sright >> (31 - TRANSPORT_STOP_ZOOM);
request.top = stop >> (31 - TRANSPORT_STOP_ZOOM);
request.bottom = sbottom >> (31 - TRANSPORT_STOP_ZOOM);
request.zoom = zoom;
return request;
}
public void close() throws IOException{ public void close() throws IOException{
if(codedIS != null){ if(codedIS != null){
raf.close(); raf.close();
@ -1498,7 +1680,7 @@ public class BinaryMapIndexReader {
} }
private static class IndexStringTable { private static class IndexStringTable {
private static final int SIZE_OFFSET_ARRAY = 4; private static final int SIZE_OFFSET_ARRAY = 100;
private static final int WINDOW_SIZE = 25; private static final int WINDOW_SIZE = 25;
int fileOffset = 0; int fileOffset = 0;
int length = 0; int length = 0;
@ -1532,6 +1714,8 @@ public class BinaryMapIndexReader {
System.out.println("VERSION " + reader.getVersion()); System.out.println("VERSION " + reader.getVersion());
long time = System.currentTimeMillis(); long time = System.currentTimeMillis();
// test search // test search
// int sleft = MapUtils.get31TileNumberX(27.596); // int sleft = MapUtils.get31TileNumberX(27.596);
// int sright = MapUtils.get31TileNumberX(27.599); // int sright = MapUtils.get31TileNumberX(27.599);
@ -1569,10 +1753,20 @@ public class BinaryMapIndexReader {
for(TransportIndex i : reader.transportIndexes){ for(TransportIndex i : reader.transportIndexes){
System.out.println(i.left + " " + i.right + " " + i.top + " " + i.bottom); System.out.println(i.left + " " + i.right + " " + i.top + " " + i.bottom);
System.out.println(i.stringTable.cacheOfStrings); System.out.println(i.stringTable.cacheOfStrings);
System.out.println(reader.getStringFromStringTable(i.stringTable, 58));
System.out.println(i.stringTable.offsets); System.out.println(i.stringTable.offsets);
System.out.println(i.stringTable.window); System.out.println(i.stringTable.window);
} }
int sleft = MapUtils.get31TileNumberX(27.573);
int sright = MapUtils.get31TileNumberX(27.581);
int stop = MapUtils.get31TileNumberY(53.912);
int sbottom = MapUtils.get31TileNumberY(53.908);
for(TransportStop s : reader.searchTransportIndex(buildSearchTransportRequest(sleft, sright, stop, sbottom, 15))){
System.out.println(s.getName());
for (int i : s.getReferencesToRoutes()) {
TransportRoute route = reader.getTransportRoute(i);
System.out.println(" " + route.getRef() + " " + route.getName() + " " + route.getDistance() + " " + route.getAvgBothDistance());
}
}
System.out.println("MEMORY " + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())); System.out.println("MEMORY " + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()));
System.out.println("Time " + (System.currentTimeMillis() - time)); System.out.println("Time " + (System.currentTimeMillis() - time));

4
DataExtractionOSM/src/net/osmand/binary/BinaryMapIndexWriter.java
View file

@ -662,8 +662,10 @@ public class BinaryMapIndexWriter {
stackBaseIds.pop(); stackBaseIds.pop();
stackBaseIds.push(id); stackBaseIds.push(id);
} }
codedOutStream.writeTag(OsmandOdb.TransportStopsTree.LEAFS_FIELD_NUMBER, WireFormat.FieldType.MESSAGE.getWireType());
codedOutStream.flush(); codedOutStream.flush();
long fp = raf.getFilePointer(); long fp = raf.getFilePointer();
OsmandOdb.TransportStop.Builder ts = OsmandOdb.TransportStop.newBuilder(); OsmandOdb.TransportStop.Builder ts = OsmandOdb.TransportStop.newBuilder();
ts.setName(registerString(stringTable, name)); ts.setName(registerString(stringTable, name));
if(nameEn != null){ if(nameEn != null){
@ -676,7 +678,7 @@ public class BinaryMapIndexWriter {
ts.addRoutes((int)(fp - i)); ts.addRoutes((int)(fp - i));
} }
codedOutStream.writeMessage(OsmandOdb.TransportStopsTree.LEAFS_FIELD_NUMBER, ts.build()); codedOutStream.writeMessageNoTag(ts.build());
} }

2
DataExtractionOSM/src/net/osmand/data/index/DataIndexWriter.java
View file

@ -146,7 +146,7 @@ public class DataIndexWriter {
prepStops.setString(IndexTransportStop.NAME.ordinal() + 1, s.getName()); prepStops.setString(IndexTransportStop.NAME.ordinal() + 1, s.getName());
prepStops.setString(IndexTransportStop.NAME_EN.ordinal() + 1, s.getEnName()); prepStops.setString(IndexTransportStop.NAME_EN.ordinal() + 1, s.getEnName());
int x = (int) MapUtils.getTileNumberX(24, s.getLocation().getLongitude()); int x = (int) MapUtils.getTileNumberX(24, s.getLocation().getLongitude());
int y = (int) MapUtils.getTileNumberX(24, s.getLocation().getLatitude()); int y = (int) MapUtils.getTileNumberY(24, s.getLocation().getLatitude());
addBatch(count, prepStops); addBatch(count, prepStops);
try { try {
transportStopsTree.insert(new LeafElement(new Rect(x, y, x, y), s.getId())); transportStopsTree.insert(new LeafElement(new Rect(x, y, x, y), s.getId()));

4
DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java
View file

@ -1734,8 +1734,8 @@ public class IndexCreator {
selectTransportRouteStop.setLong(1, id); selectTransportRouteStop.setLong(1, id);
ResultSet rs = selectTransportStop.executeQuery(); ResultSet rs = selectTransportStop.executeQuery();
if (rs.next()) { if (rs.next()) {
int x24 = (int) MapUtils.getLongitudeFromTile(24, rs.getDouble(2)); int x24 = (int) MapUtils.getTileNumberX(24, rs.getDouble(3));
int y24 = (int) MapUtils.getLongitudeFromTile(24, rs.getDouble(3)); int y24 = (int) MapUtils.getTileNumberY(24, rs.getDouble(2));
String name = rs.getString(4); String name = rs.getString(4);
String nameEn = rs.getString(5); String nameEn = rs.getString(5);
if(nameEn != null && nameEn.equals(Junidecode.unidecode(name))){ if(nameEn != null && nameEn.equals(Junidecode.unidecode(name))){

3
DataExtractionOSM/src/osmand_odb.proto
View file

@ -204,7 +204,8 @@ message TransportStop {
optional uint32 name_en = 7; // index in message table optional uint32 name_en = 7; // index in message table
repeated uint32 routes = 16; // -shift to transport route containing that stop repeated uint32 routes = 16; // -shift to transport route containing that stop
// TransportStop.Message.start - routes[i] = TransportRoute.Message.start - sizeof(TransportRoute.Message.length) // TransportStop.Message.start - sizeof(TransportStop.Message.length) - routes[i]
// = TransportRoute.Message.start - sizeof(TransportRoute.Message.length)
// message is started when body is started // message is started when body is started
} }