fix indexing issues
git-svn-id: https://osmand.googlecode.com/svn/trunk@631 e29c36b1-1cfa-d876-8d93-3434fc2bb7b8
This commit is contained in:
Victor Shcherb
parent
a9475d2196
commit
880b5e5541
9 changed files with 3825 additions and 588 deletions
|
|
@ -8,9 +8,8 @@ package net.osmand;
|
|||
public class ToDoConstants {
|
||||
|
||||
// TODO max 101
|
||||
|
||||
// introduce bidforfix on site
|
||||
// !!! Fix progress in DataExtractionOsm
|
||||
// !!! Fix files in IndexCreator (create one file!)
|
||||
|
||||
// Outside base 0.4 release
|
||||
// 69. Add phone and site information to POI (enable call to POI and open site)
|
||||
|
|
|
|||
|
|
@ -5,20 +5,26 @@ import java.io.OutputStream;
|
|||
import java.io.RandomAccessFile;
|
||||
import java.util.Collection;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.Stack;
|
||||
|
||||
import net.osmand.Algoritms;
|
||||
import net.osmand.binary.OsmandOdb.CityIndex;
|
||||
import net.osmand.binary.OsmandOdb.InteresectedStreets;
|
||||
import net.osmand.binary.OsmandOdb.OsmAndTransportIndex;
|
||||
import net.osmand.binary.OsmandOdb.PostcodeIndex;
|
||||
import net.osmand.binary.OsmandOdb.StreetIndex;
|
||||
import net.osmand.binary.OsmandOdb.StreetIntersection;
|
||||
import net.osmand.binary.OsmandOdb.TransportRoute;
|
||||
import net.osmand.binary.OsmandOdb.TransportRouteStop;
|
||||
import net.osmand.data.Building;
|
||||
import net.osmand.data.City;
|
||||
import net.osmand.data.MapObject;
|
||||
import net.osmand.data.Street;
|
||||
import net.osmand.data.TransportStop;
|
||||
import net.osmand.data.index.IndexConstants;
|
||||
import net.osmand.osm.LatLon;
|
||||
import net.osmand.osm.MapUtils;
|
||||
|
|
@ -69,6 +75,9 @@ public class BinaryMapIndexWriter {
|
|||
private final static int POSTCODES_INDEX_INIT = 7;
|
||||
private final static int VILLAGES_INDEX_INIT = 8;
|
||||
|
||||
private final static int TRANSPORT_INDEX_INIT = 9;
|
||||
private final static int TRANSPORT_STOPS_TREE = 10;
|
||||
|
||||
public BinaryMapIndexWriter(final RandomAccessFile raf) throws IOException{
|
||||
this.raf = raf;
|
||||
codedOutStream = CodedOutputStream.newInstance(new OutputStream() {
|
||||
|
|
@ -386,24 +395,26 @@ public class BinaryMapIndexWriter {
|
|||
|
||||
if(wayNodes != null){
|
||||
InteresectedStreets.Builder sbuilders = OsmandOdb.InteresectedStreets.newBuilder();
|
||||
|
||||
Map<Long, Set<Integer>> reverseMap = new LinkedHashMap<Long, Set<Integer>>();
|
||||
for (int i = 0; i < streets.size(); i++) {
|
||||
for (int j = i + 1; j < streets.size(); j++) {
|
||||
Node intersection = null;
|
||||
List<Node> l1 = wayNodes.get(streets.get(i));
|
||||
List<Node> l2 = wayNodes.get(streets.get(j));
|
||||
if (l1 != null && l2 != null) {
|
||||
loop: for (Node n : l1) {
|
||||
for (Node n2 : l2) {
|
||||
if (n.getId() == n2.getId()) {
|
||||
intersection = n;
|
||||
break loop;
|
||||
streets.get(i).setIndexInCity(i);
|
||||
for (Node n : wayNodes.get(streets.get(i))) {
|
||||
if(!reverseMap.containsKey(n.getId())){
|
||||
reverseMap.put(n.getId(), new LinkedHashSet<Integer>(3));
|
||||
}
|
||||
reverseMap.get(n.getId()).add(i);
|
||||
}
|
||||
}
|
||||
Set<Integer> checkedStreets = new LinkedHashSet<Integer>();
|
||||
for (int i = 0; i < streets.size(); i++) {
|
||||
Street s1 = streets.get(i);
|
||||
checkedStreets.clear();
|
||||
for(Node intersection : wayNodes.get(s1)){
|
||||
for(Integer j : reverseMap.get(intersection.getId())){
|
||||
if(i >= j || checkedStreets.contains(j)){
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if(intersection != null){
|
||||
checkedStreets.add(j);
|
||||
StreetIntersection.Builder builder = OsmandOdb.StreetIntersection.newBuilder();
|
||||
builder.setIntersectedStreet1(i);
|
||||
builder.setIntersectedStreet2(j);
|
||||
|
|
@ -515,6 +526,120 @@ public class BinaryMapIndexWriter {
|
|||
return streetBuilder.build();
|
||||
}
|
||||
|
||||
public void startWriteTransportIndex() throws IOException {
|
||||
pushState(TRANSPORT_INDEX_INIT, OSMAND_STRUCTURE_INIT);
|
||||
codedOutStream.writeTag(OsmandOdb.OsmAndStructure.TRANSPORTINDEX_FIELD_NUMBER, WireFormat.WIRETYPE_FIXED32_LENGTH_DELIMITED);
|
||||
stackBounds.push(new Bounds(0, 0, 0, 0)); // for transport stops tree
|
||||
preserveInt32Size();
|
||||
}
|
||||
|
||||
public void endWriteTransportIndex() throws IOException {
|
||||
popState(TRANSPORT_INDEX_INIT);
|
||||
int len = writeInt32Size();
|
||||
stackBounds.pop();
|
||||
System.out.println("TRANSPORT INDEX SIZE : " + len);
|
||||
}
|
||||
|
||||
private int registerString(Map<String, Integer> stringTable, String s) {
|
||||
if (stringTable.containsKey(s)) {
|
||||
return stringTable.get(s);
|
||||
}
|
||||
int size = stringTable.size();
|
||||
stringTable.put(s, size);
|
||||
return size;
|
||||
}
|
||||
|
||||
public void writeTransportRoute(long idRoute, String routeName, String routeEnName, String ref, String operator, String type,
|
||||
int dist, List<TransportStop> directStops, List<TransportStop> reverseStops, Map<String, Integer> stringTable) throws IOException {
|
||||
checkPeekState(TRANSPORT_INDEX_INIT);
|
||||
TransportRoute.Builder tRoute = OsmandOdb.TransportRoute.newBuilder();
|
||||
tRoute.setRef(ref);
|
||||
tRoute.setOperator(registerString(stringTable, operator));
|
||||
tRoute.setType(registerString(stringTable, type));
|
||||
tRoute.setId(idRoute);
|
||||
tRoute.setName(registerString(stringTable, routeName));
|
||||
tRoute.setDistance(dist);
|
||||
|
||||
if(routeEnName != null){
|
||||
tRoute.setNameEn(registerString(stringTable, routeEnName));
|
||||
}
|
||||
for (int i = 0; i < 2; i++) {
|
||||
List<TransportStop> stops = i == 0 ? directStops : reverseStops;
|
||||
long id = 0;
|
||||
int x24 = 0;
|
||||
int y24 = 0;
|
||||
for (TransportStop st : stops) {
|
||||
TransportRouteStop.Builder tStop = OsmandOdb.TransportRouteStop.newBuilder();
|
||||
tStop.setId(st.getId() - id);
|
||||
id = st.getId();
|
||||
int x = (int) MapUtils.getTileNumberX(24, st.getLocation().getLongitude());
|
||||
int y = (int) MapUtils.getTileNumberY(24, st.getLocation().getLatitude());
|
||||
tStop.setDx(x - x24);
|
||||
tStop.setDy(y - y24);
|
||||
x24 = x;
|
||||
y24 = y;
|
||||
tStop.setName(registerString(stringTable, st.getName()));
|
||||
if (st.getEnName() != null) {
|
||||
tStop.setNameEn(registerString(stringTable, st.getEnName()));
|
||||
}
|
||||
if (i == 0) {
|
||||
tRoute.addDirectStops(tStop.build());
|
||||
} else {
|
||||
tRoute.addReverseStops(tStop.build());
|
||||
}
|
||||
}
|
||||
}
|
||||
codedOutStream.writeMessage(OsmandOdb.OsmAndTransportIndex.ROUTES_FIELD_NUMBER, tRoute.build());
|
||||
}
|
||||
|
||||
public void startTransportTreeElement(int leftX, int rightX, int topY, int bottomY) throws IOException {
|
||||
checkPeekState(TRANSPORT_STOPS_TREE, TRANSPORT_INDEX_INIT);
|
||||
if(state.peek() == TRANSPORT_STOPS_TREE){
|
||||
codedOutStream.writeTag(OsmandOdb.TransportStopsTree.SUBTREES_FIELD_NUMBER, WireFormat.WIRETYPE_FIXED32_LENGTH_DELIMITED);
|
||||
} else {
|
||||
codedOutStream.writeTag(OsmandOdb.OsmAndTransportIndex.STOPS_FIELD_NUMBER, WireFormat.WIRETYPE_FIXED32_LENGTH_DELIMITED);
|
||||
}
|
||||
state.push(TRANSPORT_STOPS_TREE);
|
||||
preserveInt32Size();
|
||||
|
||||
|
||||
Bounds bounds = stackBounds.peek();
|
||||
|
||||
codedOutStream.writeSInt32(OsmandOdb.TransportStopsTree.LEFT_FIELD_NUMBER, leftX - bounds.leftX);
|
||||
codedOutStream.writeSInt32(OsmandOdb.TransportStopsTree.RIGHT_FIELD_NUMBER, rightX - bounds.rightX);
|
||||
codedOutStream.writeSInt32(OsmandOdb.TransportStopsTree.TOP_FIELD_NUMBER, topY - bounds.topY);
|
||||
codedOutStream.writeSInt32(OsmandOdb.TransportStopsTree.BOTTOM_FIELD_NUMBER, bottomY - bounds.bottomY);
|
||||
stackBounds.push(new Bounds(leftX, rightX, topY, bottomY));
|
||||
stackBaseIds.push(-1L);
|
||||
|
||||
|
||||
}
|
||||
|
||||
public void endWriteTransportTreeElement() throws IOException {
|
||||
Long baseId = stackBaseIds.pop();
|
||||
if(baseId >= 0){
|
||||
codedOutStream.writeUInt64(OsmandOdb.TransportStopsTree.BASEID_FIELD_NUMBER, baseId);
|
||||
}
|
||||
popState(TRANSPORT_STOPS_TREE);
|
||||
stackBounds.pop();
|
||||
writeInt32Size();
|
||||
}
|
||||
|
||||
|
||||
public void writeTransportStringTable(Map<String, Integer> stringTable) throws IOException {
|
||||
checkPeekState(TRANSPORT_INDEX_INIT);
|
||||
// expect linked hash map
|
||||
int i = 0;
|
||||
OsmandOdb.StringTable.Builder st = OsmandOdb.StringTable.newBuilder();
|
||||
for(String s : stringTable.keySet()){
|
||||
if(stringTable.get(s) != i++){
|
||||
throw new IllegalStateException();
|
||||
}
|
||||
st.addS(s);
|
||||
}
|
||||
codedOutStream.writeMessage(OsmAndTransportIndex.STRINGTABLE_FIELD_NUMBER, st.build());
|
||||
}
|
||||
|
||||
|
||||
private void pushState(int push, int peek){
|
||||
if(state.peek() != peek){
|
||||
|
|
@ -544,4 +669,6 @@ public class BinaryMapIndexWriter {
|
|||
codedOutStream.writeInt32(OsmandOdb.OsmAndStructure.VERSIONCONFIRM_FIELD_NUMBER, IndexConstants.BINARY_MAP_VERSION);
|
||||
codedOutStream.flush();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -134,7 +134,7 @@ public class DataIndexWriter {
|
|||
}
|
||||
|
||||
|
||||
private static void writeRouteStops(PreparedStatement prepRouteStops, PreparedStatement prepStops, Map<PreparedStatement, Integer> count,
|
||||
private static void writeRouteStops(RTree transportStopsTree, PreparedStatement prepRouteStops, PreparedStatement prepStops, Map<PreparedStatement, Integer> count,
|
||||
Set<Long> writtenStops, TransportRoute r, List<TransportStop> stops, boolean direction) throws SQLException {
|
||||
int i = 0;
|
||||
for(TransportStop s : stops){
|
||||
|
|
@ -145,7 +145,16 @@ public class DataIndexWriter {
|
|||
prepStops.setDouble(IndexTransportStop.LONGITUDE.ordinal() + 1, s.getLocation().getLongitude());
|
||||
prepStops.setString(IndexTransportStop.NAME.ordinal() + 1, s.getName());
|
||||
prepStops.setString(IndexTransportStop.NAME_EN.ordinal() + 1, s.getEnName());
|
||||
int x = (int) MapUtils.getTileNumberX(24, s.getLocation().getLongitude());
|
||||
int y = (int) MapUtils.getTileNumberX(24, s.getLocation().getLatitude());
|
||||
addBatch(count, prepStops);
|
||||
try {
|
||||
transportStopsTree.insert(new LeafElement(new Rect(x, y, x, y), s.getId()));
|
||||
} catch (RTreeInsertException e) {
|
||||
throw new IllegalArgumentException(e);
|
||||
} catch (IllegalValueException e) {
|
||||
throw new IllegalArgumentException(e);
|
||||
}
|
||||
writtenStops.add(s.getId());
|
||||
}
|
||||
assert IndexTransportRouteStop.values().length == 4;
|
||||
|
|
@ -159,7 +168,8 @@ public class DataIndexWriter {
|
|||
|
||||
|
||||
public static void insertTransportIntoIndex(PreparedStatement prepRoute, PreparedStatement prepRouteStops,
|
||||
PreparedStatement prepStops, Set<Long> writtenStops, TransportRoute route, Map<PreparedStatement, Integer> statements,
|
||||
PreparedStatement prepStops, RTree transportStopsTree,
|
||||
Set<Long> writtenStops, TransportRoute route, Map<PreparedStatement, Integer> statements,
|
||||
int batchSize) throws SQLException {
|
||||
assert IndexTransportRoute.values().length == 7;
|
||||
prepRoute.setLong(IndexTransportRoute.ID.ordinal() + 1, route.getId());
|
||||
|
|
@ -171,8 +181,8 @@ public class DataIndexWriter {
|
|||
prepRoute.setInt(IndexTransportRoute.DIST.ordinal() + 1, route.getAvgBothDistance());
|
||||
addBatch(statements, prepRoute);
|
||||
|
||||
writeRouteStops(prepRouteStops, prepStops, statements, writtenStops, route, route.getForwardStops(), true);
|
||||
writeRouteStops(prepRouteStops, prepStops, statements, writtenStops, route, route.getBackwardStops(), false);
|
||||
writeRouteStops(transportStopsTree, prepRouteStops, prepStops, statements, writtenStops, route, route.getForwardStops(), true);
|
||||
writeRouteStops(transportStopsTree, prepRouteStops, prepStops, statements, writtenStops, route, route.getBackwardStops(), false);
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -268,8 +278,6 @@ public class DataIndexWriter {
|
|||
mapBinaryStat.setInt(IndexBinaryMapRenderObject.HIGHWAY.ordinal() + 1, highwayAttributes);
|
||||
mapBinaryStat.setString(IndexBinaryMapRenderObject.NAME.ordinal() + 1, name);
|
||||
addBatch(statements, mapBinaryStat);
|
||||
|
||||
|
||||
try {
|
||||
mapTree.insert(new LeafElement(new Rect(minX, minY, maxX, maxY), id));
|
||||
} catch (RTreeInsertException e1) {
|
||||
|
|
|
|||
|
|
@ -282,8 +282,6 @@ public class IndexBatchCreator {
|
|||
indexCreator.setNormalizeStreets(true);
|
||||
indexCreator.setSaveAddressWays(writeWayNodes);
|
||||
|
||||
String transportFileName = regionName + "_" + IndexConstants.TRANSPORT_TABLE_VERSION + IndexConstants.TRANSPORT_INDEX_EXT;
|
||||
indexCreator.setTransportFileName(transportFileName);
|
||||
String poiFileName = regionName + "_" + IndexConstants.POI_TABLE_VERSION + IndexConstants.POI_INDEX_EXT;
|
||||
indexCreator.setPoiFileName(poiFileName);
|
||||
String mapFileName = regionName + "_" + IndexConstants.BINARY_MAP_VERSION + IndexConstants.BINARY_MAP_INDEX_EXT;
|
||||
|
|
@ -294,10 +292,7 @@ public class IndexBatchCreator {
|
|||
if (indexPOI) {
|
||||
uploadIndex(new File(indexDirFiles, poiFileName), alreadyUploadedFiles);
|
||||
}
|
||||
if (indexTransport) {
|
||||
uploadIndex(new File(indexDirFiles, transportFileName), alreadyUploadedFiles);
|
||||
}
|
||||
if (indexMap || indexAddress) {
|
||||
if (indexMap || indexAddress || indexTransport) {
|
||||
uploadIndex(new File(indexDirFiles, mapFileName), alreadyUploadedFiles);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
|
|
@ -355,10 +350,21 @@ public class IndexBatchCreator {
|
|||
summary = "Transport index for ";
|
||||
} else if(f.getName().endsWith(IndexConstants.BINARY_MAP_INDEX_EXT) || f.getName().endsWith(IndexConstants.BINARY_MAP_INDEX_EXT_ZIP)){
|
||||
regionName = f.getName().substring(0, f.getName().length() - IndexConstants.BINARY_MAP_INDEX_EXT.length() - 2);
|
||||
summary = "Map index for ";
|
||||
summary = " index for ";
|
||||
boolean fir = true;
|
||||
if (indexAddress) {
|
||||
summary = "Address, " + summary;
|
||||
summary = "Address" + (fir ? "" : ", ") + summary;
|
||||
fir = false;
|
||||
}
|
||||
if (indexTransport) {
|
||||
summary = "Transport" + (fir ? "" : ", ") + summary;
|
||||
fir = false;
|
||||
}
|
||||
if (indexMap) {
|
||||
summary = "Map" + (fir ? "" : ", ") + summary;
|
||||
fir = false;
|
||||
}
|
||||
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -411,46 +411,6 @@ public class IndexConstants {
|
|||
}
|
||||
|
||||
|
||||
public static String indexMapLocationsTable = "map_locations"; //$NON-NLS-1$
|
||||
public static String indexMapLocationsTable2 = "map_locations_2"; //$NON-NLS-1$
|
||||
public static String indexMapLocationsTable3 = "map_locations_3"; //$NON-NLS-1$
|
||||
|
||||
public enum IndexMapRenderObject implements IndexColumn {
|
||||
ID("long", true), TYPE("integer"), NAME, NODES("BLOB"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
|
||||
boolean index = false;
|
||||
String type = null;
|
||||
|
||||
private IndexMapRenderObject() {
|
||||
}
|
||||
|
||||
private IndexMapRenderObject(String type) {
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
private IndexMapRenderObject(String type, boolean index) {
|
||||
this(type);
|
||||
this.index = index;
|
||||
}
|
||||
|
||||
public static String getTable() {
|
||||
return "map_objects"; //$NON-NLS-1$
|
||||
}
|
||||
|
||||
public String getTableName() {
|
||||
return getTable();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getType() {
|
||||
return type;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isIndex() {
|
||||
return index;
|
||||
}
|
||||
}
|
||||
|
||||
public enum IndexBinaryMapRenderObject implements IndexColumn {
|
||||
ID("long", true), NAME, TYPES("BLOB"), RESTRICTIONS("BLOB"), NODES("BLOB"), HIGHWAY("INT"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$
|
||||
boolean index = false;
|
||||
|
|
|
|||
|
|
@ -112,10 +112,9 @@ public class IndexCreator {
|
|||
|
||||
private boolean normalizeStreets = true; // true by default
|
||||
private boolean saveAddressWays = true; // true by default
|
||||
private boolean recreateOnlyBinaryFile = false; //false;
|
||||
|
||||
private String regionName;
|
||||
|
||||
private String transportFileName = null;
|
||||
private String poiFileName = null;
|
||||
private String mapFileName = null;
|
||||
private Long lastModifiedDate = null;
|
||||
|
|
@ -136,23 +135,23 @@ public class IndexCreator {
|
|||
private PreparedStatement poiPreparedStatement;
|
||||
|
||||
|
||||
Set<Long> visitedStops = new HashSet<Long>();
|
||||
private File transportIndexFile;
|
||||
private Connection transportConnection;
|
||||
private PreparedStatement transRouteStat;
|
||||
private PreparedStatement transRouteStopsStat;
|
||||
private PreparedStatement transStopsStat;
|
||||
|
||||
private File mapFile;
|
||||
private RandomAccessFile mapRAFile;
|
||||
private Connection mapConnection;
|
||||
|
||||
private PreparedStatement mapBinaryStat;
|
||||
|
||||
private PreparedStatement addressCityStat;
|
||||
private PreparedStatement addressStreetStat;
|
||||
private PreparedStatement addressBuildingStat;
|
||||
private PreparedStatement addressStreetNodeStat;
|
||||
|
||||
private PreparedStatement mapBinaryStat;
|
||||
|
||||
private Set<Long> visitedStops = new HashSet<Long>();
|
||||
private PreparedStatement transRouteStat;
|
||||
private PreparedStatement transRouteStopsStat;
|
||||
private PreparedStatement transStopsStat;
|
||||
private RTree transportStopsTree;
|
||||
|
||||
|
||||
private RTree[] mapTree = null;
|
||||
|
|
@ -473,10 +472,6 @@ public class IndexCreator {
|
|||
this.poiFileName = poiFileName;
|
||||
}
|
||||
|
||||
public void setTransportFileName(String transportFileName) {
|
||||
this.transportFileName = transportFileName;
|
||||
}
|
||||
|
||||
public void setNodesDBFile(File file){
|
||||
dbFile = file;
|
||||
}
|
||||
|
|
@ -495,13 +490,6 @@ public class IndexCreator {
|
|||
return getMapFileName() + ".tmp";
|
||||
}
|
||||
|
||||
public String getTransportFileName() {
|
||||
if(transportFileName == null){
|
||||
return IndexConstants.TRANSPORT_INDEX_DIR + getRegionName() + IndexConstants.TRANSPORT_INDEX_EXT;
|
||||
}
|
||||
return transportFileName;
|
||||
}
|
||||
|
||||
public Long getLastModifiedDate() {
|
||||
return lastModifiedDate;
|
||||
}
|
||||
|
|
@ -967,7 +955,7 @@ public class IndexCreator {
|
|||
loadEntityData(e, true);
|
||||
TransportRoute route = indexTransportRoute((Relation) e);
|
||||
if (route != null) {
|
||||
DataIndexWriter.insertTransportIntoIndex(transRouteStat, transRouteStopsStat, transStopsStat, visitedStops, route,
|
||||
DataIndexWriter.insertTransportIntoIndex(transStopsStat, transRouteStat, transRouteStopsStat, transportStopsTree, visitedStops, route,
|
||||
pStatements, BATCH_SIZE);
|
||||
}
|
||||
}
|
||||
|
|
@ -1621,6 +1609,141 @@ public class IndexCreator {
|
|||
}
|
||||
}
|
||||
|
||||
private int registerString(Map<String, Integer> stringTable, String s) {
|
||||
if (stringTable.containsKey(s)) {
|
||||
return stringTable.get(s);
|
||||
}
|
||||
int size = stringTable.size();
|
||||
stringTable.put(s, size);
|
||||
return size;
|
||||
}
|
||||
|
||||
private Map<String, Integer> createStringTableForTransport(){
|
||||
Map<String, Integer> stringTable = new LinkedHashMap<String, Integer>();
|
||||
registerString(stringTable, "bus");
|
||||
registerString(stringTable, "trolleybus");
|
||||
registerString(stringTable, "subway");
|
||||
registerString(stringTable, "tram");
|
||||
registerString(stringTable, "share_taxi");
|
||||
registerString(stringTable, "taxi");
|
||||
registerString(stringTable, "train");
|
||||
registerString(stringTable, "ferry");
|
||||
return stringTable;
|
||||
}
|
||||
|
||||
public void writeBinaryTransportIndex(BinaryMapIndexWriter writer) throws IOException, SQLException {
|
||||
try {
|
||||
visitedStops = null; // allow gc to collect it
|
||||
PreparedStatement selectTransportRouteData = mapConnection.prepareStatement("SELECT * FROM " + IndexTransportRoute.getTable());
|
||||
PreparedStatement selectTransportData = mapConnection.prepareStatement("SELECT S.stop, S.direction," +
|
||||
" A.latitude, A.longitude, A.name, A.name_en " +
|
||||
"FROM transport_route_stop S INNER JOIN transport_stop A ON A.id = S.stop ORDER BY S.ord ASC WHERE S.route = ?");
|
||||
|
||||
writer.startWriteTransportIndex();
|
||||
|
||||
// expect that memory would be enough
|
||||
Map<String, Integer> stringTable = createStringTableForTransport();
|
||||
|
||||
|
||||
ResultSet rs = selectTransportRouteData.executeQuery();
|
||||
List<TransportStop> directStops = new ArrayList<TransportStop>();
|
||||
List<TransportStop> reverseStops = new ArrayList<TransportStop>();
|
||||
while(rs.next()){
|
||||
long idRoute = rs.getLong(IndexTransportRoute.ID.ordinal() + 1);
|
||||
int dist = rs.getInt(IndexTransportRoute.DIST.ordinal() + 1);
|
||||
String routeName = rs.getString(IndexTransportRoute.NAME.ordinal() + 1);
|
||||
String routeEnName = rs.getString(IndexTransportRoute.NAME_EN.ordinal() + 1);
|
||||
if(routeEnName != null && routeEnName.equals(Junidecode.unidecode(routeName))){
|
||||
routeEnName = null;
|
||||
}
|
||||
String ref = rs.getString(IndexTransportRoute.REF.ordinal() + 1);
|
||||
String operator = rs.getString(IndexTransportRoute.OPERATOR.ordinal() + 1);
|
||||
String type = rs.getString(IndexTransportRoute.TYPE.ordinal() + 1);
|
||||
|
||||
selectTransportData.setLong(1, idRoute);
|
||||
ResultSet rset = selectTransportData.executeQuery();
|
||||
while (rset.next()) {
|
||||
boolean dir = rset.getInt(2) != 0;
|
||||
long idStop = rset.getInt(1);
|
||||
String stopName = rset.getString(5);
|
||||
String stopEnName = rset.getString(6);
|
||||
if (stopEnName != null && stopEnName.equals(Junidecode.unidecode(stopName))) {
|
||||
stopEnName = null;
|
||||
}
|
||||
TransportStop st = new TransportStop();
|
||||
st.setId(idStop);
|
||||
st.setName(stopName);
|
||||
st.setLocation(rset.getDouble(3), rset.getDouble(4));
|
||||
if (stopEnName != null) {
|
||||
st.setEnName(stopEnName);
|
||||
}
|
||||
if(dir){
|
||||
directStops.add(st);
|
||||
} else {
|
||||
reverseStops.add(st);
|
||||
}
|
||||
}
|
||||
writer.writeTransportRoute(idRoute, routeName, routeEnName, ref, operator, type, dist, directStops, reverseStops, stringTable);
|
||||
}
|
||||
rs.close();
|
||||
selectTransportRouteData.close();
|
||||
|
||||
|
||||
// TODO prepare transportStopsTree!!!
|
||||
|
||||
PreparedStatement selectTransportStop = mapConnection.prepareStatement(
|
||||
"SELECT A.id, A.latitude, A.longitude, A.name, A.name_en FROM transport_stop A where A.id = ?");
|
||||
PreparedStatement selectTransportRouteStop = mapConnection.prepareStatement(
|
||||
"SELECT S.route FROM transport_route_stop S WHERE S.stop = ? ");
|
||||
long rootIndex = transportStopsTree.getFileHdr().getRootIndex();
|
||||
rtree.Node root = transportStopsTree.getReadNode(rootIndex);
|
||||
Rect rootBounds = calcBounds(root);
|
||||
if (rootBounds != null) {
|
||||
writer.startTransportTreeElement(rootBounds.getMinX(), rootBounds.getMaxX(), rootBounds.getMinY(), rootBounds.getMaxY());
|
||||
writeBinaryTransportTree(root, transportStopsTree, writer, selectTransportStop, selectTransportRouteStop);
|
||||
writer.endWriteTransportTreeElement();
|
||||
}
|
||||
|
||||
writer.writeTransportStringTable(stringTable);
|
||||
|
||||
writer.endWriteTransportIndex();
|
||||
} catch (RTreeException e) {
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public void writeBinaryTransportTree(rtree.Node parent, RTree r, BinaryMapIndexWriter writer,
|
||||
PreparedStatement selectTransportStop, PreparedStatement selectTransportRouteStop) throws IOException, RTreeException, SQLException {
|
||||
Element[] e = parent.getAllElements();
|
||||
for (int i = 0; i < parent.getTotalElements(); i++) {
|
||||
Rect re = e[i].getRect();
|
||||
if (e[i].getElementType() == rtree.Node.LEAF_NODE) {
|
||||
long id = ((LeafElement) e[i]).getPtr();
|
||||
selectTransportStop.setLong(1, id);
|
||||
selectTransportRouteStop.setLong(1, id);
|
||||
ResultSet rs = selectTransportStop.executeQuery();
|
||||
if (rs.next()) {
|
||||
|
||||
ResultSet rset = selectTransportRouteStop.executeQuery();
|
||||
while(rset.next()){
|
||||
|
||||
}
|
||||
rset.close();
|
||||
} else {
|
||||
log.error("Something goes wrong with id = " + id);
|
||||
}
|
||||
} else {
|
||||
long ptr = ((NonLeafElement) e[i]).getPtr();
|
||||
rtree.Node ns = r.getReadNode(ptr);
|
||||
|
||||
writer.startTransportTreeElement(re.getMinX(), re.getMaxX(), re.getMinY(), re.getMaxY());
|
||||
writeBinaryTransportTree(ns, r, writer, selectTransportStop, selectTransportRouteStop);
|
||||
writer.endWriteTransportTreeElement();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
public Rect calcBounds(rtree.Node n) {
|
||||
Rect r = null;
|
||||
|
|
@ -1651,6 +1774,9 @@ public class IndexCreator {
|
|||
public String getRTreeMapIndexNonPackFileName(){
|
||||
return mapFile.getAbsolutePath()+".rtree";
|
||||
}
|
||||
public String getRTreeTransportStopsFileName(){
|
||||
return mapFile.getAbsolutePath()+".trans";
|
||||
}
|
||||
|
||||
public String getRTreeMapIndexPackFileName(){
|
||||
return mapFile.getAbsolutePath()+".prtree";
|
||||
|
|
@ -1683,6 +1809,7 @@ public class IndexCreator {
|
|||
normalizeSuffixes = DataExtractionSettings.getSettings().getSuffixesToNormalizeStreets();
|
||||
}
|
||||
|
||||
boolean success = false;
|
||||
// Main generation method
|
||||
try {
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
|
|
@ -1716,6 +1843,23 @@ public class IndexCreator {
|
|||
pselectRelation = dbConn.prepareStatement("select * from relations where id = ? order by ord");
|
||||
pselectTags = dbConn.prepareStatement("select key, value from tags where id = ? and type = ?");
|
||||
|
||||
// do not create temp map file and rtree files
|
||||
if (recreateOnlyBinaryFile) {
|
||||
mapFile = new File(workingDir, getMapFileName());
|
||||
File tempDBMapFile = new File(workingDir, getTempMapDBFileName());
|
||||
mapConnection = DriverManager.getConnection("jdbc:sqlite:" + tempDBMapFile.getAbsolutePath());
|
||||
mapConnection.setAutoCommit(false);
|
||||
mapTree = new RTree[MAP_ZOOMS.length - 1];
|
||||
try {
|
||||
for (int i = 0; i < MAP_ZOOMS.length - 1; i++) {
|
||||
mapTree[i] = new RTree(getRTreeMapIndexPackFileName() + i);
|
||||
}
|
||||
} catch (RTreeException e) {
|
||||
log.error("Error flushing", e);
|
||||
throw new IOException(e);
|
||||
}
|
||||
} else {
|
||||
|
||||
// 2. Create index connections and index structure
|
||||
createDatabaseIndexesStructure();
|
||||
|
||||
|
|
@ -1745,7 +1889,8 @@ public class IndexCreator {
|
|||
if (indexAddress || indexMap) {
|
||||
progress.setGeneralProgress("[40 of 100]");
|
||||
progress.startTask("Preindexing address and map ways...", allRelations);
|
||||
allRelations = iterateOverEntities(progress, EntityType.RELATION, allRelations, STEP_ADDRESS_RELATIONS_AND_MULTYPOLYGONS);
|
||||
allRelations = iterateOverEntities(progress, EntityType.RELATION, allRelations,
|
||||
STEP_ADDRESS_RELATIONS_AND_MULTYPOLYGONS);
|
||||
// commit to put all cities
|
||||
if (indexAddress) {
|
||||
if (pStatements.get(addressBuildingStat) > 0) {
|
||||
|
|
@ -1765,13 +1910,12 @@ public class IndexCreator {
|
|||
progress.startTask("Processing osm nodes...", allNodes);
|
||||
iterateOverEntities(progress, EntityType.NODE, allNodes, STEP_MAIN);
|
||||
progress.setGeneralProgress("[70 of 100]");
|
||||
progress.startTask("Processing osm nodes...", allWays);
|
||||
progress.startTask("Processing osm ways...", allWays);
|
||||
iterateOverEntities(progress, EntityType.WAY, allWays, STEP_MAIN);
|
||||
progress.setGeneralProgress("[85 of 100]");
|
||||
progress.startTask("Processing osm nodes...", allRelations);
|
||||
progress.startTask("Processing osm relations...", allRelations);
|
||||
iterateOverEntities(progress, EntityType.RELATION, allRelations, STEP_MAIN);
|
||||
|
||||
|
||||
// 3.4 update all postal codes from relations
|
||||
if (indexAddress && !postalCodeRelations.isEmpty()) {
|
||||
progress.setGeneralProgress("[90 of 100]");
|
||||
|
|
@ -1791,9 +1935,10 @@ public class IndexCreator {
|
|||
packingRtreeMapIndexes();
|
||||
log.info("Finish packing RTree files");
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Writing binary file
|
||||
if(indexMap || indexAddress){
|
||||
if(indexMap || indexAddress || indexTransport){
|
||||
if(mapFile.exists()){
|
||||
mapFile.delete();
|
||||
}
|
||||
|
|
@ -1814,10 +1959,17 @@ public class IndexCreator {
|
|||
mapConnection.commit();
|
||||
writeBinaryAddressIndex(writer, progress);
|
||||
}
|
||||
if(indexTransport){
|
||||
progress.setGeneralProgress("[95 of 100]");
|
||||
progress.startTask("Writing transport index to binary file...", -1);
|
||||
closePreparedStatements(transRouteStat, transRouteStopsStat, transStopsStat);
|
||||
mapConnection.commit();
|
||||
}
|
||||
progress.finishTask();
|
||||
writer.close();
|
||||
log.info("Finish writing binary file");
|
||||
}
|
||||
success = true;
|
||||
} finally {
|
||||
try {
|
||||
if (pselectNode != null) {
|
||||
|
|
@ -1847,21 +1999,6 @@ public class IndexCreator {
|
|||
poiIndexFile.setLastModified(lastModifiedDate);
|
||||
}
|
||||
}
|
||||
if (transportConnection != null) {
|
||||
transportConnection.commit();
|
||||
transportConnection.close();
|
||||
transportConnection = null;
|
||||
if (lastModifiedDate != null) {
|
||||
transportIndexFile.setLastModified(lastModifiedDate);
|
||||
}
|
||||
}
|
||||
if (mapRAFile != null) {
|
||||
mapRAFile.close();
|
||||
if (lastModifiedDate != null && mapFile.exists()) {
|
||||
mapFile.setLastModified(lastModifiedDate);
|
||||
}
|
||||
}
|
||||
|
||||
if (mapConnection != null) {
|
||||
mapConnection.commit();
|
||||
mapConnection.close();
|
||||
|
|
@ -1882,12 +2019,12 @@ public class IndexCreator {
|
|||
}
|
||||
for (int i = 0; i < mapTree.length; i++) {
|
||||
File f = new File(getRTreeMapIndexNonPackFileName() + i);
|
||||
if (f.exists()) {
|
||||
f.delete();
|
||||
if (f.exists() && success) {
|
||||
// f.delete();
|
||||
}
|
||||
f = new File(getRTreeMapIndexPackFileName() + i);
|
||||
if (f.exists()) {
|
||||
f.delete();
|
||||
if (f.exists() && success) {
|
||||
// f.delete();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1997,7 +2134,7 @@ public class IndexCreator {
|
|||
|
||||
private void createDatabaseIndexesStructure() throws SQLException, IOException {
|
||||
// 2.1 create temporary sqlite database to put temporary results to it
|
||||
if (indexMap || indexTransport) {
|
||||
if (indexMap || indexAddress || indexTransport) {
|
||||
mapFile = new File(workingDir, getMapFileName());
|
||||
// to save space
|
||||
mapFile.getParentFile().mkdirs();
|
||||
|
|
@ -2075,37 +2212,39 @@ public class IndexCreator {
|
|||
}
|
||||
|
||||
if (indexTransport) {
|
||||
transportIndexFile = new File(workingDir, getTransportFileName());
|
||||
// to save space
|
||||
if (transportIndexFile.exists()) {
|
||||
transportIndexFile.delete();
|
||||
DataIndexWriter.createTransportIndexStructure(mapConnection);
|
||||
try {
|
||||
File file = new File(getRTreeTransportStopsFileName());
|
||||
if (file.exists()) {
|
||||
file.delete();
|
||||
}
|
||||
transportIndexFile.getParentFile().mkdirs();
|
||||
// creating nodes db to fast access for all nodes
|
||||
transportConnection = DriverManager.getConnection("jdbc:sqlite:" + transportIndexFile.getAbsolutePath());
|
||||
|
||||
DataIndexWriter.createTransportIndexStructure(transportConnection);
|
||||
transRouteStat = transportConnection.prepareStatement(IndexConstants.generatePrepareStatementToInsert(IndexTransportRoute
|
||||
transportStopsTree = new RTree(file.getAbsolutePath());
|
||||
} catch (RTreeException e) {
|
||||
throw new IOException(e);
|
||||
}
|
||||
transRouteStat = mapConnection.prepareStatement(IndexConstants.generatePrepareStatementToInsert(IndexTransportRoute
|
||||
.getTable(), IndexTransportRoute.values().length));
|
||||
transRouteStopsStat = transportConnection.prepareStatement(IndexConstants.generatePrepareStatementToInsert(
|
||||
transRouteStopsStat = mapConnection.prepareStatement(IndexConstants.generatePrepareStatementToInsert(
|
||||
IndexTransportRouteStop.getTable(), IndexTransportRouteStop.values().length));
|
||||
transStopsStat = transportConnection.prepareStatement(IndexConstants.generatePrepareStatementToInsert(IndexTransportStop
|
||||
transStopsStat = mapConnection.prepareStatement(IndexConstants.generatePrepareStatementToInsert(IndexTransportStop
|
||||
.getTable(), IndexTransportStop.values().length));
|
||||
pStatements.put(transRouteStat, 0);
|
||||
pStatements.put(transRouteStopsStat, 0);
|
||||
pStatements.put(transStopsStat, 0);
|
||||
transportConnection.setAutoCommit(false);
|
||||
mapConnection.setAutoCommit(false);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
protected void closePreparedStatements(PreparedStatement... preparedStatements) throws SQLException{
|
||||
for(PreparedStatement p : preparedStatements){
|
||||
if (p != null) {
|
||||
p.executeBatch();
|
||||
p.close();
|
||||
pStatements.remove(p);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static void removeWayNodes(File sqlitedb) throws SQLException{
|
||||
|
|
@ -2127,7 +2266,7 @@ public class IndexCreator {
|
|||
IndexCreator creator = new IndexCreator(new File("e:/Information/OSM maps/osmand/"));
|
||||
creator.setIndexMap(true);
|
||||
creator.setIndexAddress(true);
|
||||
creator.setSaveAddressWays(true);
|
||||
creator.setSaveAddressWays(false);
|
||||
creator.setNormalizeStreets(true);
|
||||
// creator.setIndexPOI(true);
|
||||
// creator.setIndexTransport(true);
|
||||
|
|
|
|||
|
|
@ -14,11 +14,20 @@ message OsmAndStructure {
|
|||
repeated OsmAndMapIndex mapIndex = 2;
|
||||
// encoded as fixed32 length delimited
|
||||
repeated OsmAndAddressIndex addressIndex = 3;
|
||||
// encoded as fixed32 length delimited
|
||||
repeated OsmAndTransportIndex transportIndex = 4;
|
||||
|
||||
// last field should version again (to check consistency)
|
||||
required uint32 versionConfirm = 32;
|
||||
}
|
||||
|
||||
/**
|
||||
String table, contains the common strings in each block.
|
||||
*/
|
||||
message StringTable {
|
||||
repeated string s = 1;
|
||||
}
|
||||
|
||||
|
||||
message OsmAndMapIndex {
|
||||
// encoded as fixed32 length delimited
|
||||
|
|
@ -51,16 +60,9 @@ message MapTree {
|
|||
repeated MapTree subtrees = 7;
|
||||
|
||||
repeated MapData leafs = 8;
|
||||
|
||||
}
|
||||
|
||||
// These messages could be read directly
|
||||
/**
|
||||
String table, contains the common strings in each block.
|
||||
*/
|
||||
message StringTable {
|
||||
repeated string s = 1;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/// Simple messages
|
||||
|
|
@ -76,7 +78,7 @@ message MapData {
|
|||
optional int32 highwayMeta = 6;
|
||||
}
|
||||
|
||||
|
||||
/// Address messages
|
||||
|
||||
message OsmAndAddressIndex {
|
||||
|
||||
|
|
@ -120,7 +122,6 @@ message CityIndex {
|
|||
}
|
||||
|
||||
message InteresectedStreets {
|
||||
|
||||
repeated StreetIntersection intersections = 5;
|
||||
}
|
||||
|
||||
|
|
@ -163,3 +164,67 @@ message BuildingIndex {
|
|||
required sint32 y = 4; // delta encoded to street 24 zoom
|
||||
|
||||
}
|
||||
|
||||
// transport messages
|
||||
|
||||
message TransportRoute {
|
||||
required uint64 id = 1;
|
||||
|
||||
optional uint32 type = 3; // reference in string table
|
||||
optional uint32 operator = 4; // reference in string table
|
||||
optional string ref = 5;
|
||||
optional uint32 name = 6; // reference in string table
|
||||
optional uint32 name_en = 7; // reference in string table
|
||||
optional uint32 distance = 8; // distance in meters
|
||||
|
||||
repeated TransportRouteStop directStops = 15;
|
||||
repeated TransportRouteStop reverseStops = 16;
|
||||
}
|
||||
|
||||
message TransportRouteStop {
|
||||
required sint64 id = 1; // delta encoded to previous stop (first stop is delta to 0)
|
||||
required sint32 dx = 2; // delta encoded to previous stop (24 zoom)
|
||||
required sint32 dy = 3; // delta encoded to previous stop (24 zoom)
|
||||
|
||||
required uint32 name = 6; // index in message table
|
||||
optional uint32 name_en = 7; // index in message table
|
||||
}
|
||||
|
||||
message TransportStop {
|
||||
required sint64 id = 1; // delta encoded to parent base id
|
||||
required sint32 dx = 2; // delta encoded to parent (24 zoom)
|
||||
required sint32 dy = 3; // delta encoded to parent (24 zoom)
|
||||
|
||||
required uint32 name = 6; // index in message table
|
||||
optional uint32 name_en = 7; // index in message table
|
||||
|
||||
repeated uint32 routes = 16; // -shift to transport route containing that stop
|
||||
// (TransportStop.Message.start - routes[i] = TransportRoute.Message.start)
|
||||
|
||||
}
|
||||
|
||||
message TransportStopsTree {
|
||||
required sint32 left = 1; // delta encoded (24 zoom)
|
||||
required sint32 right = 2; // delta encoded (24 zoom)
|
||||
required sint32 top = 3; // delta encoded (24 zoom)
|
||||
required sint32 bottom = 4; // delta encoded (24 zoom)
|
||||
|
||||
// encoded as fixed32 length delimited
|
||||
repeated TransportStopsTree subtrees = 7;
|
||||
|
||||
repeated TransportStop leafs = 8;
|
||||
|
||||
// written as last
|
||||
optional uint64 baseId = 16;
|
||||
}
|
||||
|
||||
message OsmAndTransportIndex {
|
||||
|
||||
repeated TransportRoute routes = 3; // routes
|
||||
|
||||
// encoded as fixed32 length delimited
|
||||
optional TransportStopsTree stops = 6;
|
||||
|
||||
|
||||
required StringTable stringTable = 9;
|
||||
}
|
||||
|
|
@ -81,6 +81,8 @@ public class Pack
|
|||
return 2;
|
||||
}
|
||||
}
|
||||
|
||||
public final int BUFFER_SIZE = 8192;
|
||||
private int packTree(Element[] elmts, RTree rtree, String newFile)
|
||||
{
|
||||
try{
|
||||
|
|
@ -112,10 +114,11 @@ public class Pack
|
|||
//overwrite the old rtree file with the temp file
|
||||
FileInputStream fis=new FileInputStream(tmpPckFile);
|
||||
FileOutputStream fos=new FileOutputStream(fo);
|
||||
int i=fis.available();
|
||||
byte b[]=new byte[i];
|
||||
while((i=fis.read(b))!=-1)
|
||||
fos.write(b);
|
||||
byte b[]=new byte[BUFFER_SIZE];
|
||||
int i;
|
||||
while((i=fis.read(b))!=-1){
|
||||
fos.write(b, 0, i);
|
||||
}
|
||||
fos.close();
|
||||
fis.close();
|
||||
rFile.close();
|
||||
|
|
|
|||
Loading…
Reference in a new issue