Android-builders-club/OsmAnd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Refactoring IndexCreator (2)

Browse source
This commit is contained in:
Victor Shcherb 2011-05-02 00:36:19 +02:00
parent a45a479be2
commit bcbee2496c
8 changed files with 779 additions and 647 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

76
DataExtractionOSM/src/net/osmand/data/MapAlgorithms.java Normal file
View file

@ -0,0 +1,76 @@
package net.osmand.data;
import java.util.ArrayList;
import java.util.List;
import net.osmand.osm.MapUtils;
import net.osmand.osm.Node;
import net.osmand.osm.Way;
public class MapAlgorithms {
public static void simplifyDouglasPeucker(List<Node> n, int zoom, int epsilon, Way w){
ArrayList<Integer> l = new ArrayList<Integer>();
int first = 0;
while(first < n.size()){
if(n.get(first) != null){
break;
}
first++;
}
int last = n.size() - 1;
while (last >= 0) {
if (n.get(last) != null) {
break;
}
last--;
}
if(last - first < 1){
return;
}
boolean cycle = n.get(first).getId() == n.get(last).getId();
simplifyDouglasPeucker(n, zoom, epsilon, l, first, cycle ? last - 1: last);
w.addNode(n.get(first));
for (int i = 0; i < l.size(); i++) {
w.addNode(n.get(l.get(i)));
}
if (cycle) {
w.addNode(n.get(first));
}
}
private static void simplifyDouglasPeucker(List<Node> n, int zoom, int epsilon, List<Integer> ints, int start, int end){
double dmax = -1;
int index = -1;
for (int i = start + 1; i <= end - 1; i++) {
if(n.get(i) == null){
continue;
}
double d = orthogonalDistance(zoom, n.get(start), n.get(end), n.get(i));// calculate distance from line
if (d > dmax) {
dmax = d;
index = i;
}
}
if(dmax >= epsilon){
simplifyDouglasPeucker(n, zoom, epsilon, ints, start, index);
simplifyDouglasPeucker(n, zoom, epsilon, ints, index, end);
} else {
ints.add(end);
}
}
private static double orthogonalDistance(int zoom, Node nodeLineStart, Node nodeLineEnd, Node node) {
double x1 = MapUtils.getTileNumberX(zoom, nodeLineStart.getLongitude());
double y1 = MapUtils.getTileNumberY(zoom, nodeLineStart.getLatitude());
double x2 = MapUtils.getTileNumberX(zoom, nodeLineEnd.getLongitude());
double y2 = MapUtils.getTileNumberY(zoom, nodeLineEnd.getLatitude());
double x = MapUtils.getTileNumberX(zoom, node.getLongitude());
double y = MapUtils.getTileNumberY(zoom, node.getLatitude());
double A = x - x1;
double B = y - y1;
double C = x2 - x1;
double D = y2 - y1;
return Math.abs(A * D - C * B) / Math.sqrt(C * C + D * D);
}
}

37
DataExtractionOSM/src/net/osmand/data/index/DataIndexWriter.java
View file

@ -38,41 +38,6 @@ public class DataIndexWriter {
private static final int BATCH_SIZE = 1000; private static final int BATCH_SIZE = 1000;
public static void insertAmenityIntoPoi(PreparedStatement prep, Map<PreparedStatement, Integer> map, Amenity amenity, int batchSize) throws SQLException {
assert IndexConstants.POI_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
prep.setLong(1, amenity.getId());
prep.setInt(2, MapUtils.get31TileNumberX(amenity.getLocation().getLongitude()));
prep.setInt(3, MapUtils.get31TileNumberY(amenity.getLocation().getLatitude()));
prep.setString(4, amenity.getEnName());
prep.setString(5, amenity.getName());
prep.setString(6, AmenityType.valueToString(amenity.getType()));
prep.setString(7, amenity.getSubType());
prep.setString(8, amenity.getOpeningHours());
prep.setString(9, amenity.getSite());
prep.setString(10, amenity.getPhone());
addBatch(map, prep, batchSize);
}
public static PreparedStatement createStatementAmenityInsert(Connection conn) throws SQLException{
return conn.prepareStatement("INSERT INTO " + IndexConstants.POI_TABLE + "(id, x, y, name_en, name, type, subtype, opening_hours, site, phone) " + //$NON-NLS-1$//$NON-NLS-2$
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
}
public static void createPoiIndexStructure(Connection conn, DBDialect dialect) throws SQLException{
Statement stat = conn.createStatement();
stat.executeUpdate("create table " + IndexConstants.POI_TABLE + //$NON-NLS-1$
"(id bigint, x int, y int, name_en varchar(255), name varchar(255), " +
"type varchar(255), subtype varchar(255), opening_hours varchar(255), phone varchar(255), site varchar(255)," +
"primary key(id, type, subtype))");
stat.executeUpdate("create index poi_loc on poi (x, y, type, subtype)");
stat.executeUpdate("create index poi_id on poi (id, type, subtype)");
if(dialect == DBDialect.SQLITE){
stat.execute("PRAGMA user_version = " + IndexConstants.POI_TABLE_VERSION); //$NON-NLS-1$
}
stat.close();
}
public static PreparedStatement getStreetNodeInsertPreparedStatement(Connection conn) throws SQLException { public static PreparedStatement getStreetNodeInsertPreparedStatement(Connection conn) throws SQLException {
assert IndexConstants.STREET_NODE_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ assert IndexConstants.STREET_NODE_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
@ -349,7 +314,7 @@ public class DataIndexWriter {
} }
} }
} }
private static void addBatch(Map<PreparedStatement, Integer> count, PreparedStatement p) throws SQLException { public static void addBatch(Map<PreparedStatement, Integer> count, PreparedStatement p) throws SQLException {
addBatch(count, p, BATCH_SIZE, true); addBatch(count, p, BATCH_SIZE, true);
} }

475
DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java
View file

@ -21,7 +21,6 @@ import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator; import java.util.Comparator;
import java.util.HashSet; import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.LinkedHashSet; import java.util.LinkedHashSet;
import java.util.List; import java.util.List;
@ -32,17 +31,17 @@ import java.util.TreeMap;
import net.osmand.Algoritms; import net.osmand.Algoritms;
import net.osmand.IProgress; import net.osmand.IProgress;
import net.osmand.binary.BinaryMapIndexWriter; import net.osmand.binary.BinaryMapIndexWriter;
import net.osmand.data.Amenity;
import net.osmand.data.Boundary; import net.osmand.data.Boundary;
import net.osmand.data.Building; import net.osmand.data.Building;
import net.osmand.data.City; import net.osmand.data.City;
import net.osmand.data.DataTileManager; import net.osmand.data.DataTileManager;
import net.osmand.data.MapAlgorithms;
import net.osmand.data.Street; import net.osmand.data.Street;
import net.osmand.data.TransportRoute;
import net.osmand.data.City.CityType; import net.osmand.data.City.CityType;
import net.osmand.data.index.DataIndexReader; import net.osmand.data.index.DataIndexReader;
import net.osmand.data.index.DataIndexWriter; import net.osmand.data.index.DataIndexWriter;
import net.osmand.data.index.IndexConstants; import net.osmand.data.index.IndexConstants;
import net.osmand.data.preparation.OsmDbAccessor.OsmDbVisitor;
import net.osmand.impl.ConsoleProgressImplementation; import net.osmand.impl.ConsoleProgressImplementation;
import net.osmand.osm.Entity; import net.osmand.osm.Entity;
import net.osmand.osm.LatLon; import net.osmand.osm.LatLon;
@ -93,9 +92,6 @@ public class IndexCreator {
public static final int BATCH_SIZE_OSM = 10000; public static final int BATCH_SIZE_OSM = 10000;
public static final String TEMP_NODES_DB = "nodes.tmp.odb"; public static final String TEMP_NODES_DB = "nodes.tmp.odb";
public static final int STEP_CITY_NODES = 1;
public static final int STEP_ADDRESS_RELATIONS_AND_MULTYPOLYGONS = 2;
public static final int STEP_BORDER_CITY_WAYS = 3;
public static final int STEP_MAIN = 4; public static final int STEP_MAIN = 4;
private File workingDir = null; private File workingDir = null;
@ -113,13 +109,10 @@ public class IndexCreator {
private String mapFileName = null; private String mapFileName = null;
private Long lastModifiedDate = null; private Long lastModifiedDate = null;
private PreparedStatement pselectNode;
private PreparedStatement pselectWay;
private PreparedStatement pselectRelation;
private PreparedStatement pselectTags;
private IndexTransportCreator indexTransportCreator; private IndexTransportCreator indexTransportCreator;
private IndexPoiCreator indexPoiCreator;
private OsmDbAccessor accessor;
// constants to start process from the middle and save temporary results // constants to start process from the middle and save temporary results
private boolean recreateOnlyBinaryFile = false; // false; private boolean recreateOnlyBinaryFile = false; // false;
private boolean deleteOsmDB = false; private boolean deleteOsmDB = false;
@ -130,10 +123,6 @@ public class IndexCreator {
Map<PreparedStatement, Integer> pStatements = new LinkedHashMap<PreparedStatement, Integer>(); Map<PreparedStatement, Integer> pStatements = new LinkedHashMap<PreparedStatement, Integer>();
private Connection poiConnection;
private File poiIndexFile;
private PreparedStatement poiPreparedStatement;
private File mapFile; private File mapFile;
private RandomAccessFile mapRAFile; private RandomAccessFile mapRAFile;
private Connection mapConnection; private Connection mapConnection;
@ -189,6 +178,8 @@ public class IndexCreator {
public IndexCreator(File workingDir) { public IndexCreator(File workingDir) {
this.workingDir = workingDir; this.workingDir = workingDir;
this.indexTransportCreator = new IndexTransportCreator(this); this.indexTransportCreator = new IndexTransportCreator(this);
this.indexPoiCreator = new IndexPoiCreator(this);
this.accessor = new OsmDbAccessor(this);
} }
public void setIndexAddress(boolean indexAddress) { public void setIndexAddress(boolean indexAddress) {
@ -241,12 +232,12 @@ public class IndexCreator {
} }
private Connection getDatabaseConnection(String fileName) throws SQLException { private Connection getDatabaseConnection(String fileName) throws SQLException {
return getDatabaseConnection(fileName, false); return getDatabaseConnection(fileName, dialect);
} }
private Connection getDatabaseConnection(String fileName, boolean forceSqLite) throws SQLException { protected Connection getDatabaseConnection(String fileName, DBDialect dialect) throws SQLException {
if (DBDialect.SQLITE == dialect || forceSqLite) { if (DBDialect.SQLITE == dialect) {
try { try {
Class.forName("org.sqlite.JDBC"); Class.forName("org.sqlite.JDBC");
} catch (ClassNotFoundException e) { } catch (ClassNotFoundException e) {
@ -283,111 +274,6 @@ public class IndexCreator {
} }
public void loadEntityData(Entity e, boolean loadTags) throws SQLException {
if (e instanceof Node || (e instanceof Way && !((Way) e).getNodes().isEmpty())) {
// do not load tags for nodes inside way
return;
}
Map<EntityId, Entity> map = new LinkedHashMap<EntityId, Entity>();
if (e instanceof Relation && ((Relation) e).getMemberIds().isEmpty()) {
pselectRelation.setLong(1, e.getId());
if (pselectRelation.execute()) {
ResultSet rs = pselectRelation.getResultSet();
boolean first = true;
while (rs.next()) {
int ord = rs.getInt(4);
if (ord > 0 || first) {
first = false;
((Relation) e).addMember(rs.getLong(1), EntityType.values()[rs.getInt(2)], rs.getString(3));
}
}
rs.close();
}
} else if (e instanceof Way && ((Way) e).getEntityIds().isEmpty()) {
pselectWay.setLong(1, e.getId());
if (pselectWay.execute()) {
ResultSet rs = pselectWay.getResultSet();
boolean first = true;
while (rs.next()) {
int ord = rs.getInt(2);
if (ord > 0 || first) {
first = false;
((Way) e).addNode(new Node(rs.getDouble(5), rs.getDouble(6), rs.getLong(1)));
}
}
rs.close();
}
}
Collection<EntityId> ids = e instanceof Relation ? ((Relation) e).getMemberIds() : ((Way) e).getEntityIds();
for (EntityId i : ids) {
// pselectNode = dbConn.prepareStatement("select n.latitude, n.longitude, t.skeys, t.value from node n left join tags t on n.id = t.id and t.type = 0 where n.id = ?");
if (i.getType() == EntityType.NODE) {
pselectNode.setLong(1, i.getId());
if (pselectNode.execute()) {
ResultSet rs = pselectNode.getResultSet();
Node n = null;
while (rs.next()) {
if (n == null) {
n = new Node(rs.getDouble(1), rs.getDouble(2), i.getId());
}
if (rs.getObject(3) != null) {
n.putTag(rs.getString(3), rs.getString(4));
}
}
map.put(i, n);
rs.close();
}
} else if (i.getType() == EntityType.WAY) {
// pselectWay = dbConn.prepareStatement("select w.node, w.ord, t.skeys, t.value, n.latitude, n.longitude " +
// "from ways w left join tags t on w.id = t.id and t.type = 1 and w.ord = 0 inner join node n on w.node = n.id " +
// "where w.id = ? order by w.ord");
pselectWay.setLong(1, i.getId());
if (pselectWay.execute()) {
ResultSet rs = pselectWay.getResultSet();
Way way = new Way(i.getId());
map.put(i, way);
boolean first = true;
while (rs.next()) {
int ord = rs.getInt(2);
if (ord > 0 || first) {
first = false;
way.addNode(new Node(rs.getDouble(5), rs.getDouble(6), rs.getLong(1)));
}
if (ord == 0 && rs.getObject(3) != null) {
way.putTag(rs.getString(3), rs.getString(4));
}
}
rs.close();
}
} else if (i.getType() == EntityType.RELATION) {
pselectRelation.setLong(1, i.getId());
// pselectRelation = dbConn.prepareStatement("select r.member, r.type, r.role, r.ord, t.skeys, t.value" +
// "from relations r left join tags t on r.id = t.id and t.type = 2 and r.ord = 0 " +
// "where r.id = ? order by r.ord");
if (pselectRelation.execute()) {
ResultSet rs = pselectRelation.getResultSet();
Relation rel = new Relation(i.getId());
map.put(i, rel);
boolean first = true;
while (rs.next()) {
int ord = rs.getInt(4);
if (ord > 0 || first) {
first = false;
rel.addMember(rs.getLong(1), EntityType.values()[rs.getInt(2)], rs.getString(3));
}
if (ord == 0 && rs.getObject(5) != null) {
rel.putTag(rs.getString(5), rs.getString(6));
}
}
// do not load relation members recursively ? It is not needed for transport, address, poi before
rs.close();
}
}
}
e.initializeLinks(map);
}
public void setPoiFileName(String poiFileName) { public void setPoiFileName(String poiFileName) {
this.poiFileName = poiFileName; this.poiFileName = poiFileName;
@ -427,85 +313,6 @@ public class IndexCreator {
return poiFileName; return poiFileName;
} }
public int iterateOverEntities(IProgress progress, EntityType type, int allCount, int step) throws SQLException {
Statement statement = dbConn.createStatement();
String select;
int count = 0;
// stat.executeUpdate("create table tags (id "+longType+", type smallint, skeys varchar(255), value varchar(255))");
// stat.executeUpdate("create table ways (id "+longType+", node "+longType+", ord smallint)");
// stat.executeUpdate("create table relations (id "+longType+", member "+longType+", type smallint, role varchar(255), ord smallint)");
if (type == EntityType.NODE) {
// filter out all nodes without tags
select = "select n.id, n.latitude, n.longitude, t.skeys, t.value from node n inner join tags t on n.id = t.id and t.type = 0 order by n.id"; //$NON-NLS-1$
} else if (type == EntityType.WAY) {
select = "select w.id, w.node, w.ord, t.skeys, t.value, n.latitude, n.longitude " + //$NON-NLS-1$
"from ways w left join tags t on w.id = t.id and t.type = 1 and w.ord = 0 inner join node n on w.node = n.id " + //$NON-NLS-1$
"order by w.id, w.ord"; //$NON-NLS-1$
} else {
select = "select r.id, t.skeys, t.value from relations r inner join tags t on t.id = r.id and t.type = 2 and r.ord = 0"; //$NON-NLS-1$
}
ResultSet rs = statement.executeQuery(select);
Entity prevEntity = null;
long prevId = -1;
while (rs.next()) {
long curId = rs.getLong(1);
boolean newEntity = curId != prevId;
Entity e = prevEntity;
if (type == EntityType.NODE) {
if (newEntity) {
e = new Node(rs.getDouble(2), rs.getDouble(3), curId);
}
e.putTag(rs.getString(4), rs.getString(5));
} else if (type == EntityType.WAY) {
if (newEntity) {
e = new Way(curId);
}
int ord = rs.getInt(3);
if (ord == 0 && rs.getObject(4) != null) {
e.putTag(rs.getString(4), rs.getString(5));
}
if (newEntity || ord > 0) {
((Way) e).addNode(new Node(rs.getDouble(6), rs.getDouble(7), rs.getLong(2)));
}
} else {
if (newEntity) {
e = new Relation(curId);
}
e.putTag(rs.getString(2), rs.getString(3));
}
if (newEntity) {
count++;
if (progress != null) {
progress.progress(1);
}
if (prevEntity != null) {
iterateEntity(prevEntity, step);
}
prevEntity = e;
}
prevId = curId;
}
if (prevEntity != null) {
count++;
iterateEntity(prevEntity, step);
}
rs.close();
return count;
}
protected void loadEntityTags(EntityType type, Entity e) throws SQLException {
pselectTags.setLong(1, e.getId());
pselectTags.setByte(2, (byte) type.ordinal());
ResultSet rsTags = pselectTags.executeQuery();
while (rsTags.next()) {
e.putTag(rsTags.getString(1), rsTags.getString(2));
}
rsTags.close();
}
public String getCityAdminLevel() { public String getCityAdminLevel() {
@ -516,13 +323,13 @@ public class IndexCreator {
this.cityAdminLevel = cityAdminLevel; this.cityAdminLevel = cityAdminLevel;
} }
public void indexBoundariesRelation(Entity e) throws SQLException { public void indexBoundariesRelation(Entity e, OsmDbAccessorContext ctx) throws SQLException {
String adminLevel = e.getTag("admin_level"); String adminLevel = e.getTag("admin_level");
Boundary boundary = null; Boundary boundary = null;
if (cityAdminLevel.equals(adminLevel)) { if (cityAdminLevel.equals(adminLevel)) {
if (e instanceof Relation) { if (e instanceof Relation) {
Relation i = (Relation) e; Relation i = (Relation) e;
loadEntityData(i, true); ctx.loadEntityData(i, true);
boundary = new Boundary(); boundary = new Boundary();
if (i.getTag(OSMTagKey.NAME) != null) { if (i.getTag(OSMTagKey.NAME) != null) {
boundary.setName(i.getTag(OSMTagKey.NAME)); boundary.setName(i.getTag(OSMTagKey.NAME));
@ -602,7 +409,7 @@ public class IndexCreator {
} }
} }
public void indexAddressRelation(Relation i) throws SQLException { public void indexAddressRelation(Relation i, OsmDbAccessorContext ctx) throws SQLException {
String type = i.getTag(OSMTagKey.ADDRESS_TYPE); String type = i.getTag(OSMTagKey.ADDRESS_TYPE);
boolean house = "house".equals(type); //$NON-NLS-1$ boolean house = "house".equals(type); //$NON-NLS-1$
boolean street = "a6".equals(type); //$NON-NLS-1$ boolean street = "a6".equals(type); //$NON-NLS-1$
@ -610,7 +417,7 @@ public class IndexCreator {
// try to find appropriate city/street // try to find appropriate city/street
City c = null; City c = null;
// load with member ways with their nodes and tags ! // load with member ways with their nodes and tags !
loadEntityData(i, true); ctx.loadEntityData(i, true);
Collection<Entity> members = i.getMembers("is_in"); //$NON-NLS-1$ Collection<Entity> members = i.getMembers("is_in"); //$NON-NLS-1$
Relation a3 = null; Relation a3 = null;
@ -620,7 +427,7 @@ public class IndexCreator {
a6 = i; a6 = i;
} }
Entity in = members.iterator().next(); Entity in = members.iterator().next();
loadEntityData(in, true); ctx.loadEntityData(in, true);
if (in instanceof Relation) { if (in instanceof Relation) {
// go one level up for house // go one level up for house
if (house) { if (house) {
@ -628,7 +435,7 @@ public class IndexCreator {
members = ((Relation) in).getMembers("is_in"); //$NON-NLS-1$ members = ((Relation) in).getMembers("is_in"); //$NON-NLS-1$
if (!members.isEmpty()) { if (!members.isEmpty()) {
in = members.iterator().next(); in = members.iterator().next();
loadEntityData(in, true); ctx.loadEntityData(in, true);
if (in instanceof Relation) { if (in instanceof Relation) {
a3 = (Relation) in; a3 = (Relation) in;
} }
@ -884,63 +691,20 @@ public class IndexCreator {
return foundId; return foundId;
} }
private List<Amenity> tempAmenityList = new ArrayList<Amenity>();
public void checkEntity(Entity e){
String name = e.getTag(OSMTagKey.NAME);
if (name == null){
String msg = "";
Collection<String> keys = e.getTagKeySet();
int cnt = 0;
for (Iterator iter = keys.iterator(); iter.hasNext();) {
String key = (String) iter.next();
if (key.startsWith("name:") && key.length() <= 8) {
// ignore specialties like name:botanical
if (cnt == 0)
msg += "Entity misses default name tag, but it has localized name tag(s):\n";
msg += key + "=" + e.getTag(key) + "\n";
cnt++;
}
}
if (cnt > 0) {
msg += "Consider adding the name tag at " + e.getOsmUrl();
log.warn(msg);
}
}
}
private void iterateEntity(Entity e, int step) throws SQLException {
if (step == STEP_MAIN) { private void iterateMainEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException {
if (indexPOI) { if (indexPOI) {
tempAmenityList.clear(); indexPoiCreator.iterateEntity(e, pStatements);
tempAmenityList = Amenity.parseAmenities(e, tempAmenityList);
if (!tempAmenityList.isEmpty() && poiPreparedStatement != null) {
// load data for way (location etc...)
loadEntityData(e, false);
for (Amenity a : tempAmenityList) {
checkEntity(e);
a.setEntity(e);
if (a.getLocation() != null) {
// do not convert english name
// convertEnglishName(a);
DataIndexWriter.insertAmenityIntoPoi(poiPreparedStatement, pStatements, a, BATCH_SIZE);
}
}
}
} }
if (indexTransport) { if (indexTransport) {
if (e instanceof Relation && e.getTag(OSMTagKey.ROUTE) != null) { indexTransportCreator.visitEntityMainStep(e, ctx, pStatements);
loadEntityData(e, true);
TransportRoute route = indexTransportCreator.indexTransportRoute((Relation) e);
if (route != null) {
indexTransportCreator.insertTransportIntoIndex(route, pStatements);
}
}
} }
if (indexMap && (e instanceof Way || e instanceof Node)) { if (indexMap && (e instanceof Way || e instanceof Node)) {
// manipulate what kind of way to load // manipulate what kind of way to load
loadEntityData(e, false); ctx.loadEntityData(e, false);
boolean inverse = "-1".equals(e.getTag(OSMTagKey.ONEWAY)); //$NON-NLS-1$ boolean inverse = "-1".equals(e.getTag(OSMTagKey.ONEWAY)); //$NON-NLS-1$
for (int i = 0; i < mapZooms.size(); i++) { for (int i = 0; i < mapZooms.size(); i++) {
writeBinaryEntityToMapDatabase(e, e.getId(), i == 0 ? inverse : false, i); writeBinaryEntityToMapDatabase(e, e.getId(), i == 0 ? inverse : false, i);
@ -964,7 +728,7 @@ public class IndexCreator {
} }
if (!exist) { if (!exist) {
loadEntityData(e, false); ctx.loadEntityData(e, false);
LatLon l = e.getLatLon(); LatLon l = e.getLatLon();
City city = getClosestCity(l); City city = getClosestCity(l);
Long idStreet = getStreetInCity(city, e.getTag(OSMTagKey.ADDR_STREET), l, (e.getId() << 2)); Long idStreet = getStreetInCity(city, e.getTag(OSMTagKey.ADDR_STREET), l, (e.getId() << 2));
@ -994,7 +758,7 @@ public class IndexCreator {
// check that street way is not registered already // check that street way is not registered already
if (!exist) { if (!exist) {
loadEntityData(e, false); ctx.loadEntityData(e, false);
LatLon l = e.getLatLon(); LatLon l = e.getLatLon();
City city = getClosestCity(l); City city = getClosestCity(l);
Long idStreet = getStreetInCity(city, e.getTag(OSMTagKey.NAME), l, (e.getId() << 2) | 1); Long idStreet = getStreetInCity(city, e.getTag(OSMTagKey.NAME), l, (e.getId() << 2) | 1);
@ -1005,25 +769,21 @@ public class IndexCreator {
} }
if (e instanceof Relation) { if (e instanceof Relation) {
if (e.getTag(OSMTagKey.POSTAL_CODE) != null) { if (e.getTag(OSMTagKey.POSTAL_CODE) != null) {
loadEntityData(e, false); ctx.loadEntityData(e, false);
postalCodeRelations.add((Relation) e); postalCodeRelations.add((Relation) e);
} }
} }
} }
} else if(step == STEP_BORDER_CITY_WAYS) {
if (indexAddress) {
if (e instanceof Way && "administrative".equals(e.getTag(OSMTagKey.BOUNDARY))) { //$NON-NLS-1$
indexBoundariesRelation(e);
} }
}
} else if (step == STEP_ADDRESS_RELATIONS_AND_MULTYPOLYGONS) { public void indexAddressRelationsAndMultiPolygons(Entity e, OsmDbAccessorContext ctx) throws SQLException {
if (indexAddress) { if (indexAddress) {
if (e instanceof Relation && "address".equals(e.getTag(OSMTagKey.TYPE))) { //$NON-NLS-1$ if (e instanceof Relation && "address".equals(e.getTag(OSMTagKey.TYPE))) { //$NON-NLS-1$
indexAddressRelation((Relation) e); indexAddressRelation((Relation) e, ctx);
} }
if (e instanceof Relation && "administrative".equals(e.getTag(OSMTagKey.BOUNDARY))) { //$NON-NLS-1$ if (e instanceof Relation && "administrative".equals(e.getTag(OSMTagKey.BOUNDARY))) { //$NON-NLS-1$
indexBoundariesRelation((Relation) e); indexBoundariesRelation((Relation) e, ctx);
} }
} }
if (indexMap && e instanceof Relation && "restriction".equals(e.getTag(OSMTagKey.TYPE))) { //$NON-NLS-1$ if (indexMap && e instanceof Relation && "restriction".equals(e.getTag(OSMTagKey.TYPE))) { //$NON-NLS-1$
@ -1046,7 +806,7 @@ public class IndexCreator {
type = MapRenderingTypes.RESTRICTION_ONLY_STRAIGHT_ON; type = MapRenderingTypes.RESTRICTION_ONLY_STRAIGHT_ON;
} }
if (type != -1) { if (type != -1) {
loadEntityData(e, true); ctx.loadEntityData(e, true);
Collection<EntityId> fromL = ((Relation) e).getMemberIds("from"); //$NON-NLS-1$ Collection<EntityId> fromL = ((Relation) e).getMemberIds("from"); //$NON-NLS-1$
Collection<EntityId> toL = ((Relation) e).getMemberIds("to"); //$NON-NLS-1$ Collection<EntityId> toL = ((Relation) e).getMemberIds("to"); //$NON-NLS-1$
if (!fromL.isEmpty() && !toL.isEmpty()) { if (!fromL.isEmpty() && !toL.isEmpty()) {
@ -1063,7 +823,7 @@ public class IndexCreator {
} }
} }
if (indexMap && e instanceof Relation && "multipolygon".equals(e.getTag(OSMTagKey.TYPE))) { //$NON-NLS-1$ if (indexMap && e instanceof Relation && "multipolygon".equals(e.getTag(OSMTagKey.TYPE))) { //$NON-NLS-1$
loadEntityData(e, true); ctx.loadEntityData(e, true);
Map<Entity, String> entities = ((Relation) e).getMemberEntities(); Map<Entity, String> entities = ((Relation) e).getMemberEntities();
boolean outerFound = false; boolean outerFound = false;
@ -1139,9 +899,7 @@ public class IndexCreator {
} }
} }
} else if (step == STEP_CITY_NODES) {
registerCityIfNeeded(e);
}
} }
public void combineMultiPolygons(Way w, List<List<Way>> completedRings, List<List<Way>> incompletedRings) { public void combineMultiPolygons(Way w, List<List<Way>> completedRings, List<List<Way>> incompletedRings) {
@ -1464,7 +1222,7 @@ public class IndexCreator {
return null; return null;
} }
simplifyDouglasPeucker(nodes, zoom + 8, 3, way); MapAlgorithms.simplifyDouglasPeucker(nodes, zoom + 8, 3, way);
if (way.getNodes().size() < 2) { if (way.getNodes().size() < 2) {
return null; return null;
} }
@ -1592,7 +1350,7 @@ public class IndexCreator {
if (!skip) { if (!skip) {
Way newWs = new Way(id); Way newWs = new Way(id);
simplifyDouglasPeucker(wNodes, zoom - 1 + 8, 3, newWs); MapAlgorithms.simplifyDouglasPeucker(wNodes, zoom - 1 + 8, 3, newWs);
int type = decodeTypesFromOneLong(ltype); int type = decodeTypesFromOneLong(ltype);
DataIndexWriter.insertBinaryMapRenderObjectIndex(pStatements, mapBinaryStat, mapTree[level], newWs, name, DataIndexWriter.insertBinaryMapRenderObjectIndex(pStatements, mapBinaryStat, mapTree[level], newWs, name,
@ -1603,7 +1361,6 @@ public class IndexCreator {
} }
private boolean checkForSmallAreas(List<Node> nodes, int zoom, int minz, int maxz) { private boolean checkForSmallAreas(List<Node> nodes, int zoom, int minz, int maxz) {
int minX = Integer.MAX_VALUE; int minX = Integer.MAX_VALUE;
int maxX = Integer.MIN_VALUE; int maxX = Integer.MIN_VALUE;
@ -1629,70 +1386,7 @@ public class IndexCreator {
} }
private void simplifyDouglasPeucker(List<Node> n, int zoom, int epsilon, Way w){
ArrayList<Integer> l = new ArrayList<Integer>();
int first = 0;
while(first < n.size()){
if(n.get(first) != null){
break;
}
first++;
}
int last = n.size() - 1;
while (last >= 0) {
if (n.get(last) != null) {
break;
}
last--;
}
if(last - first < 1){
return;
}
boolean cycle = n.get(first).getId() == n.get(last).getId();
simplifyDouglasPeucker(n, zoom, epsilon, l, first, cycle ? last - 1: last);
w.addNode(n.get(first));
for (int i = 0; i < l.size(); i++) {
w.addNode(n.get(l.get(i)));
}
if (cycle) {
w.addNode(n.get(first));
}
}
private void simplifyDouglasPeucker(List<Node> n, int zoom, int epsilon, List<Integer> ints, int start, int end){
double dmax = -1;
int index = -1;
for (int i = start + 1; i <= end - 1; i++) {
if(n.get(i) == null){
continue;
}
double d = orthogonalDistance(zoom, n.get(start), n.get(end), n.get(i));// calculate distance from line
if (d > dmax) {
dmax = d;
index = i;
}
}
if(dmax >= epsilon){
simplifyDouglasPeucker(n, zoom, epsilon, ints, start, index);
simplifyDouglasPeucker(n, zoom, epsilon, ints, index, end);
} else {
ints.add(end);
}
}
private double orthogonalDistance(int zoom, Node nodeLineStart, Node nodeLineEnd, Node node) {
double x1 = MapUtils.getTileNumberX(zoom, nodeLineStart.getLongitude());
double y1 = MapUtils.getTileNumberY(zoom, nodeLineStart.getLatitude());
double x2 = MapUtils.getTileNumberX(zoom, nodeLineEnd.getLongitude());
double y2 = MapUtils.getTileNumberY(zoom, nodeLineEnd.getLatitude());
double x = MapUtils.getTileNumberX(zoom, node.getLongitude());
double y = MapUtils.getTileNumberY(zoom, node.getLatitude());
double A = x - x1;
double B = y - y1;
double C = x2 - x1;
double D = y2 - y1;
return Math.abs(A * D - C * B) / Math.sqrt(C * C + D * D);
}
public boolean nodeIsLastSubTree(RTree tree, long ptr) throws RTreeException { public boolean nodeIsLastSubTree(RTree tree, long ptr) throws RTreeException {
rtree.Node parent = tree.getReadNode(ptr); rtree.Node parent = tree.getReadNode(ptr);
@ -1983,15 +1677,7 @@ public class IndexCreator {
allRelations = filter.getAllRelations(); allRelations = filter.getAllRelations();
} }
} }
accessor.initDatabase(dbConn);
pselectNode = dbConn.prepareStatement("select n.latitude, n.longitude, t.skeys, t.value from node n left join tags t on n.id = t.id and t.type = 0 where n.id = ?"); //$NON-NLS-1$
pselectWay = dbConn.prepareStatement("select w.node, w.ord, t.skeys, t.value, n.latitude, n.longitude " + //$NON-NLS-1$
"from ways w left join tags t on w.id = t.id and t.type = 1 and w.ord = 0 inner join node n on w.node = n.id " + //$NON-NLS-1$
"where w.id = ? order by w.ord"); //$NON-NLS-1$
pselectRelation = dbConn.prepareStatement("select r.member, r.type, r.role, r.ord, t.skeys, t.value " + //$NON-NLS-1$
"from relations r left join tags t on r.id = t.id and t.type = 2 and r.ord = 0 " + //$NON-NLS-1$
"where r.id = ? order by r.ord"); //$NON-NLS-1$
pselectTags = dbConn.prepareStatement("select skeys, value from tags where id = ? and type = ?"); //$NON-NLS-1$
// do not create temp map file and rtree files // do not create temp map file and rtree files
if (recreateOnlyBinaryFile) { if (recreateOnlyBinaryFile) {
@ -2026,7 +1712,13 @@ public class IndexCreator {
progress.setGeneralProgress("[40 / 100]"); //$NON-NLS-1$ progress.setGeneralProgress("[40 / 100]"); //$NON-NLS-1$
progress.startTask(Messages.getString("IndexCreator.INDEX_CITIES"), allNodes); //$NON-NLS-1$ progress.startTask(Messages.getString("IndexCreator.INDEX_CITIES"), allNodes); //$NON-NLS-1$
if (!loadFromPath) { if (!loadFromPath) {
allNodes = iterateOverEntities(progress, EntityType.NODE, allNodes, STEP_CITY_NODES); // load cities names
allNodes = accessor.iterateOverEntities(progress, EntityType.NODE, allNodes, new OsmDbVisitor() {
@Override
public void iterateEntity(Entity e, OsmDbAccessorContext ctx) {
registerCityIfNeeded(e);
}
});
} }
for (City c : cities.values()) { for (City c : cities.values()) {
@ -2045,12 +1737,23 @@ public class IndexCreator {
if (indexAddress || indexMap) { if (indexAddress || indexMap) {
progress.setGeneralProgress("[30 / 100]"); //$NON-NLS-1$ progress.setGeneralProgress("[30 / 100]"); //$NON-NLS-1$
progress.startTask(Messages.getString("IndexCreator.PREINDEX_ADRESS_MAP"), allRelations); //$NON-NLS-1$ progress.startTask(Messages.getString("IndexCreator.PREINDEX_ADRESS_MAP"), allRelations); //$NON-NLS-1$
allRelations = iterateOverEntities(progress, EntityType.RELATION, allRelations, allRelations = accessor.iterateOverEntities(progress, EntityType.RELATION, allRelations, new OsmDbVisitor() {
STEP_ADDRESS_RELATIONS_AND_MULTYPOLYGONS); @Override
public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException {
indexAddressRelationsAndMultiPolygons(e, ctx);
}
});
if (indexAddress) { if (indexAddress) {
progress.setGeneralProgress("[40 / 100]"); //$NON-NLS-1$ progress.setGeneralProgress("[40 / 100]"); //$NON-NLS-1$
progress.startTask(Messages.getString("IndexCreator.PREINDEX_ADRESS_MAP"), allWays); //$NON-NLS-1$ progress.startTask(Messages.getString("IndexCreator.PREINDEX_ADRESS_MAP"), allWays); //$NON-NLS-1$
allWays = iterateOverEntities(progress, EntityType.WAY, allWays, STEP_BORDER_CITY_WAYS); allWays = accessor.iterateOverEntities(progress, EntityType.WAY, allWays, new OsmDbVisitor() {
@Override
public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException {
if (e instanceof Way && "administrative".equals(e.getTag(OSMTagKey.BOUNDARY))) { //$NON-NLS-1$
indexBoundariesRelation(e ,ctx);
}
}
});
} }
// commit to put all cities // commit to put all cities
@ -2072,14 +1775,29 @@ public class IndexCreator {
if (indexPOI || indexAddress || indexMap) { if (indexPOI || indexAddress || indexMap) {
progress.setGeneralProgress("[50 / 100]"); progress.setGeneralProgress("[50 / 100]");
progress.startTask(Messages.getString("IndexCreator.PROCESS_OSM_NODES"), allNodes); progress.startTask(Messages.getString("IndexCreator.PROCESS_OSM_NODES"), allNodes);
iterateOverEntities(progress, EntityType.NODE, allNodes, STEP_MAIN); accessor.iterateOverEntities(progress, EntityType.NODE, allNodes, new OsmDbVisitor() {
@Override
public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException {
iterateMainEntity(e, ctx);
}
});
progress.setGeneralProgress("[70 / 100]"); progress.setGeneralProgress("[70 / 100]");
progress.startTask(Messages.getString("IndexCreator.PROCESS_OSM_WAYS"), allWays); progress.startTask(Messages.getString("IndexCreator.PROCESS_OSM_WAYS"), allWays);
iterateOverEntities(progress, EntityType.WAY, allWays, STEP_MAIN); accessor.iterateOverEntities(progress, EntityType.WAY, allWays, new OsmDbVisitor() {
@Override
public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException {
iterateMainEntity(e, ctx);
}
});
} }
progress.setGeneralProgress("[85 / 100]"); progress.setGeneralProgress("[85 / 100]");
progress.startTask(Messages.getString("IndexCreator.PROCESS_OSM_REL"), allRelations); progress.startTask(Messages.getString("IndexCreator.PROCESS_OSM_REL"), allRelations);
iterateOverEntities(progress, EntityType.RELATION, allRelations, STEP_MAIN); accessor.iterateOverEntities(progress, EntityType.RELATION, allRelations, new OsmDbVisitor() {
@Override
public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException {
iterateMainEntity(e, ctx);
}
});
// 3.4 combine all low level ways and simplify them // 3.4 combine all low level ways and simplify them
if(indexMap){ if(indexMap){
@ -2164,18 +1882,8 @@ public class IndexCreator {
throw e; throw e;
} finally { } finally {
try { try {
if (pselectNode != null) { accessor.closeReadingConnection();
pselectNode.close();
}
if (pselectWay != null) {
pselectWay.close();
}
if (pselectRelation != null) {
pselectRelation.close();
}
if (pselectTags != null) {
pselectTags.close();
}
for (PreparedStatement p : pStatements.keySet()) { for (PreparedStatement p : pStatements.keySet()) {
if (pStatements.get(p) > 0) { if (pStatements.get(p) > 0) {
p.executeBatch(); p.executeBatch();
@ -2183,14 +1891,7 @@ public class IndexCreator {
p.close(); p.close();
} }
if (poiConnection != null) { indexPoiCreator.commitAndClosePoiFile(lastModifiedDate);
poiConnection.commit();
poiConnection.close();
poiConnection = null;
if (lastModifiedDate != null) {
poiIndexFile.setLastModified(lastModifiedDate);
}
}
if (mapConnection != null) { if (mapConnection != null) {
mapConnection.commit(); mapConnection.commit();
@ -2224,19 +1925,6 @@ public class IndexCreator {
} }
} }
// delete transport rtree files
if (indexTransportCreator.transportStopsTree != null) {
indexTransportCreator.transportStopsTree.getFileHdr().getFile().close();
File f = new File(getRTreeTransportStopsFileName());
if (f.exists() && deleteDatabaseIndexes) {
f.delete();
}
f = new File(getRTreeTransportStopsPackFileName());
if (f.exists() && deleteDatabaseIndexes) {
f.delete();
}
}
// do not delete first db connection // do not delete first db connection
if (dbConn != null) { if (dbConn != null) {
if (DBDialect.H2 == dialect) { if (DBDialect.H2 == dialect) {
@ -2409,18 +2097,7 @@ public class IndexCreator {
} }
if (indexPOI) { if (indexPOI) {
poiIndexFile = new File(workingDir, getPoiFileName()); indexPoiCreator.createDatabaseStructure(new File(workingDir, getPoiFileName()), pStatements);
// to save space
if (poiIndexFile.exists()) {
Algoritms.removeAllFiles(poiIndexFile);
}
poiIndexFile.getParentFile().mkdirs();
// creating nodes db to fast access for all nodes
poiConnection = getDatabaseConnection(poiIndexFile.getAbsolutePath(), true);
DataIndexWriter.createPoiIndexStructure(poiConnection, dialect);
poiPreparedStatement = DataIndexWriter.createStatementAmenityInsert(poiConnection);
pStatements.put(poiPreparedStatement, 0);
poiConnection.setAutoCommit(false);
} }
if (indexTransport) { if (indexTransport) {

144
DataExtractionOSM/src/net/osmand/data/preparation/IndexPoiCreator.java Normal file
View file

@ -0,0 +1,144 @@
package net.osmand.data.preparation;
import java.io.File;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import net.osmand.Algoritms;
import net.osmand.data.Amenity;
import net.osmand.data.AmenityType;
import net.osmand.data.index.DataIndexWriter;
import net.osmand.data.index.IndexConstants;
import net.osmand.osm.Entity;
import net.osmand.osm.MapUtils;
import net.osmand.osm.OSMSettings.OSMTagKey;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
public class IndexPoiCreator {
private static final int BATCH_SIZE = 1000;
private static final Log log = LogFactory.getLog(IndexPoiCreator.class);
private final IndexCreator creator;
private Connection poiConnection;
private File poiIndexFile;
private PreparedStatement poiPreparedStatement;
private List<Amenity> tempAmenityList = new ArrayList<Amenity>();
public IndexPoiCreator(IndexCreator creator){
this.creator = creator;
}
public void iterateEntity(Entity e, Map<PreparedStatement, Integer> pStatements) throws SQLException{
tempAmenityList.clear();
tempAmenityList = Amenity.parseAmenities(e, tempAmenityList);
if (!tempAmenityList.isEmpty() && poiPreparedStatement != null) {
// load data for way (location etc...)
creator.loadEntityData(e, false);
for (Amenity a : tempAmenityList) {
checkEntity(e);
a.setEntity(e);
if (a.getLocation() != null) {
// do not convert english name
// convertEnglishName(a);
insertAmenityIntoPoi(pStatements, a);
}
}
}
}
public void commitAndClosePoiFile(Long lastModifiedDate) throws SQLException {
if (poiConnection != null) {
poiConnection.commit();
poiConnection.close();
poiConnection = null;
if (lastModifiedDate != null) {
poiIndexFile.setLastModified(lastModifiedDate);
}
}
}
public void insertAmenityIntoPoi( Map<PreparedStatement, Integer> map, Amenity amenity) throws SQLException {
assert IndexConstants.POI_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
poiPreparedStatement.setLong(1, amenity.getId());
poiPreparedStatement.setInt(2, MapUtils.get31TileNumberX(amenity.getLocation().getLongitude()));
poiPreparedStatement.setInt(3, MapUtils.get31TileNumberY(amenity.getLocation().getLatitude()));
poiPreparedStatement.setString(4, amenity.getEnName());
poiPreparedStatement.setString(5, amenity.getName());
poiPreparedStatement.setString(6, AmenityType.valueToString(amenity.getType()));
poiPreparedStatement.setString(7, amenity.getSubType());
poiPreparedStatement.setString(8, amenity.getOpeningHours());
poiPreparedStatement.setString(9, amenity.getSite());
poiPreparedStatement.setString(10, amenity.getPhone());
DataIndexWriter.addBatch(map, poiPreparedStatement, BATCH_SIZE);
}
public void checkEntity(Entity e){
String name = e.getTag(OSMTagKey.NAME);
if (name == null){
String msg = "";
Collection<String> keys = e.getTagKeySet();
int cnt = 0;
for (Iterator iter = keys.iterator(); iter.hasNext();) {
String key = (String) iter.next();
if (key.startsWith("name:") && key.length() <= 8) {
// ignore specialties like name:botanical
if (cnt == 0)
msg += "Entity misses default name tag, but it has localized name tag(s):\n";
msg += key + "=" + e.getTag(key) + "\n";
cnt++;
}
}
if (cnt > 0) {
msg += "Consider adding the name tag at " + e.getOsmUrl();
log.warn(msg);
}
}
}
public void createDatabaseStructure(File poiIndexFile, Map<PreparedStatement, Integer> pStatements) throws SQLException {
this.poiIndexFile = poiIndexFile;
// to save space
if (poiIndexFile.exists()) {
Algoritms.removeAllFiles(poiIndexFile);
}
poiIndexFile.getParentFile().mkdirs();
// creating nodes db to fast access for all nodes
poiConnection = creator.getDatabaseConnection(poiIndexFile.getAbsolutePath(), DBDialect.SQLITE);
createPoiIndexStructure(poiConnection);
poiPreparedStatement = createStatementAmenityInsert(poiConnection);
pStatements.put(poiPreparedStatement, 0);
poiConnection.setAutoCommit(false);
}
public void createPoiIndexStructure(Connection conn) throws SQLException{
Statement stat = conn.createStatement();
stat.executeUpdate("create table " + IndexConstants.POI_TABLE + //$NON-NLS-1$
"(id bigint, x int, y int, name_en varchar(255), name varchar(255), " +
"type varchar(255), subtype varchar(255), opening_hours varchar(255), phone varchar(255), site varchar(255)," +
"primary key(id, type, subtype))");
stat.executeUpdate("create index poi_loc on poi (x, y, type, subtype)");
stat.executeUpdate("create index poi_id on poi (id, type, subtype)");
stat.execute("PRAGMA user_version = " + IndexConstants.POI_TABLE_VERSION); //$NON-NLS-1$
stat.close();
}
public PreparedStatement createStatementAmenityInsert(Connection conn) throws SQLException{
return conn.prepareStatement("INSERT INTO " + IndexConstants.POI_TABLE + "(id, x, y, name_en, name, type, subtype, opening_hours, site, phone) " + //$NON-NLS-1$//$NON-NLS-2$
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
}
}

78
DataExtractionOSM/src/net/osmand/data/preparation/IndexTransportCreator.java
View file

@ -17,18 +17,6 @@ import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.Map.Entry; import java.util.Map.Entry;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import rtree.Element;
import rtree.IllegalValueException;
import rtree.LeafElement;
import rtree.NonLeafElement;
import rtree.RTree;
import rtree.RTreeException;
import rtree.RTreeInsertException;
import rtree.Rect;
import net.osmand.binary.BinaryMapIndexWriter; import net.osmand.binary.BinaryMapIndexWriter;
import net.osmand.data.TransportRoute; import net.osmand.data.TransportRoute;
import net.osmand.data.TransportStop; import net.osmand.data.TransportStop;
@ -42,6 +30,18 @@ import net.osmand.osm.Way;
import net.osmand.osm.OSMSettings.OSMTagKey; import net.osmand.osm.OSMSettings.OSMTagKey;
import net.sf.junidecode.Junidecode; import net.sf.junidecode.Junidecode;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import rtree.Element;
import rtree.IllegalValueException;
import rtree.LeafElement;
import rtree.NonLeafElement;
import rtree.RTree;
import rtree.RTreeException;
import rtree.RTreeInsertException;
import rtree.Rect;
public class IndexTransportCreator { public class IndexTransportCreator {
private final IndexCreator creator; private final IndexCreator creator;
@ -119,10 +119,16 @@ public class IndexTransportCreator {
} }
public void insertTransportIntoIndex(TransportRoute route, Map<PreparedStatement, Integer> pStatements) throws SQLException { public void visitEntityMainStep(Entity e, OsmDbAccessorContext ctx, Map<PreparedStatement, Integer> pStatements) throws SQLException {
if (e instanceof Relation && e.getTag(OSMTagKey.ROUTE) != null) {
ctx.loadEntityData(e, true);
TransportRoute route = indexTransportRoute((Relation) e);
if (route != null) {
insertTransportIntoIndex(transRouteStat, transRouteStopsStat, transStopsStat, transportStopsTree, insertTransportIntoIndex(transRouteStat, transRouteStopsStat, transStopsStat, transportStopsTree,
visitedStops, route, pStatements, BATCH_SIZE); visitedStops, route, pStatements, BATCH_SIZE);
} }
}
}
public void insertTransportIntoIndex(PreparedStatement prepRoute, PreparedStatement prepRouteStops, public void insertTransportIntoIndex(PreparedStatement prepRoute, PreparedStatement prepRouteStops,
PreparedStatement prepStops, RTree transportStopsTree, PreparedStatement prepStops, RTree transportStopsTree,
@ -136,7 +142,7 @@ public class IndexTransportCreator {
prepRoute.setString(5, route.getName()); prepRoute.setString(5, route.getName());
prepRoute.setString(6, route.getEnName()); prepRoute.setString(6, route.getEnName());
prepRoute.setInt(7, route.getAvgBothDistance()); prepRoute.setInt(7, route.getAvgBothDistance());
addBatch(statements, prepRoute); DataIndexWriter.addBatch(statements, prepRoute);
writeRouteStops(transportStopsTree, prepRouteStops, prepStops, statements, writtenStops, route, route.getForwardStops(), true); writeRouteStops(transportStopsTree, prepRouteStops, prepStops, statements, writtenStops, route, route.getForwardStops(), true);
writeRouteStops(transportStopsTree, prepRouteStops, prepStops, statements, writtenStops, route, route.getBackwardStops(), false); writeRouteStops(transportStopsTree, prepRouteStops, prepStops, statements, writtenStops, route, route.getBackwardStops(), false);
@ -166,7 +172,7 @@ public class IndexTransportCreator {
prepStops.setString(5, s.getEnName()); prepStops.setString(5, s.getEnName());
int x = (int) MapUtils.getTileNumberX(24, s.getLocation().getLongitude()); int x = (int) MapUtils.getTileNumberX(24, s.getLocation().getLongitude());
int y = (int) MapUtils.getTileNumberY(24, s.getLocation().getLatitude()); int y = (int) MapUtils.getTileNumberY(24, s.getLocation().getLatitude());
addBatch(count, prepStops); DataIndexWriter.addBatch(count, prepStops);
try { try {
transportStopsTree.insert(new LeafElement(new Rect(x, y, x, y), s.getId())); transportStopsTree.insert(new LeafElement(new Rect(x, y, x, y), s.getId()));
} catch (RTreeInsertException e) { } catch (RTreeInsertException e) {
@ -180,7 +186,7 @@ public class IndexTransportCreator {
prepRouteStops.setLong(2, s.getId()); prepRouteStops.setLong(2, s.getId());
prepRouteStops.setInt(3, direction ? 1 : 0); prepRouteStops.setInt(3, direction ? 1 : 0);
prepRouteStops.setInt(4, i++); prepRouteStops.setInt(4, i++);
addBatch(count, prepRouteStops); DataIndexWriter.addBatch(count, prepRouteStops);
} }
} }
@ -326,6 +332,21 @@ public class IndexTransportCreator {
} }
public void commitAndCloseFiles(String rtreeStopsFileName, String rtreeStopsPackFileName, boolean deleteDatabaseIndexes) throws IOException {
// delete transport rtree files
if (transportStopsTree != null) {
transportStopsTree.getFileHdr().getFile().close();
File f = new File(rtreeStopsFileName);
if (f.exists() && deleteDatabaseIndexes) {
f.delete();
}
f = new File(rtreeStopsPackFileName);
if (f.exists() && deleteDatabaseIndexes) {
f.delete();
}
}
}
public void writeBinaryTransportTree(rtree.Node parent, RTree r, BinaryMapIndexWriter writer, public void writeBinaryTransportTree(rtree.Node parent, RTree r, BinaryMapIndexWriter writer,
PreparedStatement selectTransportStop, PreparedStatement selectTransportRouteStop, PreparedStatement selectTransportStop, PreparedStatement selectTransportRouteStop,
@ -474,29 +495,4 @@ public class IndexTransportCreator {
return r; return r;
} }
private static void addBatch(Map<PreparedStatement, Integer> count, PreparedStatement p) throws SQLException {
addBatch(count, p, BATCH_SIZE, true);
}
public static void addBatch(Map<PreparedStatement, Integer> count, PreparedStatement p, boolean commit) throws SQLException{
addBatch(count, p, BATCH_SIZE, commit);
}
public static void addBatch(Map<PreparedStatement, Integer> count, PreparedStatement p, int batchSize) throws SQLException{
addBatch(count, p, batchSize, true);
}
public static void addBatch(Map<PreparedStatement, Integer> count, PreparedStatement p, int batchSize, boolean commit) throws SQLException{
p.addBatch();
if(count.get(p) >= batchSize){
p.executeBatch();
if(commit){
p.getConnection().commit();
}
count.put(p, 0);
} else {
count.put(p, count.get(p) + 1);
}
}
} }

255
DataExtractionOSM/src/net/osmand/data/preparation/OsmDbAccessor.java Normal file
View file

@ -0,0 +1,255 @@
package net.osmand.data.preparation;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Map;
import net.osmand.IProgress;
import net.osmand.osm.Entity;
import net.osmand.osm.Node;
import net.osmand.osm.Relation;
import net.osmand.osm.Way;
import net.osmand.osm.Entity.EntityId;
import net.osmand.osm.Entity.EntityType;
public class OsmDbAccessor implements OsmDbAccessorContext {
private final IndexCreator indexCreator;
private PreparedStatement pselectNode;
private PreparedStatement pselectWay;
private PreparedStatement pselectRelation;
private PreparedStatement pselectTags;
private Connection dbConn;
public interface OsmDbVisitor {
public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException;
}
public OsmDbAccessor(IndexCreator indexCreator){
this.indexCreator = indexCreator;
}
public void initDatabase(Connection dbConn) throws SQLException {
this.dbConn = dbConn;
pselectNode = dbConn.prepareStatement("select n.latitude, n.longitude, t.skeys, t.value from node n left join tags t on n.id = t.id and t.type = 0 where n.id = ?"); //$NON-NLS-1$
pselectWay = dbConn.prepareStatement("select w.node, w.ord, t.skeys, t.value, n.latitude, n.longitude " + //$NON-NLS-1$
"from ways w left join tags t on w.id = t.id and t.type = 1 and w.ord = 0 inner join node n on w.node = n.id " + //$NON-NLS-1$
"where w.id = ? order by w.ord"); //$NON-NLS-1$
pselectRelation = dbConn.prepareStatement("select r.member, r.type, r.role, r.ord, t.skeys, t.value " + //$NON-NLS-1$
"from relations r left join tags t on r.id = t.id and t.type = 2 and r.ord = 0 " + //$NON-NLS-1$
"where r.id = ? order by r.ord"); //$NON-NLS-1$
pselectTags = dbConn.prepareStatement("select skeys, value from tags where id = ? and type = ?"); //$NON-NLS-1$
}
public void loadEntityData(Entity e, boolean loadTags) throws SQLException {
if (e instanceof Node || (e instanceof Way && !((Way) e).getNodes().isEmpty())) {
// do not load tags for nodes inside way
return;
}
Map<EntityId, Entity> map = new LinkedHashMap<EntityId, Entity>();
if (e instanceof Relation && ((Relation) e).getMemberIds().isEmpty()) {
pselectRelation.setLong(1, e.getId());
if (pselectRelation.execute()) {
ResultSet rs = pselectRelation.getResultSet();
boolean first = true;
while (rs.next()) {
int ord = rs.getInt(4);
if (ord > 0 || first) {
first = false;
((Relation) e).addMember(rs.getLong(1), EntityType.values()[rs.getInt(2)], rs.getString(3));
}
}
rs.close();
}
} else if (e instanceof Way && ((Way) e).getEntityIds().isEmpty()) {
pselectWay.setLong(1, e.getId());
if (pselectWay.execute()) {
ResultSet rs = pselectWay.getResultSet();
boolean first = true;
while (rs.next()) {
int ord = rs.getInt(2);
if (ord > 0 || first) {
first = false;
((Way) e).addNode(new Node(rs.getDouble(5), rs.getDouble(6), rs.getLong(1)));
}
}
rs.close();
}
}
Collection<EntityId> ids = e instanceof Relation ? ((Relation) e).getMemberIds() : ((Way) e).getEntityIds();
for (EntityId i : ids) {
// pselectNode = dbConn.prepareStatement("select n.latitude, n.longitude, t.skeys, t.value from node n left join tags t on n.id = t.id and t.type = 0 where n.id = ?");
if (i.getType() == EntityType.NODE) {
pselectNode.setLong(1, i.getId());
if (pselectNode.execute()) {
ResultSet rs = pselectNode.getResultSet();
Node n = null;
while (rs.next()) {
if (n == null) {
n = new Node(rs.getDouble(1), rs.getDouble(2), i.getId());
}
if (rs.getObject(3) != null) {
n.putTag(rs.getString(3), rs.getString(4));
}
}
map.put(i, n);
rs.close();
}
} else if (i.getType() == EntityType.WAY) {
// pselectWay = dbConn.prepareStatement("select w.node, w.ord, t.skeys, t.value, n.latitude, n.longitude " +
// "from ways w left join tags t on w.id = t.id and t.type = 1 and w.ord = 0 inner join node n on w.node = n.id " +
// "where w.id = ? order by w.ord");
pselectWay.setLong(1, i.getId());
if (pselectWay.execute()) {
ResultSet rs = pselectWay.getResultSet();
Way way = new Way(i.getId());
map.put(i, way);
boolean first = true;
while (rs.next()) {
int ord = rs.getInt(2);
if (ord > 0 || first) {
first = false;
way.addNode(new Node(rs.getDouble(5), rs.getDouble(6), rs.getLong(1)));
}
if (ord == 0 && rs.getObject(3) != null) {
way.putTag(rs.getString(3), rs.getString(4));
}
}
rs.close();
}
} else if (i.getType() == EntityType.RELATION) {
pselectRelation.setLong(1, i.getId());
// pselectRelation = dbConn.prepareStatement("select r.member, r.type, r.role, r.ord, t.skeys, t.value" +
// "from relations r left join tags t on r.id = t.id and t.type = 2 and r.ord = 0 " +
// "where r.id = ? order by r.ord");
if (pselectRelation.execute()) {
ResultSet rs = pselectRelation.getResultSet();
Relation rel = new Relation(i.getId());
map.put(i, rel);
boolean first = true;
while (rs.next()) {
int ord = rs.getInt(4);
if (ord > 0 || first) {
first = false;
rel.addMember(rs.getLong(1), EntityType.values()[rs.getInt(2)], rs.getString(3));
}
if (ord == 0 && rs.getObject(5) != null) {
rel.putTag(rs.getString(5), rs.getString(6));
}
}
// do not load relation members recursively ? It is not needed for transport, address, poi before
rs.close();
}
}
}
e.initializeLinks(map);
}
public int iterateOverEntities(IProgress progress, EntityType type, int allCount,
OsmDbVisitor visitor) throws SQLException {
Statement statement = dbConn.createStatement();
String select;
int count = 0;
// stat.executeUpdate("create table tags (id "+longType+", type smallint, skeys varchar(255), value varchar(255))");
// stat.executeUpdate("create table ways (id "+longType+", node "+longType+", ord smallint)");
// stat.executeUpdate("create table relations (id "+longType+", member "+longType+", type smallint, role varchar(255), ord smallint)");
if (type == EntityType.NODE) {
// filter out all nodes without tags
select = "select n.id, n.latitude, n.longitude, t.skeys, t.value from node n inner join tags t on n.id = t.id and t.type = 0 order by n.id"; //$NON-NLS-1$
} else if (type == EntityType.WAY) {
select = "select w.id, w.node, w.ord, t.skeys, t.value, n.latitude, n.longitude " + //$NON-NLS-1$
"from ways w left join tags t on w.id = t.id and t.type = 1 and w.ord = 0 inner join node n on w.node = n.id " + //$NON-NLS-1$
"order by w.id, w.ord"; //$NON-NLS-1$
} else {
select = "select r.id, t.skeys, t.value from relations r inner join tags t on t.id = r.id and t.type = 2 and r.ord = 0"; //$NON-NLS-1$
}
ResultSet rs = statement.executeQuery(select);
Entity prevEntity = null;
long prevId = -1;
while (rs.next()) {
long curId = rs.getLong(1);
boolean newEntity = curId != prevId;
Entity e = prevEntity;
if (type == EntityType.NODE) {
if (newEntity) {
e = new Node(rs.getDouble(2), rs.getDouble(3), curId);
}
e.putTag(rs.getString(4), rs.getString(5));
} else if (type == EntityType.WAY) {
if (newEntity) {
e = new Way(curId);
}
int ord = rs.getInt(3);
if (ord == 0 && rs.getObject(4) != null) {
e.putTag(rs.getString(4), rs.getString(5));
}
if (newEntity || ord > 0) {
((Way) e).addNode(new Node(rs.getDouble(6), rs.getDouble(7), rs.getLong(2)));
}
} else {
if (newEntity) {
e = new Relation(curId);
}
e.putTag(rs.getString(2), rs.getString(3));
}
if (newEntity) {
count++;
if (progress != null) {
progress.progress(1);
}
if (prevEntity != null) {
visitor.iterateEntity(prevEntity, this);
}
prevEntity = e;
}
prevId = curId;
}
if (prevEntity != null) {
count++;
visitor.iterateEntity(prevEntity, this);
}
rs.close();
return count;
}
public void loadEntityTags(EntityType type, Entity e) throws SQLException {
pselectTags.setLong(1, e.getId());
pselectTags.setByte(2, (byte) type.ordinal());
ResultSet rsTags = pselectTags.executeQuery();
while (rsTags.next()) {
e.putTag(rsTags.getString(1), rsTags.getString(2));
}
rsTags.close();
}
public void closeReadingConnection() throws SQLException {
if (pselectNode != null) {
pselectNode.close();
}
if (pselectWay != null) {
pselectWay.close();
}
if (pselectRelation != null) {
pselectRelation.close();
}
if (pselectTags != null) {
pselectTags.close();
}
}
}

13
DataExtractionOSM/src/net/osmand/data/preparation/OsmDbAccessorContext.java Normal file
View file

@ -0,0 +1,13 @@
package net.osmand.data.preparation;
import java.sql.SQLException;
import net.osmand.osm.Entity;
import net.osmand.osm.Entity.EntityType;
public interface OsmDbAccessorContext {
public void loadEntityTags(EntityType type, Entity e) throws SQLException;
public void loadEntityData(Entity e, boolean loadTags) throws SQLException;
}

6
DataExtractionOSM/src/net/osmand/data/preparation/OsmDbCreator.java
View file

@ -2,8 +2,12 @@ package net.osmand.data.preparation;
import java.sql.Connection; import java.sql.Connection;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.sql.Statement; import java.sql.Statement;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import net.osmand.osm.Entity; import net.osmand.osm.Entity;
@ -43,6 +47,7 @@ public class OsmDbCreator implements IOsmStorageFilter {
private Connection dbConn; private Connection dbConn;
private final IndexCreator indexCreator; private final IndexCreator indexCreator;
public OsmDbCreator(IndexCreator indexCreator) { public OsmDbCreator(IndexCreator indexCreator) {
this.indexCreator = indexCreator; this.indexCreator = indexCreator;
} }
@ -177,4 +182,5 @@ public class OsmDbCreator implements IOsmStorageFilter {
return allWays; return allWays;
} }
} }