fix slow processing of streets, boundaries
This commit is contained in:
Pavol Zibrita
parent
3b315c36ce
commit
90372dc9e7
10 changed files with 606 additions and 380 deletions
|
|
@ -34,7 +34,7 @@ public class BinaryInspector {
|
|||
inspector(args);
|
||||
// test cases show info
|
||||
//inspector(new String[]{"/home/victor/projects/OsmAnd/data/osm-gen/saved/Belarus-newzooms-new-rt.obf"});
|
||||
// inspector(new String[]{""});
|
||||
//inspector(new String[]{"-v","C:\\Users\\tpd\\osmand\\Slovakia.obf"});
|
||||
|
||||
|
||||
// test case extract parts
|
||||
|
|
|
|||
|
|
@ -17,9 +17,9 @@ public class Boundary {
|
|||
|
||||
|
||||
// not necessary ready rings
|
||||
private List<Way> outerWays = new ArrayList<Way>();
|
||||
private List<Way> innerWays = new ArrayList<Way>();
|
||||
private final boolean closedWay;
|
||||
private List<Way> outerWays = new ArrayList<Way>(1);
|
||||
private List<Way> innerWays = new ArrayList<Way>(0);
|
||||
private boolean closedWay;
|
||||
|
||||
public Boundary(boolean closedWay){
|
||||
this.closedWay = closedWay;
|
||||
|
|
@ -29,16 +29,21 @@ public class Boundary {
|
|||
return closedWay;
|
||||
}
|
||||
|
||||
public void setClosedWay(boolean closedWay) {
|
||||
this.closedWay = closedWay;
|
||||
}
|
||||
|
||||
public boolean computeIsClosedWay()
|
||||
{
|
||||
//now we try to merge the ways until we have only one
|
||||
int oldSize = 0;
|
||||
while (getOuterWays().size() != oldSize) {
|
||||
while (getOuterWays().size() != 0 && getOuterWays().size() != oldSize) {
|
||||
oldSize = getOuterWays().size();
|
||||
mergeOuterWays();
|
||||
}
|
||||
//there is one way and last element is equal to the first...
|
||||
return getOuterWays().size() == 1 && getOuterWays().get(0).getNodes().get(0).getId() == getOuterWays().get(0).getNodes().get(getOuterWays().get(0).getNodes().size()-1).getId();
|
||||
closedWay = getOuterWays().size() == 1 && getOuterWays().get(0).getNodes().get(0).getId() == getOuterWays().get(0).getNodes().get(getOuterWays().get(0).getNodes().size()-1).getId();
|
||||
return closedWay;
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -150,5 +155,9 @@ public class Boundary {
|
|||
this.adminLevel = adminLevel;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return getName() + " alevel:" + getAdminLevel() + " type: relation closed:" + isClosedWay();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
13
DataExtractionOSM/src/net/osmand/data/WayBoundary.java
Normal file
13
DataExtractionOSM/src/net/osmand/data/WayBoundary.java
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
package net.osmand.data;
|
||||
|
||||
public class WayBoundary extends Boundary {
|
||||
|
||||
public WayBoundary(boolean closedWay) {
|
||||
super(closedWay);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return getName() + " alevel:" + getAdminLevel() + " type: way closed:" + isClosedWay();
|
||||
}
|
||||
}
|
||||
|
|
@ -2,6 +2,7 @@ package net.osmand.data.preparation;
|
|||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.SQLException;
|
||||
import java.util.LinkedHashMap;
|
||||
|
|
@ -22,6 +23,13 @@ public class AbstractIndexPartCreator {
|
|||
|
||||
protected Map<PreparedStatement, Integer> pStatements = new LinkedHashMap<PreparedStatement, Integer>();
|
||||
|
||||
public PreparedStatement createPrepareStatement(Connection mapConnection,
|
||||
String string) throws SQLException {
|
||||
PreparedStatement prepareStatement = mapConnection.prepareStatement(string);
|
||||
pStatements.put(prepareStatement, 0);
|
||||
return prepareStatement;
|
||||
}
|
||||
|
||||
protected void closePreparedStatements(PreparedStatement... preparedStatements) throws SQLException {
|
||||
for (PreparedStatement p : preparedStatements) {
|
||||
if (p != null) {
|
||||
|
|
@ -41,6 +49,18 @@ public class AbstractIndexPartCreator {
|
|||
}
|
||||
}
|
||||
|
||||
protected boolean executePendingPreparedStatements() throws SQLException {
|
||||
boolean exec = false;
|
||||
for (PreparedStatement p : pStatements.keySet()) {
|
||||
if (pStatements.get(p) > 0) {
|
||||
p.executeBatch();
|
||||
pStatements.put(p, 0);
|
||||
exec = true;
|
||||
}
|
||||
}
|
||||
return exec;
|
||||
}
|
||||
|
||||
protected void addBatch(PreparedStatement p) throws SQLException {
|
||||
addBatch(p, BATCH_SIZE, true);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,84 @@
|
|||
package net.osmand.data.preparation;
|
||||
|
||||
import gnu.trove.set.hash.TLongHashSet;
|
||||
|
||||
import java.sql.SQLException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import net.osmand.data.Building;
|
||||
import net.osmand.data.City;
|
||||
import net.osmand.data.preparation.DBStreetDAO.SimpleStreet;
|
||||
import net.osmand.osm.Entity;
|
||||
import net.osmand.osm.LatLon;
|
||||
import net.osmand.osm.Way;
|
||||
|
||||
public class CachedDBStreetDAO extends DBStreetDAO
|
||||
{
|
||||
private Map<String, SimpleStreet> addressStreetLocalMap = new HashMap<String, SimpleStreet>();
|
||||
private TLongHashSet addressBuildingLocalSet = new TLongHashSet();
|
||||
private TLongHashSet addressStreetNodeLocalSet = new TLongHashSet();
|
||||
|
||||
@Override
|
||||
public SimpleStreet findStreet(String name, City city, String cityPart) {
|
||||
return addressStreetLocalMap.get(createStreetUniqueName(name, city, cityPart)); //$NON-NLS-1$
|
||||
}
|
||||
|
||||
public SimpleStreet findStreet(String name, City city) {
|
||||
return addressStreetLocalMap.get(createStreetUniqueName(name, city)); //$NON-NLS-1$
|
||||
}
|
||||
|
||||
private String createStreetUniqueName(String name, City city, String cityPart) {
|
||||
return new StringBuilder().append(name).append('_').append(city.getId()).append('_').append(cityPart).toString();
|
||||
}
|
||||
|
||||
private String createStreetUniqueName(String name, City city) {
|
||||
return new StringBuilder().append(name).append('_').append(city.getId()).toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void writeStreetWayNodes(Set<Long> streetId, Way way)
|
||||
throws SQLException {
|
||||
super.writeStreetWayNodes(streetId, way);
|
||||
addressStreetNodeLocalSet.add(way.getId());
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void writeBuilding(Set<Long> streetId, Building building)
|
||||
throws SQLException {
|
||||
super.writeBuilding(streetId, building);
|
||||
addressBuildingLocalSet.add(building.getId());
|
||||
}
|
||||
|
||||
@Override
|
||||
public long insertStreet(String name, String nameEn, LatLon location, City city, String cityPart) throws SQLException {
|
||||
//batch the insert
|
||||
long streetId = fillInsertStreetStatement(name, nameEn, location, city, cityPart);
|
||||
addBatch(addressStreetStat);
|
||||
SimpleStreet ss = new SimpleStreet(streetId,name,cityPart,location);
|
||||
addressStreetLocalMap.put(createStreetUniqueName(name, city, cityPart), ss);
|
||||
addressStreetLocalMap.put(createStreetUniqueName(name, city), ss);
|
||||
return streetId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public SimpleStreet updateStreetCityPart(SimpleStreet street, City city, String cityPart) throws SQLException {
|
||||
commit(); //we are doing batch updates, so we must commit before this update
|
||||
super.updateStreetCityPart(street, city, cityPart);
|
||||
SimpleStreet updatedSS = new SimpleStreet(street.getId(),street.getName(),cityPart,street.getLocation());
|
||||
addressStreetLocalMap.put(createStreetUniqueName(street.getName(), city), updatedSS);
|
||||
addressStreetLocalMap.put(createStreetUniqueName(street.getName(), city, cityPart), updatedSS);
|
||||
return updatedSS;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean findBuilding(Entity e) {
|
||||
return addressBuildingLocalSet.contains(e.getId());
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean findStreetNode(Entity e) {
|
||||
return addressStreetNodeLocalSet.contains(e.getId());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,208 @@
|
|||
package net.osmand.data.preparation;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.util.Set;
|
||||
|
||||
import net.osmand.data.Building;
|
||||
import net.osmand.data.City;
|
||||
import net.osmand.osm.Entity;
|
||||
import net.osmand.osm.LatLon;
|
||||
import net.osmand.osm.Node;
|
||||
import net.osmand.osm.Way;
|
||||
|
||||
public class DBStreetDAO extends AbstractIndexPartCreator
|
||||
{
|
||||
public static class SimpleStreet {
|
||||
private final long id;
|
||||
private final String name;
|
||||
private final String cityPart;
|
||||
private LatLon location;
|
||||
|
||||
public SimpleStreet(long id, String name, String cityPart, double latitude, double longitude) {
|
||||
this(id,name,cityPart, new LatLon(latitude,longitude));
|
||||
}
|
||||
public SimpleStreet(long id, String name, String cityPart,
|
||||
LatLon location) {
|
||||
this.id = id;
|
||||
this.name = name;
|
||||
this.cityPart = cityPart;
|
||||
this.location = location;
|
||||
}
|
||||
public String getCityPart() {
|
||||
return cityPart;
|
||||
}
|
||||
public long getId() {
|
||||
return id;
|
||||
}
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
public LatLon getLocation() {
|
||||
return location;
|
||||
}
|
||||
}
|
||||
|
||||
protected PreparedStatement addressStreetStat;
|
||||
private PreparedStatement addressStreetNodeStat;
|
||||
private PreparedStatement addressBuildingStat;
|
||||
private PreparedStatement addressSearchStreetStat;
|
||||
private PreparedStatement addressSearchBuildingStat;
|
||||
private PreparedStatement addressSearchStreetNodeStat;
|
||||
private PreparedStatement addressSearchStreetStatWithoutCityPart;
|
||||
|
||||
private Connection mapConnection;
|
||||
private PreparedStatement addressStreetUpdateCityPart;
|
||||
|
||||
public void createDatabaseStructure(Connection mapConnection, DBDialect dialect) throws SQLException {
|
||||
this.mapConnection = mapConnection;
|
||||
Statement stat = mapConnection.createStatement();
|
||||
stat.executeUpdate("create table street (id bigint primary key, latitude double, longitude double, " +
|
||||
"name varchar(1024), name_en varchar(1024), city bigint, citypart varchar(1024))");
|
||||
stat.executeUpdate("create index street_cnp on street (city,citypart,name,id)");
|
||||
stat.executeUpdate("create index street_city on street (city)");
|
||||
stat.executeUpdate("create index street_id on street (id)");
|
||||
|
||||
// create index on name ?
|
||||
stat.executeUpdate("create table building (id bigint, latitude double, longitude double, " +
|
||||
"name varchar(1024), name_en varchar(1024), street bigint, postcode varchar(1024), primary key(street, id))");
|
||||
stat.executeUpdate("create index building_postcode on building (postcode)");
|
||||
stat.executeUpdate("create index building_street on building (street)");
|
||||
stat.executeUpdate("create index building_id on building (id)");
|
||||
|
||||
stat.executeUpdate("create table street_node (id bigint, latitude double, longitude double, " +
|
||||
"street bigint, way bigint)");
|
||||
stat.executeUpdate("create index street_node_street on street_node (street)");
|
||||
stat.executeUpdate("create index street_node_way on street_node (way)");
|
||||
stat.close();
|
||||
|
||||
addressStreetStat = createPrepareStatement(mapConnection,"insert into street (id, latitude, longitude, name, name_en, city, citypart) values (?, ?, ?, ?, ?, ?, ?)");
|
||||
addressStreetNodeStat = createPrepareStatement(mapConnection,"insert into street_node (id, latitude, longitude, street, way) values (?, ?, ?, ?, ?)");
|
||||
addressBuildingStat = createPrepareStatement(mapConnection,"insert into building (id, latitude, longitude, name, name_en, street, postcode) values (?, ?, ?, ?, ?, ?, ?)");
|
||||
addressSearchStreetStat = createPrepareStatement(mapConnection,"SELECT id,latitude,longitude FROM street WHERE ? = city AND ? = citypart AND ? = name");
|
||||
addressSearchStreetStatWithoutCityPart = createPrepareStatement(mapConnection,"SELECT id,name,citypart,latitude,longitude FROM street WHERE ? = city AND ? = name");
|
||||
addressStreetUpdateCityPart = createPrepareStatement(mapConnection,"UPDATE street SET citypart = ? WHERE id = ?");
|
||||
addressSearchBuildingStat = createPrepareStatement(mapConnection,"SELECT id FROM building where ? = id");
|
||||
addressSearchStreetNodeStat = createPrepareStatement(mapConnection,"SELECT way FROM street_node WHERE ? = way");
|
||||
}
|
||||
|
||||
protected void writeStreetWayNodes(Set<Long> streetIds, Way way) throws SQLException {
|
||||
for (Long streetId : streetIds) {
|
||||
for (Node n : way.getNodes()) {
|
||||
if (n == null) {
|
||||
continue;
|
||||
}
|
||||
addressStreetNodeStat.setLong(1, n.getId());
|
||||
addressStreetNodeStat.setDouble(2, n.getLatitude());
|
||||
addressStreetNodeStat.setDouble(3, n.getLongitude());
|
||||
addressStreetNodeStat.setLong(5, way.getId());
|
||||
addressStreetNodeStat.setLong(4, streetId);
|
||||
addBatch(addressStreetNodeStat);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected void writeBuilding(Set<Long> streetIds, Building building) throws SQLException {
|
||||
for (Long streetId : streetIds) {
|
||||
addressBuildingStat.setLong(1, building.getId());
|
||||
addressBuildingStat.setDouble(2, building.getLocation().getLatitude());
|
||||
addressBuildingStat.setDouble(3, building.getLocation().getLongitude());
|
||||
addressBuildingStat.setString(4, building.getName());
|
||||
addressBuildingStat.setString(5, building.getEnName());
|
||||
addressBuildingStat.setLong(6, streetId);
|
||||
addressBuildingStat.setString(7, building.getPostcode() == null ? null : building.getPostcode().toUpperCase());
|
||||
addBatch(addressBuildingStat);
|
||||
}
|
||||
}
|
||||
|
||||
public DBStreetDAO.SimpleStreet findStreet(String name, City city) throws SQLException {
|
||||
addressSearchStreetStatWithoutCityPart.setLong(1, city.getId());
|
||||
addressSearchStreetStatWithoutCityPart.setString(2, name);
|
||||
ResultSet rs = addressSearchStreetStatWithoutCityPart.executeQuery();
|
||||
DBStreetDAO.SimpleStreet foundId = null;
|
||||
if (rs.next()) {
|
||||
foundId = new SimpleStreet(rs.getLong(1),rs.getString(2),rs.getString(3),rs.getDouble(4),rs.getDouble(5));
|
||||
}
|
||||
rs.close();
|
||||
return foundId;
|
||||
}
|
||||
|
||||
public DBStreetDAO.SimpleStreet findStreet(String name, City city, String cityPart) throws SQLException {
|
||||
addressSearchStreetStat.setLong(1, city.getId());
|
||||
addressSearchStreetStat.setString(2, cityPart);
|
||||
addressSearchStreetStat.setString(3, name);
|
||||
ResultSet rs = addressSearchStreetStat.executeQuery();
|
||||
DBStreetDAO.SimpleStreet foundId = null;
|
||||
if (rs.next()) {
|
||||
foundId = new SimpleStreet(rs.getLong(1),name,cityPart,rs.getDouble(2),rs.getDouble(3));
|
||||
}
|
||||
rs.close();
|
||||
return foundId;
|
||||
}
|
||||
|
||||
private long streetIdSequence = 0;
|
||||
|
||||
public long insertStreet(String name, String nameEn, LatLon location,
|
||||
City city, String cityPart) throws SQLException {
|
||||
long streetId = fillInsertStreetStatement(name, nameEn, location, city, cityPart);
|
||||
// execute the insert statement
|
||||
addressStreetStat.execute();
|
||||
// commit immediately to search after
|
||||
mapConnection.commit();
|
||||
|
||||
return streetId;
|
||||
}
|
||||
|
||||
protected long fillInsertStreetStatement(String name, String nameEn,
|
||||
LatLon location, City city, String cityPart)
|
||||
throws SQLException {
|
||||
long streetId = streetIdSequence++;
|
||||
addressStreetStat.setLong(1, streetId);
|
||||
addressStreetStat.setString(4, name);
|
||||
addressStreetStat.setString(5, nameEn);
|
||||
addressStreetStat.setDouble(2, location.getLatitude());
|
||||
addressStreetStat.setDouble(3, location.getLongitude());
|
||||
addressStreetStat.setLong(6, city.getId());
|
||||
addressStreetStat.setString(7, cityPart);
|
||||
return streetId;
|
||||
}
|
||||
|
||||
public boolean findBuilding(Entity e) throws SQLException {
|
||||
commit(); //we are doing batch adds, to search, we must commit
|
||||
addressSearchBuildingStat.setLong(1, e.getId());
|
||||
ResultSet rs = addressSearchBuildingStat.executeQuery();
|
||||
boolean exist = rs.next();
|
||||
rs.close();
|
||||
return exist;
|
||||
}
|
||||
|
||||
public boolean findStreetNode(Entity e) throws SQLException {
|
||||
commit(); //we are doing batch adds, to search, we must commit
|
||||
addressSearchStreetNodeStat.setLong(1, e.getId());
|
||||
ResultSet rs = addressSearchStreetNodeStat.executeQuery();
|
||||
boolean exist = rs.next();
|
||||
rs.close();
|
||||
return exist;
|
||||
}
|
||||
|
||||
public void commit() throws SQLException {
|
||||
if (executePendingPreparedStatements()) {
|
||||
mapConnection.commit();
|
||||
}
|
||||
}
|
||||
|
||||
public void close() throws SQLException {
|
||||
closePreparedStatements(addressStreetStat, addressStreetNodeStat, addressBuildingStat);
|
||||
}
|
||||
|
||||
public DBStreetDAO.SimpleStreet updateStreetCityPart(DBStreetDAO.SimpleStreet street, City city, String cityPart) throws SQLException {
|
||||
addressStreetUpdateCityPart.setString(1, cityPart);
|
||||
addressStreetUpdateCityPart.setLong(2, street.getId());
|
||||
addressStreetUpdateCityPart.executeUpdate();
|
||||
mapConnection.commit();
|
||||
return new SimpleStreet(street.getId(),street.getName(),cityPart,street.getLocation());
|
||||
}
|
||||
}
|
||||
|
|
@ -16,7 +16,6 @@ import java.util.Collections;
|
|||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
|
|
@ -33,6 +32,8 @@ import net.osmand.data.City;
|
|||
import net.osmand.data.City.CityType;
|
||||
import net.osmand.data.DataTileManager;
|
||||
import net.osmand.data.Street;
|
||||
import net.osmand.data.WayBoundary;
|
||||
import net.osmand.data.preparation.DBStreetDAO.SimpleStreet;
|
||||
import net.osmand.osm.Entity;
|
||||
import net.osmand.osm.Entity.EntityId;
|
||||
import net.osmand.osm.LatLon;
|
||||
|
|
@ -51,17 +52,10 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
|
|||
|
||||
private static final Log log = LogFactory.getLog(IndexAddressCreator.class);
|
||||
|
||||
|
||||
private PreparedStatement addressCityStat;
|
||||
private PreparedStatement addressStreetStat;
|
||||
private PreparedStatement addressBuildingStat;
|
||||
private PreparedStatement addressStreetNodeStat;
|
||||
|
||||
// MEMORY address : choose what to use ?
|
||||
private boolean loadInMemory = true;
|
||||
private PreparedStatement addressSearchStreetStat;
|
||||
private PreparedStatement addressSearchBuildingStat;
|
||||
private PreparedStatement addressSearchStreetNodeStat;
|
||||
|
||||
// MEMORY address : address structure
|
||||
// load it in memory
|
||||
|
|
@ -70,6 +64,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
|
|||
private DataTileManager<City> cityManager = new DataTileManager<City>(10);
|
||||
private List<Relation> postalCodeRelations = new ArrayList<Relation>();
|
||||
private Map<City, Boundary> cityBoundaries = new HashMap<City, Boundary>();
|
||||
private Map<Boundary,List<City>> boundariesToCities = new HashMap<Boundary,List<City>>();
|
||||
private Set<Boundary> allBoundaries = new HashSet<Boundary>();
|
||||
private TLongHashSet visitedBoundaryWays = new TLongHashSet();
|
||||
|
||||
|
|
@ -83,126 +78,6 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
|
|||
Connection mapConnection;
|
||||
DBStreetDAO streetDAO;
|
||||
|
||||
|
||||
|
||||
public class DBStreetDAO
|
||||
{
|
||||
protected void writeStreetWayNodes(Set<Long> streetIds, Way way) throws SQLException {
|
||||
for (Long streetId : streetIds) {
|
||||
for (Node n : way.getNodes()) {
|
||||
if (n == null) {
|
||||
continue;
|
||||
}
|
||||
addressStreetNodeStat.setLong(1, n.getId());
|
||||
addressStreetNodeStat.setDouble(2, n.getLatitude());
|
||||
addressStreetNodeStat.setDouble(3, n.getLongitude());
|
||||
addressStreetNodeStat.setLong(5, way.getId());
|
||||
addressStreetNodeStat.setLong(4, streetId);
|
||||
addBatch(addressStreetNodeStat);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected void writeBuilding(Set<Long> streetIds, Building building) throws SQLException {
|
||||
for (Long streetId : streetIds) {
|
||||
addressBuildingStat.setLong(1, building.getId());
|
||||
addressBuildingStat.setDouble(2, building.getLocation().getLatitude());
|
||||
addressBuildingStat.setDouble(3, building.getLocation().getLongitude());
|
||||
addressBuildingStat.setString(4, building.getName());
|
||||
addressBuildingStat.setString(5, building.getEnName());
|
||||
addressBuildingStat.setLong(6, streetId);
|
||||
addressBuildingStat.setString(7, building.getPostcode() == null ? null : building.getPostcode().toUpperCase());
|
||||
addBatch(addressBuildingStat);
|
||||
}
|
||||
}
|
||||
|
||||
public Long findStreet(String name, City city, String cityPart) throws SQLException {
|
||||
addressSearchStreetStat.setLong(1, city.getId());
|
||||
addressSearchStreetStat.setString(2, cityPart);
|
||||
addressSearchStreetStat.setString(3, name);
|
||||
ResultSet rs = addressSearchStreetStat.executeQuery();
|
||||
Long foundId = null;
|
||||
if (rs.next()) {
|
||||
foundId = rs.getLong(1);
|
||||
}
|
||||
rs.close();
|
||||
return foundId;
|
||||
}
|
||||
|
||||
public void insertStreet(PreparedStatement addressStreetStat,
|
||||
String name, City city, String cityPart, long initId) throws SQLException {
|
||||
//execute the insert statement
|
||||
addressStreetStat.execute();
|
||||
// commit immediately to search after
|
||||
mapConnection.commit();
|
||||
}
|
||||
|
||||
public boolean findBuilding(Entity e) throws SQLException {
|
||||
addressSearchBuildingStat.setLong(1, e.getId());
|
||||
ResultSet rs = addressSearchBuildingStat.executeQuery();
|
||||
boolean exist = rs.next();
|
||||
rs.close();
|
||||
return exist;
|
||||
}
|
||||
|
||||
public boolean findStreetNode(Entity e) throws SQLException {
|
||||
addressSearchStreetNodeStat.setLong(1, e.getId());
|
||||
ResultSet rs = addressSearchStreetNodeStat.executeQuery();
|
||||
boolean exist = rs.next();
|
||||
rs.close();
|
||||
return exist;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public class CachedDBStreetDAO extends DBStreetDAO
|
||||
{
|
||||
private Map<String, Long> addressStreetLocalMap = new HashMap<String, Long>();
|
||||
private TLongHashSet addressBuildingLocalSet = new TLongHashSet();
|
||||
private TLongHashSet addressStreetNodeLocalSet = new TLongHashSet();
|
||||
|
||||
@Override
|
||||
public Long findStreet(String name, City city, String cityPart) {
|
||||
return addressStreetLocalMap.get(createStreetUniqueName(name, city, cityPart)); //$NON-NLS-1$
|
||||
}
|
||||
|
||||
private String createStreetUniqueName(String name, City city, String cityPart) {
|
||||
return new StringBuilder().append(name).append('_').append(city.getId()).append('_').append(cityPart).toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void writeStreetWayNodes(Set<Long> streetId, Way way)
|
||||
throws SQLException {
|
||||
super.writeStreetWayNodes(streetId, way);
|
||||
addressStreetNodeLocalSet.add(way.getId());
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void writeBuilding(Set<Long> streetId, Building building)
|
||||
throws SQLException {
|
||||
super.writeBuilding(streetId, building);
|
||||
addressBuildingLocalSet.add(building.getId());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void insertStreet(PreparedStatement addressStreetStat,
|
||||
String name, City city, String cityPart, long initId) throws SQLException {
|
||||
//batch the insert
|
||||
addBatch(addressStreetStat);
|
||||
addressStreetLocalMap.put(createStreetUniqueName(name, city, cityPart), initId); //$NON-NLS-1$
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean findBuilding(Entity e) {
|
||||
return addressBuildingLocalSet.contains(e.getId());
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean findStreetNode(Entity e) {
|
||||
return addressStreetNodeLocalSet.contains(e.getId());
|
||||
}
|
||||
}
|
||||
|
||||
public static class StreetAndDistrict {
|
||||
private final Street street;
|
||||
private final String district;
|
||||
|
|
@ -254,88 +129,87 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
|
|||
|
||||
public void indexBoundariesRelation(Entity e, OsmDbAccessorContext ctx) throws SQLException {
|
||||
Boundary boundary = extractBoundary(e, ctx);
|
||||
if (boundary != null && boundary.getAdminLevel() >= 4 && boundary.getCenterPoint() != null && !Algoritms.isEmpty(boundary.getName())) {
|
||||
if (boundary != null && boundary.isClosedWay() && boundary.getAdminLevel() >= 4 && boundary.getCenterPoint() != null && !Algoritms.isEmpty(boundary.getName())) {
|
||||
LatLon boundaryCenter = boundary.getCenterPoint();
|
||||
List<City> citiesToSearch = new ArrayList<City>();
|
||||
citiesToSearch.addAll(cityManager.getClosestObjects(boundaryCenter.getLatitude(), boundaryCenter.getLongitude(), 3));
|
||||
citiesToSearch.addAll(cityVillageManager.getClosestObjects(boundaryCenter.getLatitude(), boundaryCenter.getLongitude(), 3));
|
||||
|
||||
List<City> cityFound = new ArrayList<City>();
|
||||
City cityFound = null;
|
||||
String boundaryName = boundary.getName().toLowerCase();
|
||||
for (City c : citiesToSearch) {
|
||||
if (boundary.containsPoint(c.getLocation())) {
|
||||
if (boundaryName.equalsIgnoreCase(c.getName())) {
|
||||
cityFound.add(c);
|
||||
cityFound = c;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
//We should not look for similarities, this can be 'very' wrong....
|
||||
if (cityFound.isEmpty()) {
|
||||
if (cityFound == null) {
|
||||
// try to find same name in the middle
|
||||
for (City c : citiesToSearch) {
|
||||
if (boundary.containsPoint(c.getLocation())) {
|
||||
String lower = c.getName().toLowerCase();
|
||||
if (boundaryName.startsWith(lower + " ") || boundaryName.endsWith(lower + " ") || boundaryName.contains(" " + lower + " ")) {
|
||||
cityFound.add(c);
|
||||
cityFound = c;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// if (cityFound.isEmpty()) {
|
||||
// try to find closes city to the boundary center
|
||||
// double minDist = Double.MAX_VALUE;
|
||||
// City closestCity = null;
|
||||
// for (City c : citiesToSearch) {
|
||||
// if (boundary.containsPoint(c.getLocation())) {
|
||||
//// double dist = MapUtils.getDistance(boundaryCenter, c.getLocation());
|
||||
//// if (dist < minDist) {
|
||||
//// dist = minDist;
|
||||
//// closestCity = c;
|
||||
//// }
|
||||
// cityFound.add(c);
|
||||
// }
|
||||
// }
|
||||
// cityFound = closestCity;
|
||||
// }
|
||||
if (!cityFound.isEmpty()) {
|
||||
//we can have more cities in one boundary!! (suburbs...)
|
||||
if (cityFound != null) {
|
||||
putCityBoundary(boundary, cityFound);
|
||||
}
|
||||
allBoundaries.add(boundary);
|
||||
} else if (boundary != null && !boundary.isClosedWay()){
|
||||
System.out.println("Not using opened boundary: " + boundary);
|
||||
}
|
||||
}
|
||||
|
||||
public void bindCitiesWithBoundaries() {
|
||||
Iterator<Boundary> iter = allBoundaries.iterator();
|
||||
while (iter.hasNext()) {
|
||||
Boundary b = iter.next();
|
||||
if (!b.computeIsClosedWay()) {
|
||||
System.out.println("Removing not closed boundary: " + b.getName() + " alevel:" + b.getAdminLevel());
|
||||
iter.remove();
|
||||
}
|
||||
}
|
||||
|
||||
public void bindCitiesWithBoundaries(IProgress progress) {
|
||||
progress.startWork(cities.size()*2);
|
||||
Set<Boundary> freeBoundaries = new HashSet<Boundary>(allBoundaries);
|
||||
freeBoundaries.removeAll(cityBoundaries.values());
|
||||
//for cities without boundaries, try to find the right one
|
||||
for (City c : cities.values()) {
|
||||
progress.progress(1);
|
||||
Boundary cityB = cityBoundaries.get(c);
|
||||
int smallestAdminLevel = 8; //TODO start at level 8 for now...
|
||||
if (cityB == null) {
|
||||
LatLon location = c.getLocation();
|
||||
Boundary smallestoundary = null;
|
||||
Boundary smallestBoundary = null;
|
||||
//try to found boundary
|
||||
for (Boundary b : allBoundaries) {
|
||||
if (b.containsPoint(location.getLatitude(), location.getLongitude())) {
|
||||
//the bigger the admin level, the smaller the boundary :-)
|
||||
if (b.getAdminLevel() > smallestAdminLevel) {
|
||||
smallestAdminLevel = b.getAdminLevel();
|
||||
smallestoundary = b;
|
||||
for (Boundary b : freeBoundaries) {
|
||||
if (b.getAdminLevel() > smallestAdminLevel) {
|
||||
if (b.containsPoint(location.getLatitude(), location.getLongitude())) {
|
||||
//the bigger the admin level, the smaller the boundary :-)
|
||||
smallestAdminLevel = b.getAdminLevel();
|
||||
smallestBoundary = b;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (smallestoundary != null) {
|
||||
putCityBoundary(smallestoundary, Collections.singletonList(c));
|
||||
if (smallestBoundary != null) {
|
||||
Boundary oldBoundary = putCityBoundary(smallestBoundary,c);
|
||||
freeBoundaries.remove(smallestBoundary);
|
||||
if (oldBoundary != null) {
|
||||
freeBoundaries.add(oldBoundary);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//now for each city, try to put it in all boundaries it belongs to
|
||||
for (City c : cities.values()) {
|
||||
progress.progress(1);
|
||||
for (Boundary b : allBoundaries) {
|
||||
if (b.containsPoint(c.getLocation())) {
|
||||
List<City> list = boundariesToCities.get(b);
|
||||
if (list == null) {
|
||||
list = new ArrayList<City>(1);
|
||||
boundariesToCities.put(b, list);
|
||||
}
|
||||
list.add(c);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -354,30 +228,29 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
|
|||
}
|
||||
}
|
||||
|
||||
private void putCityBoundary(Boundary boundary, List<City> cities) {
|
||||
for (City cityFound : cities) {
|
||||
final Boundary oldBoundary = cityBoundaries.get(cityFound);
|
||||
if (oldBoundary != null) {
|
||||
// try to found the biggest area (not small center district)
|
||||
if (oldBoundary.getAdminLevel() > boundary.getAdminLevel() && !oldBoundary.getName().equalsIgnoreCase(cityFound.getName())) {
|
||||
cityBoundaries.put(cityFound, boundary);
|
||||
System.out.println("City: " + cityFound.getName() + " boundary: " + boundary.getName());
|
||||
} else if(boundary.getName().equalsIgnoreCase(cityFound.getName()) &&
|
||||
!oldBoundary.getName().equalsIgnoreCase(cityFound.getName())){
|
||||
cityBoundaries.put(cityFound, boundary);
|
||||
System.out.println("City: " + cityFound.getName() + " boundary: " + boundary.getName());
|
||||
} else if(oldBoundary.getAdminLevel() == boundary.getAdminLevel() &&
|
||||
oldBoundary != boundary && boundary.getName().equalsIgnoreCase(oldBoundary.getName())){
|
||||
if(!oldBoundary.isClosedWay() && !boundary.isClosedWay()){
|
||||
oldBoundary.getInnerWays().addAll(boundary.getInnerWays());
|
||||
oldBoundary.getOuterWays().addAll(boundary.getOuterWays());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
private Boundary putCityBoundary(Boundary boundary, City cityFound) {
|
||||
final Boundary oldBoundary = cityBoundaries.get(cityFound);
|
||||
if (oldBoundary != null) {
|
||||
// try to found the biggest area (not small center district)
|
||||
if (oldBoundary.getAdminLevel() > boundary.getAdminLevel() && !oldBoundary.getName().equalsIgnoreCase(cityFound.getName())) {
|
||||
cityBoundaries.put(cityFound, boundary);
|
||||
System.out.println("City: " + cityFound.getName() + " boundary: " + boundary.getName());
|
||||
} else if(boundary.getName().equalsIgnoreCase(cityFound.getName()) &&
|
||||
!oldBoundary.getName().equalsIgnoreCase(cityFound.getName())){
|
||||
cityBoundaries.put(cityFound, boundary);
|
||||
System.out.println("City: " + cityFound.getName() + " boundary: " + boundary.getName());
|
||||
} else if(oldBoundary.getAdminLevel() == boundary.getAdminLevel() &&
|
||||
oldBoundary != boundary && boundary.getName().equalsIgnoreCase(oldBoundary.getName())){
|
||||
if(!oldBoundary.isClosedWay() && !boundary.isClosedWay()){
|
||||
oldBoundary.getInnerWays().addAll(boundary.getInnerWays());
|
||||
oldBoundary.getOuterWays().addAll(boundary.getOuterWays());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
cityBoundaries.put(cityFound, boundary);
|
||||
System.out.println("City: " + cityFound.getName() + " boundary: " + boundary.getName());
|
||||
}
|
||||
return oldBoundary;
|
||||
}
|
||||
|
||||
private boolean isBoundary(Entity e) {
|
||||
|
|
@ -390,7 +263,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
|
|||
if (e instanceof Relation) {
|
||||
Relation aRelation = (Relation) e;
|
||||
ctx.loadEntityData(aRelation);
|
||||
boundary = new Boundary(true); //TODO here should be true or false ??? or, how it should be computed in this case?
|
||||
boundary = new Boundary(true); //is computed later
|
||||
boundary.setName(aRelation.getTag(OSMTagKey.NAME));
|
||||
boundary.setBoundaryId(aRelation.getId());
|
||||
boundary.setAdminLevel(extractBoundaryAdminLevel(aRelation));
|
||||
|
|
@ -410,6 +283,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
|
|||
}
|
||||
}
|
||||
}
|
||||
boundary.computeIsClosedWay();
|
||||
} else if (e instanceof Way) {
|
||||
if (!visitedBoundaryWays.contains(e.getId())) {
|
||||
List<Long> nodeIds = ((Way) e).getNodeIds();
|
||||
|
|
@ -417,7 +291,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
|
|||
if(nodeIds.size() > 0){
|
||||
closed = Algoritms.objectEquals(nodeIds.get(0), nodeIds.get(nodeIds.size() - 1));
|
||||
}
|
||||
boundary = new Boundary(closed);
|
||||
boundary = new WayBoundary(closed);
|
||||
boundary.setName(e.getTag(OSMTagKey.NAME));
|
||||
boundary.setBoundaryId(e.getId());
|
||||
boundary.setAdminLevel(extractBoundaryAdminLevel(e));
|
||||
|
|
@ -430,7 +304,6 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
public void indexAddressRelation(Relation i, OsmDbAccessorContext ctx) throws SQLException {
|
||||
if (i instanceof Relation && "address".equals(i.getTag(OSMTagKey.TYPE))) { //$NON-NLS-1$
|
||||
String type = i.getTag(OSMTagKey.ADDRESS_TYPE);
|
||||
|
|
@ -631,14 +504,16 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
|
|||
List<City> nearestObjects = new ArrayList<City>();
|
||||
nearestObjects.addAll(cityManager.getClosestObjects(location.getLatitude(),location.getLongitude()));
|
||||
nearestObjects.addAll(cityVillageManager.getClosestObjects(location.getLatitude(),location.getLongitude()));
|
||||
//either we found a city boundary the street is in
|
||||
for (City c : nearestObjects) {
|
||||
Boundary boundary = cityBoundaries.get(c);
|
||||
if (isInNames.contains(c.getName()) || (boundary != null && boundary.containsPoint(location))) {
|
||||
result.add(c);
|
||||
}
|
||||
}
|
||||
//or we need to find closes city
|
||||
if (result.isEmpty()) {
|
||||
City city = getClosestCity(location, isInNames);
|
||||
City city = getClosestCity(location, isInNames, nearestObjects);
|
||||
if (city != null) {
|
||||
result.add(city);
|
||||
}
|
||||
|
|
@ -657,107 +532,94 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
|
|||
|
||||
Set<Long> values = new TreeSet<Long>();
|
||||
for (City city : result) {
|
||||
String cityPart = city.getName();
|
||||
boolean found = false;
|
||||
for(City subpart : result){
|
||||
if(subpart != city){
|
||||
Boundary subBoundary = cityBoundaries.get(subpart);
|
||||
Boundary b = cityBoundaries.get(city);
|
||||
if(b != null && subBoundary != null && subBoundary.getAdminLevel() > b.getAdminLevel()){
|
||||
cityPart = findNearestCityOrSuburb(subBoundary, location); //subpart.getName();
|
||||
found = true;
|
||||
}
|
||||
String cityPart = null;
|
||||
SimpleStreet foundStreet = streetDAO.findStreet(name, city);
|
||||
if (foundStreet != null) {
|
||||
//oops, same street name within one city!
|
||||
if (foundStreet.getCityPart() == null) {
|
||||
//we need to update the city part first
|
||||
String aCityPart = findCityPart(foundStreet.getLocation(), city);
|
||||
foundStreet = streetDAO.updateStreetCityPart(foundStreet, city, aCityPart != null ? aCityPart : city.getName());
|
||||
}
|
||||
//matching the nodes is done somewhere else. This is just a simple check if the streets are really close to each other
|
||||
if (MapUtils.getDistance(location, foundStreet.getLocation()) > 500) {
|
||||
//now, try to find our cityPart again
|
||||
cityPart = findCityPart(location, city);
|
||||
foundStreet = streetDAO.findStreet(name, city, cityPart);
|
||||
}
|
||||
}
|
||||
if (!found) {
|
||||
Boundary b = cityBoundaries.get(city);
|
||||
if (b != null) {
|
||||
cityPart = findNearestCityOrSuburb(b, location);
|
||||
}
|
||||
if (foundStreet == null) {
|
||||
//by default write city with cityPart of the city
|
||||
long streetId = streetDAO.insertStreet(name, nameEn, location, city, cityPart);
|
||||
values.add(streetId);
|
||||
}
|
||||
Long foundId = streetDAO.findStreet(name, city, cityPart);
|
||||
if (foundId == null) {
|
||||
long streetId = insertStreetData(addressStreetStat, name, nameEn, location.getLatitude(), location.getLongitude(), city.getId(),
|
||||
cityPart);
|
||||
streetDAO.insertStreet(addressStreetStat, name, city, cityPart, streetId);
|
||||
foundId = streetId;
|
||||
}
|
||||
values.add(foundId);
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
private String findCityPart(LatLon location, City city) {
|
||||
String cityPart = city.getName();
|
||||
boolean found = false;
|
||||
Boundary cityBoundary = cityBoundaries.get(city);
|
||||
if (cityBoundary != null) {
|
||||
for(City subpart : boundariesToCities.get(cityBoundary)){
|
||||
if(subpart != city){
|
||||
Boundary subBoundary = cityBoundaries.get(subpart);
|
||||
if(cityBoundary != null && subBoundary != null && subBoundary.getAdminLevel() > cityBoundary.getAdminLevel()){
|
||||
cityPart = findNearestCityOrSuburb(subBoundary, location); //subpart.getName();
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!found) {
|
||||
Boundary b = cityBoundaries.get(city);
|
||||
cityPart = findNearestCityOrSuburb(b, location);
|
||||
}
|
||||
return cityPart;
|
||||
}
|
||||
|
||||
private String findNearestCityOrSuburb(Boundary greatestBoundary,
|
||||
LatLon location) {
|
||||
String result = greatestBoundary.getName();
|
||||
List<City> nearestObjects = new ArrayList<City>();
|
||||
nearestObjects.addAll(cityManager.getClosestObjects(location.getLatitude(),location.getLongitude()));
|
||||
nearestObjects.addAll(cityVillageManager.getClosestObjects(location.getLatitude(),location.getLongitude()));
|
||||
String result = null;
|
||||
double dist = Double.MAX_VALUE;
|
||||
for (City c : nearestObjects) {
|
||||
if (greatestBoundary.containsPoint(c.getLocation())) {
|
||||
double actualDistance = MapUtils.getDistance(location, c.getLocation());
|
||||
if (actualDistance < dist) {
|
||||
result = c.getName();
|
||||
dist = actualDistance;
|
||||
}
|
||||
List<City> list = new ArrayList<City>();
|
||||
if(greatestBoundary != null) {
|
||||
result = greatestBoundary.getName();
|
||||
list = boundariesToCities.get(greatestBoundary);
|
||||
} else {
|
||||
list.addAll(cityManager.getClosestObjects(location.getLatitude(),location.getLongitude()));
|
||||
list.addAll(cityVillageManager.getClosestObjects(location.getLatitude(),location.getLongitude()));
|
||||
}
|
||||
for (City c : list) {
|
||||
double actualDistance = MapUtils.getDistance(location, c.getLocation());
|
||||
if (actualDistance < dist) {
|
||||
result = c.getName();
|
||||
dist = actualDistance;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
public City getClosestCity(LatLon point, Set<String> isInNames) {
|
||||
public City getClosestCity(LatLon point, Set<String> isInNames, Collection<City> nearCitiesAndVillages) {
|
||||
if (point == null) {
|
||||
return null;
|
||||
}
|
||||
// search by boundaries
|
||||
for (City c : cityManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) {
|
||||
Boundary boundary = cityBoundaries.get(c);
|
||||
if(boundary != null){
|
||||
if(boundary.containsPoint(point)){
|
||||
return c;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (City c : cityVillageManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) {
|
||||
Boundary boundary = cityBoundaries.get(c);
|
||||
if(boundary != null){
|
||||
if(boundary.containsPoint(point)){
|
||||
return c;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// search by distance considering is_in names
|
||||
City closest = null;
|
||||
double relDist = Double.POSITIVE_INFINITY;
|
||||
for (City c : cityManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) {
|
||||
double rel = MapUtils.getDistance(c.getLocation(), point) / c.getType().getRadius();
|
||||
for (City c : nearCitiesAndVillages) {
|
||||
if(isInNames.contains(c.getName())){
|
||||
return c;
|
||||
}
|
||||
double rel = MapUtils.getDistance(c.getLocation(), point) / c.getType().getRadius();
|
||||
if (rel < relDist) {
|
||||
closest = c;
|
||||
relDist = rel;
|
||||
if (relDist < 0.2d && isInNames.isEmpty()) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (relDist < 0.2d && isInNames.isEmpty()) {
|
||||
return closest;
|
||||
}
|
||||
for (City c : cityVillageManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) {
|
||||
if(isInNames.contains(c.getName())){
|
||||
return c;
|
||||
}
|
||||
double rel = MapUtils.getDistance(c.getLocation(), point) / c.getType().getRadius();
|
||||
if (rel < relDist) {
|
||||
closest = c;
|
||||
relDist = rel;
|
||||
if (relDist < 0.2d) {
|
||||
break;
|
||||
return closest;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -820,25 +682,8 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
|
|||
}
|
||||
|
||||
|
||||
private long streetIdSequence = 0;
|
||||
|
||||
private boolean debugFullNames = false; //true to see atached cityPart and boundaries to the street names
|
||||
|
||||
private long insertStreetData(PreparedStatement addressStreetStat, //long id,
|
||||
String name, String nameEn, double latitude,
|
||||
double longitude, Long cityId, String cityPart) throws SQLException {
|
||||
long streetId = streetIdSequence++;
|
||||
addressStreetStat.setLong(1, streetId);
|
||||
addressStreetStat.setString(4, name);
|
||||
addressStreetStat.setString(5, nameEn);
|
||||
addressStreetStat.setDouble(2, latitude);
|
||||
addressStreetStat.setDouble(3, longitude);
|
||||
addressStreetStat.setLong(6, cityId);
|
||||
addressStreetStat.setString(7, cityPart);
|
||||
return streetId;
|
||||
}
|
||||
|
||||
|
||||
public void writeCitiesIntoDb() throws SQLException {
|
||||
for (City c : cities.values()) {
|
||||
writeCity(c);
|
||||
|
|
@ -852,11 +697,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
|
|||
}
|
||||
|
||||
public void processingPostcodes() throws SQLException {
|
||||
if (pStatements.get(addressBuildingStat) > 0) {
|
||||
addressBuildingStat.executeBatch();
|
||||
pStatements.put(addressBuildingStat, 0);
|
||||
mapConnection.commit();
|
||||
}
|
||||
streetDAO.commit();
|
||||
PreparedStatement pstat = mapConnection.prepareStatement("UPDATE building SET postcode = ? WHERE id = ?");
|
||||
pStatements.put(pstat, 0);
|
||||
for (Relation r : postalCodeRelations) {
|
||||
|
|
@ -875,7 +716,8 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
|
|||
|
||||
|
||||
public void writeBinaryAddressIndex(BinaryMapIndexWriter writer, String regionName, IProgress progress) throws IOException, SQLException {
|
||||
closePreparedStatements(addressCityStat, addressStreetStat, addressStreetNodeStat, addressBuildingStat);
|
||||
streetDAO.close();
|
||||
closePreparedStatements(addressCityStat);
|
||||
mapConnection.commit();
|
||||
boolean readWayNodes = saveAddressWays;
|
||||
|
||||
|
|
@ -995,38 +837,16 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
|
|||
|
||||
public void commitToPutAllCities() throws SQLException {
|
||||
// commit to put all cities
|
||||
if (pStatements.get(addressBuildingStat) > 0) {
|
||||
addressBuildingStat.executeBatch();
|
||||
pStatements.put(addressBuildingStat, 0);
|
||||
}
|
||||
if (pStatements.get(addressStreetNodeStat) > 0) {
|
||||
addressStreetNodeStat.executeBatch();
|
||||
pStatements.put(addressStreetNodeStat, 0);
|
||||
}
|
||||
mapConnection.commit();
|
||||
|
||||
streetDAO.commit();
|
||||
}
|
||||
|
||||
public void createDatabaseStructure(Connection mapConnection, DBDialect dialect) throws SQLException {
|
||||
this.mapConnection = mapConnection;
|
||||
streetDAO.createDatabaseStructure(mapConnection, dialect);
|
||||
createAddressIndexStructure(mapConnection, dialect);
|
||||
addressCityStat = mapConnection.prepareStatement("insert into city (id, latitude, longitude, name, name_en, city_type) values (?, ?, ?, ?, ?, ?)");
|
||||
addressStreetStat = mapConnection.prepareStatement("insert into street (id, latitude, longitude, name, name_en, city, citypart) values (?, ?, ?, ?, ?, ?, ?)");
|
||||
addressBuildingStat = mapConnection.prepareStatement("insert into building (id, latitude, longitude, name, name_en, street, postcode) values (?, ?, ?, ?, ?, ?, ?)");
|
||||
addressStreetNodeStat = mapConnection.prepareStatement("insert into street_node (id, latitude, longitude, street, way) values (?, ?, ?, ?, ?)");
|
||||
|
||||
addressSearchStreetStat = mapConnection.prepareStatement("SELECT ID FROM street WHERE ? = city AND ? = citypart AND ? = name");
|
||||
addressSearchBuildingStat = mapConnection.prepareStatement("SELECT id FROM building where ? = id");
|
||||
addressSearchStreetNodeStat = mapConnection.prepareStatement("SELECT way FROM street_node WHERE ? = way");
|
||||
|
||||
pStatements.put(addressCityStat, 0);
|
||||
pStatements.put(addressStreetStat, 0);
|
||||
pStatements.put(addressStreetNodeStat, 0);
|
||||
pStatements.put(addressBuildingStat, 0);
|
||||
// put search statements to close them after all
|
||||
pStatements.put(addressSearchBuildingStat, 0);
|
||||
pStatements.put(addressSearchStreetNodeStat, 0);
|
||||
pStatements.put(addressSearchStreetStat, 0);
|
||||
}
|
||||
|
||||
private void createAddressIndexStructure(Connection conn, DBDialect dialect) throws SQLException{
|
||||
|
|
@ -1036,25 +856,6 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
|
|||
"name varchar(1024), name_en varchar(1024), city_type varchar(32))");
|
||||
stat.executeUpdate("create index city_ind on city (id, city_type)");
|
||||
|
||||
stat.executeUpdate("create table street (id bigint primary key, latitude double, longitude double, " +
|
||||
"name varchar(1024), name_en varchar(1024), city bigint, citypart varchar(1024))");
|
||||
stat.executeUpdate("create index street_cnp on street (city,citypart,name,id)");
|
||||
stat.executeUpdate("create index street_city on street (city)");
|
||||
stat.executeUpdate("create index street_id on street (id)");
|
||||
// create index on name ?
|
||||
|
||||
stat.executeUpdate("create table building (id bigint, latitude double, longitude double, " +
|
||||
"name varchar(1024), name_en varchar(1024), street bigint, postcode varchar(1024), primary key(street, id))");
|
||||
stat.executeUpdate("create index building_postcode on building (postcode)");
|
||||
stat.executeUpdate("create index building_street on building (street)");
|
||||
stat.executeUpdate("create index building_id on building (id)");
|
||||
|
||||
|
||||
stat.executeUpdate("create table street_node (id bigint, latitude double, longitude double, " +
|
||||
"street bigint, way bigint)");
|
||||
stat.executeUpdate("create index street_node_street on street_node (street)");
|
||||
stat.executeUpdate("create index street_node_way on street_node (way)");
|
||||
|
||||
// if(dialect == DBDialect.SQLITE){
|
||||
// stat.execute("PRAGMA user_version = " + IndexConstants.ADDRESS_TABLE_VERSION); //$NON-NLS-1$
|
||||
// }
|
||||
|
|
@ -1099,8 +900,8 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
|
|||
|
||||
//If there are more streets with same name in different districts.
|
||||
//Add district name to all other names. If sorting is right, the first street was the one in the city
|
||||
String cityPart = " (" + set.getString(12) + ")";
|
||||
String district = identifyBestDistrict(street, streetName, cityPart, uniqueNames, streetNodes);
|
||||
String defaultDistrict = set.getString(12);
|
||||
String district = identifyBestDistrict(street, streetName, " (" + defaultDistrict + ")", uniqueNames, streetNodes);
|
||||
street.setName(streetName + district);
|
||||
street.setEnName(set.getString(3) + district);
|
||||
//if for this street there is already same street, add just nodes to the street.
|
||||
|
|
|
|||
|
|
@ -18,10 +18,10 @@ import net.osmand.data.IndexConstants;
|
|||
import net.osmand.data.preparation.OsmDbAccessor.OsmDbVisitor;
|
||||
import net.osmand.impl.ConsoleProgressImplementation;
|
||||
import net.osmand.osm.Entity;
|
||||
import net.osmand.osm.MapRenderingTypes;
|
||||
import net.osmand.osm.Relation;
|
||||
import net.osmand.osm.Entity.EntityId;
|
||||
import net.osmand.osm.Entity.EntityType;
|
||||
import net.osmand.osm.MapRenderingTypes;
|
||||
import net.osmand.osm.Relation;
|
||||
import net.osmand.osm.io.IOsmStorageFilter;
|
||||
import net.osmand.osm.io.OsmBaseStorage;
|
||||
import net.osmand.swing.DataExtractionSettings;
|
||||
|
|
@ -32,8 +32,6 @@ import org.apache.commons.logging.LogFactory;
|
|||
import org.apache.tools.bzip2.CBZip2InputStream;
|
||||
import org.xml.sax.SAXException;
|
||||
|
||||
import com.anvisics.jleveldb.LevelDBAccess;
|
||||
|
||||
import rtree.RTreeException;
|
||||
|
||||
/**
|
||||
|
|
@ -287,6 +285,7 @@ public class IndexCreator {
|
|||
}
|
||||
|
||||
private boolean createPlainOsmDb(IProgress progress, File readFile, IOsmStorageFilter addFilter) throws SQLException, FileNotFoundException, IOException, SAXException{
|
||||
// dbFile = new File(workingDir, TEMP_NODES_DB);
|
||||
// initialize db file
|
||||
boolean loadFromExistingFile = dbFile != null && dialect.databaseFileExists(dbFile);
|
||||
if (dbFile == null) {
|
||||
|
|
@ -437,7 +436,7 @@ public class IndexCreator {
|
|||
// 3.2 index address relations
|
||||
if (indexAddress || indexMap) {
|
||||
progress.setGeneralProgress("[30 / 100]"); //$NON-NLS-1$
|
||||
progress.startTask(Messages.getString("IndexCreator.PREINDEX_ADRESS_MAP"), accessor.getAllRelations()); //$NON-NLS-1$
|
||||
progress.startTask(Messages.getString("IndexCreator.PREINDEX_BOUNDARIES_RELATIONS"), accessor.getAllRelations()); //$NON-NLS-1$
|
||||
accessor.iterateOverEntities(progress, EntityType.RELATION, new OsmDbVisitor() {
|
||||
@Override
|
||||
public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException {
|
||||
|
|
@ -452,16 +451,18 @@ public class IndexCreator {
|
|||
});
|
||||
if (indexAddress) {
|
||||
progress.setGeneralProgress("[40 / 100]"); //$NON-NLS-1$
|
||||
progress.startTask(Messages.getString("IndexCreator.PREINDEX_ADRESS_MAP"), accessor.getAllWays()); //$NON-NLS-1$
|
||||
accessor.iterateOverEntities(progress, EntityType.WAY, new OsmDbVisitor() {
|
||||
progress.startTask(Messages.getString("IndexCreator.PREINDEX_BOUNDARIES_WAYS"), accessor.getAllWays()); //$NON-NLS-1$
|
||||
accessor.iterateOverEntities(progress, EntityType.WAY_BOUNDARY, new OsmDbVisitor() {
|
||||
@Override
|
||||
public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException {
|
||||
indexAddressCreator.indexBoundariesRelation(e, ctx);
|
||||
}
|
||||
});
|
||||
|
||||
progress.setGeneralProgress("[42 / 100]"); //$NON-NLS-1$
|
||||
progress.startTask(Messages.getString("IndexCreator.BIND_CITIES_AND_BOUNDARIES"), 100); //$NON-NLS-1$
|
||||
//finish up the boundaries and cities
|
||||
indexAddressCreator.bindCitiesWithBoundaries();
|
||||
indexAddressCreator.bindCitiesWithBoundaries(progress);
|
||||
|
||||
progress.setGeneralProgress("[45 / 100]"); //$NON-NLS-1$
|
||||
progress.startTask(Messages.getString("IndexCreator.PREINDEX_ADRESS_MAP"), accessor.getAllRelations()); //$NON-NLS-1$
|
||||
|
|
@ -685,4 +686,4 @@ public class IndexCreator {
|
|||
System.out.println("-- MAP_DATA_AND_STRINGS SIZE " + (BinaryMapIndexWriter.MAP_DATA_SIZE + BinaryMapIndexWriter.STRING_TABLE_SIZE)); //$NON-NLS-1$
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
package net.osmand.data.preparation;
|
||||
|
||||
import gnu.trove.list.TLongList;
|
||||
import gnu.trove.list.array.TLongArrayList;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
|
|
@ -16,6 +19,7 @@ import net.osmand.osm.Entity;
|
|||
import net.osmand.osm.Entity.EntityId;
|
||||
import net.osmand.osm.Entity.EntityType;
|
||||
import net.osmand.osm.Node;
|
||||
import net.osmand.osm.OSMSettings.OSMTagKey;
|
||||
import net.osmand.osm.Relation;
|
||||
import net.osmand.osm.Way;
|
||||
|
||||
|
|
@ -37,7 +41,8 @@ public class OsmDbAccessor implements OsmDbAccessorContext {
|
|||
private PreparedStatement pselectWay;
|
||||
private PreparedStatement pselectRelation;
|
||||
private PreparedStatement pselectTags;
|
||||
private int allRelations ;
|
||||
private int allRelations;
|
||||
private int allBoundaries;
|
||||
private int allWays;
|
||||
private int allNodes;
|
||||
private boolean realCounts = false;
|
||||
|
|
@ -54,8 +59,9 @@ public class OsmDbAccessor implements OsmDbAccessorContext {
|
|||
private PreparedStatement iterateWays;
|
||||
|
||||
private PreparedStatement iterateRelations;
|
||||
|
||||
|
||||
|
||||
private PreparedStatement iterateWayBoundaries;
|
||||
|
||||
public interface OsmDbVisitor {
|
||||
public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException;
|
||||
}
|
||||
|
|
@ -88,6 +94,7 @@ public class OsmDbAccessor implements OsmDbAccessorContext {
|
|||
iterateWays = dbConn.prepareStatement("select w.id, w.node, w.ord, t.skeys, t.value, n.latitude, n.longitude " + //$NON-NLS-1$
|
||||
"from ways w left join tags t on w.id = t.id and t.type = 1 and w.ord = 0 inner join node n on w.node = n.id " + //$NON-NLS-1$
|
||||
"order by w.id, w.ord"); //$NON-NLS-1$
|
||||
iterateWayBoundaries = dbConn.prepareStatement("select t.id from tags t where t.skeys = \"" + OSMTagKey.BOUNDARY.getValue() + "\""); //$NON-NLS-1$
|
||||
iterateRelations = dbConn.prepareStatement("select r.id, t.skeys, t.value from relations r inner join tags t on t.id = r.id and t.type = 2 and r.ord = 0"); //$NON-NLS-1$
|
||||
}
|
||||
}
|
||||
|
|
@ -237,33 +244,38 @@ public class OsmDbAccessor implements OsmDbAccessorContext {
|
|||
// stat.executeUpdate("create table relations (id "+longType+", member "+longType+", type smallint, role varchar(1024), ord smallint)");
|
||||
computeRealCounts(statement);
|
||||
statement.close();
|
||||
if (type == EntityType.NODE) {
|
||||
// filter out all nodes without tags
|
||||
select = iterateNodes;
|
||||
count = allNodes;
|
||||
} else if (type == EntityType.WAY) {
|
||||
select = iterateWays;
|
||||
count = allWays;
|
||||
|
||||
BlockingQueue<Entity> toProcess = new ArrayBlockingQueue<Entity>(100000);
|
||||
AbstractProducer entityProducer = null;
|
||||
if (type == EntityType.WAY_BOUNDARY) {
|
||||
select = iterateWayBoundaries;
|
||||
count = allBoundaries;
|
||||
entityProducer = new BoundaryProducer(toProcess, select);
|
||||
} else {
|
||||
select = iterateRelations;
|
||||
count = allRelations;
|
||||
if (type == EntityType.NODE) {
|
||||
// filter out all nodes without tags
|
||||
select = iterateNodes;
|
||||
count = allNodes;
|
||||
} else if (type == EntityType.WAY) {
|
||||
select = iterateWays;
|
||||
count = allWays;
|
||||
} else {
|
||||
select = iterateRelations;
|
||||
count = allRelations;
|
||||
}
|
||||
entityProducer = new EntityProducer(toProcess, type, select);
|
||||
}
|
||||
progress.startWork(count);
|
||||
|
||||
BlockingQueue<Entity> toProcess = new ArrayBlockingQueue<Entity>(100000);
|
||||
|
||||
//produce
|
||||
EntityProducer entityProducer = new EntityProducer(toProcess, type, select);
|
||||
entityProducer.start();
|
||||
|
||||
int counter = 0;
|
||||
Entity entityToProcess = null;
|
||||
Entity endEntity = entityProducer.getEndingEntity();
|
||||
while ((entityToProcess = toProcess.take()) != endEntity) {
|
||||
if (progress != null) {
|
||||
progress.progress(1);
|
||||
}
|
||||
counter++;
|
||||
visitor.iterateEntity(entityToProcess, this);
|
||||
}
|
||||
return count;
|
||||
|
|
@ -275,8 +287,9 @@ public class OsmDbAccessor implements OsmDbAccessorContext {
|
|||
realCounts = true;
|
||||
// filter out all nodes without tags
|
||||
allNodes = statement.executeQuery("select count(distinct n.id) from node n inner join tags t on n.id = t.id and t.type = 0").getInt(1); //$NON-NLS-1$
|
||||
allWays = statement.executeQuery("select count(distinct w.id) from ways w").getInt(1); //$NON-NLS-1$
|
||||
allWays = statement.executeQuery("select count(*) from ways w where w.ord = 0").getInt(1); //$NON-NLS-1$
|
||||
allRelations = statement.executeQuery("select count(distinct r.id) from relations r inner join tags t on t.id = r.id and t.type = 2 and r.ord = 0").getInt(1); //$NON-NLS-1$
|
||||
allBoundaries = statement.executeQuery("select count(distinct t.id) from tags t where t.skeys = \"" + OSMTagKey.BOUNDARY.getValue() + "\"").getInt(1); //$NON-NLS-1$
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -458,29 +471,103 @@ public class OsmDbAccessor implements OsmDbAccessorContext {
|
|||
if (iterateWays != null) {
|
||||
iterateWays.close();
|
||||
}
|
||||
if (iterateWayBoundaries != null) {
|
||||
iterateWayBoundaries.close();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public class EntityProducer extends Thread {
|
||||
public class AbstractProducer extends Thread {
|
||||
private final Entity endingEntity = new Node(0,0,0);
|
||||
|
||||
public Entity getEndingEntity() {
|
||||
return endingEntity;
|
||||
}
|
||||
}
|
||||
|
||||
public class BoundaryProducer extends AbstractProducer {
|
||||
private final BlockingQueue<Entity> toProcess;
|
||||
private final PreparedStatement select;
|
||||
|
||||
public BoundaryProducer(BlockingQueue<Entity> toProcess, PreparedStatement selectIds) {
|
||||
this.toProcess = toProcess;
|
||||
this.select = selectIds;
|
||||
setDaemon(true);
|
||||
setName("BoundaryProducer");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
ResultSet rs;
|
||||
try {
|
||||
select.execute();
|
||||
rs = select.getResultSet();
|
||||
TLongArrayList boundariesToLoad = new TLongArrayList();
|
||||
while (rs.next()) {
|
||||
boundariesToLoad.add(rs.getLong(1));
|
||||
}
|
||||
rs.close();
|
||||
PreparedStatement iterateWaysByIds = null;
|
||||
int idsSize = 0;
|
||||
|
||||
while (!boundariesToLoad.isEmpty()) {
|
||||
int chunk = Math.min(100, boundariesToLoad.size());
|
||||
if (chunk != idsSize) {
|
||||
if (iterateWaysByIds != null) {
|
||||
iterateWaysByIds.close();
|
||||
}
|
||||
StringBuilder b = new StringBuilder();
|
||||
for (int i = 0; i <= chunk; i++) {
|
||||
b.append('?').append(',');
|
||||
}
|
||||
b.deleteCharAt(b.length()-1);
|
||||
iterateWaysByIds = dbConn.prepareStatement("select w.id, w.node, w.ord, t.skeys, t.value, n.latitude, n.longitude " + //$NON-NLS-1$
|
||||
"from ways w left join tags t on w.id = t.id and t.type = 1 and w.ord = 0 inner join node n on w.node = n.id where w.id in (" + b.toString() + //$NON-NLS-1$
|
||||
") order by w.id, w.ord"); //$NON-NLS-1$
|
||||
idsSize = chunk;
|
||||
}
|
||||
TLongList subList = boundariesToLoad.subList(0, chunk);
|
||||
for (int i = 0; i < chunk; i++) {
|
||||
iterateWaysByIds.setLong(i+1, subList.get(i));
|
||||
}
|
||||
boundariesToLoad.remove(0, chunk);
|
||||
//load the ways
|
||||
new EntityProducer(toProcess, EntityType.WAY, iterateWaysByIds,false).run();
|
||||
}
|
||||
} catch (SQLException e1) {
|
||||
e1.printStackTrace();
|
||||
} finally {
|
||||
try {
|
||||
toProcess.put(getEndingEntity());
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public class EntityProducer extends AbstractProducer {
|
||||
|
||||
private final BlockingQueue<Entity> toProcess;
|
||||
private final PreparedStatement select;
|
||||
private final EntityType type;
|
||||
private final Entity endingEntity = new Node(0,0,0);
|
||||
private final boolean putEndingEntity;
|
||||
|
||||
public EntityProducer(BlockingQueue<Entity> toProcess, EntityType type, PreparedStatement select) {
|
||||
this(toProcess,type,select,true);
|
||||
}
|
||||
|
||||
public EntityProducer(BlockingQueue<Entity> toProcess, EntityType type, PreparedStatement select2) {
|
||||
public EntityProducer(BlockingQueue<Entity> toProcess, EntityType type, PreparedStatement select, boolean putEndingEntity) {
|
||||
this.toProcess = toProcess;
|
||||
this.type = type;
|
||||
this.select = select2;
|
||||
this.select = select;
|
||||
this.putEndingEntity = putEndingEntity;
|
||||
setDaemon(true);
|
||||
setName("EntityProducer");
|
||||
}
|
||||
|
||||
public Entity getEndingEntity() {
|
||||
return endingEntity;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
ResultSet rs;
|
||||
|
|
@ -536,10 +623,12 @@ public class OsmDbAccessor implements OsmDbAccessorContext {
|
|||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
try {
|
||||
toProcess.put(getEndingEntity());
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
if (putEndingEntity) {
|
||||
try {
|
||||
toProcess.put(getEndingEntity());
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,7 +14,8 @@ public abstract class Entity {
|
|||
public enum EntityType {
|
||||
NODE,
|
||||
WAY,
|
||||
RELATION;
|
||||
RELATION,
|
||||
WAY_BOUNDARY;
|
||||
|
||||
public static EntityType valueOf(Entity e){
|
||||
if(e instanceof Node){
|
||||
|
|
|
|||
Loading…
Reference in a new issue