Fixed issue 352. Multiple streets with same names in different

districts.
This commit is contained in:
Pavol Zibrita 2011-08-28 15:15:21 +02:00
parent fa0ed6294e
commit 9675297248
11 changed files with 524 additions and 245 deletions

View file

@ -3,12 +3,12 @@
<classpathentry kind="src" path="src"/> <classpathentry kind="src" path="src"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/> <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="lib" path="lib/bzip2-20090327.jar"/> <classpathentry kind="lib" path="lib/bzip2-20090327.jar"/>
<classpathentry kind="lib" path="lib/sqlitejdbc-v056.jar"/>
<classpathentry kind="lib" path="lib/commons-logging-1.1.1.jar"/> <classpathentry kind="lib" path="lib/commons-logging-1.1.1.jar"/>
<classpathentry kind="lib" path="lib/json-20090211.jar"/> <classpathentry kind="lib" path="lib/json-20090211.jar"/>
<classpathentry kind="lib" path="lib/h2-latest.jar"/> <classpathentry kind="lib" path="lib/h2-latest.jar"/>
<classpathentry kind="lib" path="lib/bsh-core-2.0b4.jar"/> <classpathentry kind="lib" path="lib/bsh-core-2.0b4.jar"/>
<classpathentry kind="lib" path="lib/junidecode-0.1.jar"/> <classpathentry kind="lib" path="lib/junidecode-0.1.jar"/>
<classpathentry kind="lib" path="lib/jleveldb-v01.jar"/> <classpathentry kind="lib" path="lib/jleveldb-v01.jar"/>
<classpathentry kind="lib" path="lib/sqlite-jdbc-3.7.2.jar"/>
<classpathentry kind="output" path="bin"/> <classpathentry kind="output" path="bin"/>
</classpath> </classpath>

View file

@ -12,4 +12,5 @@
</listAttribute> </listAttribute>
<stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="net.osmand.swing.OsmExtractionUI"/> <stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="net.osmand.swing.OsmExtractionUI"/>
<stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="DataExtractionOSM"/> <stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="DataExtractionOSM"/>
<stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-XX:+UseParallelGC -Xmx640M -Xmn256M"/>
</launchConfiguration> </launchConfiguration>

View file

@ -1,6 +1,7 @@
package net.osmand.data; package net.osmand.data;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection;
import java.util.List; import java.util.List;
import net.osmand.osm.LatLon; import net.osmand.osm.LatLon;
@ -19,6 +20,7 @@ public class Boundary {
private List<Way> outerWays = new ArrayList<Way>(); private List<Way> outerWays = new ArrayList<Way>();
private List<Way> innerWays = new ArrayList<Way>(); private List<Way> innerWays = new ArrayList<Way>();
private List<Boundary> subboundaries = new ArrayList<Boundary>();
public boolean containsPoint(LatLon point) { public boolean containsPoint(LatLon point) {
return containsPoint(point.getLatitude(), point.getLongitude()); return containsPoint(point.getLatitude(), point.getLongitude());
@ -88,5 +90,18 @@ public class Boundary {
this.adminLevel = adminLevel; this.adminLevel = adminLevel;
} }
public List<Boundary> getSubboundaries() {
return subboundaries;
}
public void addSubBoundary(Boundary subBoundary) {
if (subBoundary != null) {
subboundaries.add(subBoundary);
}
}
public void addSubBoundaries(Collection<Boundary> subBoundaries) {
subboundaries.addAll(subBoundaries);
}
} }

View file

@ -9,6 +9,11 @@ import java.sql.Statement;
import net.osmand.Algoritms; import net.osmand.Algoritms;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.sqlite.SQLiteConfig;
import org.sqlite.SQLiteConfig.JournalMode;
import org.sqlite.SQLiteConfig.LockingMode;
import org.sqlite.SQLiteConfig.SynchronousMode;
import org.sqlite.SQLiteJDBCLoader;
import com.anvisics.jleveldb.LevelDBAccess; import com.anvisics.jleveldb.LevelDBAccess;
import com.anvisics.jleveldb.ext.DBAccessor; import com.anvisics.jleveldb.ext.DBAccessor;
@ -91,10 +96,15 @@ public enum DBDialect {
log.error("Illegal configuration", e); log.error("Illegal configuration", e);
throw new IllegalStateException(e); throw new IllegalStateException(e);
} }
SQLiteConfig config = new SQLiteConfig();
config.setCacheSize(10000); //size for number in file pages in memory, (disk cache)
config.setJournalMode(JournalMode.OFF); //no journal - on crash the db is not usabale
config.setSynchronous(SynchronousMode.OFF); // faster without synchronization
config.setLockingMode(LockingMode.EXCLUSIVE); // we are the only one using the file => no need to get/realease locks always => faster
config.setSharedCache(true); // => needed for readUncommited
config.setReadUncommited(true); // => faster queries (read also what is not commited yet, but is inserted)
Connection connection = DriverManager.getConnection("jdbc:sqlite:" + fileName); Connection connection = DriverManager.getConnection("jdbc:sqlite:" + fileName);
Statement statement = connection.createStatement(); System.out.println(String.format("SQLITE running in %s mode", SQLiteJDBCLoader.isNativeMode() ? "native" : "pure-java"));
statement.executeUpdate("PRAGMA synchronous = 0");
statement.close();
return connection; return connection;
} else if (DBDialect.DERBY == this) { } else if (DBDialect.DERBY == this) {
try { try {

View file

@ -1,5 +1,8 @@
package net.osmand.data.preparation; package net.osmand.data.preparation;
import gnu.trove.map.hash.TLongObjectHashMap;
import gnu.trove.set.hash.TLongHashSet;
import java.io.IOException; import java.io.IOException;
import java.sql.Connection; import java.sql.Connection;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
@ -25,17 +28,17 @@ import net.osmand.binary.BinaryMapIndexWriter;
import net.osmand.data.Boundary; import net.osmand.data.Boundary;
import net.osmand.data.Building; import net.osmand.data.Building;
import net.osmand.data.City; import net.osmand.data.City;
import net.osmand.data.City.CityType;
import net.osmand.data.DataTileManager; import net.osmand.data.DataTileManager;
import net.osmand.data.Street; import net.osmand.data.Street;
import net.osmand.data.City.CityType;
import net.osmand.osm.Entity; import net.osmand.osm.Entity;
import net.osmand.osm.Entity.EntityId;
import net.osmand.osm.LatLon; import net.osmand.osm.LatLon;
import net.osmand.osm.MapUtils; import net.osmand.osm.MapUtils;
import net.osmand.osm.Node; import net.osmand.osm.Node;
import net.osmand.osm.OSMSettings.OSMTagKey;
import net.osmand.osm.Relation; import net.osmand.osm.Relation;
import net.osmand.osm.Way; import net.osmand.osm.Way;
import net.osmand.osm.Entity.EntityId;
import net.osmand.osm.OSMSettings.OSMTagKey;
import net.osmand.swing.Messages; import net.osmand.swing.Messages;
import net.sf.junidecode.Junidecode; import net.sf.junidecode.Junidecode;
@ -58,18 +61,14 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
private PreparedStatement addressSearchBuildingStat; private PreparedStatement addressSearchBuildingStat;
private PreparedStatement addressSearchStreetNodeStat; private PreparedStatement addressSearchStreetNodeStat;
private Map<String, Long> addressStreetLocalMap = new LinkedHashMap<String, Long>();
private Set<Long> addressBuildingLocalSet = new LinkedHashSet<Long>();
private Set<Long> addressStreetNodeLocalSet = new LinkedHashSet<Long>();
// MEMORY address : address structure // MEMORY address : address structure
// load it in memory // load it in memory
private Map<EntityId, City> cities = new LinkedHashMap<EntityId, City>(); private Map<EntityId, City> cities = new LinkedHashMap<EntityId, City>();
private DataTileManager<City> cityVillageManager = new DataTileManager<City>(13); private DataTileManager<City> cityVillageManager = new DataTileManager<City>(13);
private DataTileManager<City> cityManager = new DataTileManager<City>(10); private DataTileManager<City> cityManager = new DataTileManager<City>(10);
private List<Relation> postalCodeRelations = new ArrayList<Relation>(); private List<Relation> postalCodeRelations = new ArrayList<Relation>();
private Map<City, Boundary> citiBoundaries = new LinkedHashMap<City, Boundary>(); private Map<City, Boundary> cityBoundaries = new LinkedHashMap<City, Boundary>();
private Set<Long> visitedBoundaryWays = new HashSet<Long>(); private TLongHashSet visitedBoundaryWays = new TLongHashSet();
private boolean normalizeStreets; private boolean normalizeStreets;
private String[] normalizeDefaultSuffixes; private String[] normalizeDefaultSuffixes;
@ -80,9 +79,126 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
// TODO // TODO
Connection mapConnection; Connection mapConnection;
DBStreetDAO streetDAO;
public class DBStreetDAO
{
protected void writeStreetWayNodes(Long streetId, Way way)
throws SQLException {
for (Node n : way.getNodes()) {
if (n == null) {
continue;
}
addressStreetNodeStat.setLong(1, n.getId());
addressStreetNodeStat.setDouble(2, n.getLatitude());
addressStreetNodeStat.setDouble(3, n.getLongitude());
addressStreetNodeStat.setLong(5, way.getId());
addressStreetNodeStat.setLong(4, streetId);
addBatch(addressStreetNodeStat);
}
}
protected void writeBuilding(Long streetId, Building building) throws SQLException {
addressBuildingStat.setLong(1, building.getId());
addressBuildingStat.setDouble(2, building.getLocation().getLatitude());
addressBuildingStat.setDouble(3, building.getLocation().getLongitude());
addressBuildingStat.setString(4, building.getName());
addressBuildingStat.setString(5, building.getEnName());
addressBuildingStat.setLong(6, streetId);
addressBuildingStat.setString(7, building.getPostcode() == null ? null : building.getPostcode().toUpperCase());
addBatch(addressBuildingStat);
}
public Long findStreet(String name, City city, String cityPart) throws SQLException {
addressSearchStreetStat.setLong(1, city.getId());
addressSearchStreetStat.setString(2, cityPart);
addressSearchStreetStat.setString(3, name);
ResultSet rs = addressSearchStreetStat.executeQuery();
Long foundId = null;
if (rs.next()) {
foundId = rs.getLong(1);
}
rs.close();
return foundId;
}
public void insertStreet(PreparedStatement addressStreetStat,
String name, City city, String cityPart, long initId) throws SQLException {
//execute the insert statement
addressStreetStat.execute();
// commit immediately to search after
mapConnection.commit();
}
public boolean findBuilding(Entity e) throws SQLException {
addressSearchBuildingStat.setLong(1, e.getId());
ResultSet rs = addressSearchBuildingStat.executeQuery();
boolean exist = rs.next();
rs.close();
return exist;
}
public boolean findStreetNode(Entity e) throws SQLException {
addressSearchStreetNodeStat.setLong(1, e.getId());
ResultSet rs = addressSearchStreetNodeStat.executeQuery();
boolean exist = rs.next();
rs.close();
return exist;
}
}
public class CachedDBStreetDAO extends DBStreetDAO
{
private Map<String, Long> addressStreetLocalMap = new LinkedHashMap<String, Long>();
private TLongHashSet addressBuildingLocalSet = new TLongHashSet();
private TLongHashSet addressStreetNodeLocalSet = new TLongHashSet();
@Override
public Long findStreet(String name, City city, String cityPart) {
return addressStreetLocalMap.get(createStreetUniqueName(name, city, cityPart)); //$NON-NLS-1$
}
private String createStreetUniqueName(String name, City city, String cityPart) {
return new StringBuilder().append(name).append('_').append(city.getId()).append('_').append(cityPart).toString();
}
@Override
protected void writeStreetWayNodes(Long streetId, Way way)
throws SQLException {
super.writeStreetWayNodes(streetId, way);
addressStreetNodeLocalSet.add(way.getId());
}
@Override
protected void writeBuilding(Long streetId, Building building)
throws SQLException {
super.writeBuilding(streetId, building);
addressBuildingLocalSet.add(building.getId());
}
@Override
public void insertStreet(PreparedStatement addressStreetStat,
String name, City city, String cityPart, long initId) throws SQLException {
//batch the insert
addBatch(addressStreetStat);
addressStreetLocalMap.put(createStreetUniqueName(name, city, cityPart), initId); //$NON-NLS-1$
}
@Override
public boolean findBuilding(Entity e) {
return addressBuildingLocalSet.contains(e.getId());
}
@Override
public boolean findStreetNode(Entity e) {
return addressStreetNodeLocalSet.contains(e.getId());
}
}
public IndexAddressCreator(){ public IndexAddressCreator(){
streetDAO = loadInMemory ? new CachedDBStreetDAO() : new DBStreetDAO();
} }
@ -91,7 +207,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
cities.clear(); cities.clear();
cityManager.clear(); cityManager.clear();
postalCodeRelations.clear(); postalCodeRelations.clear();
citiBoundaries.clear(); cityBoundaries.clear();
this.normalizeStreets = normalizeStreets; this.normalizeStreets = normalizeStreets;
this.normalizeDefaultSuffixes = normalizeDefaultSuffixes; this.normalizeDefaultSuffixes = normalizeDefaultSuffixes;
this.normalizeSuffixes = normalizeSuffixes; this.normalizeSuffixes = normalizeSuffixes;
@ -114,47 +230,8 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
} }
public void indexBoundariesRelation(Entity e, OsmDbAccessorContext ctx) throws SQLException { public void indexBoundariesRelation(Entity e, OsmDbAccessorContext ctx) throws SQLException {
if ("administrative".equals(e.getTag(OSMTagKey.BOUNDARY)) && (e instanceof Relation || e instanceof Way)) { if (isBoundary(e) && hasNeededCityAdminLevel(e)) {
String adminLevel = e.getTag("admin_level"); Boundary boundary = extractBoundary(e, ctx);
Boundary boundary = null;
if (cityAdminLevel.equals(adminLevel)) {
if (e instanceof Relation) {
Relation i = (Relation) e;
ctx.loadEntityData(i, true);
boundary = new Boundary();
if (i.getTag(OSMTagKey.NAME) != null) {
boundary.setName(i.getTag(OSMTagKey.NAME));
}
boundary.setBoundaryId(i.getId());
Map<Entity, String> entities = i.getMemberEntities();
for (Entity es : entities.keySet()) {
if (es instanceof Way) {
boolean inner = "inner".equals(entities.get(es)); //$NON-NLS-1$
if (inner) {
boundary.getInnerWays().add((Way) es);
} else {
String wName = es.getTag(OSMTagKey.NAME);
// if name are not equal keep the way for further check (it could be different suburb)
if (Algoritms.objectEquals(wName, boundary.getName()) || wName == null) {
visitedBoundaryWays.add(es.getId());
}
boundary.getOuterWays().add((Way) es);
}
}
}
} else if (e instanceof Way) {
if (!visitedBoundaryWays.contains(e.getId())) {
boundary = new Boundary();
if (e.getTag(OSMTagKey.NAME) != null) {
boundary.setName(e.getTag(OSMTagKey.NAME));
}
boundary.setBoundaryId(e.getId());
boundary.getOuterWays().add((Way) e);
}
}
}
if (boundary != null && boundary.getCenterPoint() != null) { if (boundary != null && boundary.getCenterPoint() != null) {
LatLon point = boundary.getCenterPoint(); LatLon point = boundary.getCenterPoint();
@ -163,7 +240,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
for (City c : cityManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) { for (City c : cityManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) {
if (boundary.containsPoint(c.getLocation())) { if (boundary.containsPoint(c.getLocation())) {
if (boundary.getName() == null || boundary.getName().equalsIgnoreCase(c.getName())) { if (boundary.getName() == null || boundary.getName().equalsIgnoreCase(c.getName())) {
citiBoundaries.put(c, boundary); putCityBoundary(boundary, c);
cityFound = true; cityFound = true;
containsCityInside = true; containsCityInside = true;
} }
@ -174,7 +251,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
for (City c : cityVillageManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) { for (City c : cityVillageManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) {
if (boundary.containsPoint(c.getLocation())) { if (boundary.containsPoint(c.getLocation())) {
if (boundary.getName() == null || boundary.getName().equalsIgnoreCase(c.getName())) { if (boundary.getName() == null || boundary.getName().equalsIgnoreCase(c.getName())) {
citiBoundaries.put(c, boundary); putCityBoundary(boundary, c);
cityFound = true; cityFound = true;
} }
} }
@ -187,7 +264,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
nCity.setLocation(point.getLatitude(), point.getLongitude()); nCity.setLocation(point.getLatitude(), point.getLongitude());
nCity.setId(-boundary.getBoundaryId()); nCity.setId(-boundary.getBoundaryId());
nCity.setName(boundary.getName()); nCity.setName(boundary.getName());
citiBoundaries.put(nCity, boundary); putCityBoundary(boundary, nCity);
cityVillageManager.registerObject(point.getLatitude(), point.getLongitude(), nCity); cityVillageManager.registerObject(point.getLatitude(), point.getLongitude(), nCity);
writeCity(nCity); writeCity(nCity);
@ -198,6 +275,109 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
} }
} }
} }
} else if (isBoundary(e) && hasGreaterCityAdminLevel(Integer.parseInt(cityAdminLevel),e)) {
//Any lower admin_level boundary is attached to the nearest city
Boundary boundary = extractBoundary(e, ctx);
if (boundary != null && boundary.getCenterPoint() != null) {
LatLon point = boundary.getCenterPoint();
for (City c : cityManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) {
Boundary cityB = cityBoundaries.get(c);
if (cityB == null) {
cityB = new Boundary(); //create empty boundary that is replaced with the real one for the city (if found)
putCityBoundary(cityB, c);
}
cityB.addSubBoundary(boundary);
break;
}
}
}
}
private void putCityBoundary(Boundary boundary, City c) {
final Boundary oldBoundary = cityBoundaries.get(c);
if (oldBoundary != null) {
boundary.addSubBoundaries(oldBoundary.getSubboundaries());
}
cityBoundaries.put(c, boundary);
}
private boolean isBoundary(Entity e)
{
return "administrative".equals(e.getTag(OSMTagKey.BOUNDARY)) && (e instanceof Relation || e instanceof Way);
}
private boolean hasNeededCityAdminLevel(Entity e)
{
return cityAdminLevel.equals(e.getTag(OSMTagKey.ADMIN_LEVEL));
}
private boolean hasGreaterCityAdminLevel(int admin_level, Entity e)
{
try {
return admin_level < Integer.parseInt(e.getTag(OSMTagKey.ADMIN_LEVEL));
} catch (NumberFormatException ex) {
return false;
}
}
private boolean hasGreaterCityAdminLevel(int admin_level, Boundary b)
{
try {
return admin_level < Integer.parseInt(b.getAdminLevel());
} catch (NumberFormatException ex) {
return false;
}
}
private Boundary extractBoundary(Entity e, OsmDbAccessorContext ctx)
throws SQLException {
if (isBoundary(e)) {
Boundary boundary = null;
if (e instanceof Relation) {
Relation i = (Relation) e;
ctx.loadEntityData(i, true);
boundary = new Boundary();
boundary.setAdminLevel(e.getTag(OSMTagKey.ADMIN_LEVEL));
if (i.getTag(OSMTagKey.NAME) != null) {
boundary.setName(i.getTag(OSMTagKey.NAME));
}
boundary.setBoundaryId(i.getId());
Map<Entity, String> entities = i.getMemberEntities();
for (Entity es : entities.keySet()) {
if (es instanceof Way) {
boolean inner = "inner".equals(entities.get(es)); //$NON-NLS-1$
if (inner) {
boundary.getInnerWays().add((Way) es);
} else {
String wName = es.getTag(OSMTagKey.NAME);
// if name are not equal keep the way for further check (it could be different suburb)
if (Algoritms.objectEquals(wName, boundary.getName()) || wName == null) {
visitedBoundaryWays.add(es.getId());
}
boundary.getOuterWays().add((Way) es);
}
} else if (isBoundary(es)) {
//add any sub boundaries...
boundary.addSubBoundary(extractBoundary(es, ctx));
}
}
} else if (e instanceof Way) {
if (!visitedBoundaryWays.contains(e.getId())) {
boundary = new Boundary();
boundary.setAdminLevel(e.getTag(OSMTagKey.ADMIN_LEVEL));
if (e.getTag(OSMTagKey.NAME) != null) {
boundary.setName(e.getTag(OSMTagKey.NAME));
}
boundary.setBoundaryId(e.getId());
boundary.getOuterWays().add((Way) e);
}
}
return boundary;
} else {
return null;
} }
} }
@ -270,10 +450,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
for (Map.Entry<Entity, String> r : i.getMemberEntities().entrySet()) { for (Map.Entry<Entity, String> r : i.getMemberEntities().entrySet()) {
if ("street".equals(r.getValue())) { //$NON-NLS-1$ if ("street".equals(r.getValue())) { //$NON-NLS-1$
if (r.getKey() instanceof Way && saveAddressWays) { if (r.getKey() instanceof Way && saveAddressWays) {
writeStreetWayNodes(streetId, (Way) r.getKey()); streetDAO.writeStreetWayNodes(streetId, (Way) r.getKey());
if (loadInMemory) {
addressStreetNodeLocalSet.add(r.getKey().getId());
}
} }
} else if ("house".equals(r.getValue())) { //$NON-NLS-1$ } else if ("house".equals(r.getValue())) { //$NON-NLS-1$
// will be registered further in other case // will be registered further in other case
@ -282,10 +459,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
if (hno != null) { if (hno != null) {
Building building = new Building(r.getKey()); Building building = new Building(r.getKey());
building.setName(hno); building.setName(hno);
writeBuilding(streetId, building); streetDAO.writeBuilding(streetId, building);
if (loadInMemory) {
addressBuildingLocalSet.add(r.getKey().getId());
}
} }
} }
} }
@ -308,10 +482,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
Building building = new Building(border); Building building = new Building(border);
if (building.getLocation() != null) { if (building.getLocation() != null) {
building.setName(hno); building.setName(hno);
writeBuilding(streetId, building); streetDAO.writeBuilding(streetId, building);
if (loadInMemory) {
addressBuildingLocalSet.add(id.getId());
}
} else { } else {
log.error("Strange border " + id + " location couldn't be found"); log.error("Strange border " + id + " location couldn't be found");
} }
@ -408,41 +579,69 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
Long foundId = null; Long foundId = null;
name = normalizeStreetName(name); name = normalizeStreetName(name);
if (loadInMemory) { String cityPart = findCityPart(city,location);
foundId = addressStreetLocalMap.get(name + "_" + city.getId()); //$NON-NLS-1$ foundId = streetDAO.findStreet(name,city,cityPart);
} else {
addressSearchStreetStat.setLong(1, city.getId());
addressSearchStreetStat.setString(2, name);
ResultSet rs = addressSearchStreetStat.executeQuery();
if (rs.next()) {
foundId = rs.getLong(1);
}
rs.close();
}
if (foundId == null) { if (foundId == null) {
insertStreetData(addressStreetStat, initId, name, Junidecode.unidecode(name), insertStreetData(addressStreetStat, initId, name, Junidecode.unidecode(name),
location.getLatitude(), location.getLongitude(), city.getId()); location.getLatitude(), location.getLongitude(), city.getId(), cityPart);
if (loadInMemory) { streetDAO.insertStreet(addressStreetStat, name, city, cityPart, initId);
addBatch(addressStreetStat);
addressStreetLocalMap.put(name + "_" + city.getId(), initId); //$NON-NLS-1$
} else {
addressStreetStat.execute();
// commit immediately to search after
mapConnection.commit();
}
foundId = initId; foundId = initId;
} }
return foundId; return foundId;
} }
private String findCityPart(City city, LatLon location) {
final Boundary cityBoundary = cityBoundaries.get(city);
int greatestBoudnaryLevel = Integer.parseInt(cityAdminLevel);
Boundary greatestBoundary = cityBoundary;
if (cityBoundary != null) {
for (Boundary subB : allSubBoundaries(cityBoundary)) {
if (subB.containsPoint(location) && hasGreaterCityAdminLevel(greatestBoudnaryLevel, subB)) {
greatestBoudnaryLevel = Integer.parseInt(subB.getAdminLevel());
greatestBoundary = subB;
}
}
}
return greatestBoundary != cityBoundary ? findNearestCityOrSuburb(greatestBoundary, location) : city.getName();
}
private String findNearestCityOrSuburb(Boundary greatestBoundary,
LatLon location) {
String result = greatestBoundary.getName();
List<City> nearestObjects = new ArrayList<City>();
nearestObjects.addAll(cityManager.getClosestObjects(location.getLatitude(),location.getLongitude()));
nearestObjects.addAll(cityVillageManager.getClosestObjects(location.getLatitude(),location.getLongitude()));
double dist = Double.MAX_VALUE;
for (City c : nearestObjects) {
if (greatestBoundary.containsPoint(c.getLocation())) {
double actualDistance = MapUtils.getDistance(location, c.getLocation());
if (actualDistance < dist) {
result = c.getName();
dist = actualDistance;
}
}
}
return result;
}
//TODO this is done on each city always, maybe we can just compute it once...
private Collection<Boundary> allSubBoundaries(Boundary cityBoundary) {
List<Boundary> result = new ArrayList<Boundary>();
for (Boundary subB : cityBoundary.getSubboundaries()) {
result.add(subB);
result.addAll(allSubBoundaries(subB));
}
return result;
}
public City getClosestCity(LatLon point, Set<String> isInNames) { public City getClosestCity(LatLon point, Set<String> isInNames) {
if (point == null) { if (point == null) {
return null; return null;
} }
// search by boundaries // search by boundaries
for (City c : cityManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) { for (City c : cityManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) {
Boundary boundary = citiBoundaries.get(c); Boundary boundary = cityBoundaries.get(c);
if(boundary != null){ if(boundary != null){
if(boundary.containsPoint(point)){ if(boundary.containsPoint(point)){
return c; return c;
@ -450,7 +649,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
} }
} }
for (City c : cityVillageManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) { for (City c : cityVillageManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) {
Boundary boundary = citiBoundaries.get(c); Boundary boundary = cityBoundaries.get(c);
if(boundary != null){ if(boundary != null){
if(boundary.containsPoint(point)){ if(boundary.containsPoint(point)){
return c; return c;
@ -514,16 +713,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
if (e.getTag(OSMTagKey.ADDR_HOUSE_NUMBER) != null && e.getTag(OSMTagKey.ADDR_STREET) != null) { if (e.getTag(OSMTagKey.ADDR_HOUSE_NUMBER) != null && e.getTag(OSMTagKey.ADDR_STREET) != null) {
// TODO e.getTag(OSMTagKey.ADDR_CITY) could be used to find city however many cities could have same name! // TODO e.getTag(OSMTagKey.ADDR_CITY) could be used to find city however many cities could have same name!
// check that building is not registered already // check that building is not registered already
boolean exist = false; boolean exist = streetDAO.findBuilding(e);
if (loadInMemory) {
exist = addressBuildingLocalSet.contains(e.getId());
} else {
addressSearchBuildingStat.setLong(1, e.getId());
ResultSet rs = addressSearchBuildingStat.executeQuery();
exist = rs.next();
rs.close();
}
if (!exist) { if (!exist) {
ctx.loadEntityData(e, false); ctx.loadEntityData(e, false);
LatLon l = e.getLatLon(); LatLon l = e.getLatLon();
@ -532,7 +722,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
if (idStreet != null) { if (idStreet != null) {
Building building = new Building(e); Building building = new Building(e);
building.setName(e.getTag(OSMTagKey.ADDR_HOUSE_NUMBER)); building.setName(e.getTag(OSMTagKey.ADDR_HOUSE_NUMBER));
writeBuilding(idStreet, building); streetDAO.writeBuilding(idStreet, building);
} }
} }
} else if (e instanceof Way /* && OSMSettings.wayForCar(e.getTag(OSMTagKey.HIGHWAY)) */ } else if (e instanceof Way /* && OSMSettings.wayForCar(e.getTag(OSMTagKey.HIGHWAY)) */
@ -543,14 +733,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
// if we saved address ways we could checked that we registered before // if we saved address ways we could checked that we registered before
if (saveAddressWays) { if (saveAddressWays) {
if (loadInMemory) { exist = streetDAO.findStreetNode(e);
exist = addressStreetNodeLocalSet.contains(e.getId());
} else {
addressSearchStreetNodeStat.setLong(1, e.getId());
ResultSet rs = addressSearchStreetNodeStat.executeQuery();
exist = rs.next();
rs.close();
}
} }
// check that street way is not registered already // check that street way is not registered already
@ -560,7 +743,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
City city = getClosestCity(l, getIsINames(e)); City city = getClosestCity(l, getIsINames(e));
Long idStreet = getStreetInCity(city, e.getTag(OSMTagKey.NAME), l, (e.getId() << 2) | 1); Long idStreet = getStreetInCity(city, e.getTag(OSMTagKey.NAME), l, (e.getId() << 2) | 1);
if (idStreet != null && saveAddressWays) { if (idStreet != null && saveAddressWays) {
writeStreetWayNodes(idStreet, (Way) e); streetDAO.writeStreetWayNodes(idStreet, (Way) e);
} }
} }
} }
@ -572,35 +755,6 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
} }
} }
private void writeStreetWayNodes(Long streetId, Way way)
throws SQLException {
for (Node n : way.getNodes()) {
if (n == null) {
continue;
}
addressStreetNodeStat.setLong(1, n.getId());
addressStreetNodeStat.setDouble(2, n.getLatitude());
addressStreetNodeStat.setDouble(3, n.getLongitude());
addressStreetNodeStat.setLong(5, way.getId());
addressStreetNodeStat.setLong(4, streetId);
addBatch(addressStreetNodeStat);
}
}
private void writeBuilding(Long streetId, Building building) throws SQLException {
addressBuildingStat.setLong(1, building.getId());
addressBuildingStat.setDouble(2, building.getLocation().getLatitude());
addressBuildingStat.setDouble(3, building.getLocation().getLongitude());
addressBuildingStat.setString(4, building.getName());
addressBuildingStat.setString(5, building.getEnName());
addressBuildingStat.setLong(6, streetId);
addressBuildingStat.setString(7, building.getPostcode() == null ? null : building.getPostcode().toUpperCase());
addBatch(addressBuildingStat);
}
private void writeCity(City city) throws SQLException { private void writeCity(City city) throws SQLException {
addressCityStat.setLong(1, city.getId()); addressCityStat.setLong(1, city.getId());
addressCityStat.setDouble(2, city.getLocation().getLatitude()); addressCityStat.setDouble(2, city.getLocation().getLatitude());
@ -613,13 +767,14 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
private void insertStreetData(PreparedStatement addressStreetStat, long id, String name, String nameEn, double latitude, private void insertStreetData(PreparedStatement addressStreetStat, long id, String name, String nameEn, double latitude,
double longitude, Long cityId) throws SQLException { double longitude, Long cityId, String cityPart) throws SQLException {
addressStreetStat.setLong(1, id); addressStreetStat.setLong(1, id);
addressStreetStat.setString(4, name); addressStreetStat.setString(4, name);
addressStreetStat.setString(5, nameEn); addressStreetStat.setString(5, nameEn);
addressStreetStat.setDouble(2, latitude); addressStreetStat.setDouble(2, latitude);
addressStreetStat.setDouble(3, longitude); addressStreetStat.setDouble(3, longitude);
addressStreetStat.setLong(6, cityId); addressStreetStat.setLong(6, cityId);
addressStreetStat.setString(7, cityPart);
} }
@ -676,9 +831,13 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
return Collator.getInstance().compare(o1.getName(), o2.getName()); return Collator.getInstance().compare(o1.getName(), o2.getName());
} }
}); });
PreparedStatement streetstat = mapConnection.prepareStatement("SELECT A.id, A.name, A.name_en, A.latitude, A.longitude, "+ //$NON-NLS-1$ PreparedStatement streetstat = mapConnection.prepareStatement(//
"B.id, B.name, B.name_en, B.latitude, B.longitude, B.postcode "+ //$NON-NLS-1$ "SELECT A.id, A.name, A.name_en, A.latitude, A.longitude, "+ //$NON-NLS-1$
"FROM street A left JOIN building B ON B.street = A.id WHERE A.city = ?"); //$NON-NLS-1$ "B.id, B.name, B.name_en, B.latitude, B.longitude, B.postcode, A.cityPart "+ //$NON-NLS-1$
"FROM street A left JOIN building B ON B.street = A.id JOIN city C ON A.city = C.id " + //$NON-NLS-1$
//with this order by we get the streets directly in city to not have the suffix if duplication
//TODO this order by might slow the query a little bit
"WHERE A.city = ? ORDER BY C.name == A.cityPart DESC"); //$NON-NLS-1$
PreparedStatement waynodesStat = null; PreparedStatement waynodesStat = null;
if (readWayNodes) { if (readWayNodes) {
waynodesStat = mapConnection.prepareStatement("SELECT A.id, A.latitude, A.longitude FROM street_node A WHERE A.street = ? "); //$NON-NLS-1$ waynodesStat = mapConnection.prepareStatement("SELECT A.id, A.latitude, A.longitude FROM street_node A WHERE A.street = ? "); //$NON-NLS-1$
@ -792,11 +951,11 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
this.mapConnection = mapConnection; this.mapConnection = mapConnection;
createAddressIndexStructure(mapConnection, dialect); createAddressIndexStructure(mapConnection, dialect);
addressCityStat = mapConnection.prepareStatement("insert into city (id, latitude, longitude, name, name_en, city_type) values (?, ?, ?, ?, ?, ?)"); addressCityStat = mapConnection.prepareStatement("insert into city (id, latitude, longitude, name, name_en, city_type) values (?, ?, ?, ?, ?, ?)");
addressStreetStat = mapConnection.prepareStatement("insert into street (id, latitude, longitude, name, name_en, city) values (?, ?, ?, ?, ?, ?)"); addressStreetStat = mapConnection.prepareStatement("insert into street (id, latitude, longitude, name, name_en, city, citypart) values (?, ?, ?, ?, ?, ?, ?)");
addressBuildingStat = mapConnection.prepareStatement("insert into building (id, latitude, longitude, name, name_en, street, postcode) values (?, ?, ?, ?, ?, ?, ?)"); addressBuildingStat = mapConnection.prepareStatement("insert into building (id, latitude, longitude, name, name_en, street, postcode) values (?, ?, ?, ?, ?, ?, ?)");
addressStreetNodeStat = mapConnection.prepareStatement("insert into street_node (id, latitude, longitude, street, way) values (?, ?, ?, ?, ?)"); addressStreetNodeStat = mapConnection.prepareStatement("insert into street_node (id, latitude, longitude, street, way) values (?, ?, ?, ?, ?)");
addressSearchStreetStat = mapConnection.prepareStatement("SELECT ID FROM street WHERE ? = city AND ? = name"); addressSearchStreetStat = mapConnection.prepareStatement("SELECT ID FROM street WHERE ? = city AND ? = citypart AND ? = name");
addressSearchBuildingStat = mapConnection.prepareStatement("SELECT id FROM building where ? = id"); addressSearchBuildingStat = mapConnection.prepareStatement("SELECT id FROM building where ? = id");
addressSearchStreetNodeStat = mapConnection.prepareStatement("SELECT way FROM street_node WHERE ? = way"); addressSearchStreetNodeStat = mapConnection.prepareStatement("SELECT way FROM street_node WHERE ? = way");
@ -808,7 +967,6 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
pStatements.put(addressSearchBuildingStat, 0); pStatements.put(addressSearchBuildingStat, 0);
pStatements.put(addressSearchStreetNodeStat, 0); pStatements.put(addressSearchStreetNodeStat, 0);
pStatements.put(addressSearchStreetStat, 0); pStatements.put(addressSearchStreetStat, 0);
} }
private void createAddressIndexStructure(Connection conn, DBDialect dialect) throws SQLException{ private void createAddressIndexStructure(Connection conn, DBDialect dialect) throws SQLException{
@ -819,7 +977,8 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
stat.executeUpdate("create index city_ind on city (id, city_type)"); stat.executeUpdate("create index city_ind on city (id, city_type)");
stat.executeUpdate("create table street (id bigint primary key, latitude double, longitude double, " + stat.executeUpdate("create table street (id bigint primary key, latitude double, longitude double, " +
"name varchar(1024), name_en varchar(1024), city bigint)"); "name varchar(1024), name_en varchar(1024), city bigint, citypart varchar(1024))");
stat.executeUpdate("create index street_cnp on street (city,citypart,name,id)");
stat.executeUpdate("create index street_city on street (city)"); stat.executeUpdate("create index street_city on street (city)");
stat.executeUpdate("create index street_id on street (id)"); stat.executeUpdate("create index street_id on street (id)");
// create index on name ? // create index on name ?
@ -844,14 +1003,16 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
private List<Street> readStreetsBuildings(PreparedStatement streetBuildingsStat, City city, List<Street> streets, private List<Street> readStreetsBuildings(PreparedStatement streetBuildingsStat, City city, List<Street> streets,
PreparedStatement waynodesStat, Map<Street, List<Node>> streetNodes, List<City> citySuburbs) throws SQLException { PreparedStatement waynodesStat, Map<Street, List<Node>> streetNodes, List<City> citySuburbs) throws SQLException {
Map<Long, Street> visitedStreets = new LinkedHashMap<Long, Street>(); TLongObjectHashMap<Street> visitedStreets = new TLongObjectHashMap<Street>();
HashSet<String> uniqueNames = new HashSet<String>();
//read streets for city //read streets for city
readStreatsByBuildingsForCity(streetBuildingsStat, city, streets, readStreatsByBuildingsForCity(streetBuildingsStat, city, streets,
waynodesStat, streetNodes, visitedStreets); waynodesStat, streetNodes, visitedStreets, uniqueNames);
//read streets for suburbs of the city //read streets for suburbs of the city
if (citySuburbs != null) { if (citySuburbs != null) {
for (City suburb : citySuburbs) { for (City suburb : citySuburbs) {
readStreatsByBuildingsForCity(streetBuildingsStat, suburb, streets, waynodesStat, streetNodes, visitedStreets); readStreatsByBuildingsForCity(streetBuildingsStat, suburb, streets, waynodesStat, streetNodes, visitedStreets, uniqueNames);
} }
} }
return streets; return streets;
@ -862,15 +1023,23 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
PreparedStatement streetBuildingsStat, City city, PreparedStatement streetBuildingsStat, City city,
List<Street> streets, PreparedStatement waynodesStat, List<Street> streets, PreparedStatement waynodesStat,
Map<Street, List<Node>> streetNodes, Map<Street, List<Node>> streetNodes,
Map<Long, Street> visitedStreets) throws SQLException { TLongObjectHashMap<Street> visitedStreets, HashSet<String> uniqueNames) throws SQLException {
streetBuildingsStat.setLong(1, city.getId()); streetBuildingsStat.setLong(1, city.getId());
ResultSet set = streetBuildingsStat.executeQuery(); ResultSet set = streetBuildingsStat.executeQuery();
while (set.next()) { while (set.next()) {
long streetId = set.getLong(1); long streetId = set.getLong(1);
if (!visitedStreets.containsKey(streetId)) { if (!visitedStreets.containsKey(streetId)) {
Street street = new Street(null); Street street = new Street(null);
street.setName(set.getString(2)); String streetName = set.getString(2);
street.setEnName(set.getString(3)); String district = "";
//There are more streets with same name in different districts.
//Add district name to all other names. If sorting is right, the first street was the one in the city
if (uniqueNames.contains(streetName)) {
district = " (" + set.getString(12) + ")";
}
uniqueNames.add(streetName);
street.setName(streetName + district);
street.setEnName(set.getString(3) + district);
street.setLocation(set.getDouble(4), set.getDouble(5)); street.setLocation(set.getDouble(4), set.getDouble(5));
street.setId(streetId); street.setId(streetId);
streets.add(street); streets.add(street);
@ -922,4 +1091,5 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
} }
} }

View file

@ -10,6 +10,7 @@ import java.io.RandomAccessFile;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.SQLException; import java.sql.SQLException;
import java.sql.Statement;
import net.osmand.Algoritms; import net.osmand.Algoritms;
import net.osmand.IProgress; import net.osmand.IProgress;
@ -304,6 +305,15 @@ public class IndexCreator {
allWays = dbCreator.getAllWays(); allWays = dbCreator.getAllWays();
allRelations = dbCreator.getAllRelations(); allRelations = dbCreator.getAllRelations();
} }
} else {
if (DBDialect.NOSQL != dialect) {
Connection dbc = (Connection) dbConn;
final Statement stmt = dbc.createStatement();
allRelations = stmt.executeQuery("select count(*) from relations").getInt(1);
allNodes = stmt.executeQuery("select count(*) from node").getInt(1);
allWays = stmt.executeQuery("select count(*) from ways").getInt(1);
stmt.close();
}
} }
accessor.initDatabase(dbConn, dialect, allNodes, allWays, allRelations); accessor.initDatabase(dbConn, dialect, allNodes, allWays, allRelations);
return loadFromExistingFile; return loadFromExistingFile;
@ -338,9 +348,9 @@ public class IndexCreator {
public void generateIndexes(File readFile, IProgress progress, IOsmStorageFilter addFilter, MapZooms mapZooms, public void generateIndexes(File readFile, IProgress progress, IOsmStorageFilter addFilter, MapZooms mapZooms,
MapRenderingTypes renderingTypes) throws IOException, SAXException, SQLException { MapRenderingTypes renderingTypes) throws IOException, SAXException, SQLException, InterruptedException {
if(!LevelDBAccess.load() && dialect == DBDialect.NOSQL){ if(LevelDBAccess.load()){
dialect = DBDialect.SQLITE; dialect = DBDialect.NOSQL;
} }
if (renderingTypes == null) { if (renderingTypes == null) {
@ -594,7 +604,7 @@ public class IndexCreator {
} }
public static void main(String[] args) throws IOException, SAXException, SQLException { public static void main(String[] args) throws IOException, SAXException, SQLException, InterruptedException {
long time = System.currentTimeMillis(); long time = System.currentTimeMillis();
IndexCreator creator = new IndexCreator(new File("/home/victor/projects/OsmAnd/data/osm-gen/")); //$NON-NLS-1$ IndexCreator creator = new IndexCreator(new File("/home/victor/projects/OsmAnd/data/osm-gen/")); //$NON-NLS-1$

View file

@ -8,14 +8,16 @@ import java.sql.Statement;
import java.util.Collection; import java.util.Collection;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.Map; import java.util.Map;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import net.osmand.IProgress; import net.osmand.IProgress;
import net.osmand.osm.Entity; import net.osmand.osm.Entity;
import net.osmand.osm.Entity.EntityId;
import net.osmand.osm.Entity.EntityType;
import net.osmand.osm.Node; import net.osmand.osm.Node;
import net.osmand.osm.Relation; import net.osmand.osm.Relation;
import net.osmand.osm.Way; import net.osmand.osm.Way;
import net.osmand.osm.Entity.EntityId;
import net.osmand.osm.Entity.EntityType;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
@ -28,6 +30,7 @@ import com.anvisics.jleveldb.ext.DBIterator;
import com.anvisics.jleveldb.ext.ReadOptions; import com.anvisics.jleveldb.ext.ReadOptions;
public class OsmDbAccessor implements OsmDbAccessorContext { public class OsmDbAccessor implements OsmDbAccessorContext {
private static final Log log = LogFactory.getLog(OsmDbAccessor.class); private static final Log log = LogFactory.getLog(OsmDbAccessor.class);
private PreparedStatement pselectNode; private PreparedStatement pselectNode;
@ -37,6 +40,7 @@ public class OsmDbAccessor implements OsmDbAccessorContext {
private int allRelations ; private int allRelations ;
private int allWays; private int allWays;
private int allNodes; private int allNodes;
private boolean realCounts = false;
private Connection dbConn; private Connection dbConn;
private DBDialect dialect; private DBDialect dialect;
@ -202,7 +206,7 @@ public class OsmDbAccessor implements OsmDbAccessorContext {
public int iterateOverEntities(IProgress progress, EntityType type, OsmDbVisitor visitor) throws SQLException { public int iterateOverEntities(IProgress progress, EntityType type, OsmDbVisitor visitor) throws SQLException, InterruptedException {
if(dialect == DBDialect.NOSQL){ if(dialect == DBDialect.NOSQL){
return iterateOverEntitiesNoSQL(progress, type, visitor); return iterateOverEntitiesNoSQL(progress, type, visitor);
} }
@ -213,75 +217,55 @@ public class OsmDbAccessor implements OsmDbAccessorContext {
// stat.executeUpdate("create table tags (id "+longType+", type smallint, skeys varchar(1024), value varchar(1024))"); // stat.executeUpdate("create table tags (id "+longType+", type smallint, skeys varchar(1024), value varchar(1024))");
// stat.executeUpdate("create table ways (id "+longType+", node "+longType+", ord smallint)"); // stat.executeUpdate("create table ways (id "+longType+", node "+longType+", ord smallint)");
// stat.executeUpdate("create table relations (id "+longType+", member "+longType+", type smallint, role varchar(1024), ord smallint)"); // stat.executeUpdate("create table relations (id "+longType+", member "+longType+", type smallint, role varchar(1024), ord smallint)");
computeRealCounts(statement);
if (type == EntityType.NODE) { if (type == EntityType.NODE) {
// filter out all nodes without tags // filter out all nodes without tags
select = "select n.id, n.latitude, n.longitude, t.skeys, t.value from node n inner join tags t on n.id = t.id and t.type = 0 order by n.id"; //$NON-NLS-1$ select = "select n.id, n.latitude, n.longitude, t.skeys, t.value from node n inner join tags t on n.id = t.id and t.type = 0 order by n.id"; //$NON-NLS-1$
count = allNodes;
} else if (type == EntityType.WAY) { } else if (type == EntityType.WAY) {
select = "select w.id, w.node, w.ord, t.skeys, t.value, n.latitude, n.longitude " + //$NON-NLS-1$ select = "select w.id, w.node, w.ord, t.skeys, t.value, n.latitude, n.longitude " + //$NON-NLS-1$
"from ways w left join tags t on w.id = t.id and t.type = 1 and w.ord = 0 inner join node n on w.node = n.id " + //$NON-NLS-1$ "from ways w left join tags t on w.id = t.id and t.type = 1 and w.ord = 0 inner join node n on w.node = n.id " + //$NON-NLS-1$
"order by w.id, w.ord"; //$NON-NLS-1$ "order by w.id, w.ord"; //$NON-NLS-1$
count = allWays;
} else { } else {
select = "select r.id, t.skeys, t.value from relations r inner join tags t on t.id = r.id and t.type = 2 and r.ord = 0"; //$NON-NLS-1$ select = "select r.id, t.skeys, t.value from relations r inner join tags t on t.id = r.id and t.type = 2 and r.ord = 0"; //$NON-NLS-1$
count = allRelations;
} }
progress.startWork(count);
ResultSet rs = statement.executeQuery(select); BlockingQueue<Entity> toProcess = new ArrayBlockingQueue<Entity>(100000);
Entity prevEntity = null;
long prevId = -1; //produce
while (rs.next()) { EntityProducer entityProducer = new EntityProducer(toProcess, type, statement, select);
long curId = rs.getLong(1); entityProducer.start();
boolean newEntity = curId != prevId;
Entity e = prevEntity; count = 0;
if (type == EntityType.NODE) { while (true) {
if (newEntity) { Entity entityToProcess = toProcess.take();
e = new Node(rs.getDouble(2), rs.getDouble(3), curId); if (entityToProcess == entityProducer.getEndingEntity()) {
} //the last entity...
e.putTag(rs.getString(4), rs.getString(5)); break;
} else if (type == EntityType.WAY) {
if (newEntity) {
e = new Way(curId);
}
int ord = rs.getInt(3);
if (ord == 0 && rs.getObject(4) != null) {
e.putTag(rs.getString(4), rs.getString(5));
}
if (newEntity || ord > 0) {
((Way) e).addNode(new Node(rs.getDouble(6), rs.getDouble(7), rs.getLong(2)));
}
} else {
if (newEntity) {
e = new Relation(curId);
}
e.putTag(rs.getString(2), rs.getString(3));
} }
if (newEntity) { if (progress != null) {
if (progress != null) { progress.progress(1);
progress.progress(1);
}
if (prevEntity != null) {
count++;
visitor.iterateEntity(prevEntity, this);
}
prevEntity = e;
} }
prevId = curId;
}
if (prevEntity != null) {
count++; count++;
visitor.iterateEntity(prevEntity, this); visitor.iterateEntity(entityToProcess, this);
}
rs.close();
if(EntityType.NODE == type){
allNodes = count;
} else if(EntityType.WAY == type){
allWays = count;
} else if(EntityType.RELATION == type){
allRelations = count;
} }
return count; return count;
} }
private void computeRealCounts(Statement statement) throws SQLException {
if (!realCounts) {
realCounts = true;
// filter out all nodes without tags
allNodes = statement.executeQuery("select count(*) from node n inner join tags t on n.id = t.id and t.type = 0 order by n.id").getInt(1); //$NON-NLS-1$
allWays = statement.executeQuery("select count(*) from ways w").getInt(1); //$NON-NLS-1$
allRelations = statement.executeQuery("select count(*) from relations r inner join tags t on t.id = r.id and t.type = 2 and r.ord = 0").getInt(1); //$NON-NLS-1$
}
}
private void loadEntityDataNoSQL(Entity e, boolean loadTags) { private void loadEntityDataNoSQL(Entity e, boolean loadTags) {
Collection<EntityId> ids = e instanceof Relation ? ((Relation) e).getMemberIds() : ((Way) e).getEntityIds(); Collection<EntityId> ids = e instanceof Relation ? ((Relation) e).getMemberIds() : ((Way) e).getEntityIds();
Map<EntityId, Entity> map = new LinkedHashMap<EntityId, Entity>(); Map<EntityId, Entity> map = new LinkedHashMap<EntityId, Entity>();
@ -455,4 +439,91 @@ public class OsmDbAccessor implements OsmDbAccessorContext {
} }
public class EntityProducer extends Thread {
private final BlockingQueue<Entity> toProcess;
private final Statement statement;
private final String select;
private final EntityType type;
private final Entity endingEntity = new Node(0,0,0);
public EntityProducer(BlockingQueue<Entity> toProcess, EntityType type,
Statement statement, String select) {
this.toProcess = toProcess;
this.type = type;
this.statement = statement;
this.select = select;
setDaemon(true);
setName("EntityProducer");
}
public Entity getEndingEntity() {
return endingEntity;
}
@Override
public void run() {
ResultSet rs;
try {
rs = statement.executeQuery(select);
// rs.setFetchSize(1000); !! not working for SQLite would case troubles probably
Entity prevEntity = null;
long prevId = -1;
while (rs.next()) {
long curId = rs.getLong(1);
boolean newEntity = curId != prevId;
Entity e = prevEntity;
if (type == EntityType.NODE) {
if (newEntity) {
e = new Node(rs.getDouble(2), rs.getDouble(3),
curId);
}
e.putTag(rs.getString(4), rs.getString(5));
} else if (type == EntityType.WAY) {
if (newEntity) {
e = new Way(curId);
}
int ord = rs.getInt(3);
if (ord == 0 && rs.getObject(4) != null) {
e.putTag(rs.getString(4), rs.getString(5));
}
if (newEntity || ord > 0) {
((Way) e).addNode(new Node(rs.getDouble(6), rs
.getDouble(7), rs.getLong(2)));
}
} else {
if (newEntity) {
e = new Relation(curId);
}
e.putTag(rs.getString(2), rs.getString(3));
}
if (newEntity) {
if (prevEntity != null) {
toProcess.put(prevEntity);
}
prevEntity = e;
}
prevId = curId;
}
if (prevEntity != null) {
toProcess.put(prevEntity);
}
rs.close();
} catch (SQLException e1) {
e1.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
} finally {
try {
toProcess.put(getEndingEntity());
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
}
} }

View file

@ -5,10 +5,8 @@ import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator; import java.util.Comparator;
import java.util.List; import java.util.List;
import java.util.Map.Entry;
import net.osmand.data.MapObject; import net.osmand.data.MapObject;
import net.osmand.osm.Entity.EntityType;
/** /**

View file

@ -56,6 +56,8 @@ public class OSMSettings {
WEBSITE("website"), //$NON-NLS-1$ WEBSITE("website"), //$NON-NLS-1$
URL("url"), //$NON-NLS-1$ URL("url"), //$NON-NLS-1$
WIKIPEDIA("wikipedia"), //$NON-NLS-1$ WIKIPEDIA("wikipedia"), //$NON-NLS-1$
ADMIN_LEVEL("admin_level"), //$NON-NLS-1$
; ;
private final String value; private final String value;

View file

@ -451,6 +451,8 @@ public class OsmExtractionUI implements IMapLocationListener {
throw new IllegalStateException(e); throw new IllegalStateException(e);
} catch (SQLException e) { } catch (SQLException e) {
throw new IllegalStateException(e); throw new IllegalStateException(e);
} catch (InterruptedException e) {
throw new IllegalStateException(e);
} }
regionName = creator.getRegionName(); regionName = creator.getRegionName();
StringBuilder msg = new StringBuilder(); StringBuilder msg = new StringBuilder();