diff --git a/DataExtractionOSM/.classpath b/DataExtractionOSM/.classpath index bf97e16ce8..c9586f09ca 100644 --- a/DataExtractionOSM/.classpath +++ b/DataExtractionOSM/.classpath @@ -1,14 +1,14 @@ - - - - - - - - - - - - - - + + + + + + + + + + + + + + diff --git a/DataExtractionOSM/OsmAndMapCreator.launch b/DataExtractionOSM/OsmAndMapCreator.launch index d40ec671ff..c7a3cba55a 100644 --- a/DataExtractionOSM/OsmAndMapCreator.launch +++ b/DataExtractionOSM/OsmAndMapCreator.launch @@ -1,15 +1,16 @@ - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + diff --git a/DataExtractionOSM/lib/sqlitejdbc-v056.jar b/DataExtractionOSM/lib/sqlitejdbc-v056.jar deleted file mode 100644 index f95d90eb07..0000000000 Binary files a/DataExtractionOSM/lib/sqlitejdbc-v056.jar and /dev/null differ diff --git a/DataExtractionOSM/src/net/osmand/data/Boundary.java b/DataExtractionOSM/src/net/osmand/data/Boundary.java index 203c7513d1..094caea149 100644 --- a/DataExtractionOSM/src/net/osmand/data/Boundary.java +++ b/DataExtractionOSM/src/net/osmand/data/Boundary.java @@ -1,6 +1,7 @@ package net.osmand.data; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import net.osmand.osm.LatLon; @@ -19,6 +20,7 @@ public class Boundary { private List outerWays = new ArrayList(); private List innerWays = new ArrayList(); + private List subboundaries = new ArrayList(); public boolean containsPoint(LatLon point) { return containsPoint(point.getLatitude(), point.getLongitude()); @@ -88,5 +90,18 @@ public class Boundary { this.adminLevel = adminLevel; } + public List getSubboundaries() { + return subboundaries; + } + public void addSubBoundary(Boundary subBoundary) { + if (subBoundary != null) { + subboundaries.add(subBoundary); + } + } + + public void addSubBoundaries(Collection subBoundaries) { + subboundaries.addAll(subBoundaries); + } + } diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/DBDialect.java b/DataExtractionOSM/src/net/osmand/data/preparation/DBDialect.java index 2695bef12b..599d1631e4 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/DBDialect.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/DBDialect.java @@ -9,6 +9,11 @@ import java.sql.Statement; import net.osmand.Algoritms; import org.apache.commons.logging.Log; +import org.sqlite.SQLiteConfig; +import org.sqlite.SQLiteConfig.JournalMode; +import org.sqlite.SQLiteConfig.LockingMode; +import org.sqlite.SQLiteConfig.SynchronousMode; +import org.sqlite.SQLiteJDBCLoader; import com.anvisics.jleveldb.LevelDBAccess; import com.anvisics.jleveldb.ext.DBAccessor; @@ -91,10 +96,15 @@ public enum DBDialect { log.error("Illegal configuration", e); throw new IllegalStateException(e); } + SQLiteConfig config = new SQLiteConfig(); + config.setCacheSize(10000); //size for number in file pages in memory, (disk cache) + config.setJournalMode(JournalMode.OFF); //no journal - on crash the db is not usabale + config.setSynchronous(SynchronousMode.OFF); // faster without synchronization + config.setLockingMode(LockingMode.EXCLUSIVE); // we are the only one using the file => no need to get/realease locks always => faster + config.setSharedCache(true); // => needed for readUncommited + config.setReadUncommited(true); // => faster queries (read also what is not commited yet, but is inserted) Connection connection = DriverManager.getConnection("jdbc:sqlite:" + fileName); - Statement statement = connection.createStatement(); - statement.executeUpdate("PRAGMA synchronous = 0"); - statement.close(); + System.out.println(String.format("SQLITE running in %s mode", SQLiteJDBCLoader.isNativeMode() ? "native" : "pure-java")); return connection; } else if (DBDialect.DERBY == this) { try { diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/IndexAddressCreator.java b/DataExtractionOSM/src/net/osmand/data/preparation/IndexAddressCreator.java index 04aef6374d..0c2a658d85 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/IndexAddressCreator.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/IndexAddressCreator.java @@ -1,5 +1,8 @@ package net.osmand.data.preparation; +import gnu.trove.map.hash.TLongObjectHashMap; +import gnu.trove.set.hash.TLongHashSet; + import java.io.IOException; import java.sql.Connection; import java.sql.PreparedStatement; @@ -25,17 +28,17 @@ import net.osmand.binary.BinaryMapIndexWriter; import net.osmand.data.Boundary; import net.osmand.data.Building; import net.osmand.data.City; +import net.osmand.data.City.CityType; import net.osmand.data.DataTileManager; import net.osmand.data.Street; -import net.osmand.data.City.CityType; import net.osmand.osm.Entity; +import net.osmand.osm.Entity.EntityId; import net.osmand.osm.LatLon; import net.osmand.osm.MapUtils; import net.osmand.osm.Node; +import net.osmand.osm.OSMSettings.OSMTagKey; import net.osmand.osm.Relation; import net.osmand.osm.Way; -import net.osmand.osm.Entity.EntityId; -import net.osmand.osm.OSMSettings.OSMTagKey; import net.osmand.swing.Messages; import net.sf.junidecode.Junidecode; @@ -58,18 +61,14 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ private PreparedStatement addressSearchBuildingStat; private PreparedStatement addressSearchStreetNodeStat; - private Map addressStreetLocalMap = new LinkedHashMap(); - private Set addressBuildingLocalSet = new LinkedHashSet(); - private Set addressStreetNodeLocalSet = new LinkedHashSet(); - // MEMORY address : address structure // load it in memory private Map cities = new LinkedHashMap(); private DataTileManager cityVillageManager = new DataTileManager(13); private DataTileManager cityManager = new DataTileManager(10); private List postalCodeRelations = new ArrayList(); - private Map citiBoundaries = new LinkedHashMap(); - private Set visitedBoundaryWays = new HashSet(); + private Map cityBoundaries = new LinkedHashMap(); + private TLongHashSet visitedBoundaryWays = new TLongHashSet(); private boolean normalizeStreets; private String[] normalizeDefaultSuffixes; @@ -80,9 +79,126 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ // TODO Connection mapConnection; + DBStreetDAO streetDAO; + public class DBStreetDAO + { + protected void writeStreetWayNodes(Long streetId, Way way) + throws SQLException { + for (Node n : way.getNodes()) { + if (n == null) { + continue; + } + addressStreetNodeStat.setLong(1, n.getId()); + addressStreetNodeStat.setDouble(2, n.getLatitude()); + addressStreetNodeStat.setDouble(3, n.getLongitude()); + addressStreetNodeStat.setLong(5, way.getId()); + addressStreetNodeStat.setLong(4, streetId); + addBatch(addressStreetNodeStat); + } + } + + protected void writeBuilding(Long streetId, Building building) throws SQLException { + addressBuildingStat.setLong(1, building.getId()); + addressBuildingStat.setDouble(2, building.getLocation().getLatitude()); + addressBuildingStat.setDouble(3, building.getLocation().getLongitude()); + addressBuildingStat.setString(4, building.getName()); + addressBuildingStat.setString(5, building.getEnName()); + addressBuildingStat.setLong(6, streetId); + addressBuildingStat.setString(7, building.getPostcode() == null ? null : building.getPostcode().toUpperCase()); + addBatch(addressBuildingStat); + } + public Long findStreet(String name, City city, String cityPart) throws SQLException { + addressSearchStreetStat.setLong(1, city.getId()); + addressSearchStreetStat.setString(2, cityPart); + addressSearchStreetStat.setString(3, name); + ResultSet rs = addressSearchStreetStat.executeQuery(); + Long foundId = null; + if (rs.next()) { + foundId = rs.getLong(1); + } + rs.close(); + return foundId; + } + + public void insertStreet(PreparedStatement addressStreetStat, + String name, City city, String cityPart, long initId) throws SQLException { + //execute the insert statement + addressStreetStat.execute(); + // commit immediately to search after + mapConnection.commit(); + } + + public boolean findBuilding(Entity e) throws SQLException { + addressSearchBuildingStat.setLong(1, e.getId()); + ResultSet rs = addressSearchBuildingStat.executeQuery(); + boolean exist = rs.next(); + rs.close(); + return exist; + } + + public boolean findStreetNode(Entity e) throws SQLException { + addressSearchStreetNodeStat.setLong(1, e.getId()); + ResultSet rs = addressSearchStreetNodeStat.executeQuery(); + boolean exist = rs.next(); + rs.close(); + return exist; + } + } + + + public class CachedDBStreetDAO extends DBStreetDAO + { + private Map addressStreetLocalMap = new LinkedHashMap(); + private TLongHashSet addressBuildingLocalSet = new TLongHashSet(); + private TLongHashSet addressStreetNodeLocalSet = new TLongHashSet(); + + @Override + public Long findStreet(String name, City city, String cityPart) { + return addressStreetLocalMap.get(createStreetUniqueName(name, city, cityPart)); //$NON-NLS-1$ + } + + private String createStreetUniqueName(String name, City city, String cityPart) { + return new StringBuilder().append(name).append('_').append(city.getId()).append('_').append(cityPart).toString(); + } + + @Override + protected void writeStreetWayNodes(Long streetId, Way way) + throws SQLException { + super.writeStreetWayNodes(streetId, way); + addressStreetNodeLocalSet.add(way.getId()); + } + + @Override + protected void writeBuilding(Long streetId, Building building) + throws SQLException { + super.writeBuilding(streetId, building); + addressBuildingLocalSet.add(building.getId()); + } + + @Override + public void insertStreet(PreparedStatement addressStreetStat, + String name, City city, String cityPart, long initId) throws SQLException { + //batch the insert + addBatch(addressStreetStat); + addressStreetLocalMap.put(createStreetUniqueName(name, city, cityPart), initId); //$NON-NLS-1$ + } + + @Override + public boolean findBuilding(Entity e) { + return addressBuildingLocalSet.contains(e.getId()); + } + + @Override + public boolean findStreetNode(Entity e) { + return addressStreetNodeLocalSet.contains(e.getId()); + } + } + + public IndexAddressCreator(){ + streetDAO = loadInMemory ? new CachedDBStreetDAO() : new DBStreetDAO(); } @@ -91,7 +207,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ cities.clear(); cityManager.clear(); postalCodeRelations.clear(); - citiBoundaries.clear(); + cityBoundaries.clear(); this.normalizeStreets = normalizeStreets; this.normalizeDefaultSuffixes = normalizeDefaultSuffixes; this.normalizeSuffixes = normalizeSuffixes; @@ -114,47 +230,8 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ } public void indexBoundariesRelation(Entity e, OsmDbAccessorContext ctx) throws SQLException { - if ("administrative".equals(e.getTag(OSMTagKey.BOUNDARY)) && (e instanceof Relation || e instanceof Way)) { - String adminLevel = e.getTag("admin_level"); - Boundary boundary = null; - if (cityAdminLevel.equals(adminLevel)) { - if (e instanceof Relation) { - Relation i = (Relation) e; - ctx.loadEntityData(i, true); - boundary = new Boundary(); - if (i.getTag(OSMTagKey.NAME) != null) { - boundary.setName(i.getTag(OSMTagKey.NAME)); - } - boundary.setBoundaryId(i.getId()); - Map entities = i.getMemberEntities(); - for (Entity es : entities.keySet()) { - if (es instanceof Way) { - boolean inner = "inner".equals(entities.get(es)); //$NON-NLS-1$ - if (inner) { - boundary.getInnerWays().add((Way) es); - } else { - String wName = es.getTag(OSMTagKey.NAME); - // if name are not equal keep the way for further check (it could be different suburb) - if (Algoritms.objectEquals(wName, boundary.getName()) || wName == null) { - visitedBoundaryWays.add(es.getId()); - } - boundary.getOuterWays().add((Way) es); - } - } - } - } else if (e instanceof Way) { - if (!visitedBoundaryWays.contains(e.getId())) { - boundary = new Boundary(); - if (e.getTag(OSMTagKey.NAME) != null) { - boundary.setName(e.getTag(OSMTagKey.NAME)); - } - boundary.setBoundaryId(e.getId()); - boundary.getOuterWays().add((Way) e); - - } - - } - } + if (isBoundary(e) && hasNeededCityAdminLevel(e)) { + Boundary boundary = extractBoundary(e, ctx); if (boundary != null && boundary.getCenterPoint() != null) { LatLon point = boundary.getCenterPoint(); @@ -163,7 +240,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ for (City c : cityManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) { if (boundary.containsPoint(c.getLocation())) { if (boundary.getName() == null || boundary.getName().equalsIgnoreCase(c.getName())) { - citiBoundaries.put(c, boundary); + putCityBoundary(boundary, c); cityFound = true; containsCityInside = true; } @@ -174,7 +251,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ for (City c : cityVillageManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) { if (boundary.containsPoint(c.getLocation())) { if (boundary.getName() == null || boundary.getName().equalsIgnoreCase(c.getName())) { - citiBoundaries.put(c, boundary); + putCityBoundary(boundary, c); cityFound = true; } } @@ -187,7 +264,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ nCity.setLocation(point.getLatitude(), point.getLongitude()); nCity.setId(-boundary.getBoundaryId()); nCity.setName(boundary.getName()); - citiBoundaries.put(nCity, boundary); + putCityBoundary(boundary, nCity); cityVillageManager.registerObject(point.getLatitude(), point.getLongitude(), nCity); writeCity(nCity); @@ -198,6 +275,109 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ } } } + } else if (isBoundary(e) && hasGreaterCityAdminLevel(Integer.parseInt(cityAdminLevel),e)) { + //Any lower admin_level boundary is attached to the nearest city + Boundary boundary = extractBoundary(e, ctx); + if (boundary != null && boundary.getCenterPoint() != null) { + LatLon point = boundary.getCenterPoint(); + for (City c : cityManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) { + Boundary cityB = cityBoundaries.get(c); + if (cityB == null) { + cityB = new Boundary(); //create empty boundary that is replaced with the real one for the city (if found) + putCityBoundary(cityB, c); + } + cityB.addSubBoundary(boundary); + break; + } + } + } + } + + private void putCityBoundary(Boundary boundary, City c) { + final Boundary oldBoundary = cityBoundaries.get(c); + if (oldBoundary != null) { + boundary.addSubBoundaries(oldBoundary.getSubboundaries()); + } + cityBoundaries.put(c, boundary); + } + + + private boolean isBoundary(Entity e) + { + return "administrative".equals(e.getTag(OSMTagKey.BOUNDARY)) && (e instanceof Relation || e instanceof Way); + } + + private boolean hasNeededCityAdminLevel(Entity e) + { + return cityAdminLevel.equals(e.getTag(OSMTagKey.ADMIN_LEVEL)); + } + + private boolean hasGreaterCityAdminLevel(int admin_level, Entity e) + { + try { + return admin_level < Integer.parseInt(e.getTag(OSMTagKey.ADMIN_LEVEL)); + } catch (NumberFormatException ex) { + return false; + } + } + + private boolean hasGreaterCityAdminLevel(int admin_level, Boundary b) + { + try { + return admin_level < Integer.parseInt(b.getAdminLevel()); + } catch (NumberFormatException ex) { + return false; + } + } + + private Boundary extractBoundary(Entity e, OsmDbAccessorContext ctx) + throws SQLException { + if (isBoundary(e)) { + Boundary boundary = null; + if (e instanceof Relation) { + Relation i = (Relation) e; + ctx.loadEntityData(i, true); + boundary = new Boundary(); + boundary.setAdminLevel(e.getTag(OSMTagKey.ADMIN_LEVEL)); + if (i.getTag(OSMTagKey.NAME) != null) { + boundary.setName(i.getTag(OSMTagKey.NAME)); + } + boundary.setBoundaryId(i.getId()); + Map entities = i.getMemberEntities(); + for (Entity es : entities.keySet()) { + if (es instanceof Way) { + boolean inner = "inner".equals(entities.get(es)); //$NON-NLS-1$ + if (inner) { + boundary.getInnerWays().add((Way) es); + } else { + String wName = es.getTag(OSMTagKey.NAME); + // if name are not equal keep the way for further check (it could be different suburb) + if (Algoritms.objectEquals(wName, boundary.getName()) || wName == null) { + visitedBoundaryWays.add(es.getId()); + } + boundary.getOuterWays().add((Way) es); + } + } else if (isBoundary(es)) { + //add any sub boundaries... + boundary.addSubBoundary(extractBoundary(es, ctx)); + } + } + } else if (e instanceof Way) { + if (!visitedBoundaryWays.contains(e.getId())) { + boundary = new Boundary(); + boundary.setAdminLevel(e.getTag(OSMTagKey.ADMIN_LEVEL)); + if (e.getTag(OSMTagKey.NAME) != null) { + boundary.setName(e.getTag(OSMTagKey.NAME)); + } + boundary.setBoundaryId(e.getId()); + boundary.getOuterWays().add((Way) e); + + } + + } + return boundary; + } else { + return null; } } @@ -270,10 +450,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ for (Map.Entry r : i.getMemberEntities().entrySet()) { if ("street".equals(r.getValue())) { //$NON-NLS-1$ if (r.getKey() instanceof Way && saveAddressWays) { - writeStreetWayNodes(streetId, (Way) r.getKey()); - if (loadInMemory) { - addressStreetNodeLocalSet.add(r.getKey().getId()); - } + streetDAO.writeStreetWayNodes(streetId, (Way) r.getKey()); } } else if ("house".equals(r.getValue())) { //$NON-NLS-1$ // will be registered further in other case @@ -282,10 +459,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ if (hno != null) { Building building = new Building(r.getKey()); building.setName(hno); - writeBuilding(streetId, building); - if (loadInMemory) { - addressBuildingLocalSet.add(r.getKey().getId()); - } + streetDAO.writeBuilding(streetId, building); } } } @@ -308,10 +482,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ Building building = new Building(border); if (building.getLocation() != null) { building.setName(hno); - writeBuilding(streetId, building); - if (loadInMemory) { - addressBuildingLocalSet.add(id.getId()); - } + streetDAO.writeBuilding(streetId, building); } else { log.error("Strange border " + id + " location couldn't be found"); } @@ -408,41 +579,69 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ Long foundId = null; name = normalizeStreetName(name); - if (loadInMemory) { - foundId = addressStreetLocalMap.get(name + "_" + city.getId()); //$NON-NLS-1$ - } else { - addressSearchStreetStat.setLong(1, city.getId()); - addressSearchStreetStat.setString(2, name); - ResultSet rs = addressSearchStreetStat.executeQuery(); - if (rs.next()) { - foundId = rs.getLong(1); - } - rs.close(); - } + String cityPart = findCityPart(city,location); + foundId = streetDAO.findStreet(name,city,cityPart); if (foundId == null) { insertStreetData(addressStreetStat, initId, name, Junidecode.unidecode(name), - location.getLatitude(), location.getLongitude(), city.getId()); - if (loadInMemory) { - addBatch(addressStreetStat); - addressStreetLocalMap.put(name + "_" + city.getId(), initId); //$NON-NLS-1$ - } else { - addressStreetStat.execute(); - // commit immediately to search after - mapConnection.commit(); - } + location.getLatitude(), location.getLongitude(), city.getId(), cityPart); + streetDAO.insertStreet(addressStreetStat, name, city, cityPart, initId); foundId = initId; } return foundId; } + private String findCityPart(City city, LatLon location) { + final Boundary cityBoundary = cityBoundaries.get(city); + int greatestBoudnaryLevel = Integer.parseInt(cityAdminLevel); + Boundary greatestBoundary = cityBoundary; + if (cityBoundary != null) { + for (Boundary subB : allSubBoundaries(cityBoundary)) { + if (subB.containsPoint(location) && hasGreaterCityAdminLevel(greatestBoudnaryLevel, subB)) { + greatestBoudnaryLevel = Integer.parseInt(subB.getAdminLevel()); + greatestBoundary = subB; + } + } + } + return greatestBoundary != cityBoundary ? findNearestCityOrSuburb(greatestBoundary, location) : city.getName(); + } + + private String findNearestCityOrSuburb(Boundary greatestBoundary, + LatLon location) { + String result = greatestBoundary.getName(); + List nearestObjects = new ArrayList(); + nearestObjects.addAll(cityManager.getClosestObjects(location.getLatitude(),location.getLongitude())); + nearestObjects.addAll(cityVillageManager.getClosestObjects(location.getLatitude(),location.getLongitude())); + double dist = Double.MAX_VALUE; + for (City c : nearestObjects) { + if (greatestBoundary.containsPoint(c.getLocation())) { + double actualDistance = MapUtils.getDistance(location, c.getLocation()); + if (actualDistance < dist) { + result = c.getName(); + dist = actualDistance; + } + } + } + return result; + } + + //TODO this is done on each city always, maybe we can just compute it once... + private Collection allSubBoundaries(Boundary cityBoundary) { + List result = new ArrayList(); + for (Boundary subB : cityBoundary.getSubboundaries()) { + result.add(subB); + result.addAll(allSubBoundaries(subB)); + } + return result; + } + public City getClosestCity(LatLon point, Set isInNames) { if (point == null) { return null; } // search by boundaries for (City c : cityManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) { - Boundary boundary = citiBoundaries.get(c); + Boundary boundary = cityBoundaries.get(c); if(boundary != null){ if(boundary.containsPoint(point)){ return c; @@ -450,7 +649,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ } } for (City c : cityVillageManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) { - Boundary boundary = citiBoundaries.get(c); + Boundary boundary = cityBoundaries.get(c); if(boundary != null){ if(boundary.containsPoint(point)){ return c; @@ -514,16 +713,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ if (e.getTag(OSMTagKey.ADDR_HOUSE_NUMBER) != null && e.getTag(OSMTagKey.ADDR_STREET) != null) { // TODO e.getTag(OSMTagKey.ADDR_CITY) could be used to find city however many cities could have same name! // check that building is not registered already - boolean exist = false; - if (loadInMemory) { - exist = addressBuildingLocalSet.contains(e.getId()); - } else { - addressSearchBuildingStat.setLong(1, e.getId()); - ResultSet rs = addressSearchBuildingStat.executeQuery(); - exist = rs.next(); - rs.close(); - - } + boolean exist = streetDAO.findBuilding(e); if (!exist) { ctx.loadEntityData(e, false); LatLon l = e.getLatLon(); @@ -532,7 +722,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ if (idStreet != null) { Building building = new Building(e); building.setName(e.getTag(OSMTagKey.ADDR_HOUSE_NUMBER)); - writeBuilding(idStreet, building); + streetDAO.writeBuilding(idStreet, building); } } } else if (e instanceof Way /* && OSMSettings.wayForCar(e.getTag(OSMTagKey.HIGHWAY)) */ @@ -543,14 +733,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ // if we saved address ways we could checked that we registered before if (saveAddressWays) { - if (loadInMemory) { - exist = addressStreetNodeLocalSet.contains(e.getId()); - } else { - addressSearchStreetNodeStat.setLong(1, e.getId()); - ResultSet rs = addressSearchStreetNodeStat.executeQuery(); - exist = rs.next(); - rs.close(); - } + exist = streetDAO.findStreetNode(e); } // check that street way is not registered already @@ -560,7 +743,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ City city = getClosestCity(l, getIsINames(e)); Long idStreet = getStreetInCity(city, e.getTag(OSMTagKey.NAME), l, (e.getId() << 2) | 1); if (idStreet != null && saveAddressWays) { - writeStreetWayNodes(idStreet, (Way) e); + streetDAO.writeStreetWayNodes(idStreet, (Way) e); } } } @@ -572,35 +755,6 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ } } - - private void writeStreetWayNodes(Long streetId, Way way) - throws SQLException { - for (Node n : way.getNodes()) { - if (n == null) { - continue; - } - addressStreetNodeStat.setLong(1, n.getId()); - addressStreetNodeStat.setDouble(2, n.getLatitude()); - addressStreetNodeStat.setDouble(3, n.getLongitude()); - addressStreetNodeStat.setLong(5, way.getId()); - addressStreetNodeStat.setLong(4, streetId); - addBatch(addressStreetNodeStat); - } - } - - - private void writeBuilding(Long streetId, Building building) throws SQLException { - addressBuildingStat.setLong(1, building.getId()); - addressBuildingStat.setDouble(2, building.getLocation().getLatitude()); - addressBuildingStat.setDouble(3, building.getLocation().getLongitude()); - addressBuildingStat.setString(4, building.getName()); - addressBuildingStat.setString(5, building.getEnName()); - addressBuildingStat.setLong(6, streetId); - addressBuildingStat.setString(7, building.getPostcode() == null ? null : building.getPostcode().toUpperCase()); - addBatch(addressBuildingStat); - } - - private void writeCity(City city) throws SQLException { addressCityStat.setLong(1, city.getId()); addressCityStat.setDouble(2, city.getLocation().getLatitude()); @@ -613,13 +767,14 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ private void insertStreetData(PreparedStatement addressStreetStat, long id, String name, String nameEn, double latitude, - double longitude, Long cityId) throws SQLException { + double longitude, Long cityId, String cityPart) throws SQLException { addressStreetStat.setLong(1, id); addressStreetStat.setString(4, name); addressStreetStat.setString(5, nameEn); addressStreetStat.setDouble(2, latitude); addressStreetStat.setDouble(3, longitude); addressStreetStat.setLong(6, cityId); + addressStreetStat.setString(7, cityPart); } @@ -676,9 +831,13 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ return Collator.getInstance().compare(o1.getName(), o2.getName()); } }); - PreparedStatement streetstat = mapConnection.prepareStatement("SELECT A.id, A.name, A.name_en, A.latitude, A.longitude, "+ //$NON-NLS-1$ - "B.id, B.name, B.name_en, B.latitude, B.longitude, B.postcode "+ //$NON-NLS-1$ - "FROM street A left JOIN building B ON B.street = A.id WHERE A.city = ?"); //$NON-NLS-1$ + PreparedStatement streetstat = mapConnection.prepareStatement(// + "SELECT A.id, A.name, A.name_en, A.latitude, A.longitude, "+ //$NON-NLS-1$ + "B.id, B.name, B.name_en, B.latitude, B.longitude, B.postcode, A.cityPart "+ //$NON-NLS-1$ + "FROM street A left JOIN building B ON B.street = A.id JOIN city C ON A.city = C.id " + //$NON-NLS-1$ + //with this order by we get the streets directly in city to not have the suffix if duplication + //TODO this order by might slow the query a little bit + "WHERE A.city = ? ORDER BY C.name == A.cityPart DESC"); //$NON-NLS-1$ PreparedStatement waynodesStat = null; if (readWayNodes) { waynodesStat = mapConnection.prepareStatement("SELECT A.id, A.latitude, A.longitude FROM street_node A WHERE A.street = ? "); //$NON-NLS-1$ @@ -792,11 +951,11 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ this.mapConnection = mapConnection; createAddressIndexStructure(mapConnection, dialect); addressCityStat = mapConnection.prepareStatement("insert into city (id, latitude, longitude, name, name_en, city_type) values (?, ?, ?, ?, ?, ?)"); - addressStreetStat = mapConnection.prepareStatement("insert into street (id, latitude, longitude, name, name_en, city) values (?, ?, ?, ?, ?, ?)"); + addressStreetStat = mapConnection.prepareStatement("insert into street (id, latitude, longitude, name, name_en, city, citypart) values (?, ?, ?, ?, ?, ?, ?)"); addressBuildingStat = mapConnection.prepareStatement("insert into building (id, latitude, longitude, name, name_en, street, postcode) values (?, ?, ?, ?, ?, ?, ?)"); addressStreetNodeStat = mapConnection.prepareStatement("insert into street_node (id, latitude, longitude, street, way) values (?, ?, ?, ?, ?)"); - addressSearchStreetStat = mapConnection.prepareStatement("SELECT ID FROM street WHERE ? = city AND ? = name"); + addressSearchStreetStat = mapConnection.prepareStatement("SELECT ID FROM street WHERE ? = city AND ? = citypart AND ? = name"); addressSearchBuildingStat = mapConnection.prepareStatement("SELECT id FROM building where ? = id"); addressSearchStreetNodeStat = mapConnection.prepareStatement("SELECT way FROM street_node WHERE ? = way"); @@ -808,7 +967,6 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ pStatements.put(addressSearchBuildingStat, 0); pStatements.put(addressSearchStreetNodeStat, 0); pStatements.put(addressSearchStreetStat, 0); - } private void createAddressIndexStructure(Connection conn, DBDialect dialect) throws SQLException{ @@ -819,7 +977,8 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ stat.executeUpdate("create index city_ind on city (id, city_type)"); stat.executeUpdate("create table street (id bigint primary key, latitude double, longitude double, " + - "name varchar(1024), name_en varchar(1024), city bigint)"); + "name varchar(1024), name_en varchar(1024), city bigint, citypart varchar(1024))"); + stat.executeUpdate("create index street_cnp on street (city,citypart,name,id)"); stat.executeUpdate("create index street_city on street (city)"); stat.executeUpdate("create index street_id on street (id)"); // create index on name ? @@ -844,14 +1003,16 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ private List readStreetsBuildings(PreparedStatement streetBuildingsStat, City city, List streets, PreparedStatement waynodesStat, Map> streetNodes, List citySuburbs) throws SQLException { - Map visitedStreets = new LinkedHashMap(); + TLongObjectHashMap visitedStreets = new TLongObjectHashMap(); + HashSet uniqueNames = new HashSet(); + //read streets for city readStreatsByBuildingsForCity(streetBuildingsStat, city, streets, - waynodesStat, streetNodes, visitedStreets); + waynodesStat, streetNodes, visitedStreets, uniqueNames); //read streets for suburbs of the city if (citySuburbs != null) { for (City suburb : citySuburbs) { - readStreatsByBuildingsForCity(streetBuildingsStat, suburb, streets, waynodesStat, streetNodes, visitedStreets); + readStreatsByBuildingsForCity(streetBuildingsStat, suburb, streets, waynodesStat, streetNodes, visitedStreets, uniqueNames); } } return streets; @@ -862,15 +1023,23 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ PreparedStatement streetBuildingsStat, City city, List streets, PreparedStatement waynodesStat, Map> streetNodes, - Map visitedStreets) throws SQLException { + TLongObjectHashMap visitedStreets, HashSet uniqueNames) throws SQLException { streetBuildingsStat.setLong(1, city.getId()); ResultSet set = streetBuildingsStat.executeQuery(); while (set.next()) { long streetId = set.getLong(1); if (!visitedStreets.containsKey(streetId)) { Street street = new Street(null); - street.setName(set.getString(2)); - street.setEnName(set.getString(3)); + String streetName = set.getString(2); + String district = ""; + //There are more streets with same name in different districts. + //Add district name to all other names. If sorting is right, the first street was the one in the city + if (uniqueNames.contains(streetName)) { + district = " (" + set.getString(12) + ")"; + } + uniqueNames.add(streetName); + street.setName(streetName + district); + street.setEnName(set.getString(3) + district); street.setLocation(set.getDouble(4), set.getDouble(5)); street.setId(streetId); streets.add(street); @@ -919,7 +1088,8 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ set.close(); stat.close(); return cities; - } + } + } diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java b/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java index 4d2b97a845..cd1390d967 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java @@ -10,6 +10,7 @@ import java.io.RandomAccessFile; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; +import java.sql.Statement; import net.osmand.Algoritms; import net.osmand.IProgress; @@ -304,6 +305,15 @@ public class IndexCreator { allWays = dbCreator.getAllWays(); allRelations = dbCreator.getAllRelations(); } + } else { + if (DBDialect.NOSQL != dialect) { + Connection dbc = (Connection) dbConn; + final Statement stmt = dbc.createStatement(); + allRelations = stmt.executeQuery("select count(*) from relations").getInt(1); + allNodes = stmt.executeQuery("select count(*) from node").getInt(1); + allWays = stmt.executeQuery("select count(*) from ways").getInt(1); + stmt.close(); + } } accessor.initDatabase(dbConn, dialect, allNodes, allWays, allRelations); return loadFromExistingFile; @@ -338,9 +348,9 @@ public class IndexCreator { public void generateIndexes(File readFile, IProgress progress, IOsmStorageFilter addFilter, MapZooms mapZooms, - MapRenderingTypes renderingTypes) throws IOException, SAXException, SQLException { - if(!LevelDBAccess.load() && dialect == DBDialect.NOSQL){ - dialect = DBDialect.SQLITE; + MapRenderingTypes renderingTypes) throws IOException, SAXException, SQLException, InterruptedException { + if(LevelDBAccess.load()){ + dialect = DBDialect.NOSQL; } if (renderingTypes == null) { @@ -594,7 +604,7 @@ public class IndexCreator { } - public static void main(String[] args) throws IOException, SAXException, SQLException { + public static void main(String[] args) throws IOException, SAXException, SQLException, InterruptedException { long time = System.currentTimeMillis(); IndexCreator creator = new IndexCreator(new File("/home/victor/projects/OsmAnd/data/osm-gen/")); //$NON-NLS-1$ diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/OsmDbAccessor.java b/DataExtractionOSM/src/net/osmand/data/preparation/OsmDbAccessor.java index 7ca8caf5eb..d3af3a20af 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/OsmDbAccessor.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/OsmDbAccessor.java @@ -8,14 +8,16 @@ import java.sql.Statement; import java.util.Collection; import java.util.LinkedHashMap; import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; import net.osmand.IProgress; import net.osmand.osm.Entity; +import net.osmand.osm.Entity.EntityId; +import net.osmand.osm.Entity.EntityType; import net.osmand.osm.Node; import net.osmand.osm.Relation; import net.osmand.osm.Way; -import net.osmand.osm.Entity.EntityId; -import net.osmand.osm.Entity.EntityType; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -28,6 +30,7 @@ import com.anvisics.jleveldb.ext.DBIterator; import com.anvisics.jleveldb.ext.ReadOptions; public class OsmDbAccessor implements OsmDbAccessorContext { + private static final Log log = LogFactory.getLog(OsmDbAccessor.class); private PreparedStatement pselectNode; @@ -37,6 +40,7 @@ public class OsmDbAccessor implements OsmDbAccessorContext { private int allRelations ; private int allWays; private int allNodes; + private boolean realCounts = false; private Connection dbConn; private DBDialect dialect; @@ -202,86 +206,66 @@ public class OsmDbAccessor implements OsmDbAccessorContext { - public int iterateOverEntities(IProgress progress, EntityType type, OsmDbVisitor visitor) throws SQLException { + public int iterateOverEntities(IProgress progress, EntityType type, OsmDbVisitor visitor) throws SQLException, InterruptedException { if(dialect == DBDialect.NOSQL){ return iterateOverEntitiesNoSQL(progress, type, visitor); } Statement statement = dbConn.createStatement(); String select; int count = 0; - + // stat.executeUpdate("create table tags (id "+longType+", type smallint, skeys varchar(1024), value varchar(1024))"); // stat.executeUpdate("create table ways (id "+longType+", node "+longType+", ord smallint)"); // stat.executeUpdate("create table relations (id "+longType+", member "+longType+", type smallint, role varchar(1024), ord smallint)"); + computeRealCounts(statement); if (type == EntityType.NODE) { // filter out all nodes without tags select = "select n.id, n.latitude, n.longitude, t.skeys, t.value from node n inner join tags t on n.id = t.id and t.type = 0 order by n.id"; //$NON-NLS-1$ + count = allNodes; } else if (type == EntityType.WAY) { select = "select w.id, w.node, w.ord, t.skeys, t.value, n.latitude, n.longitude " + //$NON-NLS-1$ "from ways w left join tags t on w.id = t.id and t.type = 1 and w.ord = 0 inner join node n on w.node = n.id " + //$NON-NLS-1$ "order by w.id, w.ord"; //$NON-NLS-1$ + count = allWays; } else { select = "select r.id, t.skeys, t.value from relations r inner join tags t on t.id = r.id and t.type = 2 and r.ord = 0"; //$NON-NLS-1$ + count = allRelations; } - - ResultSet rs = statement.executeQuery(select); - Entity prevEntity = null; - - long prevId = -1; - while (rs.next()) { - long curId = rs.getLong(1); - boolean newEntity = curId != prevId; - Entity e = prevEntity; - if (type == EntityType.NODE) { - if (newEntity) { - e = new Node(rs.getDouble(2), rs.getDouble(3), curId); - } - e.putTag(rs.getString(4), rs.getString(5)); - } else if (type == EntityType.WAY) { - if (newEntity) { - e = new Way(curId); - } - int ord = rs.getInt(3); - if (ord == 0 && rs.getObject(4) != null) { - e.putTag(rs.getString(4), rs.getString(5)); - } - if (newEntity || ord > 0) { - ((Way) e).addNode(new Node(rs.getDouble(6), rs.getDouble(7), rs.getLong(2))); - } - } else { - if (newEntity) { - e = new Relation(curId); - } - e.putTag(rs.getString(2), rs.getString(3)); + progress.startWork(count); + + BlockingQueue toProcess = new ArrayBlockingQueue(100000); + + //produce + EntityProducer entityProducer = new EntityProducer(toProcess, type, statement, select); + entityProducer.start(); + + count = 0; + while (true) { + Entity entityToProcess = toProcess.take(); + if (entityToProcess == entityProducer.getEndingEntity()) { + //the last entity... + break; } - if (newEntity) { - if (progress != null) { - progress.progress(1); - } - if (prevEntity != null) { - count++; - visitor.iterateEntity(prevEntity, this); - } - prevEntity = e; + if (progress != null) { + progress.progress(1); } - prevId = curId; - } - if (prevEntity != null) { count++; - visitor.iterateEntity(prevEntity, this); - } - rs.close(); - if(EntityType.NODE == type){ - allNodes = count; - } else if(EntityType.WAY == type){ - allWays = count; - } else if(EntityType.RELATION == type){ - allRelations = count; + visitor.iterateEntity(entityToProcess, this); } return count; } + private void computeRealCounts(Statement statement) throws SQLException { + if (!realCounts) { + realCounts = true; + // filter out all nodes without tags + allNodes = statement.executeQuery("select count(*) from node n inner join tags t on n.id = t.id and t.type = 0 order by n.id").getInt(1); //$NON-NLS-1$ + allWays = statement.executeQuery("select count(*) from ways w").getInt(1); //$NON-NLS-1$ + allRelations = statement.executeQuery("select count(*) from relations r inner join tags t on t.id = r.id and t.type = 2 and r.ord = 0").getInt(1); //$NON-NLS-1$ + } + } + private void loadEntityDataNoSQL(Entity e, boolean loadTags) { Collection ids = e instanceof Relation ? ((Relation) e).getMemberIds() : ((Way) e).getEntityIds(); Map map = new LinkedHashMap(); @@ -455,4 +439,91 @@ public class OsmDbAccessor implements OsmDbAccessorContext { } + public class EntityProducer extends Thread { + + private final BlockingQueue toProcess; + private final Statement statement; + private final String select; + private final EntityType type; + private final Entity endingEntity = new Node(0,0,0); + + public EntityProducer(BlockingQueue toProcess, EntityType type, + Statement statement, String select) { + this.toProcess = toProcess; + this.type = type; + this.statement = statement; + this.select = select; + setDaemon(true); + setName("EntityProducer"); + } + + public Entity getEndingEntity() { + return endingEntity; + } + + @Override + public void run() { + ResultSet rs; + try { + rs = statement.executeQuery(select); +// rs.setFetchSize(1000); !! not working for SQLite would case troubles probably + Entity prevEntity = null; + + long prevId = -1; + while (rs.next()) { + long curId = rs.getLong(1); + boolean newEntity = curId != prevId; + Entity e = prevEntity; + if (type == EntityType.NODE) { + if (newEntity) { + e = new Node(rs.getDouble(2), rs.getDouble(3), + curId); + } + e.putTag(rs.getString(4), rs.getString(5)); + } else if (type == EntityType.WAY) { + if (newEntity) { + e = new Way(curId); + } + int ord = rs.getInt(3); + if (ord == 0 && rs.getObject(4) != null) { + e.putTag(rs.getString(4), rs.getString(5)); + } + if (newEntity || ord > 0) { + ((Way) e).addNode(new Node(rs.getDouble(6), rs + .getDouble(7), rs.getLong(2))); + } + } else { + if (newEntity) { + e = new Relation(curId); + } + e.putTag(rs.getString(2), rs.getString(3)); + } + if (newEntity) { + if (prevEntity != null) { + toProcess.put(prevEntity); + } + prevEntity = e; + } + prevId = curId; + } + if (prevEntity != null) { + toProcess.put(prevEntity); + } + rs.close(); + } catch (SQLException e1) { + e1.printStackTrace(); + } catch (InterruptedException e) { + e.printStackTrace(); + } finally { + try { + toProcess.put(getEndingEntity()); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + } + + } diff --git a/DataExtractionOSM/src/net/osmand/osm/MapUtils.java b/DataExtractionOSM/src/net/osmand/osm/MapUtils.java index efcdf3deb6..19598d5538 100644 --- a/DataExtractionOSM/src/net/osmand/osm/MapUtils.java +++ b/DataExtractionOSM/src/net/osmand/osm/MapUtils.java @@ -5,10 +5,8 @@ import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.List; -import java.util.Map.Entry; import net.osmand.data.MapObject; -import net.osmand.osm.Entity.EntityType; /** diff --git a/DataExtractionOSM/src/net/osmand/osm/OSMSettings.java b/DataExtractionOSM/src/net/osmand/osm/OSMSettings.java index c6f39dbd45..3942d0d8bc 100644 --- a/DataExtractionOSM/src/net/osmand/osm/OSMSettings.java +++ b/DataExtractionOSM/src/net/osmand/osm/OSMSettings.java @@ -56,6 +56,8 @@ public class OSMSettings { WEBSITE("website"), //$NON-NLS-1$ URL("url"), //$NON-NLS-1$ WIKIPEDIA("wikipedia"), //$NON-NLS-1$ + + ADMIN_LEVEL("admin_level"), //$NON-NLS-1$ ; private final String value; diff --git a/DataExtractionOSM/src/net/osmand/swing/OsmExtractionUI.java b/DataExtractionOSM/src/net/osmand/swing/OsmExtractionUI.java index 8a7114eb49..d228244eca 100644 --- a/DataExtractionOSM/src/net/osmand/swing/OsmExtractionUI.java +++ b/DataExtractionOSM/src/net/osmand/swing/OsmExtractionUI.java @@ -451,6 +451,8 @@ public class OsmExtractionUI implements IMapLocationListener { throw new IllegalStateException(e); } catch (SQLException e) { throw new IllegalStateException(e); + } catch (InterruptedException e) { + throw new IllegalStateException(e); } regionName = creator.getRegionName(); StringBuilder msg = new StringBuilder();