From 2093ac93dcee27b755fb5aff3a8ac028cc6563b0 Mon Sep 17 00:00:00 2001 From: Pavol Zibrita Date: Fri, 23 Sep 2011 11:09:13 +0200 Subject: [PATCH] More changes address creation without cityAdminLevel. Some small refactoring also. Still, the streets are not generated correctly. --- .../net/osmand/binary/BinaryInspector.java | 3 +- .../data/preparation/IndexAddressCreator.java | 233 +++++++++--------- .../osmand/data/preparation/IndexCreator.java | 23 +- .../data/preparation/IndexPoiCreator.java | 2 +- .../preparation/IndexTransportCreator.java | 2 +- .../preparation/IndexVectorMapCreator.java | 6 +- .../data/preparation/OsmDbAccessor.java | 95 ++++--- .../preparation/OsmDbAccessorContext.java | 2 +- .../src/net/osmand/osm/Entity.java | 26 ++ 9 files changed, 230 insertions(+), 162 deletions(-) diff --git a/DataExtractionOSM/src/net/osmand/binary/BinaryInspector.java b/DataExtractionOSM/src/net/osmand/binary/BinaryInspector.java index 8d6d1f01ad..9387798e4c 100644 --- a/DataExtractionOSM/src/net/osmand/binary/BinaryInspector.java +++ b/DataExtractionOSM/src/net/osmand/binary/BinaryInspector.java @@ -32,7 +32,8 @@ public class BinaryInspector { public static void main(String[] args) throws IOException { inspector(args); // test cases show info - inspector(new String[]{"/home/victor/projects/OsmAnd/data/osm-gen/saved/Belarus-newzooms-new-rt.obf"}); + //inspector(new String[]{"/home/victor/projects/OsmAnd/data/osm-gen/saved/Belarus-newzooms-new-rt.obf"}); + //inspector(new String[]{"-v","C:\\Users\\tpd\\osmand\\Slovakia.obf "}); // test case extract parts diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/IndexAddressCreator.java b/DataExtractionOSM/src/net/osmand/data/preparation/IndexAddressCreator.java index 58bce4cbba..fe51708339 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/IndexAddressCreator.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/IndexAddressCreator.java @@ -69,7 +69,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ private DataTileManager cityVillageManager = new DataTileManager(13); private DataTileManager cityManager = new DataTileManager(10); private List postalCodeRelations = new ArrayList(); - private Map cityBoundaries = new LinkedHashMap(); + private Map cityBoundaries = new HashMap(); private TLongHashSet visitedBoundaryWays = new TLongHashSet(); private boolean normalizeStreets; @@ -81,7 +81,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ // TODO Connection mapConnection; DBStreetDAO streetDAO; - + public class DBStreetDAO { protected void writeStreetWayNodes(Set streetIds, Way way) throws SQLException { @@ -154,7 +154,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ public class CachedDBStreetDAO extends DBStreetDAO { - private Map addressStreetLocalMap = new LinkedHashMap(); + private Map addressStreetLocalMap = new HashMap(); private TLongHashSet addressBuildingLocalSet = new TLongHashSet(); private TLongHashSet addressStreetNodeLocalSet = new TLongHashSet(); @@ -250,76 +250,95 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ } public void indexBoundariesRelation(Entity e, OsmDbAccessorContext ctx) throws SQLException { - if (isBoundary(e)) { - int adminLevel = 8; - try { - String tag = e.getTag(OSMTagKey.ADMIN_LEVEL); - if (tag == null) { - return; + Boundary boundary = extractBoundary(e, ctx); + if (boundary != null && boundary.getAdminLevel() >= 4 && boundary.getCenterPoint() != null && !Algoritms.isEmpty(boundary.getName())) { + LatLon boundaryCenter = boundary.getCenterPoint(); + List citiesToSearch = new ArrayList(); + citiesToSearch.addAll(cityManager.getClosestObjects(boundaryCenter.getLatitude(), boundaryCenter.getLongitude(), 3)); + citiesToSearch.addAll(cityVillageManager.getClosestObjects(boundaryCenter.getLatitude(), boundaryCenter.getLongitude(), 3)); + + List cityFound = new ArrayList(); + String boundaryName = boundary.getName().toLowerCase(); + for (City c : citiesToSearch) { + if (boundary.containsPoint(c.getLocation())) { + if (boundaryName.equalsIgnoreCase(c.getName())) { + cityFound.add(c); + break; + } } - adminLevel = Integer.parseInt(tag); - } catch (NumberFormatException ex) { - return; } - if (adminLevel < 4) { - return; - } - - Boundary boundary = extractBoundary(e, ctx); - if (boundary != null && boundary.getCenterPoint() != null && !Algoritms.isEmpty(boundary.getName())) { - boundary.setAdminLevel(adminLevel); - LatLon point = boundary.getCenterPoint(); - City cityFound = null; - List citiesToSearch = new ArrayList(); - citiesToSearch.addAll(cityManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)); - citiesToSearch.addAll(cityVillageManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)); - - String boundaryName = boundary.getName().toLowerCase(); + if (cityFound.isEmpty()) { + // try to find same name in the middle for (City c : citiesToSearch) { if (boundary.containsPoint(c.getLocation())) { - if (boundaryName.equalsIgnoreCase(c.getName())) { - cityFound = c; + String lower = c.getName().toLowerCase(); + if (boundaryName.startsWith(lower) || boundaryName.endsWith(lower) || boundaryName.contains(" " + lower + " ")) { + cityFound.add(c); break; } } } - if (cityFound != null) { - // try to find same name in the middle - for (City c : citiesToSearch) { - if (boundary.containsPoint(c.getLocation())) { - String lower = c.getName().toLowerCase(); - if (boundaryName.startsWith(lower) || boundaryName.endsWith(lower) || boundaryName.contains(" " + lower + " ")) { - cityFound = c; - break; - } - } - } - } - if (cityFound != null) { - putCityBoundary(boundary, cityFound); - } + } +// if (cityFound.isEmpty()) { + // try to find closes city to the boundary center +// double minDist = Double.MAX_VALUE; +// City closestCity = null; +// for (City c : citiesToSearch) { +// if (boundary.containsPoint(c.getLocation())) { +//// double dist = MapUtils.getDistance(boundaryCenter, c.getLocation()); +//// if (dist < minDist) { +//// dist = minDist; +//// closestCity = c; +//// } +// cityFound.add(c); +// } +// } +// cityFound = closestCity; +// } + if (!cityFound.isEmpty()) { + //we can have more cities in one boundary!! (suburbs...) + putCityBoundary(boundary, cityFound); } } } - private void putCityBoundary(Boundary boundary, City c) { - final Boundary oldBoundary = cityBoundaries.get(c); - if (oldBoundary != null) { - // try to found the biggest area (not small center district) - if (oldBoundary.getAdminLevel() > boundary.getAdminLevel()) { - cityBoundaries.put(c, boundary); - } else if(boundary.getName().equalsIgnoreCase(c.getName()) && - !oldBoundary.getName().equalsIgnoreCase(c.getName())){ - cityBoundaries.put(c, boundary); - } else if(oldBoundary.getAdminLevel() == boundary.getAdminLevel() && - oldBoundary != boundary && boundary.getName().equalsIgnoreCase(oldBoundary.getName())){ - if(!oldBoundary.isClosedWay() && !boundary.isClosedWay()){ - oldBoundary.getInnerWays().addAll(boundary.getInnerWays()); - oldBoundary.getOuterWays().addAll(boundary.getOuterWays()); - } + + private int extractBoundaryAdminLevel(Entity e) { + int adminLevel = -1; + try { + String tag = e.getTag(OSMTagKey.ADMIN_LEVEL); + if (tag == null) { + return adminLevel; + } + return Integer.parseInt(tag); + } catch (NumberFormatException ex) { + return adminLevel; + } + } + + private void putCityBoundary(Boundary boundary, List cities) { + for (City cityFound : cities) { + final Boundary oldBoundary = cityBoundaries.get(cityFound); + if (oldBoundary != null) { + // try to found the biggest area (not small center district) + if (oldBoundary.getAdminLevel() > boundary.getAdminLevel() && !oldBoundary.getName().equalsIgnoreCase(cityFound.getName())) { + cityBoundaries.put(cityFound, boundary); + System.out.println("City: " + cityFound.getName() + " boundary: " + boundary.getName()); + } else if(boundary.getName().equalsIgnoreCase(cityFound.getName()) && + !oldBoundary.getName().equalsIgnoreCase(cityFound.getName())){ + cityBoundaries.put(cityFound, boundary); + System.out.println("City: " + cityFound.getName() + " boundary: " + boundary.getName()); + } else if(oldBoundary.getAdminLevel() == boundary.getAdminLevel() && + oldBoundary != boundary && boundary.getName().equalsIgnoreCase(oldBoundary.getName())){ + if(!oldBoundary.isClosedWay() && !boundary.isClosedWay()){ + oldBoundary.getInnerWays().addAll(boundary.getInnerWays()); + oldBoundary.getOuterWays().addAll(boundary.getOuterWays()); + } + } + } else { + cityBoundaries.put(cityFound, boundary); + System.out.println("City: " + cityFound.getName() + " boundary: " + boundary.getName()); } - } else { - cityBoundaries.put(c, boundary); } } @@ -331,14 +350,13 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ if (isBoundary(e)) { Boundary boundary = null; if (e instanceof Relation) { - Relation i = (Relation) e; - ctx.loadEntityData(i, true); - boundary = new Boundary(true); - if (i.getTag(OSMTagKey.NAME) != null) { - boundary.setName(i.getTag(OSMTagKey.NAME)); - } - boundary.setBoundaryId(i.getId()); - Map entities = i.getMemberEntities(); + Relation aRelation = (Relation) e; + ctx.loadEntityData(aRelation); + boundary = new Boundary(true); //TODO here should be true or false ??? or, how it should be computed in this case? + boundary.setName(aRelation.getTag(OSMTagKey.NAME)); + boundary.setBoundaryId(aRelation.getId()); + boundary.setAdminLevel(extractBoundaryAdminLevel(aRelation)); + Map entities = aRelation.getMemberEntities(); for (Entity es : entities.keySet()) { if (es instanceof Way) { boolean inner = "inner".equals(entities.get(es)); //$NON-NLS-1$ @@ -362,10 +380,9 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ closed = Algoritms.objectEquals(nodeIds.get(0), nodeIds.get(nodeIds.size() - 1)); } boundary = new Boundary(closed); - if (e.getTag(OSMTagKey.NAME) != null) { - boundary.setName(e.getTag(OSMTagKey.NAME)); - } + boundary.setName(e.getTag(OSMTagKey.NAME)); boundary.setBoundaryId(e.getId()); + boundary.setAdminLevel(extractBoundaryAdminLevel(e)); boundary.getOuterWays().add((Way) e); } } @@ -385,7 +402,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ // try to find appropriate city/street City c = null; // load with member ways with their nodes and tags ! - ctx.loadEntityData(i, true); + ctx.loadEntityData(i); Collection members = i.getMembers("is_in"); //$NON-NLS-1$ Relation a3 = null; @@ -395,7 +412,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ a6 = i; } Entity in = members.iterator().next(); - ctx.loadEntityData(in, true); + ctx.loadEntityData(in); if (in instanceof Relation) { // go one level up for house if (house) { @@ -403,7 +420,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ members = ((Relation) in).getMembers("is_in"); //$NON-NLS-1$ if (!members.isEmpty()) { in = members.iterator().next(); - ctx.loadEntityData(in, true); + ctx.loadEntityData(in); if (in instanceof Relation) { a3 = (Relation) in; } @@ -436,7 +453,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ } } - Set streetId = registerStreetInCity(name, null, location, (a6.getId() << 2) | 2, Collections.singletonList(c)); + Set streetId = registerStreetInCities(name, null, location, Collections.singletonList(c)); if (streetId == null) { return; } @@ -566,13 +583,13 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ return newName.trim(); } - public Set getStreetInCity(Set isInNames, String name, String nameEn, LatLon location, long initId) throws SQLException { + public Set getStreetInCity(Set isInNames, String name, String nameEn, LatLon location) throws SQLException { if (name == null || location == null) { return Collections.emptySet(); } name = normalizeStreetName(name); - List result = new ArrayList(); + Set result = new HashSet(); List nearestObjects = new ArrayList(); nearestObjects.addAll(cityManager.getClosestObjects(location.getLatitude(),location.getLongitude())); nearestObjects.addAll(cityVillageManager.getClosestObjects(location.getLatitude(),location.getLongitude())); @@ -588,13 +605,17 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ result.add(city); } } +// for (Entry cb : cityBoundaries.entrySet()) { +// if (cb.getValue().containsPoint(location)) { +// result.add(cb.getKey()); +// } +// } - return registerStreetInCity(name, nameEn, location, initId, result); + return registerStreetInCities(name, nameEn, location, result); } - private Set registerStreetInCity(String name, String nameEn, LatLon location, long initId, - List result) throws SQLException { + private Set registerStreetInCities(String name, String nameEn, LatLon location, Collection result) throws SQLException { if (result.isEmpty()) { return null; } @@ -614,12 +635,12 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ } } } - Long foundId = streetDAO.findStreet(name, result.get(0), cityPart); + Long foundId = streetDAO.findStreet(name, city, cityPart); if (foundId == null) { - insertStreetData(addressStreetStat, initId, name, nameEn, location.getLatitude(), location.getLongitude(), city.getId(), + long streetId = insertStreetData(addressStreetStat, name, nameEn, location.getLatitude(), location.getLongitude(), city.getId(), cityPart); - streetDAO.insertStreet(addressStreetStat, name, city, cityPart, initId); - foundId = initId; + streetDAO.insertStreet(addressStreetStat, name, city, cityPart, streetId); + foundId = streetId; } values.add(foundId); } @@ -685,22 +706,6 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ return closest; } - private Set getIsINames(Entity e) { - String values = e.getTag(OSMTagKey.IS_IN); - if (values == null) { - return Collections.emptySet(); - } - if (values.indexOf(';') != -1) { - String[] splitted = values.split(";"); - Set set = new HashSet(); - for (int i = 0; i < splitted.length; i++) { - set.add(splitted[i].trim()); - } - return set; - } - return Collections.singleton(values.trim()); - } - public void iterateMainEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException { // index not only buildings but also nodes that belongs to addr:interpolation ways if (e.getTag(OSMTagKey.ADDR_HOUSE_NUMBER) != null && e.getTag(OSMTagKey.ADDR_STREET) != null) { @@ -708,13 +713,13 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ // check that building is not registered already boolean exist = streetDAO.findBuilding(e); if (!exist) { - ctx.loadEntityData(e, false); + ctx.loadEntityData(e); LatLon l = e.getLatLon(); - Set idStreet = getStreetInCity(getIsINames(e), e.getTag(OSMTagKey.ADDR_STREET), null, l, (e.getId() << 2)); - if (!idStreet.isEmpty()) { + Set idsOfStreet = getStreetInCity(e.getIsInNames(), e.getTag(OSMTagKey.ADDR_STREET), null, l); + if (!idsOfStreet.isEmpty()) { Building building = new Building(e); building.setName(e.getTag(OSMTagKey.ADDR_HOUSE_NUMBER)); - streetDAO.writeBuilding(idStreet, building); + streetDAO.writeBuilding(idsOfStreet, building); } } } else if (e instanceof Way /* && OSMSettings.wayForCar(e.getTag(OSMTagKey.HIGHWAY)) */ @@ -730,17 +735,17 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ // check that street way is not registered already if (!exist) { - ctx.loadEntityData(e, false); + ctx.loadEntityData(e); LatLon l = e.getLatLon(); - Set idStreet = getStreetInCity(getIsINames(e), e.getTag(OSMTagKey.NAME), e.getTag(OSMTagKey.NAME_EN), l, (e.getId() << 2) | 1); - if (!idStreet.isEmpty() && saveAddressWays) { - streetDAO.writeStreetWayNodes(idStreet, (Way) e); + Set idsOfStreet = getStreetInCity(e.getIsInNames(), e.getTag(OSMTagKey.NAME), e.getTag(OSMTagKey.NAME_EN), l); + if (!idsOfStreet.isEmpty() && saveAddressWays) { + streetDAO.writeStreetWayNodes(idsOfStreet, (Way) e); } } } if (e instanceof Relation) { if (e.getTag(OSMTagKey.POSTAL_CODE) != null) { - ctx.loadEntityData(e, false); + ctx.loadEntityData(e); postalCodeRelations.add((Relation) e); } } @@ -757,15 +762,21 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{ } - private void insertStreetData(PreparedStatement addressStreetStat, long id, String name, String nameEn, double latitude, + private long streetIdSequence = 0; + + private long insertStreetData(PreparedStatement addressStreetStat, //long id, + String name, String nameEn, double latitude, double longitude, Long cityId, String cityPart) throws SQLException { - addressStreetStat.setLong(1, id); + long streetId = streetIdSequence++; +// addressStreetStat.setLong(1, id); + addressStreetStat.setLong(1, streetId); addressStreetStat.setString(4, name); addressStreetStat.setString(5, nameEn); addressStreetStat.setDouble(2, latitude); addressStreetStat.setDouble(3, longitude); addressStreetStat.setLong(6, cityId); addressStreetStat.setString(7, cityPart); + return streetId; } diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java b/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java index 014aa356de..ac2365e971 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java @@ -261,7 +261,7 @@ public class IndexCreator { // 1. Loading osm file OsmDbCreator dbCreator = new OsmDbCreator(this); try { - progress.setGeneralProgress("[35 / 100]"); //$NON-NLS-1$ + progress.setGeneralProgress("[15 / 100]"); //$NON-NLS-1$ progress.startTask(Messages.getString("IndexCreator.LOADING_FILE") + readFile.getAbsolutePath(), -1); //$NON-NLS-1$ // 1 init database to store temporary data dbCreator.initDatabase(dialect, dbConn); @@ -311,9 +311,10 @@ public class IndexCreator { if (DBDialect.NOSQL != dialect) { Connection dbc = (Connection) dbConn; final Statement stmt = dbc.createStatement(); - allRelations = stmt.executeQuery("select count(*) from relations").getInt(1); - allNodes = stmt.executeQuery("select count(*) from node").getInt(1); - allWays = stmt.executeQuery("select count(*) from ways").getInt(1); + accessor.computeRealCounts(stmt); + allRelations = accessor.getAllRelations(); + allNodes = accessor.getAllNodes(); + allWays = accessor.getAllWays(); stmt.close(); } } @@ -419,7 +420,7 @@ public class IndexCreator { // 3.1 write all cities if (indexAddress) { - progress.setGeneralProgress("[40 / 100]"); //$NON-NLS-1$ + progress.setGeneralProgress("[20 / 100]"); //$NON-NLS-1$ progress.startTask(Messages.getString("IndexCreator.INDEX_CITIES"), accessor.getAllNodes()); //$NON-NLS-1$ if (loadFromExistingFile) { // load cities names @@ -441,7 +442,7 @@ public class IndexCreator { @Override public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException { if (indexAddress) { - indexAddressCreator.indexAddressRelation((Relation) e, ctx); + //indexAddressCreator.indexAddressRelation((Relation) e, ctx); streets needs loaded boundaries !!! indexAddressCreator.indexBoundariesRelation((Relation) e, ctx); } if (indexMap) { @@ -459,9 +460,17 @@ public class IndexCreator { } }); + progress.setGeneralProgress("[45 / 100]"); //$NON-NLS-1$ + progress.startTask(Messages.getString("IndexCreator.PREINDEX_ADRESS_MAP"), accessor.getAllRelations()); //$NON-NLS-1$ + accessor.iterateOverEntities(progress, EntityType.RELATION, new OsmDbVisitor() { + @Override + public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException { + indexAddressCreator.indexAddressRelation((Relation) e, ctx); + } + }); + indexAddressCreator.commitToPutAllCities(); } - } // 3.3 MAIN iterate over all entities diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/IndexPoiCreator.java b/DataExtractionOSM/src/net/osmand/data/preparation/IndexPoiCreator.java index ee63977624..357c757de0 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/IndexPoiCreator.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/IndexPoiCreator.java @@ -53,7 +53,7 @@ public class IndexPoiCreator extends AbstractIndexPartCreator { tempAmenityList = Amenity.parseAmenities(e, tempAmenityList); if (!tempAmenityList.isEmpty() && poiPreparedStatement != null) { // load data for way (location etc...) - ctx.loadEntityData(e, false); + ctx.loadEntityData(e); for (Amenity a : tempAmenityList) { // do not add that check because it is too much printing for batch creation // by statistic < 1% creates maps manually diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/IndexTransportCreator.java b/DataExtractionOSM/src/net/osmand/data/preparation/IndexTransportCreator.java index f2a38451d7..4da7d0d0f9 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/IndexTransportCreator.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/IndexTransportCreator.java @@ -130,7 +130,7 @@ public class IndexTransportCreator extends AbstractIndexPartCreator { public void visitEntityMainStep(Entity e, OsmDbAccessorContext ctx) throws SQLException { if (e instanceof Relation && e.getTag(OSMTagKey.ROUTE) != null) { - ctx.loadEntityData(e, true); + ctx.loadEntityData(e); TransportRoute route = indexTransportRoute((Relation) e); if (route != null) { insertTransportIntoIndex(route); diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/IndexVectorMapCreator.java b/DataExtractionOSM/src/net/osmand/data/preparation/IndexVectorMapCreator.java index 7e08851432..adbb434208 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/IndexVectorMapCreator.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/IndexVectorMapCreator.java @@ -84,7 +84,7 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator { private void indexMultiPolygon(Entity e, OsmDbAccessorContext ctx) throws SQLException { if (e instanceof Relation && "multipolygon".equals(e.getTag(OSMTagKey.TYPE))) { //$NON-NLS-1$ - ctx.loadEntityData(e, true); + ctx.loadEntityData(e); Map entities = ((Relation) e).getMemberEntities(); boolean outerFound = false; @@ -222,7 +222,7 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator { type = MapRenderingTypes.RESTRICTION_ONLY_STRAIGHT_ON; } if (type != -1) { - ctx.loadEntityData(e, true); + ctx.loadEntityData(e); Collection fromL = ((Relation) e).getMemberIds("from"); //$NON-NLS-1$ Collection toL = ((Relation) e).getMemberIds("to"); //$NON-NLS-1$ if (!fromL.isEmpty() && !toL.isEmpty()) { @@ -643,7 +643,7 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator { public void iterateMainEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException { if (e instanceof Way || e instanceof Node) { // manipulate what kind of way to load - ctx.loadEntityData(e, false); + ctx.loadEntityData(e); boolean oneway = "-1".equals(e.getTag(OSMTagKey.ONEWAY)); //$NON-NLS-1$ for (int i = 0; i < mapZooms.size(); i++) { boolean inverse = i == 0 ? oneway : false; diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/OsmDbAccessor.java b/DataExtractionOSM/src/net/osmand/data/preparation/OsmDbAccessor.java index 1a7c42d5d7..d72c94021c 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/OsmDbAccessor.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/OsmDbAccessor.java @@ -48,6 +48,12 @@ public class OsmDbAccessor implements OsmDbAccessorContext { private ReadOptions randomAccessOptions; private DBAccessor accessor; + + private PreparedStatement iterateNodes; + + private PreparedStatement iterateWays; + + private PreparedStatement iterateRelations; public interface OsmDbVisitor { @@ -77,6 +83,12 @@ public class OsmDbAccessor implements OsmDbAccessorContext { "from relations r left join tags t on r.id = t.id and t.type = 2 and r.ord = 0 " + //$NON-NLS-1$ "where r.id = ? order by r.ord"); //$NON-NLS-1$ pselectTags = dbConn.prepareStatement("select skeys, value from tags where id = ? and type = ?"); //$NON-NLS-1$ + + iterateNodes = dbConn.prepareStatement("select n.id, n.latitude, n.longitude, t.skeys, t.value from node n inner join tags t on n.id = t.id and t.type = 0 order by n.id"); //$NON-NLS-1$ + iterateWays = dbConn.prepareStatement("select w.id, w.node, w.ord, t.skeys, t.value, n.latitude, n.longitude " + //$NON-NLS-1$ + "from ways w left join tags t on w.id = t.id and t.type = 1 and w.ord = 0 inner join node n on w.node = n.id " + //$NON-NLS-1$ + "order by w.id, w.ord"); //$NON-NLS-1$ + iterateRelations = dbConn.prepareStatement("select r.id, t.skeys, t.value from relations r inner join tags t on t.id = r.id and t.type = 2 and r.ord = 0"); //$NON-NLS-1$ } } @@ -92,13 +104,17 @@ public class OsmDbAccessor implements OsmDbAccessorContext { return allWays; } - public void loadEntityData(Entity e, boolean loadTags) throws SQLException { + public void loadEntityData(Entity e) throws SQLException { + if (e.isDataLoaded()) { //data was already loaded, nothing to do + return; + } if (e instanceof Node || (e instanceof Way && !((Way) e).getNodes().isEmpty())) { // do not load tags for nodes inside way return; } if(dialect == DBDialect.NOSQL){ - loadEntityDataNoSQL(e, loadTags); + loadEntityDataNoSQL(e); + e.entityDataLoaded(); return; } @@ -172,6 +188,9 @@ public class OsmDbAccessor implements OsmDbAccessorContext { way.putTag(rs.getString(3), rs.getString(4)); } } + if (way.getNodes() == null) { + System.err.println("Strange, way with id:" + i.getId() + " has no nodes?"); + } rs.close(); } } else if (i.getType() == EntityType.RELATION) { @@ -201,34 +220,32 @@ public class OsmDbAccessor implements OsmDbAccessorContext { } e.initializeLinks(map); + e.entityDataLoaded(); } - - public int iterateOverEntities(IProgress progress, EntityType type, OsmDbVisitor visitor) throws SQLException, InterruptedException { if(dialect == DBDialect.NOSQL){ return iterateOverEntitiesNoSQL(progress, type, visitor); } Statement statement = dbConn.createStatement(); - String select; + PreparedStatement select; int count = 0; // stat.executeUpdate("create table tags (id "+longType+", type smallint, skeys varchar(1024), value varchar(1024))"); // stat.executeUpdate("create table ways (id "+longType+", node "+longType+", ord smallint)"); // stat.executeUpdate("create table relations (id "+longType+", member "+longType+", type smallint, role varchar(1024), ord smallint)"); computeRealCounts(statement); + statement.close(); if (type == EntityType.NODE) { // filter out all nodes without tags - select = "select n.id, n.latitude, n.longitude, t.skeys, t.value from node n inner join tags t on n.id = t.id and t.type = 0 order by n.id"; //$NON-NLS-1$ + select = iterateNodes; count = allNodes; } else if (type == EntityType.WAY) { - select = "select w.id, w.node, w.ord, t.skeys, t.value, n.latitude, n.longitude " + //$NON-NLS-1$ - "from ways w left join tags t on w.id = t.id and t.type = 1 and w.ord = 0 inner join node n on w.node = n.id " + //$NON-NLS-1$ - "order by w.id, w.ord"; //$NON-NLS-1$ + select = iterateWays; count = allWays; } else { - select = "select r.id, t.skeys, t.value from relations r inner join tags t on t.id = r.id and t.type = 2 and r.ord = 0"; //$NON-NLS-1$ + select = iterateRelations; count = allRelations; } progress.startWork(count); @@ -236,37 +253,34 @@ public class OsmDbAccessor implements OsmDbAccessorContext { BlockingQueue toProcess = new ArrayBlockingQueue(100000); //produce - EntityProducer entityProducer = new EntityProducer(toProcess, type, statement, select); + EntityProducer entityProducer = new EntityProducer(toProcess, type, select); entityProducer.start(); - count = 0; - while (true) { - Entity entityToProcess = toProcess.take(); - if (entityToProcess == entityProducer.getEndingEntity()) { - //the last entity... - break; - } + int counter = 0; + Entity entityToProcess = null; + Entity endEntity = entityProducer.getEndingEntity(); + while ((entityToProcess = toProcess.take()) != endEntity) { if (progress != null) { progress.progress(1); } - count++; + counter++; visitor.iterateEntity(entityToProcess, this); } return count; } - private void computeRealCounts(Statement statement) throws SQLException { + public void computeRealCounts(Statement statement) throws SQLException { if (!realCounts) { realCounts = true; // filter out all nodes without tags - allNodes = statement.executeQuery("select count(*) from node n inner join tags t on n.id = t.id and t.type = 0").getInt(1); //$NON-NLS-1$ - allWays = statement.executeQuery("select count(*) from ways w").getInt(1); //$NON-NLS-1$ - allRelations = statement.executeQuery("select count(*) from relations r inner join tags t on t.id = r.id and t.type = 2 and r.ord = 0").getInt(1); //$NON-NLS-1$ + allNodes = statement.executeQuery("select count(distinct n.id) from node n inner join tags t on n.id = t.id and t.type = 0").getInt(1); //$NON-NLS-1$ + allWays = statement.executeQuery("select count(distinct w.id) from ways w").getInt(1); //$NON-NLS-1$ + allRelations = statement.executeQuery("select count(distinct r.id) from relations r inner join tags t on t.id = r.id and t.type = 2 and r.ord = 0").getInt(1); //$NON-NLS-1$ } } - private void loadEntityDataNoSQL(Entity e, boolean loadTags) { + private void loadEntityDataNoSQL(Entity e) { Collection ids = e instanceof Relation ? ((Relation) e).getMemberIds() : ((Way) e).getEntityIds(); Map map = new LinkedHashMap(); for (EntityId i : ids) { @@ -275,7 +289,7 @@ public class OsmDbAccessor implements OsmDbAccessorContext { String value = accessor.get(randomAccessOptions, key); if (value != null && value.length() > 0) { try { - Entity es = loadEntityNoSqlFromValue(randomAccessOptions, key, value, loadTags, false); + Entity es = loadEntityNoSqlFromValue(randomAccessOptions, key, value, false); map.put(i, es); } catch (JSONException e1) { log.warn(key + " - " + e1.getMessage() + " " + value + "("+value.length()+"]", e1); @@ -292,7 +306,7 @@ public class OsmDbAccessor implements OsmDbAccessorContext { } private Entity loadEntityNoSqlFromValue(ReadOptions opts, - String key, String value, boolean loadTags, boolean skipIfEmptyTags) throws JSONException{ + String key, String value, boolean skipIfEmptyTags) throws JSONException{ if(value == null){ return null; } @@ -346,7 +360,7 @@ public class OsmDbAccessor implements OsmDbAccessorContext { while(n == ArraySerializer.ELEMENT){ String pointId = "0"+ tokenizer.value(); String pointVal = this.accessor.get(opts, pointId); - Node node = (Node) loadEntityNoSqlFromValue(opts, pointId, pointVal, false, false); + Node node = (Node) loadEntityNoSqlFromValue(opts, pointId, pointVal, false); if(node != null){ ((Way) e).addNode(node); } @@ -398,11 +412,11 @@ public class OsmDbAccessor implements OsmDbAccessorContext { try { Entity e = null; if (type == EntityType.NODE) { - e = loadEntityNoSqlFromValue(opts, key, value, true, true); + e = loadEntityNoSqlFromValue(opts, key, value, true); } else if (type == EntityType.WAY) { - e = loadEntityNoSqlFromValue(opts, key, value, true, false); + e = loadEntityNoSqlFromValue(opts, key, value, false); } else { - e = loadEntityNoSqlFromValue(opts, key, value, true, false); + e = loadEntityNoSqlFromValue(opts, key, value, false); } if(e != null){ @@ -435,6 +449,15 @@ public class OsmDbAccessor implements OsmDbAccessorContext { if (pselectTags != null) { pselectTags.close(); } + if (iterateNodes != null) { + iterateNodes.close(); + } + if (iterateRelations != null) { + iterateRelations.close(); + } + if (iterateWays != null) { + iterateWays.close(); + } } } @@ -442,17 +465,14 @@ public class OsmDbAccessor implements OsmDbAccessorContext { public class EntityProducer extends Thread { private final BlockingQueue toProcess; - private final Statement statement; - private final String select; + private final PreparedStatement select; private final EntityType type; private final Entity endingEntity = new Node(0,0,0); - public EntityProducer(BlockingQueue toProcess, EntityType type, - Statement statement, String select) { + public EntityProducer(BlockingQueue toProcess, EntityType type, PreparedStatement select2) { this.toProcess = toProcess; this.type = type; - this.statement = statement; - this.select = select; + this.select = select2; setDaemon(true); setName("EntityProducer"); } @@ -465,7 +485,8 @@ public class OsmDbAccessor implements OsmDbAccessorContext { public void run() { ResultSet rs; try { - rs = statement.executeQuery(select); + select.execute(); + rs = select.getResultSet(); // rs.setFetchSize(1000); !! not working for SQLite would case troubles probably Entity prevEntity = null; diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/OsmDbAccessorContext.java b/DataExtractionOSM/src/net/osmand/data/preparation/OsmDbAccessorContext.java index 825af43fdd..f956412b83 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/OsmDbAccessorContext.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/OsmDbAccessorContext.java @@ -6,5 +6,5 @@ import net.osmand.osm.Entity; public interface OsmDbAccessorContext { - public void loadEntityData(Entity e, boolean loadTags) throws SQLException; + public void loadEntityData(Entity e) throws SQLException; } \ No newline at end of file diff --git a/DataExtractionOSM/src/net/osmand/osm/Entity.java b/DataExtractionOSM/src/net/osmand/osm/Entity.java index c065d95837..dfdf736d70 100644 --- a/DataExtractionOSM/src/net/osmand/osm/Entity.java +++ b/DataExtractionOSM/src/net/osmand/osm/Entity.java @@ -2,8 +2,10 @@ package net.osmand.osm; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Set; import net.osmand.osm.OSMSettings.OSMTagKey; @@ -98,6 +100,7 @@ public abstract class Entity { // lazy initializing private Map tags = null; private final long id; + private boolean dataLoaded; public Entity(long id) { this.id = id; @@ -190,5 +193,28 @@ public abstract class Entity { } return true; } + + public Set getIsInNames() { + String values = getTag(OSMTagKey.IS_IN); + if (values == null) { + return Collections.emptySet(); + } + if (values.indexOf(';') != -1) { + String[] splitted = values.split(";"); + Set set = new HashSet(splitted.length); + for (int i = 0; i < splitted.length; i++) { + set.add(splitted[i].trim()); + } + return set; + } + return Collections.singleton(values.trim()); + } + + public void entityDataLoaded() { + this.dataLoaded = true; + } + public boolean isDataLoaded() { + return dataLoaded; + } }