From e5e7509a9855be91b7dae063b2b4dff301320eb2 Mon Sep 17 00:00:00 2001 From: Victor Shcherb Date: Sun, 28 Nov 2010 22:39:12 +0000 Subject: [PATCH] fix issue 221 fix issue 198 git-svn-id: https://osmand.googlecode.com/svn/trunk@715 e29c36b1-1cfa-d876-8d93-3434fc2bb7b8 --- .../osmand/data/index/DataIndexWriter.java | 48 ++++ .../net/osmand/data/index/IndexConstants.java | 1 + .../osmand/data/preparation/IndexCreator.java | 255 ++++++++++-------- .../impl/ConsoleProgressImplementation.java | 15 +- 4 files changed, 209 insertions(+), 110 deletions(-) diff --git a/DataExtractionOSM/src/net/osmand/data/index/DataIndexWriter.java b/DataExtractionOSM/src/net/osmand/data/index/DataIndexWriter.java index 60a95b5f33..3bea1ec9b4 100644 --- a/DataExtractionOSM/src/net/osmand/data/index/DataIndexWriter.java +++ b/DataExtractionOSM/src/net/osmand/data/index/DataIndexWriter.java @@ -302,9 +302,15 @@ public class DataIndexWriter { public static void createMapIndexStructure(Connection conn) throws SQLException{ Statement stat = conn.createStatement(); assert IndexConstants.BINARY_MAP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ + assert IndexConstants.LOW_LEVEL_MAP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ stat.executeUpdate("create table binary_map_objects (id bigint primary key, name varchar(255), " + "types binary, restrictions binary, nodes binary, highway int)"); stat.executeUpdate("create index binary_map_objects_ind on binary_map_objects (id)"); + + stat.executeUpdate("create table low_level_map_objects (id bigint primary key, start_node bigint, " + + "end_node bigint, name varchar(255), nodes binary, type bigint, level smallint)"); + stat.executeUpdate("create index low_level_map_objects_ind on low_level_map_objects (id)"); + stat.executeUpdate("create index low_level_map_objects_ind_loc on low_level_map_objects (start_node, end_node)"); stat.close(); } @@ -312,6 +318,48 @@ public class DataIndexWriter { assert IndexConstants.BINARY_MAP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ return conn.prepareStatement("insert into binary_map_objects(id, name, types, restrictions, nodes, highway) values(?, ?, ?, ?, ?, ?)"); } + + public static PreparedStatement createStatementLowLevelMapBinaryInsert(Connection conn) throws SQLException{ + assert IndexConstants.LOW_LEVEL_MAP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ + return conn.prepareStatement("insert into low_level_map_objects(id, start_node, end_node, name, nodes, type, level) values(?, ?, ?, ?, ?, ?, ?)"); + } + + public static void insertLowLevelMapBinaryObject(Map statements, + PreparedStatement mapLowLevelBinaryStat, int level,long types, long id, List nodes, String name) throws SQLException{ + assert IndexConstants.LOW_LEVEL_MAP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ + boolean first = true; + long firstId = -1; + long lastId = -1; + ByteArrayOutputStream bnodes = new ByteArrayOutputStream(); + try { + for (Node n : nodes) { + if (n != null) { + if (first) { + firstId = n.getId(); + first = false; + } + lastId = n.getId(); + Algoritms.writeInt(bnodes, Float.floatToRawIntBits((float) n.getLatitude())); + Algoritms.writeInt(bnodes, Float.floatToRawIntBits((float) n.getLongitude())); + } + } + } catch (IOException e) { + throw new IllegalStateException(e); + } + if(firstId == -1){ + return; + } + // conn.prepareStatement("insert into binary_map_objects(id, name, types, restrictions, nodes, highway) values(?, ?, ?, ?, ?, ?)"); + mapLowLevelBinaryStat.setLong(1, id); + mapLowLevelBinaryStat.setLong(2, firstId); + mapLowLevelBinaryStat.setLong(3, lastId); + mapLowLevelBinaryStat.setString(4, name); + mapLowLevelBinaryStat.setBytes(5, bnodes.toByteArray()); + mapLowLevelBinaryStat.setLong(6, types); + mapLowLevelBinaryStat.setShort(7, (short) level); + + addBatch(statements, mapLowLevelBinaryStat); + } public static void insertBinaryMapRenderObjectIndex(Map statements, PreparedStatement mapBinaryStat, RTree mapTree, Entity e, String name, long id, int type, List typeUse, int highwayAttributes, List restrictions, diff --git a/DataExtractionOSM/src/net/osmand/data/index/IndexConstants.java b/DataExtractionOSM/src/net/osmand/data/index/IndexConstants.java index bf1f3fdd2b..fa91382f60 100644 --- a/DataExtractionOSM/src/net/osmand/data/index/IndexConstants.java +++ b/DataExtractionOSM/src/net/osmand/data/index/IndexConstants.java @@ -45,6 +45,7 @@ public class IndexConstants { public final static String POI_TABLE = "poi"; //$NON-NLS-1$ public final static String BINARY_MAP_TABLE = "binary_map_objects"; //$NON-NLS-1$ + public final static String LOW_LEVEL_MAP_TABLE = "low_level_map_objects"; //$NON-NLS-1$ public final static String TRANSPORT_STOP_TABLE = "transport_stop"; //$NON-NLS-1$ public final static String TRANSPORT_ROUTE_STOP_TABLE = "transport_route_stop"; //$NON-NLS-1$ diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java b/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java index ea4374a7c6..e7b302f526 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java @@ -90,7 +90,7 @@ public class IndexCreator { private static final String SQLITE_DIALECT = "SQLITE"; private static final String CURRENT_DB = SQLITE_DIALECT; - public static final int BATCH_SIZE = 1000; + public static final int BATCH_SIZE = 5000; public static final int BATCH_SIZE_OSM = 10000; public static final String TEMP_NODES_DB = "nodes.tmp.odb"; @@ -137,6 +137,8 @@ public class IndexCreator { private Connection mapConnection; private PreparedStatement mapBinaryStat; + private PreparedStatement mapLowLevelBinaryStat; + private int lowLevelWays = -1; private PreparedStatement addressCityStat; private PreparedStatement addressStreetStat; @@ -163,17 +165,6 @@ public class IndexCreator { } private Map multiPolygonsNames = new LinkedHashMap(); private Map> highwayRestrictions = new LinkedHashMap>(); - - private Map>[] lowLevelSt = new Map[MAP_ZOOMS.length - 1]; - private Map>[] lowLevelEnd = new Map[MAP_ZOOMS.length - 1]; - { - for (int i = 0; i < lowLevelSt.length; i++) { - lowLevelSt[i] = new LinkedHashMap>(); - } - for (int i = 0; i < lowLevelEnd.length; i++) { - lowLevelEnd[i] = new LinkedHashMap>(); - } - } // MEMORY address : choose what to use ? private boolean loadInMemory = true; @@ -476,7 +467,11 @@ public class IndexCreator { log.error("Illegal configuration", e); throw new IllegalStateException(e); } - return DriverManager.getConnection("jdbc:sqlite:" + fileName); + Connection connection = DriverManager.getConnection("jdbc:sqlite:" + fileName); + Statement statement = connection.createStatement(); + statement.executeUpdate("PRAGMA synchronous = 0"); + statement.close(); + return connection; } else if (usingDerby()) { try { Class.forName("org.apache.derby.jdbc.EmbeddedDriver"); @@ -1626,7 +1621,7 @@ public class IndexCreator { } - protected Way simplifyWay(Way originalE, long id, boolean hasMulti, int zoom, String name, int type, int level) { + protected Way simplifyWay(Way originalE, long id, boolean hasMulti, int zoom, String name, int type, int level) throws SQLException { List nodes = originalE.getNodes(); Way way = new Way(id); for (String t : originalE.getTagKeySet()) { @@ -1635,8 +1630,8 @@ public class IndexCreator { boolean cycle = originalE.getNodeIds().get(0).longValue() == originalE.getNodeIds().get(nodes.size() - 1).longValue(); long longType = encodeTypesToOneLong(type); - boolean skip = checkForSmallAreas(nodes, zoom, 3); - if (skip && (cycle || !hasMulti)) { + boolean skip = checkForSmallAreas(nodes, zoom, 3, 3); + if (skip && cycle/* || !hasMulti)*/) { return null; } @@ -1648,98 +1643,137 @@ public class IndexCreator { // nothing to do return way; } else { - way.putTag(OSMTagKey.NAME.getValue(), name); - way.putTag("long_type", longType + ""); - long stId = way.getNodes().get(0).getId(); - long endId = way.getNodes().get(way.getNodes().size() - 1).getId(); - if (!lowLevelSt[level].containsKey(stId)) { - lowLevelSt[level].put(stId, new ArrayList(2)); - } - lowLevelSt[level].get(stId).add(way); - if (!lowLevelEnd[level].containsKey(endId)) { - lowLevelEnd[level].put(endId, new ArrayList(2)); - } - lowLevelEnd[level].get(endId).add(way); + lowLevelWays ++; + DataIndexWriter.insertLowLevelMapBinaryObject(pStatements, mapLowLevelBinaryStat, level, longType, id, way.getNodes(), name); return null; } } - private void processingLowLevelWays() throws SQLException { - for (int level = 1; level < MAP_ZOOMS.length - 1; level++) { - Set visitedWays = new LinkedHashSet(); - int zoom = MAP_ZOOMS[MAP_ZOOMS.length - level - 1]; - for(Long stId : lowLevelSt[level].keySet()){ - for (Way w : lowLevelSt[level].get(stId)) { - if (visitedWays.contains(w.getId())) { - continue; - } - visitedWays.add(w.getId()); - // combine startPoint with EndPoint - boolean combined = true; - while (combined) { - combined = false; - List list = lowLevelEnd[level].get(w.getNodeIds().get(0)); - if (list != null) { - for (Way ws : list) { - if (!visitedWays.contains(ws.getId()) - && Algoritms.objectEquals(ws.getTag(OSMTagKey.NAME), w.getTag(OSMTagKey.NAME)) - && Algoritms.objectEquals(ws.getTag("long_type"), w.getTag("long_type"))) { - visitedWays.add(ws.getId()); - for (int i = 1; i < w.getNodes().size(); i++) { - ws.addNode(w.getNodes().get(i)); - } - w = ws; - combined = true; - break; - } - } - } - } - - - // combined end point - combined = true; - while (combined) { - combined = false; - List list = lowLevelSt[level].get(w.getNodeIds().get(w.getNodeIds().size() - 1)); - if (list != null) { - for (Way ws : list) { - if (!visitedWays.contains(ws.getId()) - && Algoritms.objectEquals(ws.getTag(OSMTagKey.NAME), w.getTag(OSMTagKey.NAME)) - && Algoritms.objectEquals(ws.getTag("long_type"), w.getTag("long_type"))) { - visitedWays.add(ws.getId()); - for (int i = 1; i < ws.getNodes().size(); i++) { - w.addNode(ws.getNodes().get(i)); - } - combined = true; - break; - } - } - } - } - boolean skip = false; - boolean cycle = w.getNodes().get(0).getId() == w.getNodes().get(w.getNodes().size() - 1).getId(); - boolean hasMulti = multiPolygonsWays[level].containsKey(w.getId() >> 3); - if(cycle || !hasMulti){ - skip = checkForSmallAreas(w.getNodes(), zoom - 1, 4); - } - - if (!skip) { - Way newWs = new Way(w.getId()); - simplifyDouglasPeucker(w.getNodes(), zoom - 1 + 8, 3, newWs); - - Long l = Long.parseLong(w.getTag("long_type")); - int type = decodeTypesFromOneLong(l); - DataIndexWriter.insertBinaryMapRenderObjectIndex(pStatements, mapBinaryStat, mapTree[level], newWs, w - .getTag(OSMTagKey.NAME), w.getId(), type, typeUse, 0, restrictionsUse, false, false, BATCH_SIZE); - } - } - } + private void loadNodes(byte[] nodes, List toPut){ + toPut.clear(); + for (int i = 0; i < nodes.length;) { + int lat = Algoritms.parseIntFromBytes(nodes, i); + i += 4; + int lon = Algoritms.parseIntFromBytes(nodes, i); + i += 4; + toPut.add(Float.intBitsToFloat(lat)); + toPut.add(Float.intBitsToFloat(lon)); } } - private boolean checkForSmallAreas(List nodes, int zoom, int sz) { + private void processingLowLevelWays(IProgress progress) throws SQLException { + restrictionsUse.clear(); + mapLowLevelBinaryStat.executeBatch(); + pStatements.put(mapLowLevelBinaryStat, 0); + mapConnection.commit(); + + PreparedStatement startStat = mapConnection.prepareStatement("SELECT id, start_node, end_node, nodes FROM " + + IndexConstants.LOW_LEVEL_MAP_TABLE + " WHERE start_node = ? AND type=? AND level = ? AND name=?"); + PreparedStatement endStat = mapConnection.prepareStatement("SELECT id, start_node, end_node, nodes FROM " + + IndexConstants.LOW_LEVEL_MAP_TABLE + " WHERE end_node = ? AND type=? AND level = ? AND name=?"); + Statement selectStatement = mapConnection.createStatement(); + ResultSet rs = selectStatement.executeQuery("SELECT id, start_node, end_node, name, nodes, type, level FROM " + IndexConstants.LOW_LEVEL_MAP_TABLE); + Set visitedWays = new LinkedHashSet(); + ArrayList list = new ArrayList(100); + while(rs.next()){ + if(lowLevelWays != -1){ + progress.progress(1); + } + long id = rs.getLong(1); + if(visitedWays.contains(id)){ + continue; + } + + visitedWays.add(id); + int level = rs.getInt(7); + int zoom = MAP_ZOOMS[MAP_ZOOMS.length - level - 1]; + + long startNode = rs.getLong(2); + long endNode = rs.getLong(3); + + String name = rs.getString(4); + long ltype = rs.getLong(6); + loadNodes(rs.getBytes(5), list); + ArrayList wayNodes = new ArrayList(list); + + + // combine startPoint with EndPoint + boolean combined = true; + while (combined) { + combined = false; + endStat.setLong(1, startNode); + endStat.setLong(2, ltype); + endStat.setShort(3, (short) level); + endStat.setString(4, name); + ResultSet fs = endStat.executeQuery(); + while (fs.next()) { + if (!visitedWays.contains(fs.getLong(1))) { + combined = true; + long lid = fs.getLong(1); + startNode = fs.getLong(2); + visitedWays.add(lid); + loadNodes(fs.getBytes(4), list); + ArrayList li = new ArrayList(list); + // remove first lat/lon point + wayNodes.remove(0); + wayNodes.remove(0); + li.addAll(wayNodes); + wayNodes = li; + } + } + fs.close(); + } + + // combined end point + combined = true; + while (combined) { + combined = false; + startStat.setLong(1, endNode); + startStat.setLong(2, ltype); + startStat.setShort(3, (short) level); + startStat.setString(4, name); + ResultSet fs = startStat.executeQuery(); + while (fs.next()) { + if (!visitedWays.contains(fs.getLong(1))) { + combined = true; + long lid = fs.getLong(1); + endNode = fs.getLong(3); + visitedWays.add(lid); + loadNodes(fs.getBytes(4), list); + for (int i = 2; i < list.size(); i++) { + wayNodes.add(list.get(i)); + } + } + } + fs.close(); + } + List wNodes = new ArrayList(); + for (int i = 0; i < wayNodes.size(); i += 2) { + wNodes.add(new Node(wayNodes.get(i), wayNodes.get(i + 1), i == 0 ? startNode : endNode)); + } + boolean skip = false; + boolean cycle = startNode == endNode; + boolean hasMulti = multiPolygonsWays[level].containsKey(id >> 3); + if(cycle || !hasMulti){ + skip = checkForSmallAreas(wNodes, zoom - 1, 1, 4); + } + + if (!skip) { + Way newWs = new Way(id); + simplifyDouglasPeucker(wNodes, zoom - 1 + 8, 3, newWs); + + int type = decodeTypesFromOneLong(ltype); + DataIndexWriter.insertBinaryMapRenderObjectIndex(pStatements, mapBinaryStat, mapTree[level], newWs, name, + id, type, typeUse, 0, restrictionsUse, false, false, BATCH_SIZE); + } + + } + + } + + + private boolean checkForSmallAreas(List nodes, int zoom, int minz, int maxz) { int minX = Integer.MAX_VALUE; int maxX = Integer.MIN_VALUE; int minY = Integer.MAX_VALUE; @@ -1759,8 +1793,8 @@ public class IndexCreator { if (c < 2) { return true; } - return ((maxX - minX) <= (sz - 1) && (maxY - minY) <= (sz + 1)) || - ((maxX - minX) <= (sz + 1) && (maxY - minY) <= (sz - 1)); + return ((maxX - minX) <= minz && (maxY - minY) <= maxz) || + ((maxX - minX) <= maxz && (maxY - minY) <= minz); } @@ -2203,6 +2237,7 @@ public class IndexCreator { cities.clear(); cityManager.clear(); + lowLevelWays = -1; postalCodeRelations.clear(); if (normalizeStreets) { normalizeDefaultSuffixes = DataExtractionSettings.getSettings().getDefaultSuffixesToNormalizeStreets(); @@ -2331,8 +2366,8 @@ public class IndexCreator { // 3.4 combine all low level ways and simplify them if(indexMap){ progress.setGeneralProgress("[90 of 100]"); - progress.startTask("Indexing low level ways...", -1); - processingLowLevelWays(); + progress.startTask("Indexing low level ways...", lowLevelWays); + processingLowLevelWays(progress); } // 3.5 update all postal codes from relations @@ -2373,7 +2408,7 @@ public class IndexCreator { if (indexMap) { progress.setGeneralProgress("[95 of 100]"); progress.startTask("Writing map index to binary file...", -1); - closePreparedStatements(mapBinaryStat); + closePreparedStatements(mapBinaryStat, mapLowLevelBinaryStat); mapConnection.commit(); writeBinaryMapIndex(writer); } @@ -2615,6 +2650,7 @@ public class IndexCreator { if (indexMap) { DataIndexWriter.createMapIndexStructure(mapConnection); mapBinaryStat = DataIndexWriter.createStatementMapBinaryInsert(mapConnection); + mapLowLevelBinaryStat = DataIndexWriter.createStatementLowLevelMapBinaryInsert(mapConnection); try { mapTree = new RTree[MAP_ZOOMS.length - 1]; for (int i = 0; i < MAP_ZOOMS.length - 1; i++) { @@ -2630,6 +2666,7 @@ public class IndexCreator { throw new IOException(e); } pStatements.put(mapBinaryStat, 0); + pStatements.put(mapLowLevelBinaryStat, 0); } if (indexAddress) { @@ -2728,12 +2765,12 @@ public class IndexCreator { creator.recreateOnlyBinaryFile = false; creator.deleteDatabaseIndexes = true; -// creator.setNodesDBFile(new File("e:/Information/OSM maps/osmand/minsk.tmp.odb")); -// creator.generateIndexes(new File("e:/Information/OSM maps/belarus osm/minsk.osm"), new ConsoleProgressImplementation(3), null); + creator.setNodesDBFile(new File("e:/Information/OSM maps/osmand/minsk.tmp.odb")); + creator.generateIndexes(new File("e:/Information/OSM maps/belarus osm/minsk.osm"), new ConsoleProgressImplementation(3), null); - creator.setNodesDBFile(new File("e:/Information/OSM maps/osmand/belarus_nodes.tmp.odb")); - creator.generateIndexes(new File("e:/Information/OSM maps/belarus osm/belarus.osm.pbf"), new ConsoleProgressImplementation(3), null); +// creator.setNodesDBFile(new File("e:/Information/OSM maps/osmand/belarus_nodes.tmp.odb")); +// creator.generateIndexes(new File("e:/Information/OSM maps/belarus osm/belarus.osm.pbf"), new ConsoleProgressImplementation(3), null); // creator.setNodesDBFile(new File("e:/Information/OSM maps/osmand/ams.tmp.odb")); // creator.generateIndexes(new File("e:/Information/OSM maps/osm_map/ams_part_map.osm"), new ConsoleProgressImplementation(3), null); diff --git a/DataExtractionOSM/src/net/osmand/impl/ConsoleProgressImplementation.java b/DataExtractionOSM/src/net/osmand/impl/ConsoleProgressImplementation.java index 0d728b86d9..cfc1d8e1f8 100644 --- a/DataExtractionOSM/src/net/osmand/impl/ConsoleProgressImplementation.java +++ b/DataExtractionOSM/src/net/osmand/impl/ConsoleProgressImplementation.java @@ -11,13 +11,16 @@ import org.apache.commons.logging.Log; public class ConsoleProgressImplementation implements IProgress { public static double deltaPercentsToPrint = 3.5; + public static long deltaTimeToPrint = 1000; private static Log log = LogUtil.getLog(ConsoleProgressImplementation.class); String currentTask; int work; int currentDone; double delta; + long deltaTime = deltaTimeToPrint; private long previousTaskStarted = 0; + private long lastTimePrinted = 0; double lastPercentPrint = 0; public ConsoleProgressImplementation(){ @@ -28,6 +31,11 @@ public class ConsoleProgressImplementation implements IProgress { delta = deltaToPrint; } + public ConsoleProgressImplementation(double deltaToPrint, int deltaTime){ + delta = deltaToPrint; + deltaToPrint = deltaTime; + } + @Override public void finishTask() { log.info("Task " + currentTask + " is finished "); //$NON-NLS-1$ //$NON-NLS-2$ @@ -48,8 +56,13 @@ public class ConsoleProgressImplementation implements IProgress { private void printIfNeeded() { if(getCurrentPercent() - lastPercentPrint >= delta){ - System.out.println(MessageFormat.format("Done {0} %.", getCurrentPercent())); //$NON-NLS-1$ this.lastPercentPrint = getCurrentPercent(); + long now = System.currentTimeMillis(); + if(now - lastTimePrinted >= deltaTimeToPrint || deltaTime < 0){ + System.out.println(MessageFormat.format("Done {0} %.", getCurrentPercent())); //$NON-NLS-1$ + lastTimePrinted = now; + } + } }