From a45a479be21a697d94a9da5db6393276786303e5 Mon Sep 17 00:00:00 2001 From: Victor Shcherb Date: Sun, 1 May 2011 21:59:13 +0200 Subject: [PATCH] Refactoring IndexCreator --- .../osmand/data/index/DataIndexWriter.java | 99 +-- .../osmand/data/preparation/DBDialect.java | 48 ++ .../osmand/data/preparation/IndexCreator.java | 600 ++---------------- .../preparation/IndexTransportCreator.java | 502 +++++++++++++++ .../osmand/data/preparation/OsmDbCreator.java | 180 ++++++ OsmAnd/.classpath | 22 +- 6 files changed, 786 insertions(+), 665 deletions(-) create mode 100644 DataExtractionOSM/src/net/osmand/data/preparation/DBDialect.java create mode 100644 DataExtractionOSM/src/net/osmand/data/preparation/IndexTransportCreator.java create mode 100644 DataExtractionOSM/src/net/osmand/data/preparation/OsmDbCreator.java diff --git a/DataExtractionOSM/src/net/osmand/data/index/DataIndexWriter.java b/DataExtractionOSM/src/net/osmand/data/index/DataIndexWriter.java index 8b677850be..6631198aa9 100644 --- a/DataExtractionOSM/src/net/osmand/data/index/DataIndexWriter.java +++ b/DataExtractionOSM/src/net/osmand/data/index/DataIndexWriter.java @@ -11,17 +11,14 @@ import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Set; import net.osmand.Algoritms; import net.osmand.data.Amenity; import net.osmand.data.AmenityType; import net.osmand.data.Building; import net.osmand.data.City; -import net.osmand.data.TransportRoute; -import net.osmand.data.TransportStop; import net.osmand.data.City.CityType; -import net.osmand.data.preparation.IndexCreator; +import net.osmand.data.preparation.DBDialect; import net.osmand.osm.Entity; import net.osmand.osm.LatLon; import net.osmand.osm.MapUtils; @@ -62,7 +59,7 @@ public class DataIndexWriter { "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); } - public static void createPoiIndexStructure(Connection conn) throws SQLException{ + public static void createPoiIndexStructure(Connection conn, DBDialect dialect) throws SQLException{ Statement stat = conn.createStatement(); stat.executeUpdate("create table " + IndexConstants.POI_TABLE + //$NON-NLS-1$ "(id bigint, x int, y int, name_en varchar(255), name varchar(255), " + @@ -70,7 +67,7 @@ public class DataIndexWriter { "primary key(id, type, subtype))"); stat.executeUpdate("create index poi_loc on poi (x, y, type, subtype)"); stat.executeUpdate("create index poi_id on poi (id, type, subtype)"); - if(IndexCreator.usingSQLite()){ + if(dialect == DBDialect.SQLITE){ stat.execute("PRAGMA user_version = " + IndexConstants.POI_TABLE_VERSION); //$NON-NLS-1$ } stat.close(); @@ -174,7 +171,7 @@ public class DataIndexWriter { } - public static void createAddressIndexStructure(Connection conn) throws SQLException{ + public static void createAddressIndexStructure(Connection conn, DBDialect dialect) throws SQLException{ Statement stat = conn.createStatement(); assert IndexConstants.CITY_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ assert IndexConstants.STREET_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ @@ -203,101 +200,15 @@ public class DataIndexWriter { stat.executeUpdate("create index street_node_street on street_node (street)"); stat.executeUpdate("create index street_node_way on street_node (way)"); - if(IndexCreator.usingSQLite()){ + if(dialect == DBDialect.SQLITE){ stat.execute("PRAGMA user_version = " + IndexConstants.ADDRESS_TABLE_VERSION); //$NON-NLS-1$ } stat.close(); } - public static PreparedStatement createStatementTransportStopInsert(Connection conn) throws SQLException{ - assert IndexConstants.TRANSPORT_STOP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - return conn.prepareStatement("insert into transport_stop(id, latitude, longitude, name, name_en) values(?, ?, ?, ?, ?)"); - } - public static PreparedStatement createStatementTransportRouteStopInsert(Connection conn) throws SQLException{ - assert IndexConstants.TRANSPORT_ROUTE_STOP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - return conn.prepareStatement("insert into transport_route_stop(route, stop, direction, ord) values(?, ?, ?, ?)"); - } - private static void writeRouteStops(RTree transportStopsTree, PreparedStatement prepRouteStops, PreparedStatement prepStops, Map count, - Set writtenStops, TransportRoute r, List stops, boolean direction) throws SQLException { - assert IndexConstants.TRANSPORT_STOP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - assert IndexConstants.TRANSPORT_ROUTE_STOP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - int i = 0; - for(TransportStop s : stops){ - if (!writtenStops.contains(s.getId())) { - prepStops.setLong(1, s.getId()); - prepStops.setDouble(2, s.getLocation().getLatitude()); - prepStops.setDouble(3, s.getLocation().getLongitude()); - prepStops.setString(4, s.getName()); - prepStops.setString(5, s.getEnName()); - int x = (int) MapUtils.getTileNumberX(24, s.getLocation().getLongitude()); - int y = (int) MapUtils.getTileNumberY(24, s.getLocation().getLatitude()); - addBatch(count, prepStops); - try { - transportStopsTree.insert(new LeafElement(new Rect(x, y, x, y), s.getId())); - } catch (RTreeInsertException e) { - throw new IllegalArgumentException(e); - } catch (IllegalValueException e) { - throw new IllegalArgumentException(e); - } - writtenStops.add(s.getId()); - } - prepRouteStops.setLong(1, r.getId()); - prepRouteStops.setLong(2, s.getId()); - prepRouteStops.setInt(3, direction ? 1 : 0); - prepRouteStops.setInt(4, i++); - addBatch(count, prepRouteStops); - } - } - public static PreparedStatement createStatementTransportRouteInsert(Connection conn) throws SQLException{ - assert IndexConstants.TRANSPORT_ROUTE_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - return conn.prepareStatement("insert into transport_route(id, type, operator, ref, name, name_en, dist) values(?, ?, ?, ?, ?, ?, ?)"); - } - public static void insertTransportIntoIndex(PreparedStatement prepRoute, PreparedStatement prepRouteStops, - PreparedStatement prepStops, RTree transportStopsTree, - Set writtenStops, TransportRoute route, Map statements, - int batchSize) throws SQLException { - assert IndexConstants.TRANSPORT_ROUTE_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - prepRoute.setLong(1, route.getId()); - prepRoute.setString(2, route.getType()); - prepRoute.setString(3, route.getOperator()); - prepRoute.setString(4, route.getRef()); - prepRoute.setString(5, route.getName()); - prepRoute.setString(6, route.getEnName()); - prepRoute.setInt(7, route.getAvgBothDistance()); - addBatch(statements, prepRoute); - - writeRouteStops(transportStopsTree, prepRouteStops, prepStops, statements, writtenStops, route, route.getForwardStops(), true); - writeRouteStops(transportStopsTree, prepRouteStops, prepStops, statements, writtenStops, route, route.getBackwardStops(), false); - - } - - public static void createTransportIndexStructure(Connection conn) throws SQLException{ - Statement stat = conn.createStatement(); - assert IndexConstants.TRANSPORT_ROUTE_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - assert IndexConstants.TRANSPORT_STOP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - assert IndexConstants.TRANSPORT_ROUTE_STOP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - - stat.executeUpdate("create table transport_route (id bigint primary key, type varchar(255), operator varchar(255)," + - "ref varchar(255), name varchar(255), name_en varchar(255), dist int)"); - stat.executeUpdate("create index transport_route_id on transport_route (id)"); - - stat.executeUpdate("create table transport_route_stop (stop bigint, route bigint, ord int, direction smallint, primary key (route, ord, direction))"); - stat.executeUpdate("create index transport_route_stop_stop on transport_route_stop (stop)"); - stat.executeUpdate("create index transport_route_stop_route on transport_route_stop (route)"); - - stat.executeUpdate("create table transport_stop (id bigint primary key, latitude double, longitude double, name varchar(255), name_en varchar(255))"); - stat.executeUpdate("create index transport_stop_id on transport_stop (id)"); - stat.executeUpdate("create index transport_stop_location on transport_stop (latitude, longitude)"); - - if(IndexCreator.usingSQLite()){ - stat.execute("PRAGMA user_version = " + IndexConstants.TRANSPORT_TABLE_VERSION); //$NON-NLS-1$ - } - stat.close(); - } - public static void createMapIndexStructure(Connection conn) throws SQLException{ Statement stat = conn.createStatement(); diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/DBDialect.java b/DataExtractionOSM/src/net/osmand/data/preparation/DBDialect.java new file mode 100644 index 0000000000..eb16911330 --- /dev/null +++ b/DataExtractionOSM/src/net/osmand/data/preparation/DBDialect.java @@ -0,0 +1,48 @@ +package net.osmand.data.preparation; + +import java.io.File; +import java.sql.SQLException; +import java.sql.Statement; + +import net.osmand.Algoritms; + +public enum DBDialect { + DERBY, + H2, + SQLITE; + + public void deleteTableIfExists(String table, Statement stat) throws SQLException { + if(this == DERBY){ + try { + stat.executeUpdate("drop table " + table); //$NON-NLS-1$ + } catch (SQLException e) { + // ignore it + } + } else { + stat.executeUpdate("drop table if exists " + table); //$NON-NLS-1$ + } + + } + + public boolean databaseFileExists(File dbFile) { + if (DBDialect.H2 == this) { + return new File(dbFile.getAbsolutePath() + ".h2.db").exists(); //$NON-NLS-1$ + } else { + return dbFile.exists(); + } + } + + public void removeDatabase(File file) { + if (DBDialect.H2 == this) { + File[] list = file.getParentFile().listFiles(); + for (File f : list) { + if (f.getName().startsWith(file.getName())) { + Algoritms.removeAllFiles(f); + } + } + } else { + Algoritms.removeAllFiles(file); + } + + } +} diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java b/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java index a028f7d6fc..a28173d956 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java @@ -28,7 +28,6 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; -import java.util.Map.Entry; import net.osmand.Algoritms; import net.osmand.IProgress; @@ -40,7 +39,6 @@ import net.osmand.data.City; import net.osmand.data.DataTileManager; import net.osmand.data.Street; import net.osmand.data.TransportRoute; -import net.osmand.data.TransportStop; import net.osmand.data.City.CityType; import net.osmand.data.index.DataIndexReader; import net.osmand.data.index.DataIndexWriter; @@ -89,10 +87,7 @@ public class IndexCreator { private static final Log log = LogFactory.getLog(IndexCreator.class); // ONLY derby.jar needed for derby dialect - private static final String DERBY_DIALECT = "DERBY"; - private static final String H2_DIALECT = "H2"; - private static final String SQLITE_DIALECT = "SQLITE"; - private static final String CURRENT_DB = SQLITE_DIALECT; + private final DBDialect dialect = DBDialect.SQLITE; public static final int BATCH_SIZE = 5000; public static final int BATCH_SIZE_OSM = 10000; @@ -123,6 +118,8 @@ public class IndexCreator { private PreparedStatement pselectRelation; private PreparedStatement pselectTags; + + private IndexTransportCreator indexTransportCreator; // constants to start process from the middle and save temporary results private boolean recreateOnlyBinaryFile = false; // false; private boolean deleteOsmDB = false; @@ -150,12 +147,6 @@ public class IndexCreator { private PreparedStatement addressBuildingStat; private PreparedStatement addressStreetNodeStat; - private Set visitedStops = new HashSet(); - private PreparedStatement transRouteStat; - private PreparedStatement transRouteStopsStat; - private PreparedStatement transStopsStat; - private RTree transportStopsTree; - private RTree[] mapTree = null; private MapZooms mapZooms = MapZooms.getDefault(); private MapRenderingTypes renderingTypes = MapRenderingTypes.getDefault(); @@ -193,20 +184,11 @@ public class IndexCreator { List typeUse = new ArrayList(8); List restrictionsUse = new ArrayList(8); - public static boolean usingSQLite() { - return CURRENT_DB.equals(SQLITE_DIALECT); - } - public static boolean usingDerby() { - return CURRENT_DB.equals(DERBY_DIALECT); - } - - public static boolean usingH2() { - return CURRENT_DB.equals(H2_DIALECT); - } public IndexCreator(File workingDir) { this.workingDir = workingDir; + this.indexTransportCreator = new IndexTransportCreator(this); } public void setIndexAddress(boolean indexAddress) { @@ -246,184 +228,6 @@ public class IndexCreator { return level; } */ - protected class NewDataExtractionOsmFilter implements IOsmStorageFilter { - - int currentCountNode = 0; - private PreparedStatement prepNode; - int allNodes = 0; - - int currentRelationsCount = 0; - private PreparedStatement prepRelations; - int allRelations = 0; - - int currentWaysCount = 0; - private PreparedStatement prepWays; - int allWays = 0; - - int currentTagsCount = 0; - private PreparedStatement prepTags; - - public void initDatabase() throws SQLException { - // prepare tables - Statement stat = dbConn.createStatement(); - if (usingDerby()) { - try { - stat.executeUpdate("drop table node"); //$NON-NLS-1$ - } catch (SQLException e) { - // ignore it - } - } else { - stat.executeUpdate("drop table if exists node"); //$NON-NLS-1$ - } - stat.executeUpdate("create table node (id bigint primary key, latitude double, longitude double)"); //$NON-NLS-1$ - stat.executeUpdate("create index IdIndex ON node (id)"); //$NON-NLS-1$ - if (usingDerby()) { - try { - stat.executeUpdate("drop table ways"); //$NON-NLS-1$ - } catch (SQLException e) { - // ignore it - } - } else { - stat.executeUpdate("drop table if exists ways"); //$NON-NLS-1$ - } - stat.executeUpdate("create table ways (id bigint, node bigint, ord smallint, primary key (id, ord))"); //$NON-NLS-1$ - stat.executeUpdate("create index IdWIndex ON ways (id)"); //$NON-NLS-1$ - if (usingDerby()) { - try { - stat.executeUpdate("drop table relations"); //$NON-NLS-1$ - } catch (SQLException e) { - // ignore it - } - } else { - stat.executeUpdate("drop table if exists relations"); //$NON-NLS-1$ - } - stat - .executeUpdate("create table relations (id bigint, member bigint, type smallint, role varchar(255), ord smallint, primary key (id, ord))"); //$NON-NLS-1$ - stat.executeUpdate("create index IdRIndex ON relations (id)"); //$NON-NLS-1$ - if (usingDerby()) { - try { - stat.executeUpdate("drop table tags"); //$NON-NLS-1$ - } catch (SQLException e) { - // ignore it - } - } else { - stat.executeUpdate("drop table if exists tags"); //$NON-NLS-1$ - } - stat.executeUpdate("create table tags (id bigint, type smallint, skeys varchar(255), value varchar(255), primary key (id, type, skeys))"); //$NON-NLS-1$ - stat.executeUpdate("create index IdTIndex ON tags (id, type)"); //$NON-NLS-1$ - stat.close(); - - prepNode = dbConn.prepareStatement("insert into node values (?, ?, ?)"); //$NON-NLS-1$ - prepWays = dbConn.prepareStatement("insert into ways values (?, ?, ?)"); //$NON-NLS-1$ - prepRelations = dbConn.prepareStatement("insert into relations values (?, ?, ?, ?, ?)"); //$NON-NLS-1$ - prepTags = dbConn.prepareStatement("insert into tags values (?, ?, ?, ?)"); //$NON-NLS-1$ - dbConn.setAutoCommit(false); - } - - public void finishLoading() throws SQLException { - if (currentCountNode > 0) { - prepNode.executeBatch(); - } - prepNode.close(); - if (currentWaysCount > 0) { - prepWays.executeBatch(); - } - prepWays.close(); - if (currentRelationsCount > 0) { - prepRelations.executeBatch(); - } - prepRelations.close(); - if (currentTagsCount > 0) { - prepTags.executeBatch(); - } - prepTags.close(); - } - - @Override - public boolean acceptEntityToLoad(OsmBaseStorage storage, EntityId entityId, Entity e) { - // Register all city labelbs - registerCityIfNeeded(e); - // put all nodes into temporary db to get only required nodes after loading all data - try { - if (e instanceof Node) { - currentCountNode++; - if (!e.getTags().isEmpty()) { - allNodes++; - } - prepNode.setLong(1, e.getId()); - prepNode.setDouble(2, ((Node) e).getLatitude()); - prepNode.setDouble(3, ((Node) e).getLongitude()); - prepNode.addBatch(); - if (currentCountNode >= BATCH_SIZE_OSM) { - prepNode.executeBatch(); - dbConn.commit(); // clear memory - currentCountNode = 0; - } - } else if (e instanceof Way) { - allWays++; - short ord = 0; - for (Long i : ((Way) e).getNodeIds()) { - currentWaysCount++; - prepWays.setLong(1, e.getId()); - prepWays.setLong(2, i); - prepWays.setLong(3, ord++); - prepWays.addBatch(); - } - if (currentWaysCount >= BATCH_SIZE_OSM) { - prepWays.executeBatch(); - dbConn.commit(); // clear memory - currentWaysCount = 0; - } - } else { - allRelations++; - short ord = 0; - for (Entry i : ((Relation) e).getMembersMap().entrySet()) { - currentRelationsCount++; - prepRelations.setLong(1, e.getId()); - prepRelations.setLong(2, i.getKey().getId()); - prepRelations.setLong(3, i.getKey().getType().ordinal()); - prepRelations.setString(4, i.getValue()); - prepRelations.setLong(5, ord++); - prepRelations.addBatch(); - } - if (currentRelationsCount >= BATCH_SIZE_OSM) { - prepRelations.executeBatch(); - dbConn.commit(); // clear memory - currentRelationsCount = 0; - } - } - for (Entry i : e.getTags().entrySet()) { - currentTagsCount++; - prepTags.setLong(1, e.getId()); - prepTags.setLong(2, EntityType.valueOf(e).ordinal()); - prepTags.setString(3, i.getKey()); - prepTags.setString(4, i.getValue()); - prepTags.addBatch(); - } - if (currentTagsCount >= BATCH_SIZE_OSM) { - prepTags.executeBatch(); - dbConn.commit(); // clear memory - currentTagsCount = 0; - } - } catch (SQLException ex) { - log.error("Could not save in db", ex); //$NON-NLS-1$ - } - // do not add to storage - return false; - } - - public int getAllNodes() { - return allNodes; - } - - public int getAllRelations() { - return allRelations; - } - - public int getAllWays() { - return allWays; - } - } public String getRegionName() { if (regionName == null) { @@ -440,30 +244,9 @@ public class IndexCreator { return getDatabaseConnection(fileName, false); } - public static void removeDatabase(File file) throws SQLException { - if (usingH2()) { - File[] list = file.getParentFile().listFiles(); - for (File f : list) { - if (f.getName().startsWith(file.getName())) { - Algoritms.removeAllFiles(f); - } - } - } else { - Algoritms.removeAllFiles(file); - } - - } - - public static boolean databaseFileExists(File dbFile) { - if (usingH2()) { - return new File(dbFile.getAbsolutePath() + ".h2.db").exists(); //$NON-NLS-1$ - } else { - return dbFile.exists(); - } - } private Connection getDatabaseConnection(String fileName, boolean forceSqLite) throws SQLException { - if (usingSQLite() || forceSqLite) { + if (DBDialect.SQLITE == dialect || forceSqLite) { try { Class.forName("org.sqlite.JDBC"); } catch (ClassNotFoundException e) { @@ -475,7 +258,7 @@ public class IndexCreator { statement.executeUpdate("PRAGMA synchronous = 0"); statement.close(); return connection; - } else if (usingDerby()) { + } else if (DBDialect.DERBY == dialect) { try { Class.forName("org.apache.derby.jdbc.EmbeddedDriver"); } catch (ClassNotFoundException e) { @@ -485,7 +268,7 @@ public class IndexCreator { Connection conn = DriverManager.getConnection("jdbc:derby:" + fileName + ";create=true"); conn.setAutoCommit(false); return conn; - } else if (usingH2()) { + } else if (DBDialect.H2 == dialect) { try { Class.forName("org.h2.Driver"); } catch (ClassNotFoundException e) { @@ -723,116 +506,7 @@ public class IndexCreator { rsTags.close(); } - private static Set acceptedRoutes = new HashSet(); - static { - acceptedRoutes.add("bus"); //$NON-NLS-1$ - acceptedRoutes.add("trolleybus"); //$NON-NLS-1$ - acceptedRoutes.add("share_taxi"); //$NON-NLS-1$ - - acceptedRoutes.add("subway"); //$NON-NLS-1$ - acceptedRoutes.add("train"); //$NON-NLS-1$ - - acceptedRoutes.add("tram"); //$NON-NLS-1$ - - acceptedRoutes.add("ferry"); //$NON-NLS-1$ - } - - private TransportRoute indexTransportRoute(Relation rel) { - String ref = rel.getTag(OSMTagKey.REF); - String route = rel.getTag(OSMTagKey.ROUTE); - String operator = rel.getTag(OSMTagKey.OPERATOR); - if (route == null || ref == null) { - return null; - } - if (!acceptedRoutes.contains(route)) { - return null; - } - TransportRoute r = new TransportRoute(rel, ref); - r.setOperator(operator); - r.setType(route); - - if (operator != null) { - route = operator + " : " + route; //$NON-NLS-1$ - } - - final Map forwardStops = new LinkedHashMap(); - final Map backwardStops = new LinkedHashMap(); - int currentStop = 0; - int forwardStop = 0; - int backwardStop = 0; - for (Entry e : rel.getMemberEntities().entrySet()) { - if (e.getValue().contains("stop")) { //$NON-NLS-1$ - if (e.getKey() instanceof Node) { - TransportStop stop = new TransportStop(e.getKey()); - boolean forward = e.getValue().contains("forward"); //$NON-NLS-1$ - boolean backward = e.getValue().contains("backward"); //$NON-NLS-1$ - currentStop++; - if (forward || !backward) { - forwardStop++; - } - if (backward) { - backwardStop++; - } - boolean common = !forward && !backward; - int index = -1; - int i = e.getValue().length() - 1; - int accum = 1; - while (i >= 0 && Character.isDigit(e.getValue().charAt(i))) { - if (index < 0) { - index = 0; - } - index = accum * Character.getNumericValue(e.getValue().charAt(i)) + index; - accum *= 10; - i--; - } - if (index < 0) { - index = forward ? forwardStop : (backward ? backwardStop : currentStop); - } - if (forward || common) { - forwardStops.put(stop, index); - r.getForwardStops().add(stop); - } - if (backward || common) { - if (common) { - // put with negative index - backwardStops.put(stop, -index); - } else { - backwardStops.put(stop, index); - } - - r.getBackwardStops().add(stop); - } - - } - - } else if (e.getKey() instanceof Way) { - r.addWay((Way) e.getKey()); - } - } - if (forwardStops.isEmpty() && backwardStops.isEmpty()) { - return null; - } - Collections.sort(r.getForwardStops(), new Comparator() { - @Override - public int compare(TransportStop o1, TransportStop o2) { - return forwardStops.get(o1) - forwardStops.get(o2); - } - }); - // all common stops are with negative index (reeval them) - for (TransportStop s : new ArrayList(backwardStops.keySet())) { - if (backwardStops.get(s) < 0) { - backwardStops.put(s, backwardStops.size() + backwardStops.get(s) - 1); - } - } - Collections.sort(r.getBackwardStops(), new Comparator() { - @Override - public int compare(TransportStop o1, TransportStop o2) { - return backwardStops.get(o1) - backwardStops.get(o2); - } - }); - - return r; - } + public String getCityAdminLevel() { return cityAdminLevel; @@ -1257,10 +931,9 @@ public class IndexCreator { if (indexTransport) { if (e instanceof Relation && e.getTag(OSMTagKey.ROUTE) != null) { loadEntityData(e, true); - TransportRoute route = indexTransportRoute((Relation) e); + TransportRoute route = indexTransportCreator.indexTransportRoute((Relation) e); if (route != null) { - DataIndexWriter.insertTransportIntoIndex(transRouteStat, transRouteStopsStat, transStopsStat, transportStopsTree, - visitedStops, route, pStatements, BATCH_SIZE); + indexTransportCreator.insertTransportIntoIndex(route, pStatements); } } } @@ -1641,7 +1314,7 @@ public class IndexCreator { return ang; } - private void registerCityIfNeeded(Entity e) { + public void registerCityIfNeeded(Entity e) { if (e instanceof Node && e.getTag(OSMTagKey.PLACE) != null) { City city = new City((Node) e); if (city.getType() != null && !Algoritms.isEmpty(city.getName())) { @@ -2211,170 +1884,6 @@ public class IndexCreator { } } - private int registerString(Map stringTable, String s) { - if (stringTable.containsKey(s)) { - return stringTable.get(s); - } - int size = stringTable.size(); - stringTable.put(s, size); - return size; - } - - private Map createStringTableForTransport() { - Map stringTable = new LinkedHashMap(); - registerString(stringTable, "bus"); //$NON-NLS-1$ - registerString(stringTable, "trolleybus"); //$NON-NLS-1$ - registerString(stringTable, "subway"); //$NON-NLS-1$ - registerString(stringTable, "tram"); //$NON-NLS-1$ - registerString(stringTable, "share_taxi"); //$NON-NLS-1$ - registerString(stringTable, "taxi"); //$NON-NLS-1$ - registerString(stringTable, "train"); //$NON-NLS-1$ - registerString(stringTable, "ferry"); //$NON-NLS-1$ - return stringTable; - } - - public void writeBinaryTransportIndex(BinaryMapIndexWriter writer) throws IOException, SQLException { - try { - transportStopsTree.flush(); - visitedStops = null; // allow gc to collect it - PreparedStatement selectTransportRouteData = mapConnection.prepareStatement( - "SELECT id, dist, name, name_en, ref, operator, type FROM transport_route"); //$NON-NLS-1$ - PreparedStatement selectTransportData = mapConnection.prepareStatement("SELECT S.stop, S.direction," + //$NON-NLS-1$ - " A.latitude, A.longitude, A.name, A.name_en " + //$NON-NLS-1$ - "FROM transport_route_stop S INNER JOIN transport_stop A ON A.id = S.stop WHERE S.route = ? ORDER BY S.ord asc"); //$NON-NLS-1$ - - writer.startWriteTransportIndex(regionName); - - writer.startWriteTransportRoutes(); - - // expect that memory would be enough - Map stringTable = createStringTableForTransport(); - Map transportRoutes = new LinkedHashMap(); - - ResultSet rs = selectTransportRouteData.executeQuery(); - List directStops = new ArrayList(); - List reverseStops = new ArrayList(); - while (rs.next()) { - - long idRoute = rs.getLong(1); - int dist = rs.getInt(2); - String routeName = rs.getString(3); - String routeEnName = rs.getString(4); - if (routeEnName != null && routeEnName.equals(Junidecode.unidecode(routeName))) { - routeEnName = null; - } - String ref = rs.getString(5); - String operator = rs.getString(6); - String type = rs.getString(7); - - selectTransportData.setLong(1, idRoute); - ResultSet rset = selectTransportData.executeQuery(); - reverseStops.clear(); - directStops.clear(); - while (rset.next()) { - boolean dir = rset.getInt(2) != 0; - long idStop = rset.getInt(1); - String stopName = rset.getString(5); - String stopEnName = rset.getString(6); - if (stopEnName != null && stopEnName.equals(Junidecode.unidecode(stopName))) { - stopEnName = null; - } - TransportStop st = new TransportStop(); - st.setId(idStop); - st.setName(stopName); - st.setLocation(rset.getDouble(3), rset.getDouble(4)); - if (stopEnName != null) { - st.setEnName(stopEnName); - } - if (dir) { - directStops.add(st); - } else { - reverseStops.add(st); - } - } - writer.writeTransportRoute(idRoute, routeName, routeEnName, ref, operator, type, dist, directStops, reverseStops, - stringTable, transportRoutes); - } - rs.close(); - selectTransportRouteData.close(); - selectTransportData.close(); - writer.endWriteTransportRoutes(); - - PreparedStatement selectTransportStop = mapConnection.prepareStatement( - "SELECT A.id, A.latitude, A.longitude, A.name, A.name_en FROM transport_stop A where A.id = ?"); //$NON-NLS-1$ - PreparedStatement selectTransportRouteStop = mapConnection.prepareStatement( - "SELECT DISTINCT S.route FROM transport_route_stop S WHERE S.stop = ? "); //$NON-NLS-1$ - long rootIndex = transportStopsTree.getFileHdr().getRootIndex(); - rtree.Node root = transportStopsTree.getReadNode(rootIndex); - Rect rootBounds = calcBounds(root); - if (rootBounds != null) { - writer.startTransportTreeElement(rootBounds.getMinX(), rootBounds.getMaxX(), rootBounds.getMinY(), rootBounds.getMaxY()); - writeBinaryTransportTree(root, transportStopsTree, writer, selectTransportStop, selectTransportRouteStop, - transportRoutes, stringTable); - writer.endWriteTransportTreeElement(); - } - selectTransportStop.close(); - selectTransportRouteStop.close(); - - writer.writeTransportStringTable(stringTable); - - writer.endWriteTransportIndex(); - writer.flush(); - } catch (RTreeException e) { - throw new IllegalStateException(e); - } - } - - public void writeBinaryTransportTree(rtree.Node parent, RTree r, BinaryMapIndexWriter writer, - PreparedStatement selectTransportStop, PreparedStatement selectTransportRouteStop, - Map transportRoutes, Map stringTable) throws IOException, RTreeException, SQLException { - Element[] e = parent.getAllElements(); - List routes = null; - for (int i = 0; i < parent.getTotalElements(); i++) { - Rect re = e[i].getRect(); - if (e[i].getElementType() == rtree.Node.LEAF_NODE) { - long id = ((LeafElement) e[i]).getPtr(); - selectTransportStop.setLong(1, id); - selectTransportRouteStop.setLong(1, id); - ResultSet rs = selectTransportStop.executeQuery(); - if (rs.next()) { - int x24 = (int) MapUtils.getTileNumberX(24, rs.getDouble(3)); - int y24 = (int) MapUtils.getTileNumberY(24, rs.getDouble(2)); - String name = rs.getString(4); - String nameEn = rs.getString(5); - if (nameEn != null && nameEn.equals(Junidecode.unidecode(name))) { - nameEn = null; - } - ResultSet rset = selectTransportRouteStop.executeQuery(); - if (routes == null) { - routes = new ArrayList(); - } else { - routes.clear(); - } - while (rset.next()) { - Long route = transportRoutes.get(rset.getLong(1)); - if (route == null) { - log.error("Something goes wrong with transport route id = " + rset.getLong(1)); //$NON-NLS-1$ - } else { - routes.add(route); - } - } - rset.close(); - writer.writeTransportStop(id, x24, y24, name, nameEn, stringTable, routes); - } else { - log.error("Something goes wrong with transport id = " + id); //$NON-NLS-1$ - } - } else { - long ptr = ((NonLeafElement) e[i]).getPtr(); - rtree.Node ns = r.getReadNode(ptr); - - writer.startTransportTreeElement(re.getMinX(), re.getMaxX(), re.getMinY(), re.getMaxY()); - writeBinaryTransportTree(ns, r, writer, selectTransportStop, selectTransportRouteStop, transportRoutes, stringTable); - writer.endWriteTransportTreeElement(); - } - } - } - public Rect calcBounds(rtree.Node n) { Rect r = null; Element[] e = n.getAllElements(); @@ -2450,12 +1959,12 @@ public class IndexCreator { try { // //////////////////////////////////////////////////////////////////////// // 1. creating nodes db to fast access for all nodes and simply import all relations, ways, nodes to it - boolean loadFromPath = dbFile == null || !databaseFileExists(dbFile); + boolean loadFromPath = dbFile == null || !dialect.databaseFileExists(dbFile); if (dbFile == null) { dbFile = new File(workingDir, TEMP_NODES_DB); // to save space - if (databaseFileExists(dbFile)) { - removeDatabase(dbFile); + if (dialect.databaseFileExists(dbFile)) { + dialect.removeDatabase(dbFile); } } dbConn = getDatabaseConnection(dbFile.getAbsolutePath()); @@ -2467,7 +1976,7 @@ public class IndexCreator { progress.setGeneralProgress("[35 / 100]"); //$NON-NLS-1$ progress.startTask(Messages.getString("IndexCreator.LOADING_FILE") + readFile.getAbsolutePath(), -1); //$NON-NLS-1$ - NewDataExtractionOsmFilter filter = extractOsmToNodesDB(readFile, progress, addFilter); + OsmDbCreator filter = extractOsmToNodesDB(readFile, progress, addFilter); if (filter != null) { allNodes = filter.getAllNodes(); allWays = filter.getAllWays(); @@ -2499,7 +2008,7 @@ public class IndexCreator { } if (indexTransport) { - transportStopsTree = new RTree(getRTreeTransportStopsPackFileName()); + indexTransportCreator.createRTreeFile(getRTreeTransportStopsPackFileName()); } } catch (RTreeException e) { log.error("Error flushing", e); //$NON-NLS-1$ @@ -2603,7 +2112,7 @@ public class IndexCreator { if (indexTransport) { progress.setGeneralProgress("[90 / 100]"); //$NON-NLS-1$ progress.startTask(Messages.getString("IndexCreator.PACK_RTREE_TRANSP"), -1); //$NON-NLS-1$ - transportStopsTree = packRtreeFile(transportStopsTree, getRTreeTransportStopsFileName(), getRTreeTransportStopsPackFileName()); + indexTransportCreator.transportStopsTree = packRtreeFile(indexTransportCreator.transportStopsTree, getRTreeTransportStopsFileName(), getRTreeTransportStopsPackFileName()); } } @@ -2629,12 +2138,12 @@ public class IndexCreator { mapConnection.commit(); writeBinaryAddressIndex(writer, progress); } + + if (indexTransport) { progress.setGeneralProgress("[95 of 100]"); progress.startTask("Writing transport index to binary file...", -1); - closePreparedStatements(transRouteStat, transRouteStopsStat, transStopsStat); - mapConnection.commit(); - writeBinaryTransportIndex(writer); + indexTransportCreator.writeBinaryTransportIndex(writer, regionName, mapConnection); } progress.finishTask(); writer.close(); @@ -2688,9 +2197,9 @@ public class IndexCreator { mapConnection.close(); mapConnection = null; File tempDBFile = new File(workingDir, getTempMapDBFileName()); - if (databaseFileExists(tempDBFile) && deleteDatabaseIndexes) { + if (dialect.databaseFileExists(tempDBFile) && deleteDatabaseIndexes) { // do not delete it for now - removeDatabase(tempDBFile); + dialect.removeDatabase(tempDBFile); } } @@ -2716,8 +2225,8 @@ public class IndexCreator { } // delete transport rtree files - if (transportStopsTree != null) { - transportStopsTree.getFileHdr().getFile().close(); + if (indexTransportCreator.transportStopsTree != null) { + indexTransportCreator.transportStopsTree.getFileHdr().getFile().close(); File f = new File(getRTreeTransportStopsFileName()); if (f.exists() && deleteDatabaseIndexes) { f.delete(); @@ -2730,20 +2239,20 @@ public class IndexCreator { // do not delete first db connection if (dbConn != null) { - if (usingH2()) { + if (DBDialect.H2 == dialect) { dbConn.createStatement().execute("SHUTDOWN COMPACT"); //$NON-NLS-1$ } dbConn.close(); } if (deleteOsmDB) { - if (usingDerby()) { + if (DBDialect.DERBY == dialect) { try { DriverManager.getConnection("jdbc:derby:;shutdown=true"); //$NON-NLS-1$ } catch (SQLException e) { // ignore exception } } - removeDatabase(dbFile); + dialect.removeDatabase(dbFile); } } catch (SQLException e) { e.printStackTrace(); @@ -2769,7 +2278,7 @@ public class IndexCreator { pStatements.remove(pstat); } - private RTree packRtreeFile(RTree tree, String nonPackFileName, String packFileName) throws IOException { + public RTree packRtreeFile(RTree tree, String nonPackFileName, String packFileName) throws IOException { try { assert rtree.Node.MAX < 50 : "It is better for search performance"; //$NON-NLS-1$ tree.flush(); @@ -2795,7 +2304,7 @@ public class IndexCreator { } - private NewDataExtractionOsmFilter extractOsmToNodesDB(File readFile, IProgress progress, IOsmStorageFilter addFilter) throws FileNotFoundException, + private OsmDbCreator extractOsmToNodesDB(File readFile, IProgress progress, IOsmStorageFilter addFilter) throws FileNotFoundException, IOException, SQLException, SAXException { boolean pbfFile = false; InputStream stream = new BufferedInputStream(new FileInputStream(readFile), 8192*4);; @@ -2818,10 +2327,10 @@ public class IndexCreator { } // 1. Loading osm file - NewDataExtractionOsmFilter filter = new NewDataExtractionOsmFilter(); + OsmDbCreator filter = new OsmDbCreator(this); try { // 1 init database to store temporary data - filter.initDatabase(); + filter.initDatabase(dialect, dbConn); storage.getFilters().add(filter); if (pbfFile) { storage.parseOSMPbf(stream, progress, false); @@ -2850,7 +2359,7 @@ public class IndexCreator { // to save space mapFile.getParentFile().mkdirs(); File tempDBMapFile = new File(workingDir, getTempMapDBFileName()); - removeDatabase(tempDBMapFile); + dialect.removeDatabase(tempDBMapFile); mapConnection = getDatabaseConnection(tempDBMapFile.getAbsolutePath()); mapConnection.setAutoCommit(false); } @@ -2879,7 +2388,7 @@ public class IndexCreator { } if (indexAddress) { - DataIndexWriter.createAddressIndexStructure(mapConnection); + DataIndexWriter.createAddressIndexStructure(mapConnection, dialect); addressCityStat = DataIndexWriter.getCityInsertPreparedStatement(mapConnection); addressStreetStat = DataIndexWriter.getStreetInsertPreparedStatement(mapConnection); addressBuildingStat = DataIndexWriter.getBuildingInsertPreparedStatement(mapConnection); @@ -2908,31 +2417,14 @@ public class IndexCreator { poiIndexFile.getParentFile().mkdirs(); // creating nodes db to fast access for all nodes poiConnection = getDatabaseConnection(poiIndexFile.getAbsolutePath(), true); - DataIndexWriter.createPoiIndexStructure(poiConnection); + DataIndexWriter.createPoiIndexStructure(poiConnection, dialect); poiPreparedStatement = DataIndexWriter.createStatementAmenityInsert(poiConnection); pStatements.put(poiPreparedStatement, 0); poiConnection.setAutoCommit(false); } if (indexTransport) { - DataIndexWriter.createTransportIndexStructure(mapConnection); - try { - File file = new File(getRTreeTransportStopsFileName()); - if (file.exists()) { - file.delete(); - } - transportStopsTree = new RTree(file.getAbsolutePath()); - } catch (RTreeException e) { - throw new IOException(e); - } - transRouteStat = DataIndexWriter.createStatementTransportRouteInsert(mapConnection); - transRouteStopsStat = DataIndexWriter.createStatementTransportRouteStopInsert(mapConnection); - transStopsStat = DataIndexWriter.createStatementTransportStopInsert(mapConnection); - pStatements.put(transRouteStat, 0); - pStatements.put(transRouteStopsStat, 0); - pStatements.put(transStopsStat, 0); - mapConnection.setAutoCommit(false); - + indexTransportCreator.createTransportIndexStructure(mapConnection, dialect, getRTreeTransportStopsFileName(), pStatements); } } @@ -2946,26 +2438,11 @@ public class IndexCreator { } } - public static void removeWayNodes(File sqlitedb) throws SQLException { - Connection dbConn = DriverManager.getConnection("jdbc:sqlite:" + sqlitedb.getAbsolutePath()); //$NON-NLS-1$ - dbConn.setAutoCommit(false); - Statement st = dbConn.createStatement(); - st.execute("DELETE FROM street_node WHERE 1=1"); //$NON-NLS-1$ - st.close(); - dbConn.commit(); - st = dbConn.createStatement(); - if (usingSQLite()) { - st.execute("VACUUM"); //$NON-NLS-1$ - } - st.close(); - dbConn.close(); - } - public static void main(String[] args) throws IOException, SAXException, SQLException { long time = System.currentTimeMillis(); - IndexCreator creator = new IndexCreator(new File("/home/victor/projects/OsmAnd/download/384/")); //$NON-NLS-1$ + IndexCreator creator = new IndexCreator(new File("/home/victor/projects/OsmAnd/data/osm-gen/")); //$NON-NLS-1$ creator.setIndexMap(true); creator.setIndexAddress(true); creator.setIndexPOI(true); @@ -2976,10 +2453,13 @@ public class IndexCreator { creator.recreateOnlyBinaryFile = false; creator.deleteDatabaseIndexes = true; - creator.generateIndexes(new File("/home/victor/projects/OsmAnd/download/384/spain.osm.pbf"), +// creator.generateIndexes(new File("/home/victor/projects/OsmAnd/download/384/spain.osm.pbf"), +// new ConsoleProgressImplementation(1), null, MapZooms.getDefault(), null); + creator.generateIndexes(new File("/home/victor/projects/OsmAnd/data/osm-maps/minsk_around.osm"), new ConsoleProgressImplementation(1), null, MapZooms.getDefault(), null); + // creator.setNodesDBFile(new File("e:/Information/OSM maps/osmand/minsk.tmp.odb")); // creator.generateIndexes(new File("e:/Information/OSM maps/belarus osm/minsk.osm"), new ConsoleProgressImplementation(3), null, MapZooms.getDefault(), null); diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/IndexTransportCreator.java b/DataExtractionOSM/src/net/osmand/data/preparation/IndexTransportCreator.java new file mode 100644 index 0000000000..6c700dda2c --- /dev/null +++ b/DataExtractionOSM/src/net/osmand/data/preparation/IndexTransportCreator.java @@ -0,0 +1,502 @@ +package net.osmand.data.preparation; + +import java.io.File; +import java.io.IOException; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.Map.Entry; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import rtree.Element; +import rtree.IllegalValueException; +import rtree.LeafElement; +import rtree.NonLeafElement; +import rtree.RTree; +import rtree.RTreeException; +import rtree.RTreeInsertException; +import rtree.Rect; + +import net.osmand.binary.BinaryMapIndexWriter; +import net.osmand.data.TransportRoute; +import net.osmand.data.TransportStop; +import net.osmand.data.index.DataIndexWriter; +import net.osmand.data.index.IndexConstants; +import net.osmand.osm.Entity; +import net.osmand.osm.MapUtils; +import net.osmand.osm.Node; +import net.osmand.osm.Relation; +import net.osmand.osm.Way; +import net.osmand.osm.OSMSettings.OSMTagKey; +import net.sf.junidecode.Junidecode; + +public class IndexTransportCreator { + + private final IndexCreator creator; + private static final int BATCH_SIZE = 1000; + private static final Log log = LogFactory.getLog(IndexTransportCreator.class); + + private Set visitedStops = new HashSet(); + private PreparedStatement transRouteStat; + private PreparedStatement transRouteStopsStat; + private PreparedStatement transStopsStat; + protected RTree transportStopsTree; + + + private static Set acceptedRoutes = new HashSet(); + static { + acceptedRoutes.add("bus"); //$NON-NLS-1$ + acceptedRoutes.add("trolleybus"); //$NON-NLS-1$ + acceptedRoutes.add("share_taxi"); //$NON-NLS-1$ + + acceptedRoutes.add("subway"); //$NON-NLS-1$ + acceptedRoutes.add("train"); //$NON-NLS-1$ + + acceptedRoutes.add("tram"); //$NON-NLS-1$ + + acceptedRoutes.add("ferry"); //$NON-NLS-1$ + } + + public IndexTransportCreator(IndexCreator creator){ + this.creator = creator; + } + + + public void createRTreeFile(String rtreeTransportStopFile) throws RTreeException{ + transportStopsTree = new RTree(rtreeTransportStopFile); + } + + public void createTransportIndexStructure(Connection conn, DBDialect dialect, String rtreeStopsFileName, Map pStatements) throws SQLException, IOException{ + Statement stat = conn.createStatement(); + assert IndexConstants.TRANSPORT_ROUTE_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ + assert IndexConstants.TRANSPORT_STOP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ + assert IndexConstants.TRANSPORT_ROUTE_STOP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ + + stat.executeUpdate("create table transport_route (id bigint primary key, type varchar(255), operator varchar(255)," + + "ref varchar(255), name varchar(255), name_en varchar(255), dist int)"); + stat.executeUpdate("create index transport_route_id on transport_route (id)"); + + stat.executeUpdate("create table transport_route_stop (stop bigint, route bigint, ord int, direction smallint, primary key (route, ord, direction))"); + stat.executeUpdate("create index transport_route_stop_stop on transport_route_stop (stop)"); + stat.executeUpdate("create index transport_route_stop_route on transport_route_stop (route)"); + + stat.executeUpdate("create table transport_stop (id bigint primary key, latitude double, longitude double, name varchar(255), name_en varchar(255))"); + stat.executeUpdate("create index transport_stop_id on transport_stop (id)"); + stat.executeUpdate("create index transport_stop_location on transport_stop (latitude, longitude)"); + + if(dialect == DBDialect.SQLITE){ + stat.execute("PRAGMA user_version = " + IndexConstants.TRANSPORT_TABLE_VERSION); //$NON-NLS-1$ + } + stat.close(); + + try { + File file = new File(rtreeStopsFileName); + if (file.exists()) { + file.delete(); + } + transportStopsTree = new RTree(file.getAbsolutePath()); + } catch (RTreeException e) { + throw new IOException(e); + } + transRouteStat = createStatementTransportRouteInsert(conn); + transRouteStopsStat = createStatementTransportRouteStopInsert(conn); + transStopsStat = createStatementTransportStopInsert(conn); + pStatements.put(transRouteStat, 0); + pStatements.put(transRouteStopsStat, 0); + pStatements.put(transStopsStat, 0); + } + + + public void insertTransportIntoIndex(TransportRoute route, Map pStatements) throws SQLException { + insertTransportIntoIndex(transRouteStat, transRouteStopsStat, transStopsStat, transportStopsTree, + visitedStops, route, pStatements, BATCH_SIZE); + } + + public void insertTransportIntoIndex(PreparedStatement prepRoute, PreparedStatement prepRouteStops, + PreparedStatement prepStops, RTree transportStopsTree, + Set writtenStops, TransportRoute route, Map statements, + int batchSize) throws SQLException { + assert IndexConstants.TRANSPORT_ROUTE_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ + prepRoute.setLong(1, route.getId()); + prepRoute.setString(2, route.getType()); + prepRoute.setString(3, route.getOperator()); + prepRoute.setString(4, route.getRef()); + prepRoute.setString(5, route.getName()); + prepRoute.setString(6, route.getEnName()); + prepRoute.setInt(7, route.getAvgBothDistance()); + addBatch(statements, prepRoute); + + writeRouteStops(transportStopsTree, prepRouteStops, prepStops, statements, writtenStops, route, route.getForwardStops(), true); + writeRouteStops(transportStopsTree, prepRouteStops, prepStops, statements, writtenStops, route, route.getBackwardStops(), false); + + } + + public static PreparedStatement createStatementTransportStopInsert(Connection conn) throws SQLException{ + assert IndexConstants.TRANSPORT_STOP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ + return conn.prepareStatement("insert into transport_stop(id, latitude, longitude, name, name_en) values(?, ?, ?, ?, ?)"); + } + public static PreparedStatement createStatementTransportRouteStopInsert(Connection conn) throws SQLException{ + assert IndexConstants.TRANSPORT_ROUTE_STOP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ + return conn.prepareStatement("insert into transport_route_stop(route, stop, direction, ord) values(?, ?, ?, ?)"); + } + + private static void writeRouteStops(RTree transportStopsTree, PreparedStatement prepRouteStops, PreparedStatement prepStops, Map count, + Set writtenStops, TransportRoute r, List stops, boolean direction) throws SQLException { + assert IndexConstants.TRANSPORT_STOP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ + assert IndexConstants.TRANSPORT_ROUTE_STOP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ + int i = 0; + for(TransportStop s : stops){ + if (!writtenStops.contains(s.getId())) { + prepStops.setLong(1, s.getId()); + prepStops.setDouble(2, s.getLocation().getLatitude()); + prepStops.setDouble(3, s.getLocation().getLongitude()); + prepStops.setString(4, s.getName()); + prepStops.setString(5, s.getEnName()); + int x = (int) MapUtils.getTileNumberX(24, s.getLocation().getLongitude()); + int y = (int) MapUtils.getTileNumberY(24, s.getLocation().getLatitude()); + addBatch(count, prepStops); + try { + transportStopsTree.insert(new LeafElement(new Rect(x, y, x, y), s.getId())); + } catch (RTreeInsertException e) { + throw new IllegalArgumentException(e); + } catch (IllegalValueException e) { + throw new IllegalArgumentException(e); + } + writtenStops.add(s.getId()); + } + prepRouteStops.setLong(1, r.getId()); + prepRouteStops.setLong(2, s.getId()); + prepRouteStops.setInt(3, direction ? 1 : 0); + prepRouteStops.setInt(4, i++); + addBatch(count, prepRouteStops); + } + } + + public static PreparedStatement createStatementTransportRouteInsert(Connection conn) throws SQLException{ + assert IndexConstants.TRANSPORT_ROUTE_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ + return conn.prepareStatement("insert into transport_route(id, type, operator, ref, name, name_en, dist) values(?, ?, ?, ?, ?, ?, ?)"); + } + + + public void writeBinaryTransportIndex(BinaryMapIndexWriter writer, String regionName, + Connection mapConnection) throws IOException, SQLException { + try { + creator.closePreparedStatements(transRouteStat, transRouteStopsStat, transStopsStat); + mapConnection.commit(); + transportStopsTree.flush(); + + visitedStops = null; // allow gc to collect it + PreparedStatement selectTransportRouteData = mapConnection.prepareStatement( + "SELECT id, dist, name, name_en, ref, operator, type FROM transport_route"); //$NON-NLS-1$ + PreparedStatement selectTransportData = mapConnection.prepareStatement("SELECT S.stop, S.direction," + //$NON-NLS-1$ + " A.latitude, A.longitude, A.name, A.name_en " + //$NON-NLS-1$ + "FROM transport_route_stop S INNER JOIN transport_stop A ON A.id = S.stop WHERE S.route = ? ORDER BY S.ord asc"); //$NON-NLS-1$ + + writer.startWriteTransportIndex(regionName); + + writer.startWriteTransportRoutes(); + + // expect that memory would be enough + Map stringTable = createStringTableForTransport(); + Map transportRoutes = new LinkedHashMap(); + + ResultSet rs = selectTransportRouteData.executeQuery(); + List directStops = new ArrayList(); + List reverseStops = new ArrayList(); + while (rs.next()) { + + long idRoute = rs.getLong(1); + int dist = rs.getInt(2); + String routeName = rs.getString(3); + String routeEnName = rs.getString(4); + if (routeEnName != null && routeEnName.equals(Junidecode.unidecode(routeName))) { + routeEnName = null; + } + String ref = rs.getString(5); + String operator = rs.getString(6); + String type = rs.getString(7); + + selectTransportData.setLong(1, idRoute); + ResultSet rset = selectTransportData.executeQuery(); + reverseStops.clear(); + directStops.clear(); + while (rset.next()) { + boolean dir = rset.getInt(2) != 0; + long idStop = rset.getInt(1); + String stopName = rset.getString(5); + String stopEnName = rset.getString(6); + if (stopEnName != null && stopEnName.equals(Junidecode.unidecode(stopName))) { + stopEnName = null; + } + TransportStop st = new TransportStop(); + st.setId(idStop); + st.setName(stopName); + st.setLocation(rset.getDouble(3), rset.getDouble(4)); + if (stopEnName != null) { + st.setEnName(stopEnName); + } + if (dir) { + directStops.add(st); + } else { + reverseStops.add(st); + } + } + writer.writeTransportRoute(idRoute, routeName, routeEnName, ref, operator, type, dist, directStops, reverseStops, + stringTable, transportRoutes); + } + rs.close(); + selectTransportRouteData.close(); + selectTransportData.close(); + writer.endWriteTransportRoutes(); + + PreparedStatement selectTransportStop = mapConnection.prepareStatement( + "SELECT A.id, A.latitude, A.longitude, A.name, A.name_en FROM transport_stop A where A.id = ?"); //$NON-NLS-1$ + PreparedStatement selectTransportRouteStop = mapConnection.prepareStatement( + "SELECT DISTINCT S.route FROM transport_route_stop S WHERE S.stop = ? "); //$NON-NLS-1$ + long rootIndex = transportStopsTree.getFileHdr().getRootIndex(); + rtree.Node root = transportStopsTree.getReadNode(rootIndex); + Rect rootBounds = calcBounds(root); + if (rootBounds != null) { + writer.startTransportTreeElement(rootBounds.getMinX(), rootBounds.getMaxX(), rootBounds.getMinY(), rootBounds.getMaxY()); + writeBinaryTransportTree(root, transportStopsTree, writer, selectTransportStop, selectTransportRouteStop, + transportRoutes, stringTable); + writer.endWriteTransportTreeElement(); + } + selectTransportStop.close(); + selectTransportRouteStop.close(); + + writer.writeTransportStringTable(stringTable); + + writer.endWriteTransportIndex(); + writer.flush(); + } catch (RTreeException e) { + throw new IllegalStateException(e); + } + } + + public Rect calcBounds(rtree.Node n) { + Rect r = null; + Element[] e = n.getAllElements(); + for (int i = 0; i < n.getTotalElements(); i++) { + Rect re = e[i].getRect(); + if (r == null) { + try { + r = new Rect(re.getMinX(), re.getMinY(), re.getMaxX(), re.getMaxY()); + } catch (IllegalValueException ex) { + } + } else { + r.expandToInclude(re); + } + } + return r; + } + + private int registerString(Map stringTable, String s) { + if (stringTable.containsKey(s)) { + return stringTable.get(s); + } + int size = stringTable.size(); + stringTable.put(s, size); + return size; + } + + private Map createStringTableForTransport() { + Map stringTable = new LinkedHashMap(); + registerString(stringTable, "bus"); //$NON-NLS-1$ + registerString(stringTable, "trolleybus"); //$NON-NLS-1$ + registerString(stringTable, "subway"); //$NON-NLS-1$ + registerString(stringTable, "tram"); //$NON-NLS-1$ + registerString(stringTable, "share_taxi"); //$NON-NLS-1$ + registerString(stringTable, "taxi"); //$NON-NLS-1$ + registerString(stringTable, "train"); //$NON-NLS-1$ + registerString(stringTable, "ferry"); //$NON-NLS-1$ + return stringTable; + } + + + + public void writeBinaryTransportTree(rtree.Node parent, RTree r, BinaryMapIndexWriter writer, + PreparedStatement selectTransportStop, PreparedStatement selectTransportRouteStop, + Map transportRoutes, Map stringTable) throws IOException, RTreeException, SQLException { + Element[] e = parent.getAllElements(); + List routes = null; + for (int i = 0; i < parent.getTotalElements(); i++) { + Rect re = e[i].getRect(); + if (e[i].getElementType() == rtree.Node.LEAF_NODE) { + long id = ((LeafElement) e[i]).getPtr(); + selectTransportStop.setLong(1, id); + selectTransportRouteStop.setLong(1, id); + ResultSet rs = selectTransportStop.executeQuery(); + if (rs.next()) { + int x24 = (int) MapUtils.getTileNumberX(24, rs.getDouble(3)); + int y24 = (int) MapUtils.getTileNumberY(24, rs.getDouble(2)); + String name = rs.getString(4); + String nameEn = rs.getString(5); + if (nameEn != null && nameEn.equals(Junidecode.unidecode(name))) { + nameEn = null; + } + ResultSet rset = selectTransportRouteStop.executeQuery(); + if (routes == null) { + routes = new ArrayList(); + } else { + routes.clear(); + } + while (rset.next()) { + Long route = transportRoutes.get(rset.getLong(1)); + if (route == null) { + log.error("Something goes wrong with transport route id = " + rset.getLong(1)); //$NON-NLS-1$ + } else { + routes.add(route); + } + } + rset.close(); + writer.writeTransportStop(id, x24, y24, name, nameEn, stringTable, routes); + } else { + log.error("Something goes wrong with transport id = " + id); //$NON-NLS-1$ + } + } else { + long ptr = ((NonLeafElement) e[i]).getPtr(); + rtree.Node ns = r.getReadNode(ptr); + + writer.startTransportTreeElement(re.getMinX(), re.getMaxX(), re.getMinY(), re.getMaxY()); + writeBinaryTransportTree(ns, r, writer, selectTransportStop, selectTransportRouteStop, transportRoutes, stringTable); + writer.endWriteTransportTreeElement(); + } + } + } + + public TransportRoute indexTransportRoute(Relation rel) { + String ref = rel.getTag(OSMTagKey.REF); + String route = rel.getTag(OSMTagKey.ROUTE); + String operator = rel.getTag(OSMTagKey.OPERATOR); + if (route == null || ref == null) { + return null; + } + if (!acceptedRoutes.contains(route)) { + return null; + } + TransportRoute r = new TransportRoute(rel, ref); + r.setOperator(operator); + r.setType(route); + + if (operator != null) { + route = operator + " : " + route; //$NON-NLS-1$ + } + + final Map forwardStops = new LinkedHashMap(); + final Map backwardStops = new LinkedHashMap(); + int currentStop = 0; + int forwardStop = 0; + int backwardStop = 0; + for (Entry e : rel.getMemberEntities().entrySet()) { + if (e.getValue().contains("stop")) { //$NON-NLS-1$ + if (e.getKey() instanceof Node) { + TransportStop stop = new TransportStop(e.getKey()); + boolean forward = e.getValue().contains("forward"); //$NON-NLS-1$ + boolean backward = e.getValue().contains("backward"); //$NON-NLS-1$ + currentStop++; + if (forward || !backward) { + forwardStop++; + } + if (backward) { + backwardStop++; + } + boolean common = !forward && !backward; + int index = -1; + int i = e.getValue().length() - 1; + int accum = 1; + while (i >= 0 && Character.isDigit(e.getValue().charAt(i))) { + if (index < 0) { + index = 0; + } + index = accum * Character.getNumericValue(e.getValue().charAt(i)) + index; + accum *= 10; + i--; + } + if (index < 0) { + index = forward ? forwardStop : (backward ? backwardStop : currentStop); + } + if (forward || common) { + forwardStops.put(stop, index); + r.getForwardStops().add(stop); + } + if (backward || common) { + if (common) { + // put with negative index + backwardStops.put(stop, -index); + } else { + backwardStops.put(stop, index); + } + + r.getBackwardStops().add(stop); + } + + } + + } else if (e.getKey() instanceof Way) { + r.addWay((Way) e.getKey()); + } + } + if (forwardStops.isEmpty() && backwardStops.isEmpty()) { + return null; + } + Collections.sort(r.getForwardStops(), new Comparator() { + @Override + public int compare(TransportStop o1, TransportStop o2) { + return forwardStops.get(o1) - forwardStops.get(o2); + } + }); + // all common stops are with negative index (reeval them) + for (TransportStop s : new ArrayList(backwardStops.keySet())) { + if (backwardStops.get(s) < 0) { + backwardStops.put(s, backwardStops.size() + backwardStops.get(s) - 1); + } + } + Collections.sort(r.getBackwardStops(), new Comparator() { + @Override + public int compare(TransportStop o1, TransportStop o2) { + return backwardStops.get(o1) - backwardStops.get(o2); + } + }); + + return r; + } + + private static void addBatch(Map count, PreparedStatement p) throws SQLException { + addBatch(count, p, BATCH_SIZE, true); + } + + public static void addBatch(Map count, PreparedStatement p, boolean commit) throws SQLException{ + addBatch(count, p, BATCH_SIZE, commit); + } + + public static void addBatch(Map count, PreparedStatement p, int batchSize) throws SQLException{ + addBatch(count, p, batchSize, true); + } + + public static void addBatch(Map count, PreparedStatement p, int batchSize, boolean commit) throws SQLException{ + p.addBatch(); + if(count.get(p) >= batchSize){ + p.executeBatch(); + if(commit){ + p.getConnection().commit(); + } + count.put(p, 0); + } else { + count.put(p, count.get(p) + 1); + } + } + +} diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/OsmDbCreator.java b/DataExtractionOSM/src/net/osmand/data/preparation/OsmDbCreator.java new file mode 100644 index 0000000000..c135e8ac31 --- /dev/null +++ b/DataExtractionOSM/src/net/osmand/data/preparation/OsmDbCreator.java @@ -0,0 +1,180 @@ +package net.osmand.data.preparation; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Map.Entry; + +import net.osmand.osm.Entity; +import net.osmand.osm.Node; +import net.osmand.osm.Relation; +import net.osmand.osm.Way; +import net.osmand.osm.Entity.EntityId; +import net.osmand.osm.Entity.EntityType; +import net.osmand.osm.io.IOsmStorageFilter; +import net.osmand.osm.io.OsmBaseStorage; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class OsmDbCreator implements IOsmStorageFilter { + + private static final Log log = LogFactory.getLog(OsmDbCreator.class); + + public static final int BATCH_SIZE_OSM = 10000; + + + DBDialect dialect; + int currentCountNode = 0; + private PreparedStatement prepNode; + int allNodes = 0; + + int currentRelationsCount = 0; + private PreparedStatement prepRelations; + int allRelations = 0; + + int currentWaysCount = 0; + private PreparedStatement prepWays; + int allWays = 0; + + int currentTagsCount = 0; + private PreparedStatement prepTags; + private Connection dbConn; + private final IndexCreator indexCreator; + + public OsmDbCreator(IndexCreator indexCreator) { + this.indexCreator = indexCreator; + } + + public void initDatabase(DBDialect dialect, Connection dbConn) throws SQLException { + this.dbConn = dbConn; + this.dialect = dialect; + // prepare tables + Statement stat = dbConn.createStatement(); + dialect.deleteTableIfExists("node", stat); + stat.executeUpdate("create table node (id bigint primary key, latitude double, longitude double)"); //$NON-NLS-1$ + stat.executeUpdate("create index IdIndex ON node (id)"); //$NON-NLS-1$ + dialect.deleteTableIfExists("ways", stat); + stat.executeUpdate("create table ways (id bigint, node bigint, ord smallint, primary key (id, ord))"); //$NON-NLS-1$ + stat.executeUpdate("create index IdWIndex ON ways (id)"); //$NON-NLS-1$ + dialect.deleteTableIfExists("relations", stat); + stat.executeUpdate("create table relations (id bigint, member bigint, type smallint, role varchar(255), ord smallint, primary key (id, ord))"); //$NON-NLS-1$ + stat.executeUpdate("create index IdRIndex ON relations (id)"); //$NON-NLS-1$ + dialect.deleteTableIfExists("tags", stat); + stat.executeUpdate("create table tags (id bigint, type smallint, skeys varchar(255), value varchar(255), primary key (id, type, skeys))"); //$NON-NLS-1$ + stat.executeUpdate("create index IdTIndex ON tags (id, type)"); //$NON-NLS-1$ + stat.close(); + + prepNode = dbConn.prepareStatement("insert into node values (?, ?, ?)"); //$NON-NLS-1$ + prepWays = dbConn.prepareStatement("insert into ways values (?, ?, ?)"); //$NON-NLS-1$ + prepRelations = dbConn.prepareStatement("insert into relations values (?, ?, ?, ?, ?)"); //$NON-NLS-1$ + prepTags = dbConn.prepareStatement("insert into tags values (?, ?, ?, ?)"); //$NON-NLS-1$ + dbConn.setAutoCommit(false); + } + + public void finishLoading() throws SQLException { + if (currentCountNode > 0) { + prepNode.executeBatch(); + } + prepNode.close(); + if (currentWaysCount > 0) { + prepWays.executeBatch(); + } + prepWays.close(); + if (currentRelationsCount > 0) { + prepRelations.executeBatch(); + } + prepRelations.close(); + if (currentTagsCount > 0) { + prepTags.executeBatch(); + } + prepTags.close(); + } + + @Override + public boolean acceptEntityToLoad(OsmBaseStorage storage, EntityId entityId, Entity e) { + // Register all city labelbs + indexCreator.registerCityIfNeeded(e); + // put all nodes into temporary db to get only required nodes after loading all data + try { + if (e instanceof Node) { + currentCountNode++; + if (!e.getTags().isEmpty()) { + allNodes++; + } + prepNode.setLong(1, e.getId()); + prepNode.setDouble(2, ((Node) e).getLatitude()); + prepNode.setDouble(3, ((Node) e).getLongitude()); + prepNode.addBatch(); + if (currentCountNode >= BATCH_SIZE_OSM) { + prepNode.executeBatch(); + dbConn.commit(); // clear memory + currentCountNode = 0; + } + } else if (e instanceof Way) { + allWays++; + short ord = 0; + for (Long i : ((Way) e).getNodeIds()) { + currentWaysCount++; + prepWays.setLong(1, e.getId()); + prepWays.setLong(2, i); + prepWays.setLong(3, ord++); + prepWays.addBatch(); + } + if (currentWaysCount >= BATCH_SIZE_OSM) { + prepWays.executeBatch(); + dbConn.commit(); // clear memory + currentWaysCount = 0; + } + } else { + allRelations++; + short ord = 0; + for (Entry i : ((Relation) e).getMembersMap().entrySet()) { + currentRelationsCount++; + prepRelations.setLong(1, e.getId()); + prepRelations.setLong(2, i.getKey().getId()); + prepRelations.setLong(3, i.getKey().getType().ordinal()); + prepRelations.setString(4, i.getValue()); + prepRelations.setLong(5, ord++); + prepRelations.addBatch(); + } + if (currentRelationsCount >= BATCH_SIZE_OSM) { + prepRelations.executeBatch(); + dbConn.commit(); // clear memory + currentRelationsCount = 0; + } + } + for (Entry i : e.getTags().entrySet()) { + currentTagsCount++; + prepTags.setLong(1, e.getId()); + prepTags.setLong(2, EntityType.valueOf(e).ordinal()); + prepTags.setString(3, i.getKey()); + prepTags.setString(4, i.getValue()); + prepTags.addBatch(); + } + if (currentTagsCount >= BATCH_SIZE_OSM) { + prepTags.executeBatch(); + dbConn.commit(); // clear memory + currentTagsCount = 0; + } + } catch (SQLException ex) { + log.error("Could not save in db", ex); //$NON-NLS-1$ + } + // do not add to storage + return false; + } + + public int getAllNodes() { + return allNodes; + } + + public int getAllRelations() { + return allRelations; + } + + public int getAllWays() { + return allWays; + } + +} diff --git a/OsmAnd/.classpath b/OsmAnd/.classpath index cfba26feb3..9d8c50188e 100644 --- a/OsmAnd/.classpath +++ b/OsmAnd/.classpath @@ -1,11 +1,11 @@ - - - - - - - - - - - + + + + + + + + + + +