diff --git a/DataExtractionOSM/.classpath b/DataExtractionOSM/.classpath
index e4fcdf539d..8d42fad2de 100644
--- a/DataExtractionOSM/.classpath
+++ b/DataExtractionOSM/.classpath
@@ -7,5 +7,6 @@
+
diff --git a/DataExtractionOSM/lib/h2-latest.jar b/DataExtractionOSM/lib/h2-latest.jar
new file mode 100644
index 0000000000..67fdcfcc55
Binary files /dev/null and b/DataExtractionOSM/lib/h2-latest.jar differ
diff --git a/DataExtractionOSM/src/net/osmand/ToDoConstants.java b/DataExtractionOSM/src/net/osmand/ToDoConstants.java
index f0a0096e9a..55266ce37d 100644
--- a/DataExtractionOSM/src/net/osmand/ToDoConstants.java
+++ b/DataExtractionOSM/src/net/osmand/ToDoConstants.java
@@ -9,6 +9,7 @@ public class ToDoConstants {
// TODO max 101
// introduce bidforfix on site
+ // 1. Verify to use query
// For 0.5 release
diff --git a/DataExtractionOSM/src/net/osmand/data/index/DataIndexReader.java b/DataExtractionOSM/src/net/osmand/data/index/DataIndexReader.java
index 3f51f9ddd1..41faf35cb9 100644
--- a/DataExtractionOSM/src/net/osmand/data/index/DataIndexReader.java
+++ b/DataExtractionOSM/src/net/osmand/data/index/DataIndexReader.java
@@ -66,7 +66,7 @@ public class DataIndexReader {
public PreparedStatement getStreetsBuildingPreparedStatement(Connection c) throws SQLException{
return c.prepareStatement("SELECT A.id, A.name, A.name_en, A.latitude, A.longitude, "+
"B.id, B.name, B.name_en, B.latitude, B.longitude, B.postcode "+
- "FROM street A INNER JOIN building B ON B.street = A.id WHERE A.city = ? "); //$NON-NLS-1$
+ "FROM street A LEFT JOIN building B ON B.street = A.id WHERE A.city = ? "); //$NON-NLS-1$
}
public PreparedStatement getStreetsWayNodesPreparedStatement(Connection c) throws SQLException{
@@ -103,14 +103,16 @@ public class DataIndexReader {
rs.close();
}
}
- Street s = visitedStreets.get(streetId);
- Building b = new Building();
- b.setId(set.getLong(6));
- b.setName(set.getString(7));
- b.setEnName(set.getString(8));
- b.setLocation(set.getDouble(9), set.getDouble(10));
- b.setPostcode(set.getString(11));
- s.registerBuilding(b);
+ if (set.getObject(6) != null) {
+ Street s = visitedStreets.get(streetId);
+ Building b = new Building();
+ b.setId(set.getLong(6));
+ b.setName(set.getString(7));
+ b.setEnName(set.getString(8));
+ b.setLocation(set.getDouble(9), set.getDouble(10));
+ b.setPostcode(set.getString(11));
+ s.registerBuilding(b);
+ }
}
set.close();
diff --git a/DataExtractionOSM/src/net/osmand/data/index/DataIndexWriter.java b/DataExtractionOSM/src/net/osmand/data/index/DataIndexWriter.java
index 6e3551b14a..f41df225a5 100644
--- a/DataExtractionOSM/src/net/osmand/data/index/DataIndexWriter.java
+++ b/DataExtractionOSM/src/net/osmand/data/index/DataIndexWriter.java
@@ -12,6 +12,7 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import com.sun.xml.internal.ws.wsdl.writer.UsingAddressing;
import net.osmand.Algoritms;
import net.osmand.data.Amenity;
@@ -30,6 +31,7 @@ import net.osmand.data.index.IndexConstants.IndexStreetTable;
import net.osmand.data.index.IndexConstants.IndexTransportRoute;
import net.osmand.data.index.IndexConstants.IndexTransportRouteStop;
import net.osmand.data.index.IndexConstants.IndexTransportStop;
+import net.osmand.data.preparation.IndexCreator;
import net.osmand.osm.Entity;
import net.osmand.osm.LatLon;
import net.osmand.osm.MapUtils;
@@ -72,7 +74,9 @@ public class DataIndexWriter {
Statement stat = conn.createStatement();
stat.executeUpdate(IndexConstants.generateCreateSQL(IndexPoiTable.values()));
stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexPoiTable.values()));
- stat.execute("PRAGMA user_version = " + IndexConstants.POI_TABLE_VERSION); //$NON-NLS-1$
+ if(IndexCreator.usingSQLite()){
+ stat.execute("PRAGMA user_version = " + IndexConstants.POI_TABLE_VERSION); //$NON-NLS-1$
+ }
stat.close();
}
@@ -129,7 +133,9 @@ public class DataIndexWriter {
stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexStreetNodeTable.values()));
stat.executeUpdate(IndexConstants.generateCreateSQL(IndexStreetTable.values()));
stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexStreetTable.values()));
- stat.execute("PRAGMA user_version = " + IndexConstants.ADDRESS_TABLE_VERSION); //$NON-NLS-1$
+ if(IndexCreator.usingSQLite()){
+ stat.execute("PRAGMA user_version = " + IndexConstants.ADDRESS_TABLE_VERSION); //$NON-NLS-1$
+ }
stat.close();
}
@@ -194,7 +200,9 @@ public class DataIndexWriter {
stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexTransportRouteStop.values()));
stat.executeUpdate(IndexConstants.generateCreateSQL(IndexTransportStop.values()));
stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexTransportStop.values()));
- stat.execute("PRAGMA user_version = " + IndexConstants.TRANSPORT_TABLE_VERSION); //$NON-NLS-1$
+ if(IndexCreator.usingSQLite()){
+ stat.execute("PRAGMA user_version = " + IndexConstants.TRANSPORT_TABLE_VERSION); //$NON-NLS-1$
+ }
stat.close();
}
@@ -295,6 +303,7 @@ public class DataIndexWriter {
p.addBatch();
if(count.get(p) >= batchSize){
p.executeBatch();
+ p.getConnection().commit();
count.put(p, 0);
} else {
count.put(p, count.get(p) + 1);
diff --git a/DataExtractionOSM/src/net/osmand/data/index/IndexConstants.java b/DataExtractionOSM/src/net/osmand/data/index/IndexConstants.java
index f5092dac58..e6d07ea938 100644
--- a/DataExtractionOSM/src/net/osmand/data/index/IndexConstants.java
+++ b/DataExtractionOSM/src/net/osmand/data/index/IndexConstants.java
@@ -1,5 +1,7 @@
package net.osmand.data.index;
+import net.osmand.data.preparation.IndexCreator;
+
public class IndexConstants {
// Important : Every time you change schema of db upgrade version!!!
@@ -57,9 +59,11 @@ public class IndexConstants {
b.append(c.toString());
if(c.getType() != null){
b.append(" ").append(c.getType()); //$NON-NLS-1$
+ } else {
+ b.append(" varchar(255)"); //$NON-NLS-1$
}
}
- b.append(" ); "); //$NON-NLS-1$
+ b.append(" )"); //$NON-NLS-1$
return b.toString();
}
@@ -83,7 +87,7 @@ public class IndexConstants {
if(where != null){
b.append(" WHERE " ).append(where); //$NON-NLS-1$
}
- b.append(" ; "); //$NON-NLS-1$
+ b.append(" "); //$NON-NLS-1$
return b.toString();
}
@@ -96,7 +100,7 @@ public class IndexConstants {
}
b.append("?"); //$NON-NLS-1$
}
- b.append(");"); //$NON-NLS-1$
+ b.append(")"); //$NON-NLS-1$
return b.toString();
}
@@ -116,7 +120,7 @@ public class IndexConstants {
}
b.append(c.toString());
}
- b.append(" ); "); //$NON-NLS-1$
+ b.append(" ) "); //$NON-NLS-1$
if(first){
return null;
}
@@ -126,7 +130,7 @@ public class IndexConstants {
// POI index
public enum IndexPoiTable implements IndexColumn {
- ID("long"), LATITUDE("double", true), LONGITUDE("double", true), OPENING_HOURS, NAME, NAME_EN, TYPE, SUBTYPE; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+ ID(IndexCreator.getCurrentLongType()), LATITUDE("double", true), LONGITUDE("double", true), OPENING_HOURS, NAME, NAME_EN, TYPE, SUBTYPE; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
boolean index = false;
String type = null;
private IndexPoiTable(){}
@@ -157,7 +161,7 @@ public class IndexConstants {
// Address index
public enum IndexCityTable implements IndexColumn {
- ID("long", true), LATITUDE("double", true), LONGITUDE("double", true), NAME, NAME_EN, CITY_TYPE(null, true); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+ ID(IndexCreator.getCurrentLongType(), true), LATITUDE("double", true), LONGITUDE("double", true), NAME, NAME_EN, CITY_TYPE(null, true); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
boolean index = false;
String type = null;
@@ -193,7 +197,7 @@ public class IndexConstants {
}
public enum IndexStreetTable implements IndexColumn {
- ID("long", true), LATITUDE("double", true), LONGITUDE("double", true), NAME(null, true), NAME_EN, CITY("long", true); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
+ ID(IndexCreator.getCurrentLongType(), true), LATITUDE("double", true), LONGITUDE("double", true), NAME(null, true), NAME_EN, CITY(IndexCreator.getCurrentLongType(), true); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
boolean index = false;
String type = null;
@@ -229,7 +233,8 @@ public class IndexConstants {
}
public enum IndexStreetNodeTable implements IndexColumn {
- ID("long", true), LATITUDE("double"), LONGITUDE("double"), STREET("long", true), WAY("long", true); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$
+ ID(IndexCreator.getCurrentLongType(), true), LATITUDE("double"), LONGITUDE("double"),
+ STREET(IndexCreator.getCurrentLongType(), true), WAY(IndexCreator.getCurrentLongType(), true); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$
boolean index = false;
String type = null;
@@ -265,7 +270,8 @@ public class IndexConstants {
}
public enum IndexBuildingTable implements IndexColumn {
- ID("long", true), LATITUDE("double"), LONGITUDE("double"), NAME, NAME_EN, STREET("long", true), POSTCODE(null, true); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
+ ID(IndexCreator.getCurrentLongType(), true), LATITUDE("double"), LONGITUDE("double"), NAME,
+ NAME_EN, STREET(IndexCreator.getCurrentLongType(), true), POSTCODE(null, true); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
boolean index = false;
String type = null;
@@ -302,7 +308,7 @@ public class IndexConstants {
// Transport Index
public enum IndexTransportStop implements IndexColumn {
- ID("long", true), LATITUDE("double", true), LONGITUDE("double", true), NAME, NAME_EN; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+ ID(IndexCreator.getCurrentLongType(), true), LATITUDE("double", true), LONGITUDE("double", true), NAME, NAME_EN; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
boolean index = false;
String type = null;
@@ -339,7 +345,7 @@ public class IndexConstants {
public enum IndexTransportRouteStop implements IndexColumn {
- STOP("long", true), ROUTE("long", true), ORD("int"), DIRECTION("short"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
+ STOP(IndexCreator.getCurrentLongType(), true), ROUTE(IndexCreator.getCurrentLongType(), true), ORD("int"), DIRECTION("smallint"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
boolean index = false;
String type = null;
@@ -375,7 +381,7 @@ public class IndexConstants {
}
public enum IndexTransportRoute implements IndexColumn {
- ID("long", true), TYPE(null, true), OPERATOR, REF(null, true), NAME, NAME_EN, DIST("int"); //$NON-NLS-1$ //$NON-NLS-2$
+ ID(IndexCreator.getCurrentLongType(), true), TYPE(null, true), OPERATOR, REF(null, true), NAME, NAME_EN, DIST("int"); //$NON-NLS-1$ //$NON-NLS-2$
boolean index = false;
String type = null;
@@ -412,7 +418,7 @@ public class IndexConstants {
public enum IndexBinaryMapRenderObject implements IndexColumn {
- ID("long", true), NAME, TYPES("BLOB"), RESTRICTIONS("BLOB"), NODES("BLOB"), HIGHWAY("INT"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$
+ ID(IndexCreator.getCurrentLongType(), true), NAME, TYPES("BLOB"), RESTRICTIONS("BLOB"), NODES("BLOB"), HIGHWAY("INT"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$
boolean index = false;
String type = null;
diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java b/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java
index 67a0a38e66..65037bdee7 100644
--- a/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java
+++ b/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java
@@ -95,8 +95,14 @@ import rtree.Rect;
public class IndexCreator {
private static final Log log = LogFactory.getLog(IndexCreator.class);
+ // ONLY derby.jar needed for derby dialect
+ private static final String DERBY_DIALECT = "DERBY";
+ private static final String H2_DIALECT = "H2";
+ private static final String SQLITE_DIALECT = "SQLITE";
+ private static final String CURRENT_DB = H2_DIALECT;
- public static final int BATCH_SIZE = 10;
+ public static final int BATCH_SIZE = 1000;
+ public static final int BATCH_SIZE_OSM = 10000;
public static final String TEMP_NODES_DB = "nodes.tmp.odb";
public static final int STEP_CITY_NODES = 1;
@@ -105,6 +111,7 @@ public class IndexCreator {
private File workingDir = null;
+
private boolean indexMap;
private boolean indexPOI;
private boolean indexTransport;
@@ -196,10 +203,30 @@ public class IndexCreator {
private String[] normalizeDefaultSuffixes;
private String[] normalizeSuffixes;
+
+
// local purpose
List typeUse= new ArrayList(8);
List restrictionsUse= new ArrayList(8);
+ public static boolean usingSQLite(){
+ return CURRENT_DB.equals(SQLITE_DIALECT);
+ }
+
+ public static boolean usingDerby(){
+ return CURRENT_DB.equals(DERBY_DIALECT);
+ }
+
+ public static boolean usingH2(){
+ return CURRENT_DB.equals(H2_DIALECT);
+ }
+
+ public static String getCurrentLongType(){
+ return usingSQLite() ? "long" : "bigint";
+ }
+
+
+
public IndexCreator(File workingDir){
this.workingDir = workingDir;
}
@@ -265,24 +292,57 @@ public class IndexCreator {
public void initDatabase() throws SQLException {
// prepare tables
Statement stat = dbConn.createStatement();
- stat.executeUpdate("drop table if exists node;");
- stat.executeUpdate("create table node (id long, latitude double, longitude double);");
- stat.executeUpdate("create index IdIndex ON node (id, latitude, longitude);");
- stat.executeUpdate("drop table if exists ways;");
- stat.executeUpdate("create table ways (id long, node long, ord smallint);");
- stat.executeUpdate("create index IdWIndex ON ways (id, node);");
- stat.executeUpdate("drop table if exists relations;");
- stat.executeUpdate("create table relations (id long, member long, type byte, role text, ord smallint);");
- stat.executeUpdate("create index IdRIndex ON relations (id, member, type);");
- stat.executeUpdate("drop table if exists tags;");
- stat.executeUpdate("create table tags (id long, type byte, key, value);");
- stat.executeUpdate("create index IdTIndex ON tags (id, type);");
+ if (usingDerby()) {
+ try {
+ stat.executeUpdate("drop table node");
+ } catch (SQLException e) {
+ // ignore it
+ }
+ } else {
+ stat.executeUpdate("drop table if exists node");
+ }
+ String longType = usingSQLite() ? "long" : "bigint";
+ stat.executeUpdate("create table node (id "+longType+", latitude double, longitude double)");
+ stat.executeUpdate("create index IdIndex ON node (id, latitude, longitude)");
+ if (usingDerby()) {
+ try {
+ stat.executeUpdate("drop table ways");
+ } catch (SQLException e) {
+ // ignore it
+ }
+ } else {
+ stat.executeUpdate("drop table if exists ways");
+ }
+ stat.executeUpdate("create table ways (id "+longType+", node "+longType+", ord smallint)");
+ stat.executeUpdate("create index IdWIndex ON ways (id, node)");
+ if (usingDerby()) {
+ try {
+ stat.executeUpdate("drop table relations");
+ } catch (SQLException e) {
+ // ignore it
+ }
+ } else {
+ stat.executeUpdate("drop table if exists relations");
+ }
+ stat.executeUpdate("create table relations (id "+longType+", member "+longType+", type smallint, role varchar(255), ord smallint)");
+ stat.executeUpdate("create index IdRIndex ON relations (id, member, type)");
+ if (usingDerby()) {
+ try {
+ stat.executeUpdate("drop table tags");
+ } catch (SQLException e) {
+ // ignore it
+ }
+ } else {
+ stat.executeUpdate("drop table if exists ефпы");
+ }
+ stat.executeUpdate("create table tags (id "+longType+", type smallint, skeys varchar(255), value varchar(255))");
+ stat.executeUpdate("create index IdTIndex ON tags (id, type)");
stat.close();
- prepNode = dbConn.prepareStatement("insert into node values (?, ?, ?);");
- prepWays = dbConn.prepareStatement("insert into ways values (?, ?, ?);");
- prepRelations = dbConn.prepareStatement("insert into relations values (?, ?, ?, ?, ?);");
- prepTags = dbConn.prepareStatement("insert into tags values (?, ?, ?, ?);");
+ prepNode = dbConn.prepareStatement("insert into node values (?, ?, ?)");
+ prepWays = dbConn.prepareStatement("insert into ways values (?, ?, ?)");
+ prepRelations = dbConn.prepareStatement("insert into relations values (?, ?, ?, ?, ?)");
+ prepTags = dbConn.prepareStatement("insert into tags values (?, ?, ?, ?)");
dbConn.setAutoCommit(false);
}
@@ -303,7 +363,6 @@ public class IndexCreator {
prepTags.executeBatch();
}
prepTags.close();
- dbConn.setAutoCommit(true);
}
@Override
@@ -319,8 +378,9 @@ public class IndexCreator {
prepNode.setDouble(2, ((Node) e).getLatitude());
prepNode.setDouble(3, ((Node) e).getLongitude());
prepNode.addBatch();
- if (currentCountNode >= BATCH_SIZE) {
+ if (currentCountNode >= BATCH_SIZE_OSM) {
prepNode.executeBatch();
+ dbConn.commit(); // clear memory
currentCountNode = 0;
}
} else if (e instanceof Way) {
@@ -333,8 +393,9 @@ public class IndexCreator {
prepWays.setLong(3, ord++);
prepWays.addBatch();
}
- if (currentWaysCount >= BATCH_SIZE) {
+ if (currentWaysCount >= BATCH_SIZE_OSM) {
prepWays.executeBatch();
+ dbConn.commit(); // clear memory
currentWaysCount = 0;
}
} else {
@@ -346,11 +407,12 @@ public class IndexCreator {
prepRelations.setLong(2, i.getKey().getId());
prepRelations.setLong(3, i.getKey().getType().ordinal());
prepRelations.setString(4, i.getValue());
- prepWays.setLong(5, ord++);
+ prepRelations.setLong(5, ord++);
prepRelations.addBatch();
}
- if (currentRelationsCount >= BATCH_SIZE) {
+ if (currentRelationsCount >= BATCH_SIZE_OSM) {
prepRelations.executeBatch();
+ dbConn.commit(); // clear memory
currentRelationsCount = 0;
}
}
@@ -362,8 +424,9 @@ public class IndexCreator {
prepTags.setString(4, i.getValue());
prepTags.addBatch();
}
- if (currentTagsCount >= BATCH_SIZE) {
+ if (currentTagsCount >= BATCH_SIZE_OSM) {
prepTags.executeBatch();
+ dbConn.commit(); // clear memory
currentTagsCount = 0;
}
} catch (SQLException ex) {
@@ -398,10 +461,68 @@ public class IndexCreator {
this.regionName = regionName;
}
+ private Connection getDatabaseConnection(String fileName) throws SQLException {
+ return getDatabaseConnection(fileName, false);
+ }
+ public static void removeDatabase(File file) throws SQLException {
+ if(usingH2()){
+ File[] list = file.getParentFile().listFiles();
+ for(File f : list){
+ if(f.getName().startsWith(file.getName())){
+ Algoritms.removeAllFiles(f);
+ }
+ }
+ } else {
+ Algoritms.removeAllFiles(file);
+ }
+
+ }
+
+ public static boolean databaseFileExists(File dbFile){
+ if(usingH2()){
+ return new File(dbFile.getAbsolutePath() + ".h2.db").exists();
+ } else {
+ return dbFile.exists();
+ }
+ }
+ private Connection getDatabaseConnection(String fileName, boolean forceSqLite) throws SQLException {
+ if (usingSQLite() || forceSqLite) {
+ try {
+ Class.forName("org.sqlite.JDBC");
+ } catch (ClassNotFoundException e) {
+ log.error("Illegal configuration", e);
+ throw new IllegalStateException(e);
+ }
+ return DriverManager.getConnection("jdbc:sqlite:" + fileName);
+ } else if (usingDerby()) {
+ try {
+ Class.forName("org.apache.derby.jdbc.EmbeddedDriver");
+ } catch (ClassNotFoundException e) {
+ log.error("Illegal configuration", e);
+ throw new IllegalStateException(e);
+ }
+ Connection conn = DriverManager.getConnection("jdbc:derby:" + fileName + ";create=true");
+ conn.setAutoCommit(false);
+ return conn;
+ } else if(usingH2()){
+ try {
+ Class.forName("org.h2.Driver");
+ } catch (ClassNotFoundException e) {
+ log.error("Illegal configuration", e);
+ throw new IllegalStateException(e);
+ }
+
+ return DriverManager. getConnection("jdbc:h2:file:"+fileName);
+ } else {
+ throw new UnsupportedOperationException();
+ }
+
+ }
public void loadEntityData(Entity e, boolean loadTags) throws SQLException {
- if(e instanceof Node){
+ if(e instanceof Node || (e instanceof Way && !((Way) e).getNodes().isEmpty())){
+ // do not load tags for nodes inside way
return;
}
Map map = new LinkedHashMap();
@@ -410,7 +531,7 @@ public class IndexCreator {
if (pselectRelation.execute()) {
ResultSet rs = pselectRelation.getResultSet();
while (rs.next()) {
- ((Relation) e).addMember(rs.getLong(2), EntityType.values()[rs.getByte(3)], rs.getString(4));
+ ((Relation) e).addMember(rs.getLong(2), EntityType.values()[rs.getInt(3)], rs.getString(4));
}
rs.close();
}
@@ -451,11 +572,11 @@ public class IndexCreator {
} else if (i.getType() == EntityType.RELATION) {
pselectRelation.setLong(1, i.getId());
if (pselectRelation.execute()) {
- ResultSet rs = pselectNode.getResultSet();
+ ResultSet rs = pselectRelation.getResultSet();
Relation rel = new Relation(i.getId());
map.put(i, rel);
while (rs.next()) {
- rel.addMember(rs.getLong(1), EntityType.values()[rs.getByte(2)], rs.getString(3));
+ rel.addMember(rs.getLong(2), EntityType.values()[rs.getInt(3)], rs.getString(4));
}
// do not load relation members recursively ? It is not needed for transport, address, poi before
rs.close();
@@ -514,30 +635,65 @@ public class IndexCreator {
String select;
int count = 0;
+// stat.executeUpdate("create table tags (id "+longType+", type smallint, skeys varchar(255), value varchar(255))");
+// stat.executeUpdate("create table ways (id "+longType+", node "+longType+", ord smallint)");
+// stat.executeUpdate("create table relations (id "+longType+", member "+longType+", type smallint, role varchar(255), ord smallint)");
if(type == EntityType.NODE){
- select = "select * from node";
+ // filter out all nodes without tags
+ select = "select n.id, n.latitude, n.longitude, t.skeys, t.value from node n inner join tags t on n.id = t.id and t.type = 0 order by n.id";
} else if(type == EntityType.WAY){
- select = "select distinct id from ways";
+ select = "select w.id, w.node, w.ord, t.skeys, t.value, n.latitude, n.longitude " +
+ "from ways w left join tags t on w.id = t.id and t.type = 1 and w.ord = 0 inner join node n on w.node = n.id " +
+ "order by w.id, w.ord";
} else {
- select = "select distinct id from relations";
+ select = "select r.id, t.skeys, t.value from relations r inner join tags t on t.id = r.id and t.type = 2 and r.ord = 0";
}
ResultSet rs = statement.executeQuery(select);
+ Entity prevEntity = null;
+
+ long prevId = -1;
while(rs.next()){
- count++;
- if(progress != null){
- progress.progress(1);
- }
- Entity e;
+ long curId = rs.getLong(1);
+ boolean newEntity = curId != prevId;
+ Entity e = prevEntity;
if(type == EntityType.NODE){
- e = new Node(rs.getDouble(2),rs.getDouble(3),rs.getLong(1));
+ if(newEntity){
+ e = new Node(rs.getDouble(2),rs.getDouble(3), curId);
+ }
+ e.putTag(rs.getString(4), rs.getString(5));
} else if(type == EntityType.WAY){
- e = new Way(rs.getLong(1));
+ if(newEntity){
+ e = new Way(curId);
+ }
+ int ord = rs.getInt(3);
+ if(ord == 0 && rs.getObject(4) != null){
+ e.putTag(rs.getString(4), rs.getString(5));
+ }
+ if(newEntity || ord > 0){
+ ((Way) e).addNode(new Node(rs.getDouble(6), rs.getDouble(7), rs.getLong(2)));
+ }
} else {
- e = new Relation(rs.getLong(1));
+ if(newEntity){
+ e = new Relation(curId);
+ }
+ e.putTag(rs.getString(2), rs.getString(3));
}
- loadEntityTags(type, e);
- iterateEntity(e, step);
+ if(newEntity){
+ count++;
+ if(progress != null){
+ progress.progress(1);
+ }
+ if(prevEntity != null){
+ iterateEntity(prevEntity, step);
+ }
+ prevEntity = e;
+ }
+ prevId = curId;
+ }
+ if(prevEntity != null){
+ count++;
+ iterateEntity(prevEntity, step);
}
rs.close();
return count;
@@ -967,7 +1123,7 @@ public class IndexCreator {
if (indexMap && (e instanceof Way || e instanceof Node)) {
// manipulate what kind of way to load
- loadEntityData(e, true);
+ loadEntityData(e, false);
boolean inverse = "-1".equals(e.getTag(OSMTagKey.ONEWAY));
for (int i = 0; i < MAP_ZOOMS.length - 1; i++) {
writeBinaryEntityToMapDatabase(e, e.getId(), i == 0 ? inverse : false, i);
@@ -1825,13 +1981,6 @@ public class IndexCreator {
}
}
- try {
- Class.forName("org.sqlite.JDBC");
- } catch (ClassNotFoundException e) {
- log.error("Illegal configuration", e);
- throw new IllegalStateException(e);
- }
-
cities.clear();
cityManager.clear();
postalCodeRelations.clear();
@@ -1844,15 +1993,15 @@ public class IndexCreator {
try {
//////////////////////////////////////////////////////////////////////////
// 1. creating nodes db to fast access for all nodes and simply import all relations, ways, nodes to it
- boolean loadFromPath = dbFile == null || !dbFile.exists();
+ boolean loadFromPath = dbFile == null || !databaseFileExists(dbFile);
if (dbFile == null) {
dbFile = new File(workingDir, TEMP_NODES_DB);
// to save space
- if (dbFile.exists()) {
- dbFile.delete();
+ if (databaseFileExists(dbFile)) {
+ removeDatabase(dbFile);
}
}
- dbConn = DriverManager.getConnection("jdbc:sqlite:" + dbFile.getAbsolutePath());
+ dbConn = getDatabaseConnection(dbFile.getAbsolutePath());
int allRelations = 100000;
int allWays = 1000000;
@@ -1871,13 +2020,13 @@ public class IndexCreator {
pselectNode = dbConn.prepareStatement("select * from node where id = ?");
pselectWay = dbConn.prepareStatement("select * from ways where id = ? order by ord");
pselectRelation = dbConn.prepareStatement("select * from relations where id = ? order by ord");
- pselectTags = dbConn.prepareStatement("select key, value from tags where id = ? and type = ?");
+ pselectTags = dbConn.prepareStatement("select skeys, value from tags where id = ? and type = ?");
// do not create temp map file and rtree files
if (recreateOnlyBinaryFile) {
mapFile = new File(workingDir, getMapFileName());
File tempDBMapFile = new File(workingDir, getTempMapDBFileName());
- mapConnection = DriverManager.getConnection("jdbc:sqlite:" + tempDBMapFile.getAbsolutePath());
+ mapConnection = getDatabaseConnection(tempDBMapFile.getAbsolutePath());
mapConnection.setAutoCommit(false);
try {
if (indexMap) {
@@ -2051,9 +2200,9 @@ public class IndexCreator {
mapConnection.close();
mapConnection = null;
File tempDBFile = new File(workingDir, getTempMapDBFileName());
- if(tempDBFile.exists() && deleteDatabaseIndexes){
+ if(databaseFileExists(tempDBFile) && deleteDatabaseIndexes){
// do not delete it for now
- tempDBFile.delete();
+ removeDatabase(tempDBFile);
}
}
@@ -2081,8 +2230,6 @@ public class IndexCreator {
// delete transport rtree files
if(transportStopsTree != null){
transportStopsTree.getFileHdr().getFile().close();
- }
- {
File f = new File(getRTreeTransportStopsFileName());
if (f.exists() && deleteDatabaseIndexes) {
f.delete();
@@ -2094,9 +2241,21 @@ public class IndexCreator {
}
// do not delete first db connection
- dbConn.close();
+ if(dbConn != null){
+ if(usingH2()){
+ dbConn.createStatement().execute("SHUTDOWN COMPACT");
+ }
+ dbConn.close();
+ }
if(deleteOsmDB){
- dbFile.delete();
+ if (usingDerby()) {
+ try {
+ DriverManager.getConnection("jdbc:derby:;shutdown=true");
+ } catch (SQLException e) {
+ // ignore exception
+ }
+ }
+ removeDatabase(dbFile);
}
} catch (SQLException e) {
e.printStackTrace();
@@ -2184,6 +2343,7 @@ public class IndexCreator {
storage.parseOSM(stream, progress, streamFile, false);
}
filter.finishLoading();
+ dbConn.commit();
if (log.isInfoEnabled()) {
log.info("File parsed : " + (System.currentTimeMillis() - st));
@@ -2205,10 +2365,8 @@ public class IndexCreator {
// to save space
mapFile.getParentFile().mkdirs();
File tempDBMapFile = new File(workingDir, getTempMapDBFileName());
- if(tempDBMapFile.exists()){
- tempDBMapFile.delete();
- }
- mapConnection = DriverManager.getConnection("jdbc:sqlite:" + tempDBMapFile.getAbsolutePath());
+ removeDatabase(tempDBMapFile);
+ mapConnection = getDatabaseConnection(tempDBMapFile.getAbsolutePath());
mapConnection.setAutoCommit(false);
}
@@ -2266,11 +2424,11 @@ public class IndexCreator {
poiIndexFile = new File(workingDir, getPoiFileName());
// to save space
if (poiIndexFile.exists()) {
- poiIndexFile.delete();
+ Algoritms.removeAllFiles(poiIndexFile);
}
poiIndexFile.getParentFile().mkdirs();
// creating nodes db to fast access for all nodes
- poiConnection = DriverManager.getConnection("jdbc:sqlite:" + poiIndexFile.getAbsolutePath());
+ poiConnection = getDatabaseConnection(poiIndexFile.getAbsolutePath(), true);
DataIndexWriter.createPoiIndexStructure(poiConnection);
poiPreparedStatement = DataIndexWriter.createStatementAmenityInsert(poiConnection);
pStatements.put(poiPreparedStatement, 0);
@@ -2320,7 +2478,6 @@ public class IndexCreator {
st.execute("DELETE FROM " + IndexStreetNodeTable.getTable() + " WHERE 1=1");
st.close();
dbConn.commit();
- dbConn.setAutoCommit(true);
st = dbConn.createStatement();
st.execute("VACUUM");
st.close();
@@ -2330,18 +2487,18 @@ public class IndexCreator {
public static void main(String[] args) throws IOException, SAXException, SQLException {
long time = System.currentTimeMillis();
IndexCreator creator = new IndexCreator(new File("e:/Information/OSM maps/osmand/"));
-// creator.setIndexMap(true);
-// creator.setIndexAddress(true);
-// creator.setSaveAddressWays(false);
-// creator.setNormalizeStreets(true);
+ creator.setIndexMap(true);
+ creator.setIndexAddress(true);
+ creator.setSaveAddressWays(true);
+ creator.setNormalizeStreets(true);
// creator.setIndexPOI(true);
creator.setIndexTransport(true);
creator.recreateOnlyBinaryFile = false;
creator.deleteDatabaseIndexes = false;
-// creator.setNodesDBFile(new File("e:/Information/OSM maps/osmand/minsk.tmp.odb"));
-// creator.generateIndexes(new File("e:/Information/OSM maps/belarus osm/minsk.osm"), new ConsoleProgressImplementation(3), null);
+ creator.setNodesDBFile(new File("e:/Information/OSM maps/osmand/minsk.tmp.odb"));
+ creator.generateIndexes(new File("e:/Information/OSM maps/belarus osm/minsk.osm"), new ConsoleProgressImplementation(3), null);
// creator.setNodesDBFile(new File("e:/Information/OSM maps/osmand/belarus_nodes.tmp.odb"));
// creator.generateIndexes(new File("e:/Information/OSM maps/belarus osm/belarus.osm.bz2"), new ConsoleProgressImplementation(3), null);
@@ -2356,12 +2513,12 @@ public class IndexCreator {
// creator.setNodesDBFile(new File("e:/Information/OSM maps/osmand/den_haag.tmp.odb"));
// creator.generateIndexes(new File("e:/Information/OSM maps/osm_map/den_haag.osm"), new ConsoleProgressImplementation(3), null);
- creator.setNodesDBFile(new File("e:/Information/OSM maps/osmand/netherlands.tmp.odb"));
- creator.generateIndexes(new File("e:/Information/OSM maps/osm_map/netherlands.osm.bz2"), new ConsoleProgressImplementation(1), null);
+// creator.setNodesDBFile(new File("e:/Information/OSM maps/osmand/netherlands.tmp.odb"));
+// creator.generateIndexes(new File("e:/Information/OSM maps/osm_map/netherlands.osm.bz2"), new ConsoleProgressImplementation(1), null);
// creator.generateIndexes(new File("e:/Information/OSM maps/osm_map/forest_complex.osm"), new ConsoleProgressImplementation(25), null);
- System.out.println(System.currentTimeMillis() - time);
+ System.out.println("WHOLE GENERATION TIME : " + (System.currentTimeMillis() - time));
System.out.println("COORDINATES_SIZE " + BinaryMapIndexWriter.COORDINATES_SIZE + " count " + BinaryMapIndexWriter.COORDINATES_COUNT);
System.out.println("TYPES_SIZE " + BinaryMapIndexWriter.TYPES_SIZE);
System.out.println("ID_SIZE " + BinaryMapIndexWriter.ID_SIZE);
@@ -2371,4 +2528,4 @@ public class IndexCreator {
System.out.println("-- MAP_DATA_AND_STRINGS SIZE " + (BinaryMapIndexWriter.MAP_DATA_SIZE + BinaryMapIndexWriter.STRING_TABLE_SIZE));
}
-}
+}
\ No newline at end of file