fix issue 156
improve performance by adding h2 database git-svn-id: https://osmand.googlecode.com/svn/trunk@651 e29c36b1-1cfa-d876-8d93-3434fc2bb7b8
This commit is contained in:
Victor Shcherb
parent
d792ac8547
commit
03b9d890b4
7 changed files with 278 additions and 102 deletions
|
|
@ -7,5 +7,6 @@
|
|||
<classpathentry kind="lib" path="lib/commons-logging-1.1.1.jar"/>
|
||||
<classpathentry kind="lib" path="lib/junidecode-0.1.jar"/>
|
||||
<classpathentry kind="lib" path="lib/json-20090211.jar"/>
|
||||
<classpathentry kind="lib" path="lib/h2-latest.jar"/>
|
||||
<classpathentry kind="output" path="bin"/>
|
||||
</classpath>
|
||||
|
|
|
|||
BIN
DataExtractionOSM/lib/h2-latest.jar
Normal file
BIN
DataExtractionOSM/lib/h2-latest.jar
Normal file
Binary file not shown.
|
|
@ -9,6 +9,7 @@ public class ToDoConstants {
|
|||
|
||||
// TODO max 101
|
||||
// introduce bidforfix on site
|
||||
// 1. Verify to use query
|
||||
|
||||
|
||||
// For 0.5 release
|
||||
|
|
|
|||
|
|
@ -66,7 +66,7 @@ public class DataIndexReader {
|
|||
public PreparedStatement getStreetsBuildingPreparedStatement(Connection c) throws SQLException{
|
||||
return c.prepareStatement("SELECT A.id, A.name, A.name_en, A.latitude, A.longitude, "+
|
||||
"B.id, B.name, B.name_en, B.latitude, B.longitude, B.postcode "+
|
||||
"FROM street A INNER JOIN building B ON B.street = A.id WHERE A.city = ? "); //$NON-NLS-1$
|
||||
"FROM street A LEFT JOIN building B ON B.street = A.id WHERE A.city = ? "); //$NON-NLS-1$
|
||||
}
|
||||
|
||||
public PreparedStatement getStreetsWayNodesPreparedStatement(Connection c) throws SQLException{
|
||||
|
|
@ -103,6 +103,7 @@ public class DataIndexReader {
|
|||
rs.close();
|
||||
}
|
||||
}
|
||||
if (set.getObject(6) != null) {
|
||||
Street s = visitedStreets.get(streetId);
|
||||
Building b = new Building();
|
||||
b.setId(set.getLong(6));
|
||||
|
|
@ -112,6 +113,7 @@ public class DataIndexReader {
|
|||
b.setPostcode(set.getString(11));
|
||||
s.registerBuilding(b);
|
||||
}
|
||||
}
|
||||
|
||||
set.close();
|
||||
return streets;
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import java.util.Collections;
|
|||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import com.sun.xml.internal.ws.wsdl.writer.UsingAddressing;
|
||||
|
||||
import net.osmand.Algoritms;
|
||||
import net.osmand.data.Amenity;
|
||||
|
|
@ -30,6 +31,7 @@ import net.osmand.data.index.IndexConstants.IndexStreetTable;
|
|||
import net.osmand.data.index.IndexConstants.IndexTransportRoute;
|
||||
import net.osmand.data.index.IndexConstants.IndexTransportRouteStop;
|
||||
import net.osmand.data.index.IndexConstants.IndexTransportStop;
|
||||
import net.osmand.data.preparation.IndexCreator;
|
||||
import net.osmand.osm.Entity;
|
||||
import net.osmand.osm.LatLon;
|
||||
import net.osmand.osm.MapUtils;
|
||||
|
|
@ -72,7 +74,9 @@ public class DataIndexWriter {
|
|||
Statement stat = conn.createStatement();
|
||||
stat.executeUpdate(IndexConstants.generateCreateSQL(IndexPoiTable.values()));
|
||||
stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexPoiTable.values()));
|
||||
if(IndexCreator.usingSQLite()){
|
||||
stat.execute("PRAGMA user_version = " + IndexConstants.POI_TABLE_VERSION); //$NON-NLS-1$
|
||||
}
|
||||
stat.close();
|
||||
}
|
||||
|
||||
|
|
@ -129,7 +133,9 @@ public class DataIndexWriter {
|
|||
stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexStreetNodeTable.values()));
|
||||
stat.executeUpdate(IndexConstants.generateCreateSQL(IndexStreetTable.values()));
|
||||
stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexStreetTable.values()));
|
||||
if(IndexCreator.usingSQLite()){
|
||||
stat.execute("PRAGMA user_version = " + IndexConstants.ADDRESS_TABLE_VERSION); //$NON-NLS-1$
|
||||
}
|
||||
stat.close();
|
||||
}
|
||||
|
||||
|
|
@ -194,7 +200,9 @@ public class DataIndexWriter {
|
|||
stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexTransportRouteStop.values()));
|
||||
stat.executeUpdate(IndexConstants.generateCreateSQL(IndexTransportStop.values()));
|
||||
stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexTransportStop.values()));
|
||||
if(IndexCreator.usingSQLite()){
|
||||
stat.execute("PRAGMA user_version = " + IndexConstants.TRANSPORT_TABLE_VERSION); //$NON-NLS-1$
|
||||
}
|
||||
stat.close();
|
||||
}
|
||||
|
||||
|
|
@ -295,6 +303,7 @@ public class DataIndexWriter {
|
|||
p.addBatch();
|
||||
if(count.get(p) >= batchSize){
|
||||
p.executeBatch();
|
||||
p.getConnection().commit();
|
||||
count.put(p, 0);
|
||||
} else {
|
||||
count.put(p, count.get(p) + 1);
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
package net.osmand.data.index;
|
||||
|
||||
import net.osmand.data.preparation.IndexCreator;
|
||||
|
||||
public class IndexConstants {
|
||||
|
||||
// Important : Every time you change schema of db upgrade version!!!
|
||||
|
|
@ -57,9 +59,11 @@ public class IndexConstants {
|
|||
b.append(c.toString());
|
||||
if(c.getType() != null){
|
||||
b.append(" ").append(c.getType()); //$NON-NLS-1$
|
||||
} else {
|
||||
b.append(" varchar(255)"); //$NON-NLS-1$
|
||||
}
|
||||
}
|
||||
b.append(" ); "); //$NON-NLS-1$
|
||||
b.append(" )"); //$NON-NLS-1$
|
||||
return b.toString();
|
||||
}
|
||||
|
||||
|
|
@ -83,7 +87,7 @@ public class IndexConstants {
|
|||
if(where != null){
|
||||
b.append(" WHERE " ).append(where); //$NON-NLS-1$
|
||||
}
|
||||
b.append(" ; "); //$NON-NLS-1$
|
||||
b.append(" "); //$NON-NLS-1$
|
||||
return b.toString();
|
||||
}
|
||||
|
||||
|
|
@ -96,7 +100,7 @@ public class IndexConstants {
|
|||
}
|
||||
b.append("?"); //$NON-NLS-1$
|
||||
}
|
||||
b.append(");"); //$NON-NLS-1$
|
||||
b.append(")"); //$NON-NLS-1$
|
||||
return b.toString();
|
||||
}
|
||||
|
||||
|
|
@ -116,7 +120,7 @@ public class IndexConstants {
|
|||
}
|
||||
b.append(c.toString());
|
||||
}
|
||||
b.append(" ); "); //$NON-NLS-1$
|
||||
b.append(" ) "); //$NON-NLS-1$
|
||||
if(first){
|
||||
return null;
|
||||
}
|
||||
|
|
@ -126,7 +130,7 @@ public class IndexConstants {
|
|||
// POI index
|
||||
|
||||
public enum IndexPoiTable implements IndexColumn {
|
||||
ID("long"), LATITUDE("double", true), LONGITUDE("double", true), OPENING_HOURS, NAME, NAME_EN, TYPE, SUBTYPE; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
|
||||
ID(IndexCreator.getCurrentLongType()), LATITUDE("double", true), LONGITUDE("double", true), OPENING_HOURS, NAME, NAME_EN, TYPE, SUBTYPE; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
|
||||
boolean index = false;
|
||||
String type = null;
|
||||
private IndexPoiTable(){}
|
||||
|
|
@ -157,7 +161,7 @@ public class IndexConstants {
|
|||
// Address index
|
||||
|
||||
public enum IndexCityTable implements IndexColumn {
|
||||
ID("long", true), LATITUDE("double", true), LONGITUDE("double", true), NAME, NAME_EN, CITY_TYPE(null, true); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
|
||||
ID(IndexCreator.getCurrentLongType(), true), LATITUDE("double", true), LONGITUDE("double", true), NAME, NAME_EN, CITY_TYPE(null, true); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
|
||||
boolean index = false;
|
||||
String type = null;
|
||||
|
||||
|
|
@ -193,7 +197,7 @@ public class IndexConstants {
|
|||
}
|
||||
|
||||
public enum IndexStreetTable implements IndexColumn {
|
||||
ID("long", true), LATITUDE("double", true), LONGITUDE("double", true), NAME(null, true), NAME_EN, CITY("long", true); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
|
||||
ID(IndexCreator.getCurrentLongType(), true), LATITUDE("double", true), LONGITUDE("double", true), NAME(null, true), NAME_EN, CITY(IndexCreator.getCurrentLongType(), true); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
|
||||
boolean index = false;
|
||||
String type = null;
|
||||
|
||||
|
|
@ -229,7 +233,8 @@ public class IndexConstants {
|
|||
}
|
||||
|
||||
public enum IndexStreetNodeTable implements IndexColumn {
|
||||
ID("long", true), LATITUDE("double"), LONGITUDE("double"), STREET("long", true), WAY("long", true); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$
|
||||
ID(IndexCreator.getCurrentLongType(), true), LATITUDE("double"), LONGITUDE("double"),
|
||||
STREET(IndexCreator.getCurrentLongType(), true), WAY(IndexCreator.getCurrentLongType(), true); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$
|
||||
boolean index = false;
|
||||
String type = null;
|
||||
|
||||
|
|
@ -265,7 +270,8 @@ public class IndexConstants {
|
|||
}
|
||||
|
||||
public enum IndexBuildingTable implements IndexColumn {
|
||||
ID("long", true), LATITUDE("double"), LONGITUDE("double"), NAME, NAME_EN, STREET("long", true), POSTCODE(null, true); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
|
||||
ID(IndexCreator.getCurrentLongType(), true), LATITUDE("double"), LONGITUDE("double"), NAME,
|
||||
NAME_EN, STREET(IndexCreator.getCurrentLongType(), true), POSTCODE(null, true); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
|
||||
boolean index = false;
|
||||
String type = null;
|
||||
|
||||
|
|
@ -302,7 +308,7 @@ public class IndexConstants {
|
|||
|
||||
// Transport Index
|
||||
public enum IndexTransportStop implements IndexColumn {
|
||||
ID("long", true), LATITUDE("double", true), LONGITUDE("double", true), NAME, NAME_EN; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
|
||||
ID(IndexCreator.getCurrentLongType(), true), LATITUDE("double", true), LONGITUDE("double", true), NAME, NAME_EN; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
|
||||
boolean index = false;
|
||||
String type = null;
|
||||
|
||||
|
|
@ -339,7 +345,7 @@ public class IndexConstants {
|
|||
|
||||
|
||||
public enum IndexTransportRouteStop implements IndexColumn {
|
||||
STOP("long", true), ROUTE("long", true), ORD("int"), DIRECTION("short"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
|
||||
STOP(IndexCreator.getCurrentLongType(), true), ROUTE(IndexCreator.getCurrentLongType(), true), ORD("int"), DIRECTION("smallint"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
|
||||
boolean index = false;
|
||||
String type = null;
|
||||
|
||||
|
|
@ -375,7 +381,7 @@ public class IndexConstants {
|
|||
}
|
||||
|
||||
public enum IndexTransportRoute implements IndexColumn {
|
||||
ID("long", true), TYPE(null, true), OPERATOR, REF(null, true), NAME, NAME_EN, DIST("int"); //$NON-NLS-1$ //$NON-NLS-2$
|
||||
ID(IndexCreator.getCurrentLongType(), true), TYPE(null, true), OPERATOR, REF(null, true), NAME, NAME_EN, DIST("int"); //$NON-NLS-1$ //$NON-NLS-2$
|
||||
boolean index = false;
|
||||
String type = null;
|
||||
|
||||
|
|
@ -412,7 +418,7 @@ public class IndexConstants {
|
|||
|
||||
|
||||
public enum IndexBinaryMapRenderObject implements IndexColumn {
|
||||
ID("long", true), NAME, TYPES("BLOB"), RESTRICTIONS("BLOB"), NODES("BLOB"), HIGHWAY("INT"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$
|
||||
ID(IndexCreator.getCurrentLongType(), true), NAME, TYPES("BLOB"), RESTRICTIONS("BLOB"), NODES("BLOB"), HIGHWAY("INT"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$
|
||||
boolean index = false;
|
||||
String type = null;
|
||||
|
||||
|
|
|
|||
|
|
@ -95,8 +95,14 @@ import rtree.Rect;
|
|||
public class IndexCreator {
|
||||
private static final Log log = LogFactory.getLog(IndexCreator.class);
|
||||
|
||||
// ONLY derby.jar needed for derby dialect
|
||||
private static final String DERBY_DIALECT = "DERBY";
|
||||
private static final String H2_DIALECT = "H2";
|
||||
private static final String SQLITE_DIALECT = "SQLITE";
|
||||
private static final String CURRENT_DB = H2_DIALECT;
|
||||
|
||||
public static final int BATCH_SIZE = 10;
|
||||
public static final int BATCH_SIZE = 1000;
|
||||
public static final int BATCH_SIZE_OSM = 10000;
|
||||
public static final String TEMP_NODES_DB = "nodes.tmp.odb";
|
||||
|
||||
public static final int STEP_CITY_NODES = 1;
|
||||
|
|
@ -105,6 +111,7 @@ public class IndexCreator {
|
|||
|
||||
private File workingDir = null;
|
||||
|
||||
|
||||
private boolean indexMap;
|
||||
private boolean indexPOI;
|
||||
private boolean indexTransport;
|
||||
|
|
@ -196,10 +203,30 @@ public class IndexCreator {
|
|||
private String[] normalizeDefaultSuffixes;
|
||||
private String[] normalizeSuffixes;
|
||||
|
||||
|
||||
|
||||
// local purpose
|
||||
List<Integer> typeUse= new ArrayList<Integer>(8);
|
||||
List<Long> restrictionsUse= new ArrayList<Long>(8);
|
||||
|
||||
public static boolean usingSQLite(){
|
||||
return CURRENT_DB.equals(SQLITE_DIALECT);
|
||||
}
|
||||
|
||||
public static boolean usingDerby(){
|
||||
return CURRENT_DB.equals(DERBY_DIALECT);
|
||||
}
|
||||
|
||||
public static boolean usingH2(){
|
||||
return CURRENT_DB.equals(H2_DIALECT);
|
||||
}
|
||||
|
||||
public static String getCurrentLongType(){
|
||||
return usingSQLite() ? "long" : "bigint";
|
||||
}
|
||||
|
||||
|
||||
|
||||
public IndexCreator(File workingDir){
|
||||
this.workingDir = workingDir;
|
||||
}
|
||||
|
|
@ -265,24 +292,57 @@ public class IndexCreator {
|
|||
public void initDatabase() throws SQLException {
|
||||
// prepare tables
|
||||
Statement stat = dbConn.createStatement();
|
||||
stat.executeUpdate("drop table if exists node;");
|
||||
stat.executeUpdate("create table node (id long, latitude double, longitude double);");
|
||||
stat.executeUpdate("create index IdIndex ON node (id, latitude, longitude);");
|
||||
stat.executeUpdate("drop table if exists ways;");
|
||||
stat.executeUpdate("create table ways (id long, node long, ord smallint);");
|
||||
stat.executeUpdate("create index IdWIndex ON ways (id, node);");
|
||||
stat.executeUpdate("drop table if exists relations;");
|
||||
stat.executeUpdate("create table relations (id long, member long, type byte, role text, ord smallint);");
|
||||
stat.executeUpdate("create index IdRIndex ON relations (id, member, type);");
|
||||
stat.executeUpdate("drop table if exists tags;");
|
||||
stat.executeUpdate("create table tags (id long, type byte, key, value);");
|
||||
stat.executeUpdate("create index IdTIndex ON tags (id, type);");
|
||||
if (usingDerby()) {
|
||||
try {
|
||||
stat.executeUpdate("drop table node");
|
||||
} catch (SQLException e) {
|
||||
// ignore it
|
||||
}
|
||||
} else {
|
||||
stat.executeUpdate("drop table if exists node");
|
||||
}
|
||||
String longType = usingSQLite() ? "long" : "bigint";
|
||||
stat.executeUpdate("create table node (id "+longType+", latitude double, longitude double)");
|
||||
stat.executeUpdate("create index IdIndex ON node (id, latitude, longitude)");
|
||||
if (usingDerby()) {
|
||||
try {
|
||||
stat.executeUpdate("drop table ways");
|
||||
} catch (SQLException e) {
|
||||
// ignore it
|
||||
}
|
||||
} else {
|
||||
stat.executeUpdate("drop table if exists ways");
|
||||
}
|
||||
stat.executeUpdate("create table ways (id "+longType+", node "+longType+", ord smallint)");
|
||||
stat.executeUpdate("create index IdWIndex ON ways (id, node)");
|
||||
if (usingDerby()) {
|
||||
try {
|
||||
stat.executeUpdate("drop table relations");
|
||||
} catch (SQLException e) {
|
||||
// ignore it
|
||||
}
|
||||
} else {
|
||||
stat.executeUpdate("drop table if exists relations");
|
||||
}
|
||||
stat.executeUpdate("create table relations (id "+longType+", member "+longType+", type smallint, role varchar(255), ord smallint)");
|
||||
stat.executeUpdate("create index IdRIndex ON relations (id, member, type)");
|
||||
if (usingDerby()) {
|
||||
try {
|
||||
stat.executeUpdate("drop table tags");
|
||||
} catch (SQLException e) {
|
||||
// ignore it
|
||||
}
|
||||
} else {
|
||||
stat.executeUpdate("drop table if exists ефпы");
|
||||
}
|
||||
stat.executeUpdate("create table tags (id "+longType+", type smallint, skeys varchar(255), value varchar(255))");
|
||||
stat.executeUpdate("create index IdTIndex ON tags (id, type)");
|
||||
stat.close();
|
||||
|
||||
prepNode = dbConn.prepareStatement("insert into node values (?, ?, ?);");
|
||||
prepWays = dbConn.prepareStatement("insert into ways values (?, ?, ?);");
|
||||
prepRelations = dbConn.prepareStatement("insert into relations values (?, ?, ?, ?, ?);");
|
||||
prepTags = dbConn.prepareStatement("insert into tags values (?, ?, ?, ?);");
|
||||
prepNode = dbConn.prepareStatement("insert into node values (?, ?, ?)");
|
||||
prepWays = dbConn.prepareStatement("insert into ways values (?, ?, ?)");
|
||||
prepRelations = dbConn.prepareStatement("insert into relations values (?, ?, ?, ?, ?)");
|
||||
prepTags = dbConn.prepareStatement("insert into tags values (?, ?, ?, ?)");
|
||||
dbConn.setAutoCommit(false);
|
||||
}
|
||||
|
||||
|
|
@ -303,7 +363,6 @@ public class IndexCreator {
|
|||
prepTags.executeBatch();
|
||||
}
|
||||
prepTags.close();
|
||||
dbConn.setAutoCommit(true);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -319,8 +378,9 @@ public class IndexCreator {
|
|||
prepNode.setDouble(2, ((Node) e).getLatitude());
|
||||
prepNode.setDouble(3, ((Node) e).getLongitude());
|
||||
prepNode.addBatch();
|
||||
if (currentCountNode >= BATCH_SIZE) {
|
||||
if (currentCountNode >= BATCH_SIZE_OSM) {
|
||||
prepNode.executeBatch();
|
||||
dbConn.commit(); // clear memory
|
||||
currentCountNode = 0;
|
||||
}
|
||||
} else if (e instanceof Way) {
|
||||
|
|
@ -333,8 +393,9 @@ public class IndexCreator {
|
|||
prepWays.setLong(3, ord++);
|
||||
prepWays.addBatch();
|
||||
}
|
||||
if (currentWaysCount >= BATCH_SIZE) {
|
||||
if (currentWaysCount >= BATCH_SIZE_OSM) {
|
||||
prepWays.executeBatch();
|
||||
dbConn.commit(); // clear memory
|
||||
currentWaysCount = 0;
|
||||
}
|
||||
} else {
|
||||
|
|
@ -346,11 +407,12 @@ public class IndexCreator {
|
|||
prepRelations.setLong(2, i.getKey().getId());
|
||||
prepRelations.setLong(3, i.getKey().getType().ordinal());
|
||||
prepRelations.setString(4, i.getValue());
|
||||
prepWays.setLong(5, ord++);
|
||||
prepRelations.setLong(5, ord++);
|
||||
prepRelations.addBatch();
|
||||
}
|
||||
if (currentRelationsCount >= BATCH_SIZE) {
|
||||
if (currentRelationsCount >= BATCH_SIZE_OSM) {
|
||||
prepRelations.executeBatch();
|
||||
dbConn.commit(); // clear memory
|
||||
currentRelationsCount = 0;
|
||||
}
|
||||
}
|
||||
|
|
@ -362,8 +424,9 @@ public class IndexCreator {
|
|||
prepTags.setString(4, i.getValue());
|
||||
prepTags.addBatch();
|
||||
}
|
||||
if (currentTagsCount >= BATCH_SIZE) {
|
||||
if (currentTagsCount >= BATCH_SIZE_OSM) {
|
||||
prepTags.executeBatch();
|
||||
dbConn.commit(); // clear memory
|
||||
currentTagsCount = 0;
|
||||
}
|
||||
} catch (SQLException ex) {
|
||||
|
|
@ -398,10 +461,68 @@ public class IndexCreator {
|
|||
this.regionName = regionName;
|
||||
}
|
||||
|
||||
private Connection getDatabaseConnection(String fileName) throws SQLException {
|
||||
return getDatabaseConnection(fileName, false);
|
||||
}
|
||||
|
||||
public static void removeDatabase(File file) throws SQLException {
|
||||
if(usingH2()){
|
||||
File[] list = file.getParentFile().listFiles();
|
||||
for(File f : list){
|
||||
if(f.getName().startsWith(file.getName())){
|
||||
Algoritms.removeAllFiles(f);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Algoritms.removeAllFiles(file);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public static boolean databaseFileExists(File dbFile){
|
||||
if(usingH2()){
|
||||
return new File(dbFile.getAbsolutePath() + ".h2.db").exists();
|
||||
} else {
|
||||
return dbFile.exists();
|
||||
}
|
||||
}
|
||||
private Connection getDatabaseConnection(String fileName, boolean forceSqLite) throws SQLException {
|
||||
if (usingSQLite() || forceSqLite) {
|
||||
try {
|
||||
Class.forName("org.sqlite.JDBC");
|
||||
} catch (ClassNotFoundException e) {
|
||||
log.error("Illegal configuration", e);
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
return DriverManager.getConnection("jdbc:sqlite:" + fileName);
|
||||
} else if (usingDerby()) {
|
||||
try {
|
||||
Class.forName("org.apache.derby.jdbc.EmbeddedDriver");
|
||||
} catch (ClassNotFoundException e) {
|
||||
log.error("Illegal configuration", e);
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
Connection conn = DriverManager.getConnection("jdbc:derby:" + fileName + ";create=true");
|
||||
conn.setAutoCommit(false);
|
||||
return conn;
|
||||
} else if(usingH2()){
|
||||
try {
|
||||
Class.forName("org.h2.Driver");
|
||||
} catch (ClassNotFoundException e) {
|
||||
log.error("Illegal configuration", e);
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
|
||||
return DriverManager. getConnection("jdbc:h2:file:"+fileName);
|
||||
} else {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public void loadEntityData(Entity e, boolean loadTags) throws SQLException {
|
||||
if(e instanceof Node){
|
||||
if(e instanceof Node || (e instanceof Way && !((Way) e).getNodes().isEmpty())){
|
||||
// do not load tags for nodes inside way
|
||||
return;
|
||||
}
|
||||
Map<EntityId, Entity> map = new LinkedHashMap<EntityId, Entity>();
|
||||
|
|
@ -410,7 +531,7 @@ public class IndexCreator {
|
|||
if (pselectRelation.execute()) {
|
||||
ResultSet rs = pselectRelation.getResultSet();
|
||||
while (rs.next()) {
|
||||
((Relation) e).addMember(rs.getLong(2), EntityType.values()[rs.getByte(3)], rs.getString(4));
|
||||
((Relation) e).addMember(rs.getLong(2), EntityType.values()[rs.getInt(3)], rs.getString(4));
|
||||
}
|
||||
rs.close();
|
||||
}
|
||||
|
|
@ -451,11 +572,11 @@ public class IndexCreator {
|
|||
} else if (i.getType() == EntityType.RELATION) {
|
||||
pselectRelation.setLong(1, i.getId());
|
||||
if (pselectRelation.execute()) {
|
||||
ResultSet rs = pselectNode.getResultSet();
|
||||
ResultSet rs = pselectRelation.getResultSet();
|
||||
Relation rel = new Relation(i.getId());
|
||||
map.put(i, rel);
|
||||
while (rs.next()) {
|
||||
rel.addMember(rs.getLong(1), EntityType.values()[rs.getByte(2)], rs.getString(3));
|
||||
rel.addMember(rs.getLong(2), EntityType.values()[rs.getInt(3)], rs.getString(4));
|
||||
}
|
||||
// do not load relation members recursively ? It is not needed for transport, address, poi before
|
||||
rs.close();
|
||||
|
|
@ -514,30 +635,65 @@ public class IndexCreator {
|
|||
String select;
|
||||
int count = 0;
|
||||
|
||||
// stat.executeUpdate("create table tags (id "+longType+", type smallint, skeys varchar(255), value varchar(255))");
|
||||
// stat.executeUpdate("create table ways (id "+longType+", node "+longType+", ord smallint)");
|
||||
// stat.executeUpdate("create table relations (id "+longType+", member "+longType+", type smallint, role varchar(255), ord smallint)");
|
||||
if(type == EntityType.NODE){
|
||||
select = "select * from node";
|
||||
// filter out all nodes without tags
|
||||
select = "select n.id, n.latitude, n.longitude, t.skeys, t.value from node n inner join tags t on n.id = t.id and t.type = 0 order by n.id";
|
||||
} else if(type == EntityType.WAY){
|
||||
select = "select distinct id from ways";
|
||||
select = "select w.id, w.node, w.ord, t.skeys, t.value, n.latitude, n.longitude " +
|
||||
"from ways w left join tags t on w.id = t.id and t.type = 1 and w.ord = 0 inner join node n on w.node = n.id " +
|
||||
"order by w.id, w.ord";
|
||||
} else {
|
||||
select = "select distinct id from relations";
|
||||
select = "select r.id, t.skeys, t.value from relations r inner join tags t on t.id = r.id and t.type = 2 and r.ord = 0";
|
||||
}
|
||||
|
||||
ResultSet rs = statement.executeQuery(select);
|
||||
Entity prevEntity = null;
|
||||
|
||||
long prevId = -1;
|
||||
while(rs.next()){
|
||||
long curId = rs.getLong(1);
|
||||
boolean newEntity = curId != prevId;
|
||||
Entity e = prevEntity;
|
||||
if(type == EntityType.NODE){
|
||||
if(newEntity){
|
||||
e = new Node(rs.getDouble(2),rs.getDouble(3), curId);
|
||||
}
|
||||
e.putTag(rs.getString(4), rs.getString(5));
|
||||
} else if(type == EntityType.WAY){
|
||||
if(newEntity){
|
||||
e = new Way(curId);
|
||||
}
|
||||
int ord = rs.getInt(3);
|
||||
if(ord == 0 && rs.getObject(4) != null){
|
||||
e.putTag(rs.getString(4), rs.getString(5));
|
||||
}
|
||||
if(newEntity || ord > 0){
|
||||
((Way) e).addNode(new Node(rs.getDouble(6), rs.getDouble(7), rs.getLong(2)));
|
||||
}
|
||||
} else {
|
||||
if(newEntity){
|
||||
e = new Relation(curId);
|
||||
}
|
||||
e.putTag(rs.getString(2), rs.getString(3));
|
||||
}
|
||||
if(newEntity){
|
||||
count++;
|
||||
if(progress != null){
|
||||
progress.progress(1);
|
||||
}
|
||||
Entity e;
|
||||
if(type == EntityType.NODE){
|
||||
e = new Node(rs.getDouble(2),rs.getDouble(3),rs.getLong(1));
|
||||
} else if(type == EntityType.WAY){
|
||||
e = new Way(rs.getLong(1));
|
||||
} else {
|
||||
e = new Relation(rs.getLong(1));
|
||||
if(prevEntity != null){
|
||||
iterateEntity(prevEntity, step);
|
||||
}
|
||||
loadEntityTags(type, e);
|
||||
iterateEntity(e, step);
|
||||
prevEntity = e;
|
||||
}
|
||||
prevId = curId;
|
||||
}
|
||||
if(prevEntity != null){
|
||||
count++;
|
||||
iterateEntity(prevEntity, step);
|
||||
}
|
||||
rs.close();
|
||||
return count;
|
||||
|
|
@ -967,7 +1123,7 @@ public class IndexCreator {
|
|||
|
||||
if (indexMap && (e instanceof Way || e instanceof Node)) {
|
||||
// manipulate what kind of way to load
|
||||
loadEntityData(e, true);
|
||||
loadEntityData(e, false);
|
||||
boolean inverse = "-1".equals(e.getTag(OSMTagKey.ONEWAY));
|
||||
for (int i = 0; i < MAP_ZOOMS.length - 1; i++) {
|
||||
writeBinaryEntityToMapDatabase(e, e.getId(), i == 0 ? inverse : false, i);
|
||||
|
|
@ -1825,13 +1981,6 @@ public class IndexCreator {
|
|||
}
|
||||
}
|
||||
|
||||
try {
|
||||
Class.forName("org.sqlite.JDBC");
|
||||
} catch (ClassNotFoundException e) {
|
||||
log.error("Illegal configuration", e);
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
|
||||
cities.clear();
|
||||
cityManager.clear();
|
||||
postalCodeRelations.clear();
|
||||
|
|
@ -1844,15 +1993,15 @@ public class IndexCreator {
|
|||
try {
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// 1. creating nodes db to fast access for all nodes and simply import all relations, ways, nodes to it
|
||||
boolean loadFromPath = dbFile == null || !dbFile.exists();
|
||||
boolean loadFromPath = dbFile == null || !databaseFileExists(dbFile);
|
||||
if (dbFile == null) {
|
||||
dbFile = new File(workingDir, TEMP_NODES_DB);
|
||||
// to save space
|
||||
if (dbFile.exists()) {
|
||||
dbFile.delete();
|
||||
if (databaseFileExists(dbFile)) {
|
||||
removeDatabase(dbFile);
|
||||
}
|
||||
}
|
||||
dbConn = DriverManager.getConnection("jdbc:sqlite:" + dbFile.getAbsolutePath());
|
||||
dbConn = getDatabaseConnection(dbFile.getAbsolutePath());
|
||||
|
||||
int allRelations = 100000;
|
||||
int allWays = 1000000;
|
||||
|
|
@ -1871,13 +2020,13 @@ public class IndexCreator {
|
|||
pselectNode = dbConn.prepareStatement("select * from node where id = ?");
|
||||
pselectWay = dbConn.prepareStatement("select * from ways where id = ? order by ord");
|
||||
pselectRelation = dbConn.prepareStatement("select * from relations where id = ? order by ord");
|
||||
pselectTags = dbConn.prepareStatement("select key, value from tags where id = ? and type = ?");
|
||||
pselectTags = dbConn.prepareStatement("select skeys, value from tags where id = ? and type = ?");
|
||||
|
||||
// do not create temp map file and rtree files
|
||||
if (recreateOnlyBinaryFile) {
|
||||
mapFile = new File(workingDir, getMapFileName());
|
||||
File tempDBMapFile = new File(workingDir, getTempMapDBFileName());
|
||||
mapConnection = DriverManager.getConnection("jdbc:sqlite:" + tempDBMapFile.getAbsolutePath());
|
||||
mapConnection = getDatabaseConnection(tempDBMapFile.getAbsolutePath());
|
||||
mapConnection.setAutoCommit(false);
|
||||
try {
|
||||
if (indexMap) {
|
||||
|
|
@ -2051,9 +2200,9 @@ public class IndexCreator {
|
|||
mapConnection.close();
|
||||
mapConnection = null;
|
||||
File tempDBFile = new File(workingDir, getTempMapDBFileName());
|
||||
if(tempDBFile.exists() && deleteDatabaseIndexes){
|
||||
if(databaseFileExists(tempDBFile) && deleteDatabaseIndexes){
|
||||
// do not delete it for now
|
||||
tempDBFile.delete();
|
||||
removeDatabase(tempDBFile);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -2081,8 +2230,6 @@ public class IndexCreator {
|
|||
// delete transport rtree files
|
||||
if(transportStopsTree != null){
|
||||
transportStopsTree.getFileHdr().getFile().close();
|
||||
}
|
||||
{
|
||||
File f = new File(getRTreeTransportStopsFileName());
|
||||
if (f.exists() && deleteDatabaseIndexes) {
|
||||
f.delete();
|
||||
|
|
@ -2094,9 +2241,21 @@ public class IndexCreator {
|
|||
}
|
||||
|
||||
// do not delete first db connection
|
||||
if(dbConn != null){
|
||||
if(usingH2()){
|
||||
dbConn.createStatement().execute("SHUTDOWN COMPACT");
|
||||
}
|
||||
dbConn.close();
|
||||
}
|
||||
if(deleteOsmDB){
|
||||
dbFile.delete();
|
||||
if (usingDerby()) {
|
||||
try {
|
||||
DriverManager.getConnection("jdbc:derby:;shutdown=true");
|
||||
} catch (SQLException e) {
|
||||
// ignore exception
|
||||
}
|
||||
}
|
||||
removeDatabase(dbFile);
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
e.printStackTrace();
|
||||
|
|
@ -2184,6 +2343,7 @@ public class IndexCreator {
|
|||
storage.parseOSM(stream, progress, streamFile, false);
|
||||
}
|
||||
filter.finishLoading();
|
||||
dbConn.commit();
|
||||
|
||||
if (log.isInfoEnabled()) {
|
||||
log.info("File parsed : " + (System.currentTimeMillis() - st));
|
||||
|
|
@ -2205,10 +2365,8 @@ public class IndexCreator {
|
|||
// to save space
|
||||
mapFile.getParentFile().mkdirs();
|
||||
File tempDBMapFile = new File(workingDir, getTempMapDBFileName());
|
||||
if(tempDBMapFile.exists()){
|
||||
tempDBMapFile.delete();
|
||||
}
|
||||
mapConnection = DriverManager.getConnection("jdbc:sqlite:" + tempDBMapFile.getAbsolutePath());
|
||||
removeDatabase(tempDBMapFile);
|
||||
mapConnection = getDatabaseConnection(tempDBMapFile.getAbsolutePath());
|
||||
mapConnection.setAutoCommit(false);
|
||||
}
|
||||
|
||||
|
|
@ -2266,11 +2424,11 @@ public class IndexCreator {
|
|||
poiIndexFile = new File(workingDir, getPoiFileName());
|
||||
// to save space
|
||||
if (poiIndexFile.exists()) {
|
||||
poiIndexFile.delete();
|
||||
Algoritms.removeAllFiles(poiIndexFile);
|
||||
}
|
||||
poiIndexFile.getParentFile().mkdirs();
|
||||
// creating nodes db to fast access for all nodes
|
||||
poiConnection = DriverManager.getConnection("jdbc:sqlite:" + poiIndexFile.getAbsolutePath());
|
||||
poiConnection = getDatabaseConnection(poiIndexFile.getAbsolutePath(), true);
|
||||
DataIndexWriter.createPoiIndexStructure(poiConnection);
|
||||
poiPreparedStatement = DataIndexWriter.createStatementAmenityInsert(poiConnection);
|
||||
pStatements.put(poiPreparedStatement, 0);
|
||||
|
|
@ -2320,7 +2478,6 @@ public class IndexCreator {
|
|||
st.execute("DELETE FROM " + IndexStreetNodeTable.getTable() + " WHERE 1=1");
|
||||
st.close();
|
||||
dbConn.commit();
|
||||
dbConn.setAutoCommit(true);
|
||||
st = dbConn.createStatement();
|
||||
st.execute("VACUUM");
|
||||
st.close();
|
||||
|
|
@ -2330,18 +2487,18 @@ public class IndexCreator {
|
|||
public static void main(String[] args) throws IOException, SAXException, SQLException {
|
||||
long time = System.currentTimeMillis();
|
||||
IndexCreator creator = new IndexCreator(new File("e:/Information/OSM maps/osmand/"));
|
||||
// creator.setIndexMap(true);
|
||||
// creator.setIndexAddress(true);
|
||||
// creator.setSaveAddressWays(false);
|
||||
// creator.setNormalizeStreets(true);
|
||||
creator.setIndexMap(true);
|
||||
creator.setIndexAddress(true);
|
||||
creator.setSaveAddressWays(true);
|
||||
creator.setNormalizeStreets(true);
|
||||
// creator.setIndexPOI(true);
|
||||
creator.setIndexTransport(true);
|
||||
|
||||
creator.recreateOnlyBinaryFile = false;
|
||||
creator.deleteDatabaseIndexes = false;
|
||||
|
||||
// creator.setNodesDBFile(new File("e:/Information/OSM maps/osmand/minsk.tmp.odb"));
|
||||
// creator.generateIndexes(new File("e:/Information/OSM maps/belarus osm/minsk.osm"), new ConsoleProgressImplementation(3), null);
|
||||
creator.setNodesDBFile(new File("e:/Information/OSM maps/osmand/minsk.tmp.odb"));
|
||||
creator.generateIndexes(new File("e:/Information/OSM maps/belarus osm/minsk.osm"), new ConsoleProgressImplementation(3), null);
|
||||
|
||||
// creator.setNodesDBFile(new File("e:/Information/OSM maps/osmand/belarus_nodes.tmp.odb"));
|
||||
// creator.generateIndexes(new File("e:/Information/OSM maps/belarus osm/belarus.osm.bz2"), new ConsoleProgressImplementation(3), null);
|
||||
|
|
@ -2356,12 +2513,12 @@ public class IndexCreator {
|
|||
// creator.setNodesDBFile(new File("e:/Information/OSM maps/osmand/den_haag.tmp.odb"));
|
||||
// creator.generateIndexes(new File("e:/Information/OSM maps/osm_map/den_haag.osm"), new ConsoleProgressImplementation(3), null);
|
||||
|
||||
creator.setNodesDBFile(new File("e:/Information/OSM maps/osmand/netherlands.tmp.odb"));
|
||||
creator.generateIndexes(new File("e:/Information/OSM maps/osm_map/netherlands.osm.bz2"), new ConsoleProgressImplementation(1), null);
|
||||
// creator.setNodesDBFile(new File("e:/Information/OSM maps/osmand/netherlands.tmp.odb"));
|
||||
// creator.generateIndexes(new File("e:/Information/OSM maps/osm_map/netherlands.osm.bz2"), new ConsoleProgressImplementation(1), null);
|
||||
|
||||
// creator.generateIndexes(new File("e:/Information/OSM maps/osm_map/forest_complex.osm"), new ConsoleProgressImplementation(25), null);
|
||||
|
||||
System.out.println(System.currentTimeMillis() - time);
|
||||
System.out.println("WHOLE GENERATION TIME : " + (System.currentTimeMillis() - time));
|
||||
System.out.println("COORDINATES_SIZE " + BinaryMapIndexWriter.COORDINATES_SIZE + " count " + BinaryMapIndexWriter.COORDINATES_COUNT);
|
||||
System.out.println("TYPES_SIZE " + BinaryMapIndexWriter.TYPES_SIZE);
|
||||
System.out.println("ID_SIZE " + BinaryMapIndexWriter.ID_SIZE);
|
||||
|
|
|
|||
Loading…
Reference in a new issue