diff --git a/DataExtractionOSM/.classpath b/DataExtractionOSM/.classpath index 63183b90f4..46bbda7925 100644 --- a/DataExtractionOSM/.classpath +++ b/DataExtractionOSM/.classpath @@ -9,5 +9,7 @@ + + diff --git a/DataExtractionOSM/lib/jleveldb.jar b/DataExtractionOSM/lib/jleveldb.jar new file mode 100644 index 0000000000..39f09531ec Binary files /dev/null and b/DataExtractionOSM/lib/jleveldb.jar differ diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/DBDialect.java b/DataExtractionOSM/src/net/osmand/data/preparation/DBDialect.java index 761de54aa8..c6eaa00368 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/DBDialect.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/DBDialect.java @@ -6,13 +6,19 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; +import net.osmand.Algoritms; + import org.apache.commons.logging.Log; -import net.osmand.Algoritms; +import com.anvisics.jleveldb.LevelDBAccess; +import com.anvisics.jleveldb.ext.DBAccessor; +import com.anvisics.jleveldb.ext.Options; +import com.anvisics.jleveldb.ext.Status; public enum DBDialect { DERBY, H2, + NOSQL, SQLITE; public void deleteTableIfExists(String table, Statement stat) throws SQLException { @@ -36,7 +42,7 @@ public enum DBDialect { } } - public void removeDatabase(File file) { + public void removeDatabase(File file) { if (DBDialect.H2 == this) { File[] list = file.getParentFile().listFiles(); for (File f : list) { @@ -47,11 +53,38 @@ public enum DBDialect { } else { Algoritms.removeAllFiles(file); } - } - protected Connection getDatabaseConnection(String fileName, Log log) throws SQLException { - if (DBDialect.SQLITE == this) { + public void commitDatabase(Object connection) throws SQLException { + if(DBDialect.NOSQL != this){ + ((Connection) connection).commit(); + } else { + + } + } + + public void closeDatabase(Object dbConn) throws SQLException { + if(DBDialect.NOSQL != this){ + if (DBDialect.H2 == this) { + ((Connection) dbConn).createStatement().execute("SHUTDOWN COMPACT"); //$NON-NLS-1$ + } + ((Connection) dbConn).close(); + } else { +// ((DBAccessor) dbConn).close(); + } + } + + protected Object getDatabaseConnection(String fileName, Log log) throws SQLException { + if (DBDialect.NOSQL == this) { + DBAccessor dbAccessor = LevelDBAccess.getDBAcessor(); + Options opts = new Options(); + opts.setCreate_if_missing(true); + Status status = dbAccessor.Open(opts, fileName); + if(!status.ok()){ + throw new SQLException(status.ToString()); + } + return dbAccessor; + } else if (DBDialect.SQLITE == this) { try { Class.forName("org.sqlite.JDBC"); } catch (ClassNotFoundException e) { diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java b/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java index faa4aef845..fdba8ed0bb 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java @@ -47,6 +47,7 @@ public class IndexCreator { // ONLY derby.jar needed for derby dialect private final DBDialect dialect = DBDialect.SQLITE; + private final DBDialect mapDBDialect = DBDialect.SQLITE; public static final int BATCH_SIZE = 5000; public static final int BATCH_SIZE_OSM = 10000; @@ -83,7 +84,7 @@ public class IndexCreator { private boolean deleteOsmDB = false; private boolean deleteDatabaseIndexes = true; - private Connection dbConn; + private Object dbConn; private File dbFile; private File mapFile; @@ -135,7 +136,7 @@ public class IndexCreator { this.regionName = regionName; } - private Connection getDatabaseConnection(String fileName) throws SQLException { + private Object getDatabaseConnection(String fileName, DBDialect dialect) throws SQLException { return dialect.getDatabaseConnection(fileName, log); } @@ -267,7 +268,7 @@ public class IndexCreator { storage.parseOSM(stream, progress, streamFile, false); } dbCreator.finishLoading(); - dbConn.commit(); + dialect.commitDatabase(dbConn); if (log.isInfoEnabled()) { log.info("File parsed : " + (System.currentTimeMillis() - st)); //$NON-NLS-1$ @@ -291,7 +292,7 @@ public class IndexCreator { dialect.removeDatabase(dbFile); } } - dbConn = getDatabaseConnection(dbFile.getAbsolutePath()); + dbConn = getDatabaseConnection(dbFile.getAbsolutePath(), dialect); int allRelations = 100000; int allWays = 1000000; int allNodes = 10000000; @@ -303,7 +304,7 @@ public class IndexCreator { allRelations = dbCreator.getAllRelations(); } } - accessor.initDatabase(dbConn, allNodes, allWays, allRelations); + accessor.initDatabase(dbConn, dialect, allNodes, allWays, allRelations); return loadFromExistingFile; } @@ -314,23 +315,23 @@ public class IndexCreator { // to save space mapFile.getParentFile().mkdirs(); File tempDBMapFile = new File(workingDir, getTempMapDBFileName()); - dialect.removeDatabase(tempDBMapFile); - mapConnection = getDatabaseConnection(tempDBMapFile.getAbsolutePath()); + mapDBDialect.removeDatabase(tempDBMapFile); + mapConnection = (Connection) getDatabaseConnection(tempDBMapFile.getAbsolutePath(), mapDBDialect); mapConnection.setAutoCommit(false); } // 2.2 create rtree map if (indexMap) { - indexMapCreator.createDatabaseStructure(mapConnection, dialect, getRTreeMapIndexNonPackFileName()); + indexMapCreator.createDatabaseStructure(mapConnection, mapDBDialect, getRTreeMapIndexNonPackFileName()); } if (indexAddress) { - indexAddressCreator.createDatabaseStructure(mapConnection, dialect); + indexAddressCreator.createDatabaseStructure(mapConnection, mapDBDialect); } if (indexPOI) { indexPoiCreator.createDatabaseStructure(new File(workingDir, getPoiFileName())); } if (indexTransport) { - indexTransportCreator.createDatabaseStructure(mapConnection, dialect, getRTreeTransportStopsFileName()); + indexTransportCreator.createDatabaseStructure(mapConnection, mapDBDialect, getRTreeTransportStopsFileName()); } } @@ -379,7 +380,7 @@ public class IndexCreator { if (recreateOnlyBinaryFile) { mapFile = new File(workingDir, getMapFileName()); File tempDBMapFile = new File(workingDir, getTempMapDBFileName()); - mapConnection = getDatabaseConnection(tempDBMapFile.getAbsolutePath()); + mapConnection = (Connection) getDatabaseConnection(tempDBMapFile.getAbsolutePath(), mapDBDialect); mapConnection.setAutoCommit(false); try { if (indexMap) { @@ -559,18 +560,15 @@ public class IndexCreator { mapConnection.close(); mapConnection = null; File tempDBFile = new File(workingDir, getTempMapDBFileName()); - if (dialect.databaseFileExists(tempDBFile) && deleteDatabaseIndexes) { + if (mapDBDialect.databaseFileExists(tempDBFile) && deleteDatabaseIndexes) { // do not delete it for now - dialect.removeDatabase(tempDBFile); + mapDBDialect.removeDatabase(tempDBFile); } } // do not delete first db connection if (dbConn != null) { - if (DBDialect.H2 == dialect) { - dbConn.createStatement().execute("SHUTDOWN COMPACT"); //$NON-NLS-1$ - } - dbConn.close(); + dialect.commitDatabase(dbConn); } if (deleteOsmDB) { if (DBDialect.DERBY == dialect) { @@ -584,6 +582,8 @@ public class IndexCreator { } } catch (SQLException e) { e.printStackTrace(); + } catch (RuntimeException e) { + e.printStackTrace(); } } } @@ -594,9 +594,9 @@ public class IndexCreator { long time = System.currentTimeMillis(); IndexCreator creator = new IndexCreator(new File("/home/victor/projects/OsmAnd/data/osm-gen/")); //$NON-NLS-1$ creator.setIndexMap(true); -// creator.setIndexAddress(true); -// creator.setIndexPOI(true); -// creator.setIndexTransport(true); + creator.setIndexAddress(true); + creator.setIndexPOI(true); + creator.setIndexTransport(true); // for NL // creator.setCityAdminLevel("10"); @@ -605,9 +605,9 @@ public class IndexCreator { // creator.deleteOsmDB = true; creator.setZoomWaySmothness(2); - MapRenderingTypes rt = new MapRenderingTypes("/home/victor/projects/OsmAnd/data/testdata/roads_rendering_types.xml"); - MapZooms zooms = MapZooms.parseZooms("15-"); - creator.setNodesDBFile(new File("/home/victor/projects/OsmAnd/data/osm-gen/nodes.tmp.odb")); + MapRenderingTypes rt = MapRenderingTypes.getDefault();// new MapRenderingTypes("/home/victor/projects/OsmAnd/data/testdata/roads_rendering_types.xml"); + MapZooms zooms = MapZooms.getDefault(); // MapZooms.parseZooms("15-"); +// creator.setNodesDBFile(new File("/home/victor/projects/OsmAnd/data/osm-gen/nodes.tmp.odb")); creator.generateIndexes(new File("/home/victor/projects/OsmAnd/data/belarus-osm/belarus.osm.pbf"), new ConsoleProgressImplementation(1), null, zooms, rt); diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/IndexPoiCreator.java b/DataExtractionOSM/src/net/osmand/data/preparation/IndexPoiCreator.java index b2e9618fad..624cebd5e7 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/IndexPoiCreator.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/IndexPoiCreator.java @@ -114,7 +114,7 @@ public class IndexPoiCreator extends AbstractIndexPartCreator { } poiIndexFile.getParentFile().mkdirs(); // creating connection - poiConnection = DBDialect.SQLITE.getDatabaseConnection(poiIndexFile.getAbsolutePath(), log); + poiConnection = (Connection) DBDialect.SQLITE.getDatabaseConnection(poiIndexFile.getAbsolutePath(), log); // create database structure Statement stat = poiConnection.createStatement(); diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/OsmDbAccessor.java b/DataExtractionOSM/src/net/osmand/data/preparation/OsmDbAccessor.java index 5fda034a22..cbdf8b00b9 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/OsmDbAccessor.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/OsmDbAccessor.java @@ -17,6 +17,9 @@ import net.osmand.osm.Way; import net.osmand.osm.Entity.EntityId; import net.osmand.osm.Entity.EntityType; +import com.anvisics.jleveldb.ext.DBAccessor; +import com.anvisics.jleveldb.ext.ReadOptions; + public class OsmDbAccessor implements OsmDbAccessorContext { private PreparedStatement pselectNode; @@ -27,6 +30,9 @@ public class OsmDbAccessor implements OsmDbAccessorContext { private int allWays; private int allNodes; private Connection dbConn; + private DBDialect dialect; + private DBAccessor accessor; + private ReadOptions opts; public interface OsmDbVisitor { public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException; @@ -35,19 +41,26 @@ public class OsmDbAccessor implements OsmDbAccessorContext { public OsmDbAccessor(){ } - public void initDatabase(Connection dbConn, int allNodes, int allWays, int allRelations) throws SQLException { - this.dbConn = dbConn; - this.allNodes = allNodes; - this.allWays = allWays; - this.allRelations = allRelations; - pselectNode = dbConn.prepareStatement("select n.latitude, n.longitude, t.skeys, t.value from node n left join tags t on n.id = t.id and t.type = 0 where n.id = ?"); //$NON-NLS-1$ - pselectWay = dbConn.prepareStatement("select w.node, w.ord, t.skeys, t.value, n.latitude, n.longitude " + //$NON-NLS-1$ - "from ways w left join tags t on w.id = t.id and t.type = 1 and w.ord = 0 inner join node n on w.node = n.id " + //$NON-NLS-1$ - "where w.id = ? order by w.ord"); //$NON-NLS-1$ - pselectRelation = dbConn.prepareStatement("select r.member, r.type, r.role, r.ord, t.skeys, t.value " + //$NON-NLS-1$ - "from relations r left join tags t on r.id = t.id and t.type = 2 and r.ord = 0 " + //$NON-NLS-1$ - "where r.id = ? order by r.ord"); //$NON-NLS-1$ - pselectTags = dbConn.prepareStatement("select skeys, value from tags where id = ? and type = ?"); //$NON-NLS-1$ + public void initDatabase(Object dbConnection, DBDialect dialect, int allNodes, int allWays, int allRelations) throws SQLException { + + this.dialect = dialect; + if(this.dialect == DBDialect.NOSQL){ + opts = new ReadOptions(); + accessor = (DBAccessor) dbConnection; + } else { + this.dbConn = (Connection) dbConnection; + this.allNodes = allNodes; + this.allWays = allWays; + this.allRelations = allRelations; + pselectNode = dbConn.prepareStatement("select n.latitude, n.longitude, t.skeys, t.value from node n left join tags t on n.id = t.id and t.type = 0 where n.id = ?"); //$NON-NLS-1$ + pselectWay = dbConn.prepareStatement("select w.node, w.ord, t.skeys, t.value, n.latitude, n.longitude " + //$NON-NLS-1$ + "from ways w left join tags t on w.id = t.id and t.type = 1 and w.ord = 0 inner join node n on w.node = n.id " + //$NON-NLS-1$ + "where w.id = ? order by w.ord"); //$NON-NLS-1$ + pselectRelation = dbConn.prepareStatement("select r.member, r.type, r.role, r.ord, t.skeys, t.value " + //$NON-NLS-1$ + "from relations r left join tags t on r.id = t.id and t.type = 2 and r.ord = 0 " + //$NON-NLS-1$ + "where r.id = ? order by r.ord"); //$NON-NLS-1$ + pselectTags = dbConn.prepareStatement("select skeys, value from tags where id = ? and type = ?"); //$NON-NLS-1$ + } } public int getAllNodes() { @@ -67,6 +80,10 @@ public class OsmDbAccessor implements OsmDbAccessorContext { // do not load tags for nodes inside way return; } + if(dialect == DBDialect.NOSQL){ + loadEntityDataNoSQL(e, loadTags); + } + Map map = new LinkedHashMap(); if (e instanceof Relation && ((Relation) e).getMemberIds().isEmpty()) { pselectRelation.setLong(1, e.getId()); @@ -169,7 +186,15 @@ public class OsmDbAccessor implements OsmDbAccessorContext { } + private void loadEntityDataNoSQL(Entity e, boolean loadTags) { + // TODO Auto-generated method stub + + } + public int iterateOverEntities(IProgress progress, EntityType type, OsmDbVisitor visitor) throws SQLException { + if(dialect == DBDialect.NOSQL){ + iterateOverEntitiesNoSQL(progress, type, visitor); + } Statement statement = dbConn.createStatement(); String select; int count = 0; @@ -245,32 +270,29 @@ public class OsmDbAccessor implements OsmDbAccessorContext { return count; } - public void loadEntityTags(EntityType type, Entity e) throws SQLException { - pselectTags.setLong(1, e.getId()); - pselectTags.setByte(2, (byte) type.ordinal()); - ResultSet rsTags = pselectTags.executeQuery(); - while (rsTags.next()) { - e.putTag(rsTags.getString(1), rsTags.getString(2)); - } - rsTags.close(); + + private void iterateOverEntitiesNoSQL(IProgress progress, EntityType type, OsmDbVisitor visitor) { + // TODO Auto-generated method stub + } public void closeReadingConnection() throws SQLException { - if (pselectNode != null) { - pselectNode.close(); - } - if (pselectWay != null) { - pselectWay.close(); - } - if (pselectRelation != null) { - pselectRelation.close(); - } - if (pselectTags != null) { - pselectTags.close(); + if (dialect != DBDialect.NOSQL) { + if (pselectNode != null) { + pselectNode.close(); + } + if (pselectWay != null) { + pselectWay.close(); + } + if (pselectRelation != null) { + pselectRelation.close(); + } + if (pselectTags != null) { + pselectTags.close(); + } } } - diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/OsmDbAccessorContext.java b/DataExtractionOSM/src/net/osmand/data/preparation/OsmDbAccessorContext.java index aed31b36f0..e2780a577d 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/OsmDbAccessorContext.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/OsmDbAccessorContext.java @@ -7,7 +7,5 @@ import net.osmand.osm.Entity.EntityType; public interface OsmDbAccessorContext { - public void loadEntityTags(EntityType type, Entity e) throws SQLException; - public void loadEntityData(Entity e, boolean loadTags) throws SQLException; } \ No newline at end of file diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/OsmDbCreator.java b/DataExtractionOSM/src/net/osmand/data/preparation/OsmDbCreator.java index 95b96a9b82..2ad4c1694b 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/OsmDbCreator.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/OsmDbCreator.java @@ -2,12 +2,8 @@ package net.osmand.data.preparation; import java.sql.Connection; import java.sql.PreparedStatement; -import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.util.Collection; -import java.util.LinkedHashMap; -import java.util.Map; import java.util.Map.Entry; import net.osmand.osm.Entity; @@ -22,11 +18,15 @@ import net.osmand.osm.io.OsmBaseStorage; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import com.anvisics.jleveldb.ext.DBAccessor; +import com.anvisics.jleveldb.ext.DBWriteBatch; +import com.anvisics.jleveldb.ext.WriteOptions; + public class OsmDbCreator implements IOsmStorageFilter { private static final Log log = LogFactory.getLog(OsmDbCreator.class); - public static final int BATCH_SIZE_OSM = 10000; + public static final int BATCH_SIZE_OSM = 100000; DBDialect dialect; @@ -47,122 +47,160 @@ public class OsmDbCreator implements IOsmStorageFilter { private Connection dbConn; private final IndexCreator indexCreator; + private DBAccessor database; + private DBWriteBatch batch; + private WriteOptions options; + public OsmDbCreator(IndexCreator indexCreator) { this.indexCreator = indexCreator; } - public void initDatabase(DBDialect dialect, Connection dbConn) throws SQLException { - this.dbConn = dbConn; + public void initDatabase(DBDialect dialect, Object databaseConn) throws SQLException { + this.dialect = dialect; - // prepare tables - Statement stat = dbConn.createStatement(); - dialect.deleteTableIfExists("node", stat); - stat.executeUpdate("create table node (id bigint primary key, latitude double, longitude double)"); //$NON-NLS-1$ - stat.executeUpdate("create index IdIndex ON node (id)"); //$NON-NLS-1$ - dialect.deleteTableIfExists("ways", stat); - stat.executeUpdate("create table ways (id bigint, node bigint, ord smallint, primary key (id, ord))"); //$NON-NLS-1$ - stat.executeUpdate("create index IdWIndex ON ways (id)"); //$NON-NLS-1$ - dialect.deleteTableIfExists("relations", stat); - stat.executeUpdate("create table relations (id bigint, member bigint, type smallint, role varchar(255), ord smallint, primary key (id, ord))"); //$NON-NLS-1$ - stat.executeUpdate("create index IdRIndex ON relations (id)"); //$NON-NLS-1$ - dialect.deleteTableIfExists("tags", stat); - stat.executeUpdate("create table tags (id bigint, type smallint, skeys varchar(255), value varchar(255), primary key (id, type, skeys))"); //$NON-NLS-1$ - stat.executeUpdate("create index IdTIndex ON tags (id, type)"); //$NON-NLS-1$ - stat.close(); + if(dialect == DBDialect.NOSQL){ + database = (DBAccessor) databaseConn; + batch = new DBWriteBatch(); + options = new WriteOptions(); + } else { + this.dbConn = (Connection) databaseConn; + // prepare tables + Statement stat = dbConn.createStatement(); + dialect.deleteTableIfExists("node", stat); + stat.executeUpdate("create table node (id bigint primary key, latitude double, longitude double)"); //$NON-NLS-1$ + stat.executeUpdate("create index IdIndex ON node (id)"); //$NON-NLS-1$ + dialect.deleteTableIfExists("ways", stat); + stat.executeUpdate("create table ways (id bigint, node bigint, ord smallint, primary key (id, ord))"); //$NON-NLS-1$ + stat.executeUpdate("create index IdWIndex ON ways (id)"); //$NON-NLS-1$ + dialect.deleteTableIfExists("relations", stat); + stat.executeUpdate("create table relations (id bigint, member bigint, type smallint, role varchar(255), ord smallint, primary key (id, ord))"); //$NON-NLS-1$ + stat.executeUpdate("create index IdRIndex ON relations (id)"); //$NON-NLS-1$ + dialect.deleteTableIfExists("tags", stat); + stat.executeUpdate("create table tags (id bigint, type smallint, skeys varchar(255), value varchar(255), primary key (id, type, skeys))"); //$NON-NLS-1$ + stat.executeUpdate("create index IdTIndex ON tags (id, type)"); //$NON-NLS-1$ + stat.close(); - prepNode = dbConn.prepareStatement("insert into node values (?, ?, ?)"); //$NON-NLS-1$ - prepWays = dbConn.prepareStatement("insert into ways values (?, ?, ?)"); //$NON-NLS-1$ - prepRelations = dbConn.prepareStatement("insert into relations values (?, ?, ?, ?, ?)"); //$NON-NLS-1$ - prepTags = dbConn.prepareStatement("insert into tags values (?, ?, ?, ?)"); //$NON-NLS-1$ - dbConn.setAutoCommit(false); + prepNode = dbConn.prepareStatement("insert into node values (?, ?, ?)"); //$NON-NLS-1$ + prepWays = dbConn.prepareStatement("insert into ways values (?, ?, ?)"); //$NON-NLS-1$ + prepRelations = dbConn.prepareStatement("insert into relations values (?, ?, ?, ?, ?)"); //$NON-NLS-1$ + prepTags = dbConn.prepareStatement("insert into tags values (?, ?, ?, ?)"); //$NON-NLS-1$ + dbConn.setAutoCommit(false); + } } public void finishLoading() throws SQLException { - if (currentCountNode > 0) { - prepNode.executeBatch(); + if (dialect != DBDialect.NOSQL) { + if (currentCountNode > 0) { + prepNode.executeBatch(); + } + prepNode.close(); + if (currentWaysCount > 0) { + prepWays.executeBatch(); + } + prepWays.close(); + if (currentRelationsCount > 0) { + prepRelations.executeBatch(); + } + prepRelations.close(); + if (currentTagsCount > 0) { + prepTags.executeBatch(); + } + prepTags.close(); + } else { + database.Write(options, batch); } - prepNode.close(); - if (currentWaysCount > 0) { - prepWays.executeBatch(); - } - prepWays.close(); - if (currentRelationsCount > 0) { - prepRelations.executeBatch(); - } - prepRelations.close(); - if (currentTagsCount > 0) { - prepTags.executeBatch(); - } - prepTags.close(); } @Override public boolean acceptEntityToLoad(OsmBaseStorage storage, EntityId entityId, Entity e) { // put all nodes into temporary db to get only required nodes after loading all data - try { + if(dialect == DBDialect.NOSQL){ + batch.Put(e.getId()+"", e.getTags() +""); + + currentCountNode++; if (e instanceof Node) { - currentCountNode++; if (!e.getTags().isEmpty()) { allNodes++; } - prepNode.setLong(1, e.getId()); - prepNode.setDouble(2, ((Node) e).getLatitude()); - prepNode.setDouble(3, ((Node) e).getLongitude()); - prepNode.addBatch(); - if (currentCountNode >= BATCH_SIZE_OSM) { - prepNode.executeBatch(); - dbConn.commit(); // clear memory - currentCountNode = 0; - } } else if (e instanceof Way) { allWays++; - short ord = 0; - for (Long i : ((Way) e).getNodeIds()) { - currentWaysCount++; - prepWays.setLong(1, e.getId()); - prepWays.setLong(2, i); - prepWays.setLong(3, ord++); - prepWays.addBatch(); - } - if (currentWaysCount >= BATCH_SIZE_OSM) { - prepWays.executeBatch(); - dbConn.commit(); // clear memory - currentWaysCount = 0; - } } else { - allRelations++; - short ord = 0; - for (Entry i : ((Relation) e).getMembersMap().entrySet()) { - currentRelationsCount++; - prepRelations.setLong(1, e.getId()); - prepRelations.setLong(2, i.getKey().getId()); - prepRelations.setLong(3, i.getKey().getType().ordinal()); - prepRelations.setString(4, i.getValue()); - prepRelations.setLong(5, ord++); - prepRelations.addBatch(); + allRelations ++; + } + if(currentCountNode > BATCH_SIZE_OSM){ + database.Write(options, batch); + batch = new DBWriteBatch(); + long usedMemory = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); + System.out.println(""+Runtime.getRuntime().totalMemory()/(1024*1024) +" MB Total " + + (usedMemory / (1024*1024)) + " MB used memory"); + currentCountNode = 0; + } + } else { + try { + if (e instanceof Node) { + currentCountNode++; + if (!e.getTags().isEmpty()) { + allNodes++; + } + prepNode.setLong(1, e.getId()); + prepNode.setDouble(2, ((Node) e).getLatitude()); + prepNode.setDouble(3, ((Node) e).getLongitude()); + prepNode.addBatch(); + if (currentCountNode >= BATCH_SIZE_OSM) { + prepNode.executeBatch(); + dbConn.commit(); // clear memory + currentCountNode = 0; + } + } else if (e instanceof Way) { + allWays++; + short ord = 0; + for (Long i : ((Way) e).getNodeIds()) { + currentWaysCount++; + prepWays.setLong(1, e.getId()); + prepWays.setLong(2, i); + prepWays.setLong(3, ord++); + prepWays.addBatch(); + } + if (currentWaysCount >= BATCH_SIZE_OSM) { + prepWays.executeBatch(); + dbConn.commit(); // clear memory + currentWaysCount = 0; + } + } else { + allRelations++; + short ord = 0; + for (Entry i : ((Relation) e).getMembersMap().entrySet()) { + currentRelationsCount++; + prepRelations.setLong(1, e.getId()); + prepRelations.setLong(2, i.getKey().getId()); + prepRelations.setLong(3, i.getKey().getType().ordinal()); + prepRelations.setString(4, i.getValue()); + prepRelations.setLong(5, ord++); + prepRelations.addBatch(); + } + if (currentRelationsCount >= BATCH_SIZE_OSM) { + prepRelations.executeBatch(); + dbConn.commit(); // clear memory + currentRelationsCount = 0; + } } - if (currentRelationsCount >= BATCH_SIZE_OSM) { - prepRelations.executeBatch(); + for (Entry i : e.getTags().entrySet()) { + currentTagsCount++; + prepTags.setLong(1, e.getId()); + prepTags.setLong(2, EntityType.valueOf(e).ordinal()); + prepTags.setString(3, i.getKey()); + prepTags.setString(4, i.getValue()); + prepTags.addBatch(); + } + if (currentTagsCount >= BATCH_SIZE_OSM) { + prepTags.executeBatch(); dbConn.commit(); // clear memory - currentRelationsCount = 0; + currentTagsCount = 0; } + } catch (SQLException ex) { + log.error("Could not save in db", ex); //$NON-NLS-1$ } - for (Entry i : e.getTags().entrySet()) { - currentTagsCount++; - prepTags.setLong(1, e.getId()); - prepTags.setLong(2, EntityType.valueOf(e).ordinal()); - prepTags.setString(3, i.getKey()); - prepTags.setString(4, i.getValue()); - prepTags.addBatch(); - } - if (currentTagsCount >= BATCH_SIZE_OSM) { - prepTags.executeBatch(); - dbConn.commit(); // clear memory - currentTagsCount = 0; - } - } catch (SQLException ex) { - log.error("Could not save in db", ex); //$NON-NLS-1$ } // do not add to storage return false;