Android-builders-club/OsmAnd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Add nosql database dialect

Browse source
This commit is contained in:
Victor Shcherb 2011-08-02 01:06:31 +02:00
parent 06a2aebcbd
commit 56208ea712
8 changed files with 251 additions and 158 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
DataExtractionOSM/.classpath
View file

@ -9,5 +9,7 @@
<classpathentry kind="lib" path="lib/json-20090211.jar"/>
<classpathentry kind="lib" path="lib/h2-latest.jar"/>
<classpathentry kind="lib" path="lib/bsh-core-2.0b4.jar"/>
<classpathentry kind="lib" path="lib/orient-commons-1.0rc3.jar"/>
<classpathentry kind="lib" path="lib/jleveldb.jar" sourcepath="/JavaLevelDB"/>
<classpathentry kind="output" path="bin"/>
</classpath>

BIN
DataExtractionOSM/lib/jleveldb.jar Normal file
View file

Binary file not shown.

41
DataExtractionOSM/src/net/osmand/data/preparation/DBDialect.java
View file

@ -6,13 +6,19 @@ import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
import net.osmand.Algoritms;
import org.apache.commons.logging.Log;
import net.osmand.Algoritms;
import com.anvisics.jleveldb.LevelDBAccess;
import com.anvisics.jleveldb.ext.DBAccessor;
import com.anvisics.jleveldb.ext.Options;
import com.anvisics.jleveldb.ext.Status;
public enum DBDialect {
DERBY,
H2,
NOSQL,
SQLITE;
public void deleteTableIfExists(String table, Statement stat) throws SQLException {
@ -47,11 +53,38 @@ public enum DBDialect {
} else {
Algoritms.removeAllFiles(file);
}
}
protected Connection getDatabaseConnection(String fileName, Log log) throws SQLException {
if (DBDialect.SQLITE == this) {
public void commitDatabase(Object connection) throws SQLException {
if(DBDialect.NOSQL != this){
((Connection) connection).commit();
} else {
}
}
public void closeDatabase(Object dbConn) throws SQLException {
if(DBDialect.NOSQL != this){
if (DBDialect.H2 == this) {
((Connection) dbConn).createStatement().execute("SHUTDOWN COMPACT"); //$NON-NLS-1$
}
((Connection) dbConn).close();
} else {
// ((DBAccessor) dbConn).close();
}
}
protected Object getDatabaseConnection(String fileName, Log log) throws SQLException {
if (DBDialect.NOSQL == this) {
DBAccessor dbAccessor = LevelDBAccess.getDBAcessor();
Options opts = new Options();
opts.setCreate_if_missing(true);
Status status = dbAccessor.Open(opts, fileName);
if(!status.ok()){
throw new SQLException(status.ToString());
}
return dbAccessor;
} else if (DBDialect.SQLITE == this) {
try {
Class.forName("org.sqlite.JDBC");
} catch (ClassNotFoundException e) {

46
DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java
View file

@ -47,6 +47,7 @@ public class IndexCreator {
// ONLY derby.jar needed for derby dialect
private final DBDialect dialect = DBDialect.SQLITE;
private final DBDialect mapDBDialect = DBDialect.SQLITE;
public static final int BATCH_SIZE = 5000;
public static final int BATCH_SIZE_OSM = 10000;
@ -83,7 +84,7 @@ public class IndexCreator {
private boolean deleteOsmDB = false;
private boolean deleteDatabaseIndexes = true;
private Connection dbConn;
private Object dbConn;
private File dbFile;
private File mapFile;
@ -135,7 +136,7 @@ public class IndexCreator {
this.regionName = regionName;
}
private Connection getDatabaseConnection(String fileName) throws SQLException {
private Object getDatabaseConnection(String fileName, DBDialect dialect) throws SQLException {
return dialect.getDatabaseConnection(fileName, log);
}
@ -267,7 +268,7 @@ public class IndexCreator {
storage.parseOSM(stream, progress, streamFile, false);
}
dbCreator.finishLoading();
dbConn.commit();
dialect.commitDatabase(dbConn);
if (log.isInfoEnabled()) {
log.info("File parsed : " + (System.currentTimeMillis() - st)); //$NON-NLS-1$
@ -291,7 +292,7 @@ public class IndexCreator {
dialect.removeDatabase(dbFile);
}
}
dbConn = getDatabaseConnection(dbFile.getAbsolutePath());
dbConn = getDatabaseConnection(dbFile.getAbsolutePath(), dialect);
int allRelations = 100000;
int allWays = 1000000;
int allNodes = 10000000;
@ -303,7 +304,7 @@ public class IndexCreator {
allRelations = dbCreator.getAllRelations();
}
}
accessor.initDatabase(dbConn, allNodes, allWays, allRelations);
accessor.initDatabase(dbConn, dialect, allNodes, allWays, allRelations);
return loadFromExistingFile;
}
@ -314,23 +315,23 @@ public class IndexCreator {
// to save space
mapFile.getParentFile().mkdirs();
File tempDBMapFile = new File(workingDir, getTempMapDBFileName());
dialect.removeDatabase(tempDBMapFile);
mapConnection = getDatabaseConnection(tempDBMapFile.getAbsolutePath());
mapDBDialect.removeDatabase(tempDBMapFile);
mapConnection = (Connection) getDatabaseConnection(tempDBMapFile.getAbsolutePath(), mapDBDialect);
mapConnection.setAutoCommit(false);
}
// 2.2 create rtree map
if (indexMap) {
indexMapCreator.createDatabaseStructure(mapConnection, dialect, getRTreeMapIndexNonPackFileName());
indexMapCreator.createDatabaseStructure(mapConnection, mapDBDialect, getRTreeMapIndexNonPackFileName());
}
if (indexAddress) {
indexAddressCreator.createDatabaseStructure(mapConnection, dialect);
indexAddressCreator.createDatabaseStructure(mapConnection, mapDBDialect);
}
if (indexPOI) {
indexPoiCreator.createDatabaseStructure(new File(workingDir, getPoiFileName()));
}
if (indexTransport) {
indexTransportCreator.createDatabaseStructure(mapConnection, dialect, getRTreeTransportStopsFileName());
indexTransportCreator.createDatabaseStructure(mapConnection, mapDBDialect, getRTreeTransportStopsFileName());
}
}
@ -379,7 +380,7 @@ public class IndexCreator {
if (recreateOnlyBinaryFile) {
mapFile = new File(workingDir, getMapFileName());
File tempDBMapFile = new File(workingDir, getTempMapDBFileName());
mapConnection = getDatabaseConnection(tempDBMapFile.getAbsolutePath());
mapConnection = (Connection) getDatabaseConnection(tempDBMapFile.getAbsolutePath(), mapDBDialect);
mapConnection.setAutoCommit(false);
try {
if (indexMap) {
@ -559,18 +560,15 @@ public class IndexCreator {
mapConnection.close();
mapConnection = null;
File tempDBFile = new File(workingDir, getTempMapDBFileName());
if (dialect.databaseFileExists(tempDBFile) && deleteDatabaseIndexes) {
if (mapDBDialect.databaseFileExists(tempDBFile) && deleteDatabaseIndexes) {
// do not delete it for now
dialect.removeDatabase(tempDBFile);
mapDBDialect.removeDatabase(tempDBFile);
}
}
// do not delete first db connection
if (dbConn != null) {
if (DBDialect.H2 == dialect) {
dbConn.createStatement().execute("SHUTDOWN COMPACT"); //$NON-NLS-1$
}
dbConn.close();
dialect.commitDatabase(dbConn);
}
if (deleteOsmDB) {
if (DBDialect.DERBY == dialect) {
@ -584,6 +582,8 @@ public class IndexCreator {
}
} catch (SQLException e) {
e.printStackTrace();
} catch (RuntimeException e) {
e.printStackTrace();
}
}
}
@ -594,9 +594,9 @@ public class IndexCreator {
long time = System.currentTimeMillis();
IndexCreator creator = new IndexCreator(new File("/home/victor/projects/OsmAnd/data/osm-gen/")); //$NON-NLS-1$
creator.setIndexMap(true);
// creator.setIndexAddress(true);
// creator.setIndexPOI(true);
// creator.setIndexTransport(true);
creator.setIndexAddress(true);
creator.setIndexPOI(true);
creator.setIndexTransport(true);
// for NL
// creator.setCityAdminLevel("10");
@ -605,9 +605,9 @@ public class IndexCreator {
// creator.deleteOsmDB = true;
creator.setZoomWaySmothness(2);
MapRenderingTypes rt = new MapRenderingTypes("/home/victor/projects/OsmAnd/data/testdata/roads_rendering_types.xml");
MapZooms zooms = MapZooms.parseZooms("15-");
creator.setNodesDBFile(new File("/home/victor/projects/OsmAnd/data/osm-gen/nodes.tmp.odb"));
MapRenderingTypes rt = MapRenderingTypes.getDefault();// new MapRenderingTypes("/home/victor/projects/OsmAnd/data/testdata/roads_rendering_types.xml");
MapZooms zooms = MapZooms.getDefault(); // MapZooms.parseZooms("15-");
// creator.setNodesDBFile(new File("/home/victor/projects/OsmAnd/data/osm-gen/nodes.tmp.odb"));
creator.generateIndexes(new File("/home/victor/projects/OsmAnd/data/belarus-osm/belarus.osm.pbf"),
new ConsoleProgressImplementation(1), null, zooms, rt);

2
DataExtractionOSM/src/net/osmand/data/preparation/IndexPoiCreator.java
View file

@ -114,7 +114,7 @@ public class IndexPoiCreator extends AbstractIndexPartCreator {
}
poiIndexFile.getParentFile().mkdirs();
// creating connection
poiConnection = DBDialect.SQLITE.getDatabaseConnection(poiIndexFile.getAbsolutePath(), log);
poiConnection = (Connection) DBDialect.SQLITE.getDatabaseConnection(poiIndexFile.getAbsolutePath(), log);
// create database structure
Statement stat = poiConnection.createStatement();

44
DataExtractionOSM/src/net/osmand/data/preparation/OsmDbAccessor.java
View file

@ -17,6 +17,9 @@ import net.osmand.osm.Way;
import net.osmand.osm.Entity.EntityId;
import net.osmand.osm.Entity.EntityType;
import com.anvisics.jleveldb.ext.DBAccessor;
import com.anvisics.jleveldb.ext.ReadOptions;
public class OsmDbAccessor implements OsmDbAccessorContext {
private PreparedStatement pselectNode;
@ -27,6 +30,9 @@ public class OsmDbAccessor implements OsmDbAccessorContext {
private int allWays;
private int allNodes;
private Connection dbConn;
private DBDialect dialect;
private DBAccessor accessor;
private ReadOptions opts;
public interface OsmDbVisitor {
public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException;
@ -35,8 +41,14 @@ public class OsmDbAccessor implements OsmDbAccessorContext {
public OsmDbAccessor(){
}
public void initDatabase(Connection dbConn, int allNodes, int allWays, int allRelations) throws SQLException {
this.dbConn = dbConn;
public void initDatabase(Object dbConnection, DBDialect dialect, int allNodes, int allWays, int allRelations) throws SQLException {
this.dialect = dialect;
if(this.dialect == DBDialect.NOSQL){
opts = new ReadOptions();
accessor = (DBAccessor) dbConnection;
} else {
this.dbConn = (Connection) dbConnection;
this.allNodes = allNodes;
this.allWays = allWays;
this.allRelations = allRelations;
@ -49,6 +61,7 @@ public class OsmDbAccessor implements OsmDbAccessorContext {
"where r.id = ? order by r.ord"); //$NON-NLS-1$
pselectTags = dbConn.prepareStatement("select skeys, value from tags where id = ? and type = ?"); //$NON-NLS-1$
}
}
public int getAllNodes() {
return allNodes;
@ -67,6 +80,10 @@ public class OsmDbAccessor implements OsmDbAccessorContext {
// do not load tags for nodes inside way
return;
}
if(dialect == DBDialect.NOSQL){
loadEntityDataNoSQL(e, loadTags);
}
Map<EntityId, Entity> map = new LinkedHashMap<EntityId, Entity>();
if (e instanceof Relation && ((Relation) e).getMemberIds().isEmpty()) {
pselectRelation.setLong(1, e.getId());
@ -169,7 +186,15 @@ public class OsmDbAccessor implements OsmDbAccessorContext {
}
private void loadEntityDataNoSQL(Entity e, boolean loadTags) {
// TODO Auto-generated method stub
}
public int iterateOverEntities(IProgress progress, EntityType type, OsmDbVisitor visitor) throws SQLException {
if(dialect == DBDialect.NOSQL){
iterateOverEntitiesNoSQL(progress, type, visitor);
}
Statement statement = dbConn.createStatement();
String select;
int count = 0;
@ -245,17 +270,14 @@ public class OsmDbAccessor implements OsmDbAccessorContext {
return count;
}
public void loadEntityTags(EntityType type, Entity e) throws SQLException {
pselectTags.setLong(1, e.getId());
pselectTags.setByte(2, (byte) type.ordinal());
ResultSet rsTags = pselectTags.executeQuery();
while (rsTags.next()) {
e.putTag(rsTags.getString(1), rsTags.getString(2));
}
rsTags.close();
private void iterateOverEntitiesNoSQL(IProgress progress, EntityType type, OsmDbVisitor visitor) {
// TODO Auto-generated method stub
}
public void closeReadingConnection() throws SQLException {
if (dialect != DBDialect.NOSQL) {
if (pselectNode != null) {
pselectNode.close();
}
@ -268,10 +290,10 @@ public class OsmDbAccessor implements OsmDbAccessorContext {
if (pselectTags != null) {
pselectTags.close();
}
}
}
}

2
DataExtractionOSM/src/net/osmand/data/preparation/OsmDbAccessorContext.java
View file

@ -7,7 +7,5 @@ import net.osmand.osm.Entity.EntityType;
public interface OsmDbAccessorContext {
public void loadEntityTags(EntityType type, Entity e) throws SQLException;
public void loadEntityData(Entity e, boolean loadTags) throws SQLException;
}

52
DataExtractionOSM/src/net/osmand/data/preparation/OsmDbCreator.java
View file

@ -2,12 +2,8 @@ package net.osmand.data.preparation;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Map.Entry;
import net.osmand.osm.Entity;
@ -22,11 +18,15 @@ import net.osmand.osm.io.OsmBaseStorage;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.anvisics.jleveldb.ext.DBAccessor;
import com.anvisics.jleveldb.ext.DBWriteBatch;
import com.anvisics.jleveldb.ext.WriteOptions;
public class OsmDbCreator implements IOsmStorageFilter {
private static final Log log = LogFactory.getLog(OsmDbCreator.class);
public static final int BATCH_SIZE_OSM = 10000;
public static final int BATCH_SIZE_OSM = 100000;
DBDialect dialect;
@ -47,14 +47,24 @@ public class OsmDbCreator implements IOsmStorageFilter {
private Connection dbConn;
private final IndexCreator indexCreator;
private DBAccessor database;
private DBWriteBatch batch;
private WriteOptions options;
public OsmDbCreator(IndexCreator indexCreator) {
this.indexCreator = indexCreator;
}
public void initDatabase(DBDialect dialect, Connection dbConn) throws SQLException {
this.dbConn = dbConn;
public void initDatabase(DBDialect dialect, Object databaseConn) throws SQLException {
this.dialect = dialect;
if(dialect == DBDialect.NOSQL){
database = (DBAccessor) databaseConn;
batch = new DBWriteBatch();
options = new WriteOptions();
} else {
this.dbConn = (Connection) databaseConn;
// prepare tables
Statement stat = dbConn.createStatement();
dialect.deleteTableIfExists("node", stat);
@ -77,8 +87,10 @@ public class OsmDbCreator implements IOsmStorageFilter {
prepTags = dbConn.prepareStatement("insert into tags values (?, ?, ?, ?)"); //$NON-NLS-1$
dbConn.setAutoCommit(false);
}
}
public void finishLoading() throws SQLException {
if (dialect != DBDialect.NOSQL) {
if (currentCountNode > 0) {
prepNode.executeBatch();
}
@ -95,11 +107,36 @@ public class OsmDbCreator implements IOsmStorageFilter {
prepTags.executeBatch();
}
prepTags.close();
} else {
database.Write(options, batch);
}
}
@Override
public boolean acceptEntityToLoad(OsmBaseStorage storage, EntityId entityId, Entity e) {
// put all nodes into temporary db to get only required nodes after loading all data
if(dialect == DBDialect.NOSQL){
batch.Put(e.getId()+"", e.getTags() +"");
currentCountNode++;
if (e instanceof Node) {
if (!e.getTags().isEmpty()) {
allNodes++;
}
} else if (e instanceof Way) {
allWays++;
} else {
allRelations ++;
}
if(currentCountNode > BATCH_SIZE_OSM){
database.Write(options, batch);
batch = new DBWriteBatch();
long usedMemory = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
System.out.println(""+Runtime.getRuntime().totalMemory()/(1024*1024) +" MB Total " +
(usedMemory / (1024*1024)) + " MB used memory");
currentCountNode = 0;
}
} else {
try {
if (e instanceof Node) {
currentCountNode++;
@ -164,6 +201,7 @@ public class OsmDbCreator implements IOsmStorageFilter {
} catch (SQLException ex) {
log.error("Could not save in db", ex); //$NON-NLS-1$
}
}
// do not add to storage
return false;
}