Android-builders-club/OsmAnd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Remove deprecated scripts

Browse source
This commit is contained in:
Victor Shcherb 2012-11-15 15:08:14 +01:00
parent 2feef53dad
commit 01e49ed1db
3 changed files with 45 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
DataExtractionOSM/src/net/osmand/data/index/IndexBatchCreator.java
View file

@ -86,6 +86,9 @@ public class IndexBatchCreator {
private String wget; private String wget;
private DBDialect osmDbDialect;
private DBDialect mapDBDialect;
public static void main(String[] args) { public static void main(String[] args) {
IndexBatchCreator creator = new IndexBatchCreator(); IndexBatchCreator creator = new IndexBatchCreator();
OsmExtractionUI.configLogFile(); OsmExtractionUI.configLogFile();
@ -241,7 +244,7 @@ public class IndexBatchCreator {
String osmDbDialect = process.getAttribute("osmDbDialect"); String osmDbDialect = process.getAttribute("osmDbDialect");
if(osmDbDialect != null && osmDbDialect.length() > 0){ if(osmDbDialect != null && osmDbDialect.length() > 0){
try { try {
IndexCreator.dialect = DBDialect.valueOf(osmDbDialect.toUpperCase()); this.osmDbDialect = DBDialect.valueOf(osmDbDialect.toUpperCase());
} catch (RuntimeException e) { } catch (RuntimeException e) {
} }
} }
@ -249,7 +252,7 @@ public class IndexBatchCreator {
String mapDbDialect = process.getAttribute("mapDbDialect"); String mapDbDialect = process.getAttribute("mapDbDialect");
if (mapDbDialect != null && mapDbDialect.length() > 0) { if (mapDbDialect != null && mapDbDialect.length() > 0) {
try { try {
IndexCreator.mapDBDialect = DBDialect.valueOf(mapDbDialect.toUpperCase()); this.mapDBDialect = DBDialect.valueOf(mapDbDialect.toUpperCase());
} catch (RuntimeException e) { } catch (RuntimeException e) {
} }
} }
@ -439,8 +442,12 @@ public class IndexBatchCreator {
} else { } else {
rName = Algoritms.capitalizeFirstLetterAndLowercase(rName); rName = Algoritms.capitalizeFirstLetterAndLowercase(rName);
} }
DBDialect osmDb = this.osmDbDialect;
if(f.length() / 1024 / 1024 > 300 && osmDb == DBDialect.SQLITE_IN_MEMORY) {
osmDb = DBDialect.SQLITE;
}
IndexCreator indexCreator = new IndexCreator(workDir); IndexCreator indexCreator = new IndexCreator(workDir);
indexCreator.setDialects(osmDb, this.mapDBDialect);
indexCreator.setIndexAddress(indexAddress); indexCreator.setIndexAddress(indexAddress);
indexCreator.setIndexPOI(indexPOI); indexCreator.setIndexPOI(indexPOI);
indexCreator.setIndexTransport(indexTransport); indexCreator.setIndexTransport(indexTransport);

55
DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java
View file

@ -49,8 +49,8 @@ public class IndexCreator {
// ONLY derby.jar needed for derby dialect // ONLY derby.jar needed for derby dialect
// (NOSQL is the fastest but is supported only on linux 32) // (NOSQL is the fastest but is supported only on linux 32)
// Sqlite better to use only for 32-bit machines // Sqlite better to use only for 32-bit machines
public static DBDialect dialect = DBDialect.SQLITE; private DBDialect osmDBdialect = DBDialect.SQLITE;
public static DBDialect mapDBDialect = DBDialect.SQLITE; private DBDialect mapIndexDBDialect = DBDialect.SQLITE;
public static boolean REMOVE_POI_DB = true; public static boolean REMOVE_POI_DB = true;
public static final int BATCH_SIZE = 5000; public static final int BATCH_SIZE = 5000;
@ -164,6 +164,15 @@ public class IndexCreator {
return getMapFileName() + ".tmp"; //$NON-NLS-1$ return getMapFileName() + ".tmp"; //$NON-NLS-1$
} }
public void setDialects(DBDialect osmDBdialect, DBDialect mapIndexDBDialect) {
if(osmDBdialect != null) {
this.osmDBdialect = osmDBdialect;
}
if(mapIndexDBDialect != null) {
this.mapIndexDBDialect = mapIndexDBDialect;
}
}
public Long getLastModifiedDate() { public Long getLastModifiedDate() {
return lastModifiedDate; return lastModifiedDate;
} }
@ -268,7 +277,7 @@ public class IndexCreator {
progress.setGeneralProgress("[15 / 100]"); //$NON-NLS-1$ progress.setGeneralProgress("[15 / 100]"); //$NON-NLS-1$
progress.startTask(Messages.getString("IndexCreator.LOADING_FILE") + readFile.getAbsolutePath(), -1); //$NON-NLS-1$ progress.startTask(Messages.getString("IndexCreator.LOADING_FILE") + readFile.getAbsolutePath(), -1); //$NON-NLS-1$
// 1 init database to store temporary data // 1 init database to store temporary data
dbCreator.initDatabase(dialect, dbConn); dbCreator.initDatabase(osmDBdialect, dbConn);
storage.getFilters().add(dbCreator); storage.getFilters().add(dbCreator);
if (pbfFile) { if (pbfFile) {
storage.parseOSMPbf(stream, progress, false); storage.parseOSMPbf(stream, progress, false);
@ -276,7 +285,7 @@ public class IndexCreator {
storage.parseOSM(stream, progress, streamFile, false); storage.parseOSM(stream, progress, streamFile, false);
} }
dbCreator.finishLoading(); dbCreator.finishLoading();
dialect.commitDatabase(dbConn); osmDBdialect.commitDatabase(dbConn);
if (log.isInfoEnabled()) { if (log.isInfoEnabled()) {
log.info("File parsed : " + (System.currentTimeMillis() - st)); //$NON-NLS-1$ log.info("File parsed : " + (System.currentTimeMillis() - st)); //$NON-NLS-1$
@ -293,15 +302,15 @@ public class IndexCreator {
private boolean createPlainOsmDb(IProgress progress, File readFile, IOsmStorageFilter addFilter, boolean deletePrevious) throws SQLException, FileNotFoundException, IOException, SAXException{ private boolean createPlainOsmDb(IProgress progress, File readFile, IOsmStorageFilter addFilter, boolean deletePrevious) throws SQLException, FileNotFoundException, IOException, SAXException{
// dbFile = new File(workingDir, TEMP_NODES_DB); // dbFile = new File(workingDir, TEMP_NODES_DB);
// initialize db file // initialize db file
boolean loadFromExistingFile = dbFile != null && dialect.databaseFileExists(dbFile) && !deletePrevious; boolean loadFromExistingFile = dbFile != null && osmDBdialect.databaseFileExists(dbFile) && !deletePrevious;
if (dbFile == null || deletePrevious) { if (dbFile == null || deletePrevious) {
dbFile = new File(workingDir, TEMP_NODES_DB); dbFile = new File(workingDir, TEMP_NODES_DB);
// to save space // to save space
if (dialect.databaseFileExists(dbFile)) { if (osmDBdialect.databaseFileExists(dbFile)) {
dialect.removeDatabase(dbFile); osmDBdialect.removeDatabase(dbFile);
} }
} }
dbConn = getDatabaseConnection(dbFile.getAbsolutePath(), dialect); dbConn = getDatabaseConnection(dbFile.getAbsolutePath(), osmDBdialect);
int allRelations = 100000; int allRelations = 100000;
int allWays = 1000000; int allWays = 1000000;
int allNodes = 10000000; int allNodes = 10000000;
@ -313,7 +322,7 @@ public class IndexCreator {
allRelations = dbCreator.getAllRelations(); allRelations = dbCreator.getAllRelations();
} }
} else { } else {
if (DBDialect.NOSQL != dialect) { if (DBDialect.NOSQL != osmDBdialect) {
Connection dbc = (Connection) dbConn; Connection dbc = (Connection) dbConn;
final Statement stmt = dbc.createStatement(); final Statement stmt = dbc.createStatement();
accessor.computeRealCounts(stmt); accessor.computeRealCounts(stmt);
@ -323,7 +332,7 @@ public class IndexCreator {
stmt.close(); stmt.close();
} }
} }
accessor.initDatabase(dbConn, dialect, allNodes, allWays, allRelations); accessor.initDatabase(dbConn, osmDBdialect, allNodes, allWays, allRelations);
return loadFromExistingFile; return loadFromExistingFile;
} }
@ -333,25 +342,25 @@ public class IndexCreator {
// to save space // to save space
mapFile.getParentFile().mkdirs(); mapFile.getParentFile().mkdirs();
File tempDBMapFile = new File(workingDir, getTempMapDBFileName()); File tempDBMapFile = new File(workingDir, getTempMapDBFileName());
mapDBDialect.removeDatabase(tempDBMapFile); mapIndexDBDialect.removeDatabase(tempDBMapFile);
mapConnection = (Connection) getDatabaseConnection(tempDBMapFile.getAbsolutePath(), mapDBDialect); mapConnection = (Connection) getDatabaseConnection(tempDBMapFile.getAbsolutePath(), mapIndexDBDialect);
mapConnection.setAutoCommit(false); mapConnection.setAutoCommit(false);
// 2.2 create rtree map // 2.2 create rtree map
if (indexMap) { if (indexMap) {
indexMapCreator.createDatabaseStructure(mapConnection, mapDBDialect, getRTreeMapIndexNonPackFileName()); indexMapCreator.createDatabaseStructure(mapConnection, mapIndexDBDialect, getRTreeMapIndexNonPackFileName());
} }
if (indexRouting) { if (indexRouting) {
indexRouteCreator.createDatabaseStructure(mapConnection, mapDBDialect, getRTreeRouteIndexNonPackFileName()); indexRouteCreator.createDatabaseStructure(mapConnection, mapIndexDBDialect, getRTreeRouteIndexNonPackFileName());
} }
if (indexAddress) { if (indexAddress) {
indexAddressCreator.createDatabaseStructure(mapConnection, mapDBDialect); indexAddressCreator.createDatabaseStructure(mapConnection, mapIndexDBDialect);
} }
if (indexPOI) { if (indexPOI) {
indexPoiCreator.createDatabaseStructure(new File(workingDir, getPoiFileName())); indexPoiCreator.createDatabaseStructure(new File(workingDir, getPoiFileName()));
} }
if (indexTransport) { if (indexTransport) {
indexTransportCreator.createDatabaseStructure(mapConnection, mapDBDialect, getRTreeTransportStopsFileName()); indexTransportCreator.createDatabaseStructure(mapConnection, mapIndexDBDialect, getRTreeTransportStopsFileName());
} }
} }
@ -481,7 +490,7 @@ public class IndexCreator {
if (recreateOnlyBinaryFile) { if (recreateOnlyBinaryFile) {
mapFile = new File(workingDir, getMapFileName()); mapFile = new File(workingDir, getMapFileName());
File tempDBMapFile = new File(workingDir, getTempMapDBFileName()); File tempDBMapFile = new File(workingDir, getTempMapDBFileName());
mapConnection = (Connection) getDatabaseConnection(tempDBMapFile.getAbsolutePath(), mapDBDialect); mapConnection = (Connection) getDatabaseConnection(tempDBMapFile.getAbsolutePath(), mapIndexDBDialect);
mapConnection.setAutoCommit(false); mapConnection.setAutoCommit(false);
try { try {
if (indexMap) { if (indexMap) {
@ -707,27 +716,27 @@ public class IndexCreator {
mapConnection.close(); mapConnection.close();
mapConnection = null; mapConnection = null;
File tempDBFile = new File(workingDir, getTempMapDBFileName()); File tempDBFile = new File(workingDir, getTempMapDBFileName());
if (mapDBDialect.databaseFileExists(tempDBFile) && deleteDatabaseIndexes) { if (mapIndexDBDialect.databaseFileExists(tempDBFile) && deleteDatabaseIndexes) {
// do not delete it for now // do not delete it for now
mapDBDialect.removeDatabase(tempDBFile); mapIndexDBDialect.removeDatabase(tempDBFile);
} }
} }
// do not delete first db connection // do not delete first db connection
if (dbConn != null) { if (dbConn != null) {
dialect.commitDatabase(dbConn); osmDBdialect.commitDatabase(dbConn);
dialect.closeDatabase(dbConn); osmDBdialect.closeDatabase(dbConn);
dbConn = null; dbConn = null;
} }
if (deleteOsmDB) { if (deleteOsmDB) {
if (DBDialect.DERBY == dialect) { if (DBDialect.DERBY == osmDBdialect) {
try { try {
DriverManager.getConnection("jdbc:derby:;shutdown=true"); //$NON-NLS-1$ DriverManager.getConnection("jdbc:derby:;shutdown=true"); //$NON-NLS-1$
} catch (SQLException e) { } catch (SQLException e) {
// ignore exception // ignore exception
} }
} }
dialect.removeDatabase(dbFile); osmDBdialect.removeDatabase(dbFile);
} }
} catch (SQLException e) { } catch (SQLException e) {
e.printStackTrace(); e.printStackTrace();

6
build-scripts/indexes-roads-batch-generate.xml
View file

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<batch_process> <batch_process>
<process_attributes mapZooms="" renderingTypesFile="" zoomWaySmoothness="2" <process_attributes mapZooms="" renderingTypesFile="" zoomWaySmoothness="2"
osmDbDialect="sqlite" mapDbDialect="sqlite"/> osmDbDialect="SQLITE_IN_MEMORY" mapDbDialect="SQLITE_IN_MEMORY"/>
<!-- There are 3 subprocess : <!-- There are 3 subprocess :
1. Download fresh osm files from servers to 'directory_for_osm_files' (override existings). 1. Download fresh osm files from servers to 'directory_for_osm_files' (override existings).
@ -12,8 +12,8 @@
or you can upload any file you have to googlecode (just put into 'directory_for_index_files') or you can upload any file you have to googlecode (just put into 'directory_for_index_files')
--> -->
<!-- zoomWaySmoothness - 1-4, typical mapZooms - 8-10;11-12;13-14;15 --> <!-- zoomWaySmoothness - 1-4, typical mapZooms - 8-10;11-12;13-14;15 -->
<process directory_for_osm_files=".work/osm" directory_for_index_files="/var/www-download/road-indexes" directory_for_generation=".work" <process directory_for_osm_files=".work/osm" directory_for_index_files="/var/lib/jenkins/indexes" directory_for_generation=".work"
skipExistingIndexesAt="/var/www-download/road-indexes/" indexPOI="false" indexMap="false" indexRouting="true" indexTransport="false" indexAddress="false"> skipExistingIndexesAt="/var/lib/jenkins/indexes/road-uploaded" indexPOI="false" indexMap="false" indexRouting="true" indexTransport="false" indexAddress="false">
<!-- Add wget="C:/Program Files/GNUWin32/bin/wget.exe" to process, to use wget for download. <!-- Add wget="C:/Program Files/GNUWin32/bin/wget.exe" to process, to use wget for download.
On linux systems if wget is in your path it can be wget="wget" or you can make own script with wget command: On linux systems if wget is in your path it can be wget="wget" or you can make own script with wget command:
wget="/path/to/script/wget.sh" wget="/path/to/script/wget.sh"