Android-builders-club/OsmAnd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Fix issue 594. Add batch process parameters to use different databases

Browse source
This commit is contained in:
Victor Shcherb 2011-08-14 15:41:49 +02:00
parent 959e009560
commit dd829d8a95
3 changed files with 74 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

84
DataExtractionOSM/src/net/osmand/data/index/IndexBatchCreator.java
View file

@ -36,6 +36,8 @@ import net.osmand.Algoritms;
import net.osmand.LogUtil;
import net.osmand.binary.BinaryMapIndexReader;
import net.osmand.data.IndexConstants;
import net.osmand.data.index.ExtractGooglecodeAuthorization.GooglecodeUploadTokens;
import net.osmand.data.preparation.DBDialect;
import net.osmand.data.preparation.IndexCreator;
import net.osmand.data.preparation.MapZooms;
import net.osmand.impl.ConsoleProgressImplementation;
@ -99,6 +101,7 @@ public class IndexBatchCreator {
private String wget;
String googlePassword = "";
String cookieHSID = "";
String cookieSID = "";
String pagegen = "";
@ -125,6 +128,10 @@ public class IndexBatchCreator {
}
}
if(args.length >= 2 && args[1] != null && args[1].startsWith("-gp")){
creator.googlePassword = args[1].substring(3);
}
try {
Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(stream);
creator.runBatch(doc);
@ -161,23 +168,11 @@ public class IndexBatchCreator {
indexMap = Boolean.parseBoolean(process.getAttribute("indexMap"));
indexTransport = Boolean.parseBoolean(process.getAttribute("indexTransport"));
indexAddress = Boolean.parseBoolean(process.getAttribute("indexAddress"));
String zooms = process.getAttribute("mapZooms");
if(zooms == null || zooms.length() == 0){
mapZooms = MapZooms.getDefault();
} else {
mapZooms = MapZooms.parseZooms(zooms);
}
parseProcessAttributes(process);
String szoomWaySmoothness = process.getAttribute("zoomWaySmoothness");
if(szoomWaySmoothness != null && !szoomWaySmoothness.equals("")){
zoomWaySmoothness = Integer.parseInt(szoomWaySmoothness);
}
String f = process.getAttribute("renderingTypesFile");
if(f == null || f.length() == 0){
types = MapRenderingTypes.getDefault();
} else {
types = new MapRenderingTypes(f);
list = doc.getElementsByTagName("process_attributes");
if(list.getLength() == 1){
parseProcessAttributes((Element) list.item(0));
}
String dir = process.getAttribute("directory_for_osm_files");
@ -207,14 +202,27 @@ public class IndexBatchCreator {
throw new IllegalArgumentException("You should specify exactly 1 authorization_info element to upload indexes!");
}
Element authorization = (Element) list.item(0);
cookieHSID = authorization.getAttribute("cookieHSID");
cookieSID = authorization.getAttribute("cookieSID");
pagegen = authorization.getAttribute("pagegen");
token = authorization.getAttribute("token");
uploadToOsmandGooglecode = Boolean.parseBoolean(process.getAttribute("upload_osmand_googlecode"));
if(uploadToOsmandGooglecode){
user = authorization.getAttribute("google_code_user");
password = authorization.getAttribute("google_code_password");
cookieHSID = authorization.getAttribute("cookieHSID");
cookieSID = authorization.getAttribute("cookieSID");
pagegen = authorization.getAttribute("pagegen");
token = authorization.getAttribute("token");
if(googlePassword.length() > 0){
ExtractGooglecodeAuthorization gca = new ExtractGooglecodeAuthorization();
try {
GooglecodeUploadTokens tokens = gca.getGooglecodeTokensForUpload(user, googlePassword);
cookieHSID = tokens.getHsid();
cookieSID = tokens.getSid();
pagegen = tokens.getPagegen();
token = tokens.getToken();
} catch (IOException e) {
log.error("Error retrieving google tokens", e);
}
}
} else {
user = authorization.getAttribute("osmand_download_user");
password = authorization.getAttribute("osmand_download_password");
@ -259,6 +267,42 @@ public class IndexBatchCreator {
}
private void parseProcessAttributes(Element process) {
String zooms = process.getAttribute("mapZooms");
if(zooms == null || zooms.length() == 0){
mapZooms = MapZooms.getDefault();
} else {
mapZooms = MapZooms.parseZooms(zooms);
}
String szoomWaySmoothness = process.getAttribute("zoomWaySmoothness");
if(szoomWaySmoothness != null && !szoomWaySmoothness.equals("")){
zoomWaySmoothness = Integer.parseInt(szoomWaySmoothness);
}
String f = process.getAttribute("renderingTypesFile");
if(f == null || f.length() == 0){
types = MapRenderingTypes.getDefault();
} else {
types = new MapRenderingTypes(f);
}
String osmDbDialect = process.getAttribute("osmDbDialect");
if(osmDbDialect != null && osmDbDialect.length() > 0){
try {
IndexCreator.dialect = DBDialect.valueOf(osmDbDialect.toUpperCase());
} catch (RuntimeException e) {
}
}
String mapDbDialect = process.getAttribute("mapDbDialect");
if (mapDbDialect != null && mapDbDialect.length() > 0) {
try {
IndexCreator.mapDBDialect = DBDialect.valueOf(mapDbDialect.toUpperCase());
} catch (RuntimeException e) {
}
}
}
public void runBatch(List<RegionCountries> countriesToDownload ){
Set<String> alreadyUploadedFiles = new LinkedHashSet<String>();
Set<String> alreadyGeneratedFiles = new LinkedHashSet<String>();

9
DataExtractionOSM/src/net/osmand/data/index/batch.xml
View file

@ -13,6 +13,10 @@
pagegen="" token=""
google_code_user="" google_code_password=""
osmand_download_user="" osmand_download_password=""/>
<process_attributes mapZooms="" renderingTypesFile="" zoomWaySmoothness=""
osmDbDialect="sqlite" mapDbDialect="sqlite"/>
<!-- There are 3 subprocess :
1. Download fresh osm files from servers to 'directory_for_osm_files' (override existings).
2. Generate index files from all files in 'directory_for_osm_files' and put all indexes into 'directory_for_index_files'
@ -22,12 +26,11 @@
or you can upload any file you have to googlecode (just put into 'directory_for_index_files')
-->
<!-- zoomWaySmoothness - 1-4, typical mapZooms - 8-10;11-12;13-14;15 -->
<process directory_for_osm_files="D:/android/batch_gen_osm" directory_for_index_files="D:/android/batch_gen_index"
<process directory_for_osm_files="/home/..." directory_for_index_files="/home/..."
directory_for_uploaded_files=""
downloadOsmFiles="true" generateIndexes="true" uploadIndexes="true" upload_osmand_googlecode="true"
deleteFilesAfterUploading="true" indexPOI="true" indexMap="true"
indexTransport="true" indexAddress="true" mapZooms="" renderingTypesFile="" zoomWaySmoothness=""
>
indexTransport="true" indexAddress="true">
<!-- Add wget="C:/Program Files/GNUWin32/bin/wget.exe" to process, to use wget for download.
On linux systems if wget is in your path it can be wget="wget" or you can make own script with wget command:
wget="/path/to/script/wget.sh"

7
DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java
View file

@ -49,8 +49,9 @@ public class IndexCreator {
// ONLY derby.jar needed for derby dialect
// (NOSQL is the fastest but is supported only on linux 32)
private static DBDialect dialect = DBDialect.SQLITE;
private static DBDialect mapDBDialect = DBDialect.SQLITE;
// Sqlite better to use only for 32-bit machines
public static DBDialect dialect = DBDialect.SQLITE;
public static DBDialect mapDBDialect = DBDialect.SQLITE;
public static final int BATCH_SIZE = 5000;
public static final int BATCH_SIZE_OSM = 10000;
@ -612,7 +613,7 @@ public class IndexCreator {
MapRenderingTypes rt = MapRenderingTypes.getDefault();// new MapRenderingTypes("/home/victor/projects/OsmAnd/data/testdata/roads_rendering_types.xml");
MapZooms zooms = MapZooms.getDefault(); // MapZooms.parseZooms("15-");
// creator.setNodesDBFile(new File("/home/victor/projects/OsmAnd/data/osm-gen/nodes.tmp.odb"));
creator.generateIndexes(new File("/home/victor/projects/OsmAnd/data/belarus-osm/belarus.osm.pbf"),
creator.generateIndexes(new File("/home/victor/projects/OsmAnd/data/osm-maps/mecklenburg-vorpommern.osm.pbf"),
new ConsoleProgressImplementation(1), null, zooms, rt);
// creator.setNodesDBFile(new File("/home/victor/projects/OsmAnd/data/osm-gen/nodes3.tmp.odb"));