Android-builders-club/OsmAnd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

improve index batch creator

Browse source 
git-svn-id: https://osmand.googlecode.com/svn/trunk@301 e29c36b1-1cfa-d876-8d93-3434fc2bb7b8
This commit is contained in:
Victor Shcherb 2010-07-08 08:49:28 +00:00
parent 845fe20ca1
commit fa774610ad
1 changed files with 29 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
DataExtractionOSM/src/com/osmand/data/index/IndexBatchCreator.java
View file

@ -113,18 +113,19 @@ public class IndexBatchCreator {
public void runBatch(){ public void runBatch(){
Set<String> alreadyUploadedFiles = new LinkedHashSet<String>(); Set<String> alreadyUploadedFiles = new LinkedHashSet<String>();
Set<String> alreadyGeneratedFiles = new LinkedHashSet<String>();
if(downloadFiles){ if(downloadFiles){
downloadFiles(); downloadFiles(alreadyGeneratedFiles, alreadyUploadedFiles);
} }
if(generateIndexes){ if(generateIndexes){
generatedIndexes(alreadyUploadedFiles); generatedIndexes(alreadyGeneratedFiles, alreadyUploadedFiles);
} }
if(uploadIndexes){ if(uploadIndexes){
uploadIndexes(alreadyUploadedFiles); uploadIndexes(alreadyUploadedFiles);
} }
} }
protected void downloadFiles(){ protected void downloadFiles(Set<String> alreadyGeneratedFiles, Set<String> alreadyUploadedFiles){
// clean before downloading // clean before downloading
// for(File f : osmDirFiles.listFiles()){ // for(File f : osmDirFiles.listFiles()){
// log.info("Delete old file " + f.getName()); //$NON-NLS-1$ // log.info("Delete old file " + f.getName()); //$NON-NLS-1$
@ -133,21 +134,27 @@ public class IndexBatchCreator {
for(String country : countriesToDownload1){ for(String country : countriesToDownload1){
String url = SITE_TO_DOWNLOAD1 + country +".osm.bz2"; //$NON-NLS-1$ String url = SITE_TO_DOWNLOAD1 + country +".osm.bz2"; //$NON-NLS-1$
log.info("Downloading country " + country + " from " + url); //$NON-NLS-1$//$NON-NLS-2$ log.info("Downloading country " + country + " from " + url); //$NON-NLS-1$//$NON-NLS-2$
downloadFile(url, new File(osmDirFiles, country +".osm.bz2")); //$NON-NLS-1$ File f = new File(osmDirFiles, country +".osm.bz2");
downloadFile(url, f); //$NON-NLS-1$
generateIndex(f, alreadyGeneratedFiles, alreadyUploadedFiles);
} }
for(String country : usStates){ for(String country : usStates){
country = country.toLowerCase(); country = country.toLowerCase();
String url = SITE_TO_DOWNLOAD2 + "north_america/united_states/"+country+"/"+country +".osm.bz2"; //$NON-NLS-1$ String url = SITE_TO_DOWNLOAD2 + "north_america/united_states/"+country+"/"+country +".osm.bz2"; //$NON-NLS-1$
log.info("Downloading country " + country + " from " + url); //$NON-NLS-1$//$NON-NLS-2$ log.info("Downloading country " + country + " from " + url); //$NON-NLS-1$//$NON-NLS-2$
downloadFile(url, new File(osmDirFiles, "US_"+country +".osm.bz2")); //$NON-NLS-1$ File f = new File(osmDirFiles, "US_"+country +".osm.bz2"); //$NON-NLS-1$
downloadFile(url, f);
generateIndex(f, alreadyGeneratedFiles, alreadyUploadedFiles);
} }
for(String country : canadaStates){ for(String country : canadaStates){
country = country.toLowerCase(); country = country.toLowerCase();
String url = SITE_TO_DOWNLOAD2 + "north_america/canada/"+country+"/"+country +".osm.bz2"; //$NON-NLS-1$ String url = SITE_TO_DOWNLOAD2 + "north_america/canada/"+country+"/"+country +".osm.bz2"; //$NON-NLS-1$
log.info("Downloading country " + country + " from " + url); //$NON-NLS-1$//$NON-NLS-2$ log.info("Downloading country " + country + " from " + url); //$NON-NLS-1$//$NON-NLS-2$
downloadFile(url, new File(osmDirFiles, "Canada_"+country +".osm.bz2")); //$NON-NLS-1$ File f = new File(osmDirFiles, "Canada_"+country +".osm.bz2"); //$NON-NLS-1$
downloadFile(url, f);
generateIndex(f, alreadyGeneratedFiles, alreadyUploadedFiles);
} }
System.out.println("DOWNLOADING FILES FINISHED"); System.out.println("DOWNLOADING FILES FINISHED");
} }
@ -179,12 +186,15 @@ public class IndexBatchCreator {
} }
} }
protected void generatedIndexes(Set<String> alreadyUploadedFiles) { protected void generatedIndexes(Set<String> alreadyGeneratedFiles, Set<String> alreadyUploadedFiles) {
for (File f : getSortedFiles(osmDirFiles)) { for (File f : getSortedFiles(osmDirFiles)) {
if (alreadyGeneratedFiles.contains(f.getName())) {
continue;
}
if (f.getName().endsWith(".osm.bz2") || f.getName().endsWith(".osm")) { if (f.getName().endsWith(".osm.bz2") || f.getName().endsWith(".osm")) {
System.gc(); System.gc();
try { try {
generateIndex(f, alreadyUploadedFiles); generateIndex(f, alreadyGeneratedFiles, alreadyUploadedFiles);
} catch (OutOfMemoryError e) { } catch (OutOfMemoryError e) {
log.error("OutOfMemory", e); log.error("OutOfMemory", e);
System.gc(); System.gc();
@ -193,9 +203,13 @@ public class IndexBatchCreator {
} }
System.out.println("GENERATING INDEXES FINISHED "); System.out.println("GENERATING INDEXES FINISHED ");
} }
protected void generateIndex(File f, Set<String> alreadyUploadedFiles){ protected void generateIndex(File f, Set<String> alreadyGeneratedFiles, Set<String> alreadyUploadedFiles){
if(!generateIndexes){
return;
}
DataExtraction extr = new DataExtraction(indexAddress, indexPOI, indexTransport, indexAddress, false, false, indexDirFiles); DataExtraction extr = new DataExtraction(indexAddress, indexPOI, indexTransport, indexAddress, false, false, indexDirFiles);
try { try {
alreadyGeneratedFiles.add(f.getName());
Region country = extr.readCountry(f.getAbsolutePath(), new ConsoleProgressImplementation(9), null); Region country = extr.readCountry(f.getAbsolutePath(), new ConsoleProgressImplementation(9), null);
DataIndexWriter dataIndexWriter = new DataIndexWriter(indexDirFiles, country); DataIndexWriter dataIndexWriter = new DataIndexWriter(indexDirFiles, country);
String name = country.getName(); String name = country.getName();
@ -232,13 +246,18 @@ public class IndexBatchCreator {
protected void uploadIndexes(Set<String> alreadyUploadedFiles){ protected void uploadIndexes(Set<String> alreadyUploadedFiles){
for(File f : getSortedFiles(indexDirFiles)){ for(File f : getSortedFiles(indexDirFiles)){
uploadIndex(f, alreadyUploadedFiles); if(!alreadyUploadedFiles.contains(f.getName())){
uploadIndex(f, alreadyUploadedFiles);
}
} }
System.out.println("UPLOADING INDEXES FINISHED "); System.out.println("UPLOADING INDEXES FINISHED ");
} }
protected void uploadIndex(File f, Set<String> alreadyUploadedFiles){ protected void uploadIndex(File f, Set<String> alreadyUploadedFiles){
if(!uploadIndexes){
return;
}
MessageFormat format = new MessageFormat("{0,date,dd.MM.yyyy} : {1, number,##.#} MB", Locale.US); MessageFormat format = new MessageFormat("{0,date,dd.MM.yyyy} : {1, number,##.#} MB", Locale.US);
String summary; String summary;
double mbLengh = (double)f.length() / MB; double mbLengh = (double)f.length() / MB;