Android-builders-club/OsmAnd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

improvement of batch creator

Browse source 
git-svn-id: https://osmand.googlecode.com/svn/trunk@300 e29c36b1-1cfa-d876-8d93-3434fc2bb7b8
This commit is contained in:
Victor Shcherb 2010-07-08 07:25:21 +00:00
parent 40b655cb67
commit 845fe20ca1
1 changed files with 58 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
DataExtractionOSM/src/com/osmand/data/index/IndexBatchCreator.java
View file

@ -9,7 +9,9 @@ import java.text.MessageFormat;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Date;
import java.util.LinkedHashSet;
import java.util.Locale;
import java.util.Set;
import org.apache.commons.logging.Log;
@ -110,14 +112,15 @@ public class IndexBatchCreator {
}
public void runBatch(){
Set<String> alreadyUploadedFiles = new LinkedHashSet<String>();
if(downloadFiles){
downloadFiles();
}
if(generateIndexes){
generatedIndexes();
generatedIndexes(alreadyUploadedFiles);
}
if(uploadIndexes){
uploadIndexes();
uploadIndexes(alreadyUploadedFiles);
}
}
@ -176,12 +179,12 @@ public class IndexBatchCreator {
}
}
protected void generatedIndexes() {
protected void generatedIndexes(Set<String> alreadyUploadedFiles) {
for (File f : getSortedFiles(osmDirFiles)) {
if (f.getName().endsWith(".osm.bz2") || f.getName().endsWith(".osm")) {
System.gc();
try {
generateIndex(f);
generateIndex(f, alreadyUploadedFiles);
} catch (OutOfMemoryError e) {
log.error("OutOfMemory", e);
System.gc();
@ -190,20 +193,26 @@ public class IndexBatchCreator {
}
System.out.println("GENERATING INDEXES FINISHED ");
}
protected void generateIndex(File f){
protected void generateIndex(File f, Set<String> alreadyUploadedFiles){
DataExtraction extr = new DataExtraction(indexAddress, indexPOI, indexTransport, indexAddress, false, false, indexDirFiles);
try {
Region country = extr.readCountry(f.getAbsolutePath(), new ConsoleProgressImplementation(9), null);
DataIndexWriter dataIndexWriter = new DataIndexWriter(indexDirFiles, country);
String name = country.getName();
if(indexAddress){
dataIndexWriter.writeAddress(name + "_" + IndexConstants.ADDRESS_TABLE_VERSION + IndexConstants.ADDRESS_INDEX_EXT, f.lastModified(), writeWayNodes);
String fName = name + "_" + IndexConstants.ADDRESS_TABLE_VERSION + IndexConstants.ADDRESS_INDEX_EXT;
uploadIndex(new File(indexDirFiles, fName), alreadyUploadedFiles);
dataIndexWriter.writeAddress(fName, f.lastModified(), writeWayNodes);
}
if(indexPOI){
dataIndexWriter.writePOI(name + "_" + IndexConstants.POI_TABLE_VERSION + IndexConstants.POI_INDEX_EXT, f.lastModified());
String fName = name + "_" + IndexConstants.POI_TABLE_VERSION + IndexConstants.POI_INDEX_EXT;
uploadIndex(new File(indexDirFiles, fName), alreadyUploadedFiles);
dataIndexWriter.writePOI(fName, f.lastModified());
}
if(indexTransport){
dataIndexWriter.writeTransport(name + "_" + IndexConstants.TRANSPORT_TABLE_VERSION + IndexConstants.TRANSPORT_INDEX_EXT, f.lastModified());
String fName = name + "_" + IndexConstants.TRANSPORT_TABLE_VERSION + IndexConstants.TRANSPORT_INDEX_EXT;
uploadIndex(new File(indexDirFiles, fName), alreadyUploadedFiles);
dataIndexWriter.writeTransport(fName, f.lastModified());
}
} catch (Exception e) {
log.error("Exception generating indexes for " + f.getName()); //$NON-NLS-1$
@ -221,9 +230,16 @@ public class IndexBatchCreator {
return listFiles;
}
protected void uploadIndexes(){
MessageFormat format = new MessageFormat("{0,date,dd.MM.yyyy} : {1, number,##.#} MB", Locale.US);
protected void uploadIndexes(Set<String> alreadyUploadedFiles){
for(File f : getSortedFiles(indexDirFiles)){
uploadIndex(f, alreadyUploadedFiles);
}
System.out.println("UPLOADING INDEXES FINISHED ");
}
protected void uploadIndex(File f, Set<String> alreadyUploadedFiles){
MessageFormat format = new MessageFormat("{0,date,dd.MM.yyyy} : {1, number,##.#} MB", Locale.US);
String summary;
double mbLengh = (double)f.length() / MB;
@ -238,11 +254,13 @@ public class IndexBatchCreator {
String regionName = f.getName().substring(0, f.getName().length() - IndexConstants.TRANSPORT_INDEX_EXT.length() - 2);
summary = "Transport index for " + regionName + " " + descriptionFile;
} else {
continue;
return;
}
if(mbLengh > 90){
System.err.println("ERROR : file " + f.getName() + " exceeded 90 mb!!! Could not be uploaded.");
// return; // ? do not try?
}
alreadyUploadedFiles.add(f.getName());
GoogleCodeUploadIndex uploader = new GoogleCodeUploadIndex();
uploader.setFileName(f.getAbsolutePath());
uploader.setTargetFileName(f.getName());
@ -257,9 +275,7 @@ public class IndexBatchCreator {
log.error("Input/output exception uploading " + f.getName(), e);
}
}
System.out.println("UPLOADING INDEXES FINISHED ");
}
}