Android-builders-club/OsmAnd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

improvement of batch creator

Browse source 
git-svn-id: https://osmand.googlecode.com/svn/trunk@300 e29c36b1-1cfa-d876-8d93-3434fc2bb7b8
This commit is contained in:
Victor Shcherb 2010-07-08 07:25:21 +00:00
parent 40b655cb67
commit 845fe20ca1
1 changed files with 58 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

100
DataExtractionOSM/src/com/osmand/data/index/IndexBatchCreator.java
View file

@ -9,7 +9,9 @@ import java.text.MessageFormat;
import java.util.Arrays; import java.util.Arrays;
import java.util.Comparator; import java.util.Comparator;
import java.util.Date; import java.util.Date;
import java.util.LinkedHashSet;
import java.util.Locale; import java.util.Locale;
import java.util.Set;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
@ -110,14 +112,15 @@ public class IndexBatchCreator {
} }
public void runBatch(){ public void runBatch(){
Set<String> alreadyUploadedFiles = new LinkedHashSet<String>();
if(downloadFiles){ if(downloadFiles){
downloadFiles(); downloadFiles();
} }
if(generateIndexes){ if(generateIndexes){
generatedIndexes(); generatedIndexes(alreadyUploadedFiles);
} }
if(uploadIndexes){ if(uploadIndexes){
uploadIndexes(); uploadIndexes(alreadyUploadedFiles);
} }
} }
@ -176,12 +179,12 @@ public class IndexBatchCreator {
} }
} }
protected void generatedIndexes() { protected void generatedIndexes(Set<String> alreadyUploadedFiles) {
for (File f : getSortedFiles(osmDirFiles)) { for (File f : getSortedFiles(osmDirFiles)) {
if (f.getName().endsWith(".osm.bz2") || f.getName().endsWith(".osm")) { if (f.getName().endsWith(".osm.bz2") || f.getName().endsWith(".osm")) {
System.gc(); System.gc();
try { try {
generateIndex(f); generateIndex(f, alreadyUploadedFiles);
} catch (OutOfMemoryError e) { } catch (OutOfMemoryError e) {
log.error("OutOfMemory", e); log.error("OutOfMemory", e);
System.gc(); System.gc();
@ -190,20 +193,26 @@ public class IndexBatchCreator {
} }
System.out.println("GENERATING INDEXES FINISHED "); System.out.println("GENERATING INDEXES FINISHED ");
} }
protected void generateIndex(File f){ protected void generateIndex(File f, Set<String> alreadyUploadedFiles){
DataExtraction extr = new DataExtraction(indexAddress, indexPOI, indexTransport, indexAddress, false, false, indexDirFiles); DataExtraction extr = new DataExtraction(indexAddress, indexPOI, indexTransport, indexAddress, false, false, indexDirFiles);
try { try {
Region country = extr.readCountry(f.getAbsolutePath(), new ConsoleProgressImplementation(9), null); Region country = extr.readCountry(f.getAbsolutePath(), new ConsoleProgressImplementation(9), null);
DataIndexWriter dataIndexWriter = new DataIndexWriter(indexDirFiles, country); DataIndexWriter dataIndexWriter = new DataIndexWriter(indexDirFiles, country);
String name = country.getName(); String name = country.getName();
if(indexAddress){ if(indexAddress){
dataIndexWriter.writeAddress(name + "_" + IndexConstants.ADDRESS_TABLE_VERSION + IndexConstants.ADDRESS_INDEX_EXT, f.lastModified(), writeWayNodes); String fName = name + "_" + IndexConstants.ADDRESS_TABLE_VERSION + IndexConstants.ADDRESS_INDEX_EXT;
uploadIndex(new File(indexDirFiles, fName), alreadyUploadedFiles);
dataIndexWriter.writeAddress(fName, f.lastModified(), writeWayNodes);
} }
if(indexPOI){ if(indexPOI){
dataIndexWriter.writePOI(name + "_" + IndexConstants.POI_TABLE_VERSION + IndexConstants.POI_INDEX_EXT, f.lastModified()); String fName = name + "_" + IndexConstants.POI_TABLE_VERSION + IndexConstants.POI_INDEX_EXT;
uploadIndex(new File(indexDirFiles, fName), alreadyUploadedFiles);
dataIndexWriter.writePOI(fName, f.lastModified());
} }
if(indexTransport){ if(indexTransport){
dataIndexWriter.writeTransport(name + "_" + IndexConstants.TRANSPORT_TABLE_VERSION + IndexConstants.TRANSPORT_INDEX_EXT, f.lastModified()); String fName = name + "_" + IndexConstants.TRANSPORT_TABLE_VERSION + IndexConstants.TRANSPORT_INDEX_EXT;
uploadIndex(new File(indexDirFiles, fName), alreadyUploadedFiles);
dataIndexWriter.writeTransport(fName, f.lastModified());
} }
} catch (Exception e) { } catch (Exception e) {
log.error("Exception generating indexes for " + f.getName()); //$NON-NLS-1$ log.error("Exception generating indexes for " + f.getName()); //$NON-NLS-1$
@ -221,45 +230,52 @@ public class IndexBatchCreator {
return listFiles; return listFiles;
} }
protected void uploadIndexes(){ protected void uploadIndexes(Set<String> alreadyUploadedFiles){
MessageFormat format = new MessageFormat("{0,date,dd.MM.yyyy} : {1, number,##.#} MB", Locale.US);
for(File f : getSortedFiles(indexDirFiles)){ for(File f : getSortedFiles(indexDirFiles)){
String summary; uploadIndex(f, alreadyUploadedFiles);
double mbLengh = (double)f.length() / MB;
String descriptionFile = "{"+format.format(new Object[]{new Date(f.lastModified()), mbLengh})+"}";
if(f.getName().endsWith(IndexConstants.POI_INDEX_EXT)){
String regionName = f.getName().substring(0, f.getName().length() - IndexConstants.POI_INDEX_EXT.length() - 2);
summary = "POI index for " + regionName + " " + descriptionFile;
} else if(f.getName().endsWith(IndexConstants.ADDRESS_INDEX_EXT)){
String regionName = f.getName().substring(0, f.getName().length() - IndexConstants.ADDRESS_INDEX_EXT.length() - 2);
summary = "Adress index for " + regionName + " " + descriptionFile;
} else if(f.getName().endsWith(IndexConstants.TRANSPORT_INDEX_EXT)){
String regionName = f.getName().substring(0, f.getName().length() - IndexConstants.TRANSPORT_INDEX_EXT.length() - 2);
summary = "Transport index for " + regionName + " " + descriptionFile;
} else {
continue;
}
if(mbLengh > 90){
System.err.println("ERROR : file " + f.getName() + " exceeded 90 mb!!! Could not be uploaded.");
}
GoogleCodeUploadIndex uploader = new GoogleCodeUploadIndex();
uploader.setFileName(f.getAbsolutePath());
uploader.setTargetFileName(f.getName());
uploader.setProjectName("osmand");
uploader.setUserName(user);
uploader.setPassword(password);
uploader.setLabels("Type-Archive, Testdata");
uploader.setSummary(summary);
try {
uploader.upload();
} catch (IOException e) {
log.error("Input/output exception uploading " + f.getName(), e);
}
} }
System.out.println("UPLOADING INDEXES FINISHED "); System.out.println("UPLOADING INDEXES FINISHED ");
} }
protected void uploadIndex(File f, Set<String> alreadyUploadedFiles){
MessageFormat format = new MessageFormat("{0,date,dd.MM.yyyy} : {1, number,##.#} MB", Locale.US);
String summary;
double mbLengh = (double)f.length() / MB;
String descriptionFile = "{"+format.format(new Object[]{new Date(f.lastModified()), mbLengh})+"}";
if(f.getName().endsWith(IndexConstants.POI_INDEX_EXT)){
String regionName = f.getName().substring(0, f.getName().length() - IndexConstants.POI_INDEX_EXT.length() - 2);
summary = "POI index for " + regionName + " " + descriptionFile;
} else if(f.getName().endsWith(IndexConstants.ADDRESS_INDEX_EXT)){
String regionName = f.getName().substring(0, f.getName().length() - IndexConstants.ADDRESS_INDEX_EXT.length() - 2);
summary = "Adress index for " + regionName + " " + descriptionFile;
} else if(f.getName().endsWith(IndexConstants.TRANSPORT_INDEX_EXT)){
String regionName = f.getName().substring(0, f.getName().length() - IndexConstants.TRANSPORT_INDEX_EXT.length() - 2);
summary = "Transport index for " + regionName + " " + descriptionFile;
} else {
return;
}
if(mbLengh > 90){
System.err.println("ERROR : file " + f.getName() + " exceeded 90 mb!!! Could not be uploaded.");
// return; // ? do not try?
}
alreadyUploadedFiles.add(f.getName());
GoogleCodeUploadIndex uploader = new GoogleCodeUploadIndex();
uploader.setFileName(f.getAbsolutePath());
uploader.setTargetFileName(f.getName());
uploader.setProjectName("osmand");
uploader.setUserName(user);
uploader.setPassword(password);
uploader.setLabels("Type-Archive, Testdata");
uploader.setSummary(summary);
try {
uploader.upload();
} catch (IOException e) {
log.error("Input/output exception uploading " + f.getName(), e);
}
}
} }