From 6ab8432b8da0716179e616f5bb99e963a2f3ab18 Mon Sep 17 00:00:00 2001 From: Victor Shcherb Date: Mon, 21 Jun 2010 17:02:26 +0000 Subject: [PATCH] add console app to upload indexes to google downloads git-svn-id: https://osmand.googlecode.com/svn/trunk@192 e29c36b1-1cfa-d876-8d93-3434fc2bb7b8 --- .../osmand/data/index/DataIndexWriter.java | 15 +- .../data/index/GoogleCodeUploadIndex.java | 4 +- .../osmand/data/index/IndexBatchCreator.java | 191 ++++++++++++++++++ .../impl/ConsoleProgressImplementation.java | 20 +- OsmAnd/.classpath | 2 +- 5 files changed, 222 insertions(+), 10 deletions(-) create mode 100644 DataExtractionOSM/src/com/osmand/data/index/IndexBatchCreator.java diff --git a/DataExtractionOSM/src/com/osmand/data/index/DataIndexWriter.java b/DataExtractionOSM/src/com/osmand/data/index/DataIndexWriter.java index e04770a0ed..0bfab7d6a1 100644 --- a/DataExtractionOSM/src/com/osmand/data/index/DataIndexWriter.java +++ b/DataExtractionOSM/src/com/osmand/data/index/DataIndexWriter.java @@ -47,7 +47,7 @@ public class DataIndexWriter { protected File checkFile(String name) throws IOException { String fileName = name; File f = new File(workingDir, fileName); - f.mkdirs(); + f.getParentFile().mkdirs(); // remove existing file if (f.exists()) { log.warn("Remove existing index : " + f.getAbsolutePath()); //$NON-NLS-1$ @@ -56,9 +56,12 @@ public class DataIndexWriter { return f; } + public DataIndexWriter writePOI() throws IOException, SQLException { + return writePOI(IndexConstants.POI_INDEX_DIR+region.getName()+IndexConstants.POI_INDEX_EXT); + } - public DataIndexWriter writePOI() throws IOException, SQLException { - File file = checkFile(IndexConstants.POI_INDEX_DIR+region.getName()+IndexConstants.POI_INDEX_EXT); + public DataIndexWriter writePOI(String fileName) throws IOException, SQLException { + File file = checkFile(fileName); long now = System.currentTimeMillis(); try { Class.forName("org.sqlite.JDBC"); //$NON-NLS-1$ @@ -108,7 +111,11 @@ public class DataIndexWriter { } public DataIndexWriter writeAddress() throws IOException, SQLException{ - File file = checkFile(IndexConstants.ADDRESS_INDEX_DIR+region.getName()+IndexConstants.ADDRESS_INDEX_EXT); + return writeAddress(IndexConstants.ADDRESS_INDEX_DIR+region.getName()+IndexConstants.ADDRESS_INDEX_EXT); + } + + public DataIndexWriter writeAddress(String fileName) throws IOException, SQLException{ + File file = checkFile(fileName); long now = System.currentTimeMillis(); try { Class.forName("org.sqlite.JDBC"); //$NON-NLS-1$ diff --git a/DataExtractionOSM/src/com/osmand/data/index/GoogleCodeUploadIndex.java b/DataExtractionOSM/src/com/osmand/data/index/GoogleCodeUploadIndex.java index fb6b417e43..2b68e7de96 100644 --- a/DataExtractionOSM/src/com/osmand/data/index/GoogleCodeUploadIndex.java +++ b/DataExtractionOSM/src/com/osmand/data/index/GoogleCodeUploadIndex.java @@ -250,8 +250,8 @@ public class GoogleCodeUploadIndex { public static void main(String[] args) throws IOException { GoogleCodeUploadIndex uploadIndex = new GoogleCodeUploadIndex(); - uploadIndex.setLabels(""); - uploadIndex.setProjectName("osmand"); +// uploadIndex.setLabels(""); +// uploadIndex.setProjectName("osmand"); // uploadIndex.setFileName(_); // uploadIndex.setTargetFileName(_); // uploadIndex.setUserName(_); diff --git a/DataExtractionOSM/src/com/osmand/data/index/IndexBatchCreator.java b/DataExtractionOSM/src/com/osmand/data/index/IndexBatchCreator.java new file mode 100644 index 0000000000..e83da6b9de --- /dev/null +++ b/DataExtractionOSM/src/com/osmand/data/index/IndexBatchCreator.java @@ -0,0 +1,191 @@ +package com.osmand.data.index; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.text.MessageFormat; +import java.util.Date; +import java.util.Locale; + +import org.apache.commons.logging.Log; + +import com.osmand.LogUtil; +import com.osmand.data.Region; +import com.osmand.data.preparation.DataExtraction; +import com.osmand.impl.ConsoleProgressImplementation; + +public class IndexBatchCreator { + protected static final Log log = LogUtil.getLog(IndexBatchCreator.class); + protected static final String SITE_TO_DOWNLOAD = "http://download.geofabrik.de/osm/europe/"; //$NON-NLS-1$ + protected static final String[] countriesToDownload = new String[] { + "albania", "andorra", "austria", // 5.3, 0.4, 100 //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + "belarus", "belgium", "bosnia-herzegovina", // 39, 43, 4.1 //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + "bulgaria", "croatia", "cyprus", // 13, 12, 5 //$NON-NLS-1$//$NON-NLS-2$ //$NON-NLS-3$ + "denmark", "estonia", "faroe_islands", // 75, 38, 1.5 //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + "finland", "greece", "hungary", //80, 25, 14 //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + "iceland", "ireland", "isle_of_man", // 5.9, 27, 1.1 //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + "kosovo", "latvia", "liechtenstein", // 8.2, 6.5, 0.2 //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + "lithuania", "luxembourg", "macedonia", // 5, 5, 4 //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + "malta", "moldova", "monaco", //0.8, 5, 0.6 //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + "montenegro", "norway", "poland", // 1.2, 56, 87 //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + "portugal", "romania", "serbia", // 10, 25, 10 //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + "slovakia", "slovenia", "spain", // 69, 10, 123 //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + "sweden", "switzerland", "turkey", // 88, 83, 17 //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + "ukraine", // 19 //$NON-NLS-1$ + // TOTAL : 1129 MB + // "france", "czech_republic", "germany", // 519, 168, 860 + // "great_britain", "italy", "netherlands", // 281, 246, 375 + // ADD TO TOTAL : 2449 MB + + }; + + + boolean downloadFiles = false; + boolean generateIndexes = false; + boolean uploadIndexes = false; + File osmDirFiles; + File indexDirFiles; + String user; + String password; + + + public static void main(String[] args) { + IndexBatchCreator creator = new IndexBatchCreator(); + if(args.length < 1 || !new File(args[0]).exists()) { + throw new IllegalArgumentException("Please specify directory with .osm or .osm.bz2 files as first arg"); //$NON-NLS-1$ + } + creator.osmDirFiles = new File(args[0]); + if(args.length < 2 || !new File(args[1]).exists()) { + throw new IllegalArgumentException("Please specify directory with generated index files as second arg"); //$NON-NLS-1$ + } + creator.indexDirFiles = new File(args[1]); + for(int i=2; i DOWNLOAD_DEBUG){ + downloaded -= DOWNLOAD_DEBUG; + mbDownloaded += (DOWNLOAD_DEBUG>>20); + log.info(mbDownloaded +" megabytes downloaded of " + toSave.getName()); + } + } + ostream.close(); + stream.close(); + } catch (IOException e) { + log.error("Input/output exception " + toSave.getName() + " downloading from " + url, e); //$NON-NLS-1$ //$NON-NLS-2$ + } + } + + protected void generatedIndexes() { + for (File f : osmDirFiles.listFiles()) { + if (f.getName().endsWith(".osm.bz2") || f.getName().endsWith(".osm")) { + System.gc(); + generateIndex(f); + } + } + System.out.println("GENERATING INDEXES FINISHED "); + } + protected void generateIndex(File f){ + DataExtraction extr = new DataExtraction(true, true, false, true, false, false, indexDirFiles); + try { + Region country = extr.readCountry(f.getAbsolutePath(), new ConsoleProgressImplementation(9), null); + DataIndexWriter dataIndexWriter = new DataIndexWriter(indexDirFiles, country); + String name = country.getName(); + dataIndexWriter.writeAddress(name + "_" + IndexConstants.ADDRESS_TABLE_VERSION + IndexConstants.ADDRESS_INDEX_EXT); + dataIndexWriter.writePOI(name + "_" + IndexConstants.POI_TABLE_VERSION + IndexConstants.POI_INDEX_EXT); + } catch (Exception e) { + log.error("Exception generating indexes for " + f.getName()); //$NON-NLS-1$ + } + } + + protected void uploadIndexes(){ + MessageFormat format = new MessageFormat("{0,date,dd.MM.yyyy} : {1, number,##.#} MB", Locale.US); + for(File f : indexDirFiles.listFiles()){ + String summary; + double mbLengh = (double)f.length() / MB; + String descriptionFile = "{"+format.format(new Object[]{new Date(), mbLengh})+"}"; + if(f.getName().endsWith(IndexConstants.POI_INDEX_EXT)){ + String regionName = f.getName().substring(0, f.getName().length() - IndexConstants.POI_INDEX_EXT.length() - 2); + summary = "POI index for " + regionName + " " + descriptionFile; + } else if(f.getName().endsWith(IndexConstants.ADDRESS_INDEX_EXT)){ + String regionName = f.getName().substring(0, f.getName().length() - IndexConstants.ADDRESS_INDEX_EXT.length() - 2); + summary = "Adress index for " + regionName + " " + descriptionFile; + } else { + continue; + } + GoogleCodeUploadIndex uploader = new GoogleCodeUploadIndex(); + uploader.setFileName(f.getAbsolutePath()); + uploader.setTargetFileName(f.getName()); + uploader.setProjectName("osmand"); + uploader.setUserName(user); + uploader.setPassword(password); + uploader.setLabels("Type-Archive, Testdata"); + uploader.setSummary(summary); + try { + uploader.upload(); + } catch (IOException e) { + log.error("Input/output exception uploading " + f.getName(), e); + } + } + System.out.println("UPLOADING INDEXES FINISHED "); + + } + + +} diff --git a/DataExtractionOSM/src/com/osmand/impl/ConsoleProgressImplementation.java b/DataExtractionOSM/src/com/osmand/impl/ConsoleProgressImplementation.java index edc5a1ebf8..957770000f 100644 --- a/DataExtractionOSM/src/com/osmand/impl/ConsoleProgressImplementation.java +++ b/DataExtractionOSM/src/com/osmand/impl/ConsoleProgressImplementation.java @@ -11,8 +11,17 @@ public class ConsoleProgressImplementation implements IProgress { String currentTask; int work; int currentDone; + double delta; + private long previousTaskStarted = 0; double lastPercentPrint = 0; + public ConsoleProgressImplementation(){ + delta = deltaPercentsToPrint; + } + + public ConsoleProgressImplementation(double deltaToPrint){ + delta = deltaToPrint; + } @Override public void finishTask() { @@ -33,7 +42,7 @@ public class ConsoleProgressImplementation implements IProgress { } private void printIfNeeded() { - if(getCurrentPercent() - lastPercentPrint >= deltaPercentsToPrint){ + if(getCurrentPercent() - lastPercentPrint >= delta){ System.out.println(MessageFormat.format("Done {0} %.", getCurrentPercent())); //$NON-NLS-1$ this.lastPercentPrint = getCurrentPercent(); } @@ -53,10 +62,15 @@ public class ConsoleProgressImplementation implements IProgress { public void startTask(String taskName, int work) { if(!Algoritms.objectEquals(currentTask, taskName)){ this.currentTask = taskName; - System.out.println("Started new task : " + currentTask + " - " + work); //$NON-NLS-1$ //$NON-NLS-2$ + System.out.println("Memory before task exec: " + Runtime.getRuntime().totalMemory() + " free : " + Runtime.getRuntime().freeMemory()); //$NON-NLS-1$ //$NON-NLS-2$ + if (previousTaskStarted == 0) { + System.out.println(taskName + " started - " + work); //$NON-NLS-1$ + } else { + System.out.println(taskName + " started after " + (System.currentTimeMillis() - previousTaskStarted) + " ms" + " - " + work); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + } + previousTaskStarted = System.currentTimeMillis(); } startWork(work); - } @Override diff --git a/OsmAnd/.classpath b/OsmAnd/.classpath index 5dcc90855d..8f4539503f 100644 --- a/OsmAnd/.classpath +++ b/OsmAnd/.classpath @@ -1,7 +1,7 @@ - +