From e16371d336cdef19d76cd8c180d3a9f331c67b11 Mon Sep 17 00:00:00 2001 From: Victor Shcherb Date: Fri, 9 Jul 2010 11:15:50 +0000 Subject: [PATCH] implement zip/unzip files for downloading git-svn-id: https://osmand.googlecode.com/svn/trunk@309 e29c36b1-1cfa-d876-8d93-3434fc2bb7b8 --- .../src/com/osmand/ToDoConstants.java | 15 ++- .../index/DownloaderIndexFromGoogleCode.java | 13 ++- .../osmand/data/index/IndexBatchCreator.java | 77 ++++++++---- .../com/osmand/data/index/IndexConstants.java | 4 + OsmAnd/AndroidManifest.xml | 2 +- OsmAnd/res/values-ru-rRU/strings.xml | 1 + OsmAnd/res/values/strings.xml | 1 + .../activities/DownloadIndexActivity.java | 110 ++++++++++++------ 8 files changed, 153 insertions(+), 70 deletions(-) diff --git a/DataExtractionOSM/src/com/osmand/ToDoConstants.java b/DataExtractionOSM/src/com/osmand/ToDoConstants.java index ec0f428586..34d7cd0cb8 100644 --- a/DataExtractionOSM/src/com/osmand/ToDoConstants.java +++ b/DataExtractionOSM/src/com/osmand/ToDoConstants.java @@ -19,7 +19,10 @@ public class ToDoConstants { // 60. Audio guidance for routing ! // 68. Implement service to app work with screen offline // (introduce special settings how often update location to monitoring & audio guidance) + // 69. Multitouch zoom + // Improvement : Show stops in the transport route on the map + // Not clear if it is really needed // 43. Enable poi filter by name // 58. Upload/Download zip-index from site & unzip them on phone // 45. Get clear settings. Move that setting on top settings screen. @@ -27,6 +30,7 @@ public class ToDoConstants { // (would you like to use internet for that operation - if using internet is not checked). // Internet using now for : edit POI osm, show osm bugs layer, download tiles. + // Unscheduled (complex) // 66. Transport routing (show next stop, total distance, show stop get out) (?). // 64. Traffic information (?) // 65. Intermediate points - for better control routing, to avoid traffic jam ...(?) @@ -34,15 +38,14 @@ public class ToDoConstants { // 63. Support simple offline routing(require new index file) (?) - // FIXME BUGS Android - // 1. Improvement : Show stops in the transport route - // 2. Pinch zoom !!! + // BUGS Android + // TODO swing // 9. Fix issues with big files (such as netherlands) - save memory (!) - very slow due to transport index ! - // Current result : for big file (1 - task 60-80% time, 90% memory) - // 1. Download tiles without using dir tiles (?) - // 10. Improve address indexing (use relations). + // Current result : for big file (1 - task 60-80% time, 90% memory) (?) + // 1. Download tiles without using dir tiles (?) + // 10. Improve address indexing (use relations). (?) // use relation "a6" (to accumulate streets!), "a3" to read all cities & define boundaries for city (& define that street in city). // BUGS Swing diff --git a/DataExtractionOSM/src/com/osmand/data/index/DownloaderIndexFromGoogleCode.java b/DataExtractionOSM/src/com/osmand/data/index/DownloaderIndexFromGoogleCode.java index b7c8c8f202..241a016934 100644 --- a/DataExtractionOSM/src/com/osmand/data/index/DownloaderIndexFromGoogleCode.java +++ b/DataExtractionOSM/src/com/osmand/data/index/DownloaderIndexFromGoogleCode.java @@ -22,16 +22,19 @@ public class DownloaderIndexFromGoogleCode { * @throws IOException */ public static void main(String[] args) throws URISyntaxException, IOException { - Map indexFiles = DownloaderIndexFromGoogleCode.getIndexFiles(new String[] { IndexConstants.ADDRESS_INDEX_EXT, - IndexConstants.POI_INDEX_EXT, IndexConstants.TRANSPORT_INDEX_EXT }, new String[] { - IndexConstants.ADDRESS_TABLE_VERSION + "", IndexConstants.POI_TABLE_VERSION + "", //$NON-NLS-1$//$NON-NLS-2$ - IndexConstants.TRANSPORT_TABLE_VERSION + "" }); //$NON-NLS-1$ + Map indexFiles = DownloaderIndexFromGoogleCode.getIndexFiles( + new String[] { IndexConstants.ADDRESS_INDEX_EXT, IndexConstants.POI_INDEX_EXT, IndexConstants.TRANSPORT_INDEX_EXT, + IndexConstants.ADDRESS_INDEX_EXT_ZIP, IndexConstants.POI_INDEX_EXT_ZIP, IndexConstants.TRANSPORT_INDEX_EXT_ZIP,}, + new String[] { IndexConstants.ADDRESS_TABLE_VERSION + "", IndexConstants.POI_TABLE_VERSION + "", //$NON-NLS-1$//$NON-NLS-2$ + IndexConstants.TRANSPORT_TABLE_VERSION + "" , //$NON-NLS-1$ + IndexConstants.ADDRESS_TABLE_VERSION + "", IndexConstants.POI_TABLE_VERSION + "", //$NON-NLS-1$//$NON-NLS-2$ + IndexConstants.TRANSPORT_TABLE_VERSION + "" }); //$NON-NLS-1$ System.out.println(indexFiles); } private static StringBuilder getContent() { try { - URL url = new URL("http://code.google.com/p/osmand/downloads/list?num=500&start=0"); //$NON-NLS-1$ + URL url = new URL("http://code.google.com/p/osmand/downloads/list?num=1500&start=0"); //$NON-NLS-1$ BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream())); StringBuilder b = new StringBuilder(); diff --git a/DataExtractionOSM/src/com/osmand/data/index/IndexBatchCreator.java b/DataExtractionOSM/src/com/osmand/data/index/IndexBatchCreator.java index 11e5e2ff81..d732c38280 100644 --- a/DataExtractionOSM/src/com/osmand/data/index/IndexBatchCreator.java +++ b/DataExtractionOSM/src/com/osmand/data/index/IndexBatchCreator.java @@ -1,6 +1,7 @@ package com.osmand.data.index; import java.io.File; +import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; @@ -12,6 +13,8 @@ import java.util.Date; import java.util.LinkedHashSet; import java.util.Locale; import java.util.Set; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; import org.apache.commons.logging.Log; @@ -61,20 +64,19 @@ public class IndexBatchCreator { // us states // TODO address protected static final String[] usStates = new String[] { - "Alabama", "Alaska", "Arizona", "Arkansas", "California", "Colorado", "Connecticut", - "Delaware", "District_of_Columbia", "Florida", "Georgia", "Guantanamo_Bay", "Hawaii", - "Idaho", "Illinois", "Indiana", "Iowa", "Kansas", "Kentucky", "Louisiana", "Maine", - "Maryland", "Massachusetts", "Michigan", "Minnesota", "Mississippi", "Missouri", - "Montana", "Nebraska", "Nevada", "New_Hampshire", "New_Jersey", "New_Mexico", - "New_York", "North_Carolina", "North_Dakota", "Ohio", "Oklahoma", "Oregon", - "Pennsylvania", "Rhode Island", "South Carolina", "South Dakota", "Tennessee", - "Texas", "Utah", "Vermont", "Virginia", "Washington", "West_Virginia", "Wisconsin", "Wyoming", +// "Alabama", "Alaska", "Arizona", "Arkansas", "California", "Colorado", "Connecticut", +// "Delaware", "District_of_Columbia", "Florida", "Georgia", "Guantanamo_Bay", "Hawaii", +// "Idaho", "Illinois", "Indiana", "Iowa", "Kansas", "Kentucky", "Louisiana", "Maine", +// "Maryland", "Massachusetts", "Michigan", "Minnesota", "Mississippi", "Missouri", +// "Montana", "Nebraska", "Nevada", "New_Hampshire", "New_Jersey", "New_Mexico", +// "New_York", "North_Carolina", "North_Dakota", "Ohio", "Oklahoma", "Oregon", +// "Pennsylvania", "Rhode Island", "South Carolina", "South Dakota", "Tennessee", +// "Texas", "Utah", "Vermont", "Virginia", "Washington", "West_Virginia", "Wisconsin", "Wyoming", }; - // TODO address protected static final String[] canadaStates = new String[] { - "Alberta","British_Columbia","Manitoba","New_Brunswick","Newfoundland", - "Nova_Scotia","Nunavut", "Nw_Territories","Ontario","Pr_Edwrd_Island", - "Quebec","Saskatchewan","Yukon", +// "Alberta","British_Columbia","Manitoba","New_Brunswick","Newfoundland", +// "Nova_Scotia","Nunavut", "Nw_Territories","Ontario","Pr_Edwrd_Island", +// "Quebec","Saskatchewan","Yukon", }; @@ -266,24 +268,51 @@ public class IndexBatchCreator { MessageFormat format = new MessageFormat("{0,date,dd.MM.yyyy} : {1, number,##.#} MB", Locale.US); String summary; double mbLengh = (double)f.length() / MB; - - String descriptionFile = "{"+format.format(new Object[]{new Date(f.lastModified()), mbLengh})+"}"; - if(f.getName().endsWith(IndexConstants.POI_INDEX_EXT)){ - String regionName = f.getName().substring(0, f.getName().length() - IndexConstants.POI_INDEX_EXT.length() - 2); - summary = "POI index for " + regionName + " " + descriptionFile; - } else if(f.getName().endsWith(IndexConstants.ADDRESS_INDEX_EXT)){ - String regionName = f.getName().substring(0, f.getName().length() - IndexConstants.ADDRESS_INDEX_EXT.length() - 2); - summary = "Adress index for " + regionName + " " + descriptionFile; - } else if(f.getName().endsWith(IndexConstants.TRANSPORT_INDEX_EXT)){ - String regionName = f.getName().substring(0, f.getName().length() - IndexConstants.TRANSPORT_INDEX_EXT.length() - 2); - summary = "Transport index for " + regionName + " " + descriptionFile; + String regionName; + if(f.getName().endsWith(IndexConstants.POI_INDEX_EXT) || f.getName().endsWith(IndexConstants.POI_INDEX_EXT_ZIP)){ + regionName = f.getName().substring(0, f.getName().length() - IndexConstants.POI_INDEX_EXT.length() - 2); + summary = "POI index for " ; + } else if(f.getName().endsWith(IndexConstants.ADDRESS_INDEX_EXT) || f.getName().endsWith(IndexConstants.ADDRESS_INDEX_EXT_ZIP)){ + regionName = f.getName().substring(0, f.getName().length() - IndexConstants.ADDRESS_INDEX_EXT.length() - 2); + summary = "Adress index for " ; + } else if(f.getName().endsWith(IndexConstants.TRANSPORT_INDEX_EXT) || f.getName().endsWith(IndexConstants.TRANSPORT_INDEX_EXT_ZIP)){ + regionName = f.getName().substring(0, f.getName().length() - IndexConstants.TRANSPORT_INDEX_EXT.length() - 2); + summary = "Transport index for "; } else { return; } + if(mbLengh > 5 && f.getName().endsWith(".odb")){ + String zipFileName = f.getName().subSequence(0, f.getName().length() - 4)+".zip"; + File zFile = new File(f.getParentFile(), zipFileName); + log.info("Zipping file " + f.getName()); + try { + ZipOutputStream zout = new ZipOutputStream(new FileOutputStream(zFile)); + zout.setLevel(9); + zout.putNextEntry(new ZipEntry(f.getName())); + FileInputStream is = new FileInputStream(f); + byte[] BUFFER = new byte[8192]; + int read = 0; + while((read = is.read(BUFFER)) != -1){ + zout.write(BUFFER, 0, read); + } + zout.close(); + } catch (IOException e) { + log.error("Exception while zipping file"); + } + if(f.delete()){ + log.info("Source odb file was deleted"); + } + f = zFile; + + } + mbLengh = (double)f.length() / MB; if(mbLengh > 100){ - System.err.println("ERROR : file " + f.getName() + " exceeded 90 mb!!! Could not be uploaded."); + System.err.println("ERROR : file " + f.getName() + " exceeded 100 mb!!! Could not be uploaded."); return; // restriction for google code } + String descriptionFile = "{"+format.format(new Object[]{new Date(f.lastModified()), mbLengh})+"}"; + summary += regionName + " " + descriptionFile; + alreadyUploadedFiles.add(f.getName()); GoogleCodeUploadIndex uploader = new GoogleCodeUploadIndex(); uploader.setFileName(f.getAbsolutePath()); diff --git a/DataExtractionOSM/src/com/osmand/data/index/IndexConstants.java b/DataExtractionOSM/src/com/osmand/data/index/IndexConstants.java index bdba8f731a..106eab761a 100644 --- a/DataExtractionOSM/src/com/osmand/data/index/IndexConstants.java +++ b/DataExtractionOSM/src/com/osmand/data/index/IndexConstants.java @@ -16,6 +16,10 @@ public class IndexConstants { public static final String ADDRESS_INDEX_EXT = ".addr.odb"; //$NON-NLS-1$ public static final String TRANSPORT_INDEX_EXT = ".trans.odb"; //$NON-NLS-1$ + public static final String POI_INDEX_EXT_ZIP = ".poi.zip"; //$NON-NLS-1$ + public static final String ADDRESS_INDEX_EXT_ZIP = ".addr.zip"; //$NON-NLS-1$ + public static final String TRANSPORT_INDEX_EXT_ZIP = ".trans.zip"; //$NON-NLS-1$ + public interface IndexColumn { public boolean isIndex(); diff --git a/OsmAnd/AndroidManifest.xml b/OsmAnd/AndroidManifest.xml index ed3a2a92ca..0c2dc7c39a 100644 --- a/OsmAnd/AndroidManifest.xml +++ b/OsmAnd/AndroidManifest.xml @@ -1,6 +1,6 @@ + package="com.osmand" android:versionName="0.2.2" android:versionCode="5"> + Файл распаковывается Поверните направо и двигайтесь Поверните резко направо и двигайтесь Поверните направо и двигайтесь diff --git a/OsmAnd/res/values/strings.xml b/OsmAnd/res/values/strings.xml index a986f9e35e..92a53abe97 100644 --- a/OsmAnd/res/values/strings.xml +++ b/OsmAnd/res/values/strings.xml @@ -1,5 +1,6 @@ + File is unzipping Turn right and go Turn sharply right and go Turn slightly right and go diff --git a/OsmAnd/src/com/osmand/activities/DownloadIndexActivity.java b/OsmAnd/src/com/osmand/activities/DownloadIndexActivity.java index 56f7c423bf..05c9c2dd09 100644 --- a/OsmAnd/src/com/osmand/activities/DownloadIndexActivity.java +++ b/OsmAnd/src/com/osmand/activities/DownloadIndexActivity.java @@ -1,6 +1,7 @@ package com.osmand.activities; import java.io.File; +import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; @@ -13,6 +14,8 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Map.Entry; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import org.apache.commons.logging.Log; @@ -74,10 +77,14 @@ public class DownloadIndexActivity extends ListActivity { protected Map downloadIndex(){ try { log.debug("Start loading list of index files"); //$NON-NLS-1$ - Map indexFiles = DownloaderIndexFromGoogleCode.getIndexFiles(new String[] { IndexConstants.ADDRESS_INDEX_EXT, - IndexConstants.POI_INDEX_EXT, IndexConstants.TRANSPORT_INDEX_EXT, }, new String[] { + Map indexFiles = DownloaderIndexFromGoogleCode.getIndexFiles(new String[] { + IndexConstants.ADDRESS_INDEX_EXT, IndexConstants.POI_INDEX_EXT, IndexConstants.TRANSPORT_INDEX_EXT, + IndexConstants.ADDRESS_INDEX_EXT_ZIP, IndexConstants.POI_INDEX_EXT_ZIP, IndexConstants.TRANSPORT_INDEX_EXT_ZIP, }, + new String[] { IndexConstants.ADDRESS_TABLE_VERSION + "", IndexConstants.POI_TABLE_VERSION + "",//$NON-NLS-1$//$NON-NLS-2$ - IndexConstants.TRANSPORT_TABLE_VERSION + "",}); //$NON-NLS-1$ + IndexConstants.TRANSPORT_TABLE_VERSION + "", //$NON-NLS-1$ + IndexConstants.ADDRESS_TABLE_VERSION + "", IndexConstants.POI_TABLE_VERSION + "",//$NON-NLS-1$//$NON-NLS-2$ + IndexConstants.TRANSPORT_TABLE_VERSION + ""}); //$NON-NLS-1$ if (indexFiles != null && !indexFiles.isEmpty()) { return indexFiles; } else { @@ -129,18 +136,27 @@ public class DownloadIndexActivity extends ListActivity { if(key.endsWith(IndexConstants.ADDRESS_INDEX_EXT)){ parent = new File(Environment.getExternalStorageDirectory(), ResourceManager.ADDRESS_PATH); regionName += IndexConstants.ADDRESS_INDEX_EXT; + } else if(key.endsWith(IndexConstants.ADDRESS_INDEX_EXT_ZIP)){ + parent = new File(Environment.getExternalStorageDirectory(), ResourceManager.ADDRESS_PATH); + regionName += IndexConstants.ADDRESS_INDEX_EXT_ZIP; } else if(key.endsWith(IndexConstants.POI_INDEX_EXT)){ parent = new File(Environment.getExternalStorageDirectory(), ResourceManager.POI_PATH); regionName += IndexConstants.POI_INDEX_EXT; + } else if(key.endsWith(IndexConstants.POI_INDEX_EXT_ZIP)){ + parent = new File(Environment.getExternalStorageDirectory(), ResourceManager.POI_PATH); + regionName += IndexConstants.POI_INDEX_EXT_ZIP; } else if(key.endsWith(IndexConstants.TRANSPORT_INDEX_EXT)){ parent = new File(Environment.getExternalStorageDirectory(), ResourceManager.TRANSPORT_PATH); regionName += IndexConstants.TRANSPORT_INDEX_EXT; + } else if(key.endsWith(IndexConstants.TRANSPORT_INDEX_EXT_ZIP)){ + parent = new File(Environment.getExternalStorageDirectory(), ResourceManager.TRANSPORT_PATH); + regionName += IndexConstants.TRANSPORT_INDEX_EXT_ZIP; } if(parent != null){ parent.mkdirs(); } if(parent == null || !parent.exists()){ - Toast.makeText(DownloadIndexActivity.this, getString(R.string.download_sd_dir_not_accessible), Toast.LENGTH_LONG); + Toast.makeText(DownloadIndexActivity.this, getString(R.string.download_sd_dir_not_accessible), Toast.LENGTH_LONG).show(); return null; } File file = new File(parent, regionName); @@ -158,32 +174,55 @@ public class DownloadIndexActivity extends ListActivity { public void run() { try { FileOutputStream out = new FileOutputStream(file); - try { - URL url = DownloaderIndexFromGoogleCode.getInputStreamToLoadIndex(key); - URLConnection conn = url.openConnection(); - InputStream is = conn.getInputStream(); - impl.startTask(getString(R.string.downloading_file), conn.getContentLength()); - byte[] buffer = new byte[BUFFER_SIZE]; - int read = 0; - while((read = is.read(buffer)) != -1){ - out.write(buffer, 0, read); - impl.progress(read); + URL url = DownloaderIndexFromGoogleCode.getInputStreamToLoadIndex(key); + URLConnection conn = url.openConnection(); + InputStream is = conn.getInputStream(); + impl.startTask(getString(R.string.downloading_file), conn.getContentLength()); + byte[] buffer = new byte[BUFFER_SIZE]; + int read = 0; + while((read = is.read(buffer)) != -1){ + out.write(buffer, 0, read); + impl.progress(read); + } + out.close(); + + File toIndex = file; + if(file.getName().endsWith(".zip")){ //$NON-NLS-1$ + impl.startTask(getString(R.string.unzipping_file), -1); + toIndex = new File(file.getParentFile(), file.getName().substring(0, file.getName().length() - 3) + "odb"); //$NON-NLS-1$ + ZipInputStream zipIn = new ZipInputStream(new FileInputStream(file)); + ZipEntry entry = null; + boolean found = false; + while(!found) { + if(entry != null){ + zipIn.closeEntry(); + } + entry = zipIn.getNextEntry(); + found = entry == null || entry.getName().endsWith(".odb"); //$NON-NLS-1$ + } + if(entry != null){ + out = new FileOutputStream(toIndex); + while((read = zipIn.read(buffer)) != -1){ + out.write(buffer, 0, read); + } + out.close(); } - ArrayList warnings = new ArrayList(); - if(file.getName().endsWith(IndexConstants.ADDRESS_INDEX_EXT)){ - ResourceManager.getResourceManager().indexingAddress(impl, warnings, file); - } else if(file.getName().endsWith(IndexConstants.POI_INDEX_EXT)){ - ResourceManager.getResourceManager().indexingPoi(impl, warnings, file); - } else if(file.getName().endsWith(IndexConstants.TRANSPORT_INDEX_EXT)){ - ResourceManager.getResourceManager().indexingTransport(impl, warnings, file); - } - if(warnings.isEmpty()){ - showWarning(getString(R.string.download_index_success)); - } else { - showWarning(warnings.get(0)); - } - } finally { - out.close(); + zipIn.close(); + file.delete(); // zip is no needed more + } + + ArrayList warnings = new ArrayList(); + if(toIndex.getName().endsWith(IndexConstants.ADDRESS_INDEX_EXT)){ + ResourceManager.getResourceManager().indexingAddress(impl, warnings, toIndex); + } else if(toIndex.getName().endsWith(IndexConstants.POI_INDEX_EXT)){ + ResourceManager.getResourceManager().indexingPoi(impl, warnings, toIndex); + } else if(toIndex.getName().endsWith(IndexConstants.TRANSPORT_INDEX_EXT)){ + ResourceManager.getResourceManager().indexingTransport(impl, warnings, toIndex); + } + if(warnings.isEmpty()){ + showWarning(getString(R.string.download_index_success)); + } else { + showWarning(warnings.get(0)); } } catch (IOException e) { log.error("Exception ocurred", e); //$NON-NLS-1$ @@ -227,15 +266,18 @@ public class DownloadIndexActivity extends ListActivity { Entry e = getItem(position); int l = e.getKey().lastIndexOf('_'); String s = ""; //$NON-NLS-1$ - if(e.getKey().endsWith(IndexConstants.POI_INDEX_EXT)){ + if(e.getKey().endsWith(IndexConstants.POI_INDEX_EXT) || e.getKey().endsWith(IndexConstants.POI_INDEX_EXT_ZIP)){ s = getString(R.string.poi); - } else if(e.getKey().endsWith(IndexConstants.ADDRESS_INDEX_EXT)){ + } else if(e.getKey().endsWith(IndexConstants.ADDRESS_INDEX_EXT) || e.getKey().endsWith(IndexConstants.ADDRESS_INDEX_EXT_ZIP)){ s = getString(R.string.address); - } else if(e.getKey().endsWith(IndexConstants.TRANSPORT_INDEX_EXT)){ + } else if(e.getKey().endsWith(IndexConstants.TRANSPORT_INDEX_EXT) || e.getKey().endsWith(IndexConstants.TRANSPORT_INDEX_EXT_ZIP)){ s = getString(R.string.transport); } - - item.setText(s + "\n " + e.getKey().substring(0, l).replace('_', ' ')); //$NON-NLS-1$ + String name = e.getKey().substring(0, l).replace('_', ' '); + if(e.getKey().endsWith(".zip")){ //$NON-NLS-1$ + name += " (zip)"; //$NON-NLS-1$ + } + item.setText(s + "\n " + name); //$NON-NLS-1$ description.setText(e.getValue().replace(':', '\n')); return row; }