diff --git a/DataExtractionOSM/src/net/osmand/Version.java b/DataExtractionOSM/src/net/osmand/Version.java index 31302a2f05..5b1c079a12 100644 --- a/DataExtractionOSM/src/net/osmand/Version.java +++ b/DataExtractionOSM/src/net/osmand/Version.java @@ -1,17 +1,17 @@ -package net.osmand; - -public class Version { - - public static final String APP_NAME = "OsmAnd"; //$NON-NLS-1$ - public static final String APP_MAP_CREATOR_NAME = "OsmAndMapCreator"; //$NON-NLS-1$ - public static final String APP_VERSION = "0.6.3"; //$NON-NLS-1$ - public static final String APP_DESCRIPTION = "alpha"; //$NON-NLS-1$ - - public static final String APP_NAME_VERSION = APP_NAME + " " + APP_VERSION; //$NON-NLS-1$ - public static final String APP_FULL_NAME = APP_NAME + " " + APP_VERSION + " " +APP_DESCRIPTION; //$NON-NLS-1$ //$NON-NLS-2$ - public static final String APP_MAP_CREATOR_FULL_NAME = APP_MAP_CREATOR_NAME + " " + APP_VERSION + " " +APP_DESCRIPTION; //$NON-NLS-1$ //$NON-NLS-2$ - - - - -} +package net.osmand; + +public class Version { + + public static final String APP_NAME = "OsmAnd"; //$NON-NLS-1$ + public static final String APP_MAP_CREATOR_NAME = "OsmAndMapCreator"; //$NON-NLS-1$ + public static final String APP_VERSION = "0.6.3"; //$NON-NLS-1$ + public static final String APP_DESCRIPTION = "alpha"; //$NON-NLS-1$ + + public static final String APP_NAME_VERSION = APP_NAME + " " + APP_VERSION; //$NON-NLS-1$ + public static final String APP_FULL_NAME = APP_NAME + " " + APP_VERSION + " " +APP_DESCRIPTION; //$NON-NLS-1$ //$NON-NLS-2$ + public static final String APP_MAP_CREATOR_FULL_NAME = APP_MAP_CREATOR_NAME + " " + APP_VERSION + " " +APP_DESCRIPTION; //$NON-NLS-1$ //$NON-NLS-2$ + + + + +} diff --git a/DataExtractionOSM/src/net/osmand/data/index/IndexBatchCreator.java b/DataExtractionOSM/src/net/osmand/data/index/IndexBatchCreator.java index 7a0581db72..b55fda1e96 100644 --- a/DataExtractionOSM/src/net/osmand/data/index/IndexBatchCreator.java +++ b/DataExtractionOSM/src/net/osmand/data/index/IndexBatchCreator.java @@ -1,494 +1,494 @@ -package net.osmand.data.index; - -import java.io.BufferedReader; -import java.io.Closeable; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.io.RandomAccessFile; -import java.net.HttpURLConnection; -import java.net.URL; -import java.net.URLEncoder; -import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.Arrays; +package net.osmand.data.index; + +import java.io.BufferedReader; +import java.io.Closeable; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.RandomAccessFile; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLEncoder; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; - -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; - -import net.osmand.Algoritms; -import net.osmand.LogUtil; -import net.osmand.binary.BinaryMapIndexReader; -import net.osmand.data.preparation.IndexCreator; -import net.osmand.data.preparation.MapZooms; -import net.osmand.impl.ConsoleProgressImplementation; -import net.osmand.osm.MapRenderingTypes; - -import org.apache.commons.logging.Log; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.NodeList; -import org.xml.sax.SAXException; - -import rtree.RTree; - - - -public class IndexBatchCreator { - - - protected static final Log log = LogUtil.getLog(IndexBatchCreator.class); +import java.util.Comparator; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; + +import net.osmand.Algoritms; +import net.osmand.LogUtil; +import net.osmand.binary.BinaryMapIndexReader; +import net.osmand.data.preparation.IndexCreator; +import net.osmand.data.preparation.MapZooms; +import net.osmand.impl.ConsoleProgressImplementation; +import net.osmand.osm.MapRenderingTypes; + +import org.apache.commons.logging.Log; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; + +import rtree.RTree; + + + +public class IndexBatchCreator { + + + protected static final Log log = LogUtil.getLog(IndexBatchCreator.class); private final static double MIN_SIZE_TO_UPLOAD = 0.015d; private final static double MIN_SIZE_TO_NOT_ZIP = 2d; private final static double MAX_SIZE_TO_NOT_SPLIT = 190d; private final static double MAX_UPLOAD_SIZE = 195d; - - public static class RegionCountries { - String namePrefix = ""; // for states of the country - String nameSuffix = ""; - Map regionNames = new LinkedHashMap(); - String siteToDownload = ""; - } - - private static class RegionSpecificData { - public String cityAdminLevel; - } - - private boolean uploadToOsmandDownloads = true; - - - // process atributtes - boolean downloadFiles = false; - boolean generateIndexes = false; - boolean uploadIndexes = false; - MapZooms mapZooms = null; - MapRenderingTypes types = MapRenderingTypes.getDefault(); - boolean deleteFilesAfterUploading = true; - - File osmDirFiles; - File indexDirFiles; - - boolean indexPOI = false; - boolean indexTransport = false; - boolean indexAddress = false; - boolean indexMap = false; - - - String user; - String password; - String cookieHSID = ""; - String cookieSID = ""; - String pagegen = ""; - String token = ""; - private String wget; - - - public static void main(String[] args) { - - IndexBatchCreator creator = new IndexBatchCreator(); - if(args == null || args.length == 0){ - System.out.println("Please specify -local parameter or path to batch.xml configuration file as 1 argument."); - throw new IllegalArgumentException("Please specify -local parameter or path to batch.xml configuration file as 1 argument."); - } - String name = args[0]; - InputStream stream; - if(name.equals("-local")){ - stream = IndexBatchCreator.class.getResourceAsStream("batch.xml"); - } else { - try { - stream = new FileInputStream(name); - } catch (FileNotFoundException e) { - System.out.println("XML configuration file not found : " + name); - return; - } - } - - try { - Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(stream); - creator.runBatch(doc); - } catch (SAXException e) { - System.out.println("XML configuration file could not be read from " + name); - e.printStackTrace(); - log.error("XML configuration file could not be read from " + name, e); - } catch (IOException e) { - System.out.println("XML configuration file could not be read from " + name); - e.printStackTrace(); - log.error("XML configuration file could not be read from " + name, e); - } catch (ParserConfigurationException e) { - System.out.println("XML configuration file could not be read from " + name); - e.printStackTrace(); - log.error("XML configuration file could not be read from " + name, e); - } finally { - safeClose(stream, "Error closing stream for " + name); - } - } - - public void runBatch(Document doc){ - NodeList list = doc.getElementsByTagName("process"); - if(list.getLength() != 1){ - throw new IllegalArgumentException("You should specify exactly 1 process element!"); - } - Element process = (Element) list.item(0); - downloadFiles = Boolean.parseBoolean(process.getAttribute("downloadOsmFiles")); - generateIndexes = Boolean.parseBoolean(process.getAttribute("generateIndexes")); - uploadIndexes = Boolean.parseBoolean(process.getAttribute("uploadIndexes")); - deleteFilesAfterUploading = Boolean.parseBoolean(process.getAttribute("deleteFilesAfterUploading")); - wget = process.getAttribute("wget"); - - indexPOI = Boolean.parseBoolean(process.getAttribute("indexPOI")); - indexMap = Boolean.parseBoolean(process.getAttribute("indexMap")); - indexTransport = Boolean.parseBoolean(process.getAttribute("indexTransport")); - indexAddress = Boolean.parseBoolean(process.getAttribute("indexAddress")); - String zooms = process.getAttribute("mapZooms"); - if(zooms == null || zooms.length() == 0){ - mapZooms = MapZooms.getDefault(); - } else { - mapZooms = MapZooms.parseZooms(zooms); - } - - String f = process.getAttribute("renderingTypesFile"); - if(f == null || f.length() == 0){ - types = MapRenderingTypes.getDefault(); - } else { - types = new MapRenderingTypes(f); - } - - String dir = process.getAttribute("directory_for_osm_files"); - if(dir == null || !new File(dir).exists()) { - throw new IllegalArgumentException("Please specify directory with .osm or .osm.bz2 files as directory_for_osm_files (attribute)"); //$NON-NLS-1$ - } - osmDirFiles = new File(dir); - dir = process.getAttribute("directory_for_index_files"); - if(dir == null || !new File(dir).exists()) { - throw new IllegalArgumentException("Please specify directory with generated index files as directory_for_index_files (attribute)"); //$NON-NLS-1$ - } - indexDirFiles = new File(dir); - - if(uploadIndexes){ - list = doc.getElementsByTagName("authorization_info"); - if(list.getLength() != 1){ - throw new IllegalArgumentException("You should specify exactly 1 authorization_info element to upload indexes!"); - } - Element authorization = (Element) list.item(0); - cookieHSID = authorization.getAttribute("cookieHSID"); - cookieSID = authorization.getAttribute("cookieSID"); - pagegen = authorization.getAttribute("pagegen"); - token = authorization.getAttribute("token"); - uploadToOsmandDownloads = Boolean.parseBoolean(process.getAttribute("upload_osmand_download")); - if(uploadToOsmandDownloads){ - user = authorization.getAttribute("osmand_download_user"); - password = authorization.getAttribute("osmand_download_password"); - } else { - user = authorization.getAttribute("google_code_user"); - password = authorization.getAttribute("google_code_password"); - } - } - - List countriesToDownload = new ArrayList(); - NodeList regions = doc.getElementsByTagName("regions"); - for(int i=0; i< regions.getLength(); i++){ - Element el = (Element) regions.item(i); - if(!Boolean.parseBoolean(el.getAttribute("skip"))){ - RegionCountries countries = new RegionCountries(); - countries.siteToDownload = el.getAttribute("siteToDownload"); - if(countries.siteToDownload == null){ - continue; - } - countries.namePrefix = el.getAttribute("region_prefix"); - if(countries.namePrefix == null){ - countries.namePrefix = ""; - } - countries.nameSuffix = el.getAttribute("region_suffix"); - if(countries.nameSuffix == null){ - countries.nameSuffix = ""; - } - NodeList ncountries = el.getElementsByTagName("region"); - for(int j=0; j< ncountries.getLength(); j++){ - Element ncountry = (Element) ncountries.item(j); - String name = ncountry.getAttribute("name"); - RegionSpecificData data = new RegionSpecificData(); - data.cityAdminLevel = ncountry.getAttribute("cityAdminLevel"); - if(name != null && !Boolean.parseBoolean(ncountry.getAttribute("skip"))){ - countries.regionNames.put(name, data); - } - } - countriesToDownload.add(countries); - - } - } - - runBatch(countriesToDownload); - - - } - - public void runBatch(List countriesToDownload ){ - Set alreadyUploadedFiles = new LinkedHashSet(); - Set alreadyGeneratedFiles = new LinkedHashSet(); - if(downloadFiles){ - downloadFilesAndGenerateIndex(countriesToDownload, alreadyGeneratedFiles, alreadyUploadedFiles); - } - if(generateIndexes){ - generatedIndexes(alreadyGeneratedFiles, alreadyUploadedFiles); - } - if(uploadIndexes){ - uploadIndexes(alreadyUploadedFiles); - } - } - - - - protected void downloadFilesAndGenerateIndex(List countriesToDownload, Set alreadyGeneratedFiles, Set alreadyUploadedFiles){ - // clean before downloading -// for(File f : osmDirFiles.listFiles()){ -// log.info("Delete old file " + f.getName()); //$NON-NLS-1$ -// f.delete(); -// } - - for(RegionCountries regionCountries : countriesToDownload){ - String prefix = regionCountries.namePrefix; - String site = regionCountries.siteToDownload; - String suffix = regionCountries.nameSuffix; - for(String name : regionCountries.regionNames.keySet()){ - name = name.toLowerCase(); - RegionSpecificData regionSpecificData = regionCountries.regionNames.get(name); - String url = MessageFormat.format(site, name); - String country = prefix+name; - File toSave = downloadFile(url, country, suffix, alreadyGeneratedFiles, alreadyUploadedFiles); - if (toSave != null && generateIndexes) { - generateIndex(toSave, country, regionSpecificData, alreadyGeneratedFiles, alreadyUploadedFiles); - } - } - } - System.out.println("DOWNLOADING FILES FINISHED"); - } - - protected File downloadFile(String url, String country, String suffix, Set alreadyGeneratedFiles, Set alreadyUploadedFiles) { - String ext = ".osm"; - if(url.endsWith(".osm.bz2")){ - ext = ".osm.bz2"; - } else if(url.endsWith(".osm.pbf")){ - ext = ".osm.pbf"; - } - File toIndex = null; - File saveTo = new File(osmDirFiles, country + suffix + ext); - if (wget == null || wget.trim().length() == 0) { - toIndex = internalDownload(url, country, saveTo); - } else { - toIndex = wgetDownload(url, country, saveTo); - } - return toIndex; - } - - private File wgetDownload(String url, String country, File toSave) - { - BufferedReader wgetOutput = null; - OutputStream wgetInput = null; - Process wgetProc = null; - try { - log.info("Executing " + wget + " " + url + " -O "+ toSave.getCanonicalPath()); //$NON-NLS-1$//$NON-NLS-2$ $NON-NLS-3$ - ProcessBuilder exec = new ProcessBuilder(wget, "--read-timeout=5", "--progress=dot:binary", url, "-O", //$NON-NLS-1$//$NON-NLS-2$ $NON-NLS-3$ - toSave.getCanonicalPath()); - exec.redirectErrorStream(true); - wgetProc = exec.start(); - wgetOutput = new BufferedReader(new InputStreamReader(wgetProc.getInputStream())); - String line; - while ((line = wgetOutput.readLine()) != null) { - log.info("wget output:" + line); //$NON-NLS-1$ - } - int exitValue = wgetProc.waitFor(); - wgetProc = null; - if (exitValue != 0) { - log.error("Wget exited with error code: " + exitValue); //$NON-NLS-1$ - } else { - return toSave; - } - } catch (IOException e) { - log.error("Input/output exception " + toSave.getName() + " downloading from " + url + "using wget: " + wget, e); //$NON-NLS-1$ //$NON-NLS-2$ $NON-NLS-3$ - } catch (InterruptedException e) { - log.error("Interrupted exception " + toSave.getName() + " downloading from " + url + "using wget: " + wget, e); //$NON-NLS-1$ //$NON-NLS-2$ $NON-NLS-3$ - } finally { - safeClose(wgetOutput, ""); //$NON-NLS-1$ - safeClose(wgetInput, ""); //$NON-NLS-1$ - if (wgetProc != null) { - wgetProc.destroy(); - } - } - return null; - } - - private final static int DOWNLOAD_DEBUG = 1 << 20; - private final static int MB = 1 << 20; - private final static int BUFFER_SIZE = 1 << 15; - private File internalDownload(String url, String country, File toSave) { - int count = 0; - int downloaded = 0; - int mbDownloaded = 0; - byte[] buffer = new byte[BUFFER_SIZE]; - OutputStream ostream = null; - InputStream stream = null; - try { - ostream = new FileOutputStream(toSave); - stream = new URL(url).openStream(); - log.info("Downloading country " + country + " from " + url); //$NON-NLS-1$//$NON-NLS-2$ - while ((count = stream.read(buffer)) != -1) { - ostream.write(buffer, 0, count); - downloaded += count; - if(downloaded > DOWNLOAD_DEBUG){ - downloaded -= DOWNLOAD_DEBUG; - mbDownloaded += (DOWNLOAD_DEBUG>>20); - log.info(mbDownloaded +" megabytes downloaded of " + toSave.getName()); - } - } - return toSave; - } catch (IOException e) { - log.error("Input/output exception " + toSave.getName() + " downloading from " + url, e); //$NON-NLS-1$ //$NON-NLS-2$ - } finally { - safeClose(ostream, "Input/output exception " + toSave.getName() + " to close stream "); //$NON-NLS-1$ //$NON-NLS-2$ - safeClose(stream, "Input/output exception " + url + " to close stream "); //$NON-NLS-1$ //$NON-NLS-2$ - } - return null; - } - - private static void safeClose(Closeable ostream, String message) { - if (ostream != null) { - try { - ostream.close(); - } catch (Exception e) { - log.error(message, e); //$NON-NLS-1$ //$NON-NLS-2$ - } - } - } - - protected void generatedIndexes(Set alreadyGeneratedFiles, Set alreadyUploadedFiles) { - for (File f : getSortedFiles(osmDirFiles)) { - if (alreadyGeneratedFiles.contains(f.getName())) { - continue; - } - if (f.getName().endsWith(".osm.bz2") || f.getName().endsWith(".osm") || f.getName().endsWith(".osm.pbf")) { - generateIndex(f, null, null, alreadyGeneratedFiles, alreadyUploadedFiles); - } - } - System.out.println("GENERATING INDEXES FINISHED "); - } - - - - protected void generateIndex(File f, String rName, RegionSpecificData regionSpecificData, Set alreadyGeneratedFiles, Set alreadyUploadedFiles) { - if (!generateIndexes) { - return; - } - try { - // be independent of previous results - RTree.clearCache(); - - String regionName = f.getName(); - int i = f.getName().indexOf('.'); - if (i > -1) { - regionName = Algoritms.capitalizeFirstLetterAndLowercase(f.getName().substring(0, i)); - } - if(Algoritms.isEmpty(rName)){ - rName = regionName; - } else { - rName = Algoritms.capitalizeFirstLetterAndLowercase(rName); - } - - IndexCreator indexCreator = new IndexCreator(indexDirFiles); - indexCreator.setIndexAddress(indexAddress); - indexCreator.setIndexPOI(indexPOI); - indexCreator.setIndexTransport(indexTransport); - indexCreator.setIndexMap(indexMap); - indexCreator.setLastModifiedDate(f.lastModified()); - indexCreator.setNormalizeStreets(true); - indexCreator.setSaveAddressWays(true); - indexCreator.setRegionName(rName); - if (regionSpecificData != null && regionSpecificData.cityAdminLevel != null) { - indexCreator.setCityAdminLevel(regionSpecificData.cityAdminLevel); - } - - String poiFileName = regionName + "_" + IndexConstants.POI_TABLE_VERSION + IndexConstants.POI_INDEX_EXT; - indexCreator.setPoiFileName(poiFileName); - String mapFileName = regionName + "_" + IndexConstants.BINARY_MAP_VERSION + IndexConstants.BINARY_MAP_INDEX_EXT; - indexCreator.setMapFileName(mapFileName); - try { - alreadyGeneratedFiles.add(f.getName()); - indexCreator.generateIndexes(f, new ConsoleProgressImplementation(3), null, mapZooms, types); - if (indexPOI) { - uploadIndex(new File(indexDirFiles, poiFileName), alreadyUploadedFiles); - } - if (indexMap || indexAddress || indexTransport) { - uploadIndex(new File(indexDirFiles, mapFileName), alreadyUploadedFiles); - } - } catch (Exception e) { - log.error("Exception generating indexes for " + f.getName(), e); //$NON-NLS-1$ - } - } catch (OutOfMemoryError e) { - System.gc(); - log.error("OutOfMemory", e); - - } - System.gc(); - } - - protected File[] getSortedFiles(File dir){ - File[] listFiles = dir.listFiles(); - Arrays.sort(listFiles, new Comparator(){ - @Override - public int compare(File o1, File o2) { - return o1.getName().compareTo(o2.getName()); - } - }); - return listFiles; - } - - protected void uploadIndexes(Set alreadyUploadedFiles){ - for(File f : getSortedFiles(indexDirFiles)){ - if(!alreadyUploadedFiles.contains(f.getName())){ - uploadIndex(f, alreadyUploadedFiles); - if(!alreadyUploadedFiles.contains(f.getName())){ - System.out.println("! NOT UPLOADED " + f.getName()); - } - } - } - System.out.println("UPLOADING INDEXES FINISHED "); - - } - + + public static class RegionCountries { + String namePrefix = ""; // for states of the country + String nameSuffix = ""; + Map regionNames = new LinkedHashMap(); + String siteToDownload = ""; + } + + private static class RegionSpecificData { + public String cityAdminLevel; + } + + private boolean uploadToOsmandDownloads = true; - protected void uploadIndex(File f, Set alreadyUploadedFiles){ - if(!uploadIndexes){ - return; - } - - String summary; - double mbLengh = (double)f.length() / MB; - boolean zip = true; + // process atributtes + boolean downloadFiles = false; + boolean generateIndexes = false; + boolean uploadIndexes = false; + MapZooms mapZooms = null; + MapRenderingTypes types = MapRenderingTypes.getDefault(); + boolean deleteFilesAfterUploading = true; + + File osmDirFiles; + File indexDirFiles; + + boolean indexPOI = false; + boolean indexTransport = false; + boolean indexAddress = false; + boolean indexMap = false; + + + String user; + String password; + String cookieHSID = ""; + String cookieSID = ""; + String pagegen = ""; + String token = ""; + private String wget; + + + public static void main(String[] args) { + + IndexBatchCreator creator = new IndexBatchCreator(); + if(args == null || args.length == 0){ + System.out.println("Please specify -local parameter or path to batch.xml configuration file as 1 argument."); + throw new IllegalArgumentException("Please specify -local parameter or path to batch.xml configuration file as 1 argument."); + } + String name = args[0]; + InputStream stream; + if(name.equals("-local")){ + stream = IndexBatchCreator.class.getResourceAsStream("batch.xml"); + } else { + try { + stream = new FileInputStream(name); + } catch (FileNotFoundException e) { + System.out.println("XML configuration file not found : " + name); + return; + } + } + + try { + Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(stream); + creator.runBatch(doc); + } catch (SAXException e) { + System.out.println("XML configuration file could not be read from " + name); + e.printStackTrace(); + log.error("XML configuration file could not be read from " + name, e); + } catch (IOException e) { + System.out.println("XML configuration file could not be read from " + name); + e.printStackTrace(); + log.error("XML configuration file could not be read from " + name, e); + } catch (ParserConfigurationException e) { + System.out.println("XML configuration file could not be read from " + name); + e.printStackTrace(); + log.error("XML configuration file could not be read from " + name, e); + } finally { + safeClose(stream, "Error closing stream for " + name); + } + } + + public void runBatch(Document doc){ + NodeList list = doc.getElementsByTagName("process"); + if(list.getLength() != 1){ + throw new IllegalArgumentException("You should specify exactly 1 process element!"); + } + Element process = (Element) list.item(0); + downloadFiles = Boolean.parseBoolean(process.getAttribute("downloadOsmFiles")); + generateIndexes = Boolean.parseBoolean(process.getAttribute("generateIndexes")); + uploadIndexes = Boolean.parseBoolean(process.getAttribute("uploadIndexes")); + deleteFilesAfterUploading = Boolean.parseBoolean(process.getAttribute("deleteFilesAfterUploading")); + wget = process.getAttribute("wget"); + + indexPOI = Boolean.parseBoolean(process.getAttribute("indexPOI")); + indexMap = Boolean.parseBoolean(process.getAttribute("indexMap")); + indexTransport = Boolean.parseBoolean(process.getAttribute("indexTransport")); + indexAddress = Boolean.parseBoolean(process.getAttribute("indexAddress")); + String zooms = process.getAttribute("mapZooms"); + if(zooms == null || zooms.length() == 0){ + mapZooms = MapZooms.getDefault(); + } else { + mapZooms = MapZooms.parseZooms(zooms); + } + + String f = process.getAttribute("renderingTypesFile"); + if(f == null || f.length() == 0){ + types = MapRenderingTypes.getDefault(); + } else { + types = new MapRenderingTypes(f); + } + + String dir = process.getAttribute("directory_for_osm_files"); + if(dir == null || !new File(dir).exists()) { + throw new IllegalArgumentException("Please specify directory with .osm or .osm.bz2 files as directory_for_osm_files (attribute)"); //$NON-NLS-1$ + } + osmDirFiles = new File(dir); + dir = process.getAttribute("directory_for_index_files"); + if(dir == null || !new File(dir).exists()) { + throw new IllegalArgumentException("Please specify directory with generated index files as directory_for_index_files (attribute)"); //$NON-NLS-1$ + } + indexDirFiles = new File(dir); + + if(uploadIndexes){ + list = doc.getElementsByTagName("authorization_info"); + if(list.getLength() != 1){ + throw new IllegalArgumentException("You should specify exactly 1 authorization_info element to upload indexes!"); + } + Element authorization = (Element) list.item(0); + cookieHSID = authorization.getAttribute("cookieHSID"); + cookieSID = authorization.getAttribute("cookieSID"); + pagegen = authorization.getAttribute("pagegen"); + token = authorization.getAttribute("token"); + uploadToOsmandDownloads = Boolean.parseBoolean(process.getAttribute("upload_osmand_download")); + if(uploadToOsmandDownloads){ + user = authorization.getAttribute("osmand_download_user"); + password = authorization.getAttribute("osmand_download_password"); + } else { + user = authorization.getAttribute("google_code_user"); + password = authorization.getAttribute("google_code_password"); + } + } + + List countriesToDownload = new ArrayList(); + NodeList regions = doc.getElementsByTagName("regions"); + for(int i=0; i< regions.getLength(); i++){ + Element el = (Element) regions.item(i); + if(!Boolean.parseBoolean(el.getAttribute("skip"))){ + RegionCountries countries = new RegionCountries(); + countries.siteToDownload = el.getAttribute("siteToDownload"); + if(countries.siteToDownload == null){ + continue; + } + countries.namePrefix = el.getAttribute("region_prefix"); + if(countries.namePrefix == null){ + countries.namePrefix = ""; + } + countries.nameSuffix = el.getAttribute("region_suffix"); + if(countries.nameSuffix == null){ + countries.nameSuffix = ""; + } + NodeList ncountries = el.getElementsByTagName("region"); + for(int j=0; j< ncountries.getLength(); j++){ + Element ncountry = (Element) ncountries.item(j); + String name = ncountry.getAttribute("name"); + RegionSpecificData data = new RegionSpecificData(); + data.cityAdminLevel = ncountry.getAttribute("cityAdminLevel"); + if(name != null && !Boolean.parseBoolean(ncountry.getAttribute("skip"))){ + countries.regionNames.put(name, data); + } + } + countriesToDownload.add(countries); + + } + } + + runBatch(countriesToDownload); + + + } + + public void runBatch(List countriesToDownload ){ + Set alreadyUploadedFiles = new LinkedHashSet(); + Set alreadyGeneratedFiles = new LinkedHashSet(); + if(downloadFiles){ + downloadFilesAndGenerateIndex(countriesToDownload, alreadyGeneratedFiles, alreadyUploadedFiles); + } + if(generateIndexes){ + generatedIndexes(alreadyGeneratedFiles, alreadyUploadedFiles); + } + if(uploadIndexes){ + uploadIndexes(alreadyUploadedFiles); + } + } + + + + protected void downloadFilesAndGenerateIndex(List countriesToDownload, Set alreadyGeneratedFiles, Set alreadyUploadedFiles){ + // clean before downloading +// for(File f : osmDirFiles.listFiles()){ +// log.info("Delete old file " + f.getName()); //$NON-NLS-1$ +// f.delete(); +// } + + for(RegionCountries regionCountries : countriesToDownload){ + String prefix = regionCountries.namePrefix; + String site = regionCountries.siteToDownload; + String suffix = regionCountries.nameSuffix; + for(String name : regionCountries.regionNames.keySet()){ + name = name.toLowerCase(); + RegionSpecificData regionSpecificData = regionCountries.regionNames.get(name); + String url = MessageFormat.format(site, name); + String country = prefix+name; + File toSave = downloadFile(url, country, suffix, alreadyGeneratedFiles, alreadyUploadedFiles); + if (toSave != null && generateIndexes) { + generateIndex(toSave, country, regionSpecificData, alreadyGeneratedFiles, alreadyUploadedFiles); + } + } + } + System.out.println("DOWNLOADING FILES FINISHED"); + } + + protected File downloadFile(String url, String country, String suffix, Set alreadyGeneratedFiles, Set alreadyUploadedFiles) { + String ext = ".osm"; + if(url.endsWith(".osm.bz2")){ + ext = ".osm.bz2"; + } else if(url.endsWith(".osm.pbf")){ + ext = ".osm.pbf"; + } + File toIndex = null; + File saveTo = new File(osmDirFiles, country + suffix + ext); + if (wget == null || wget.trim().length() == 0) { + toIndex = internalDownload(url, country, saveTo); + } else { + toIndex = wgetDownload(url, country, saveTo); + } + return toIndex; + } + + private File wgetDownload(String url, String country, File toSave) + { + BufferedReader wgetOutput = null; + OutputStream wgetInput = null; + Process wgetProc = null; + try { + log.info("Executing " + wget + " " + url + " -O "+ toSave.getCanonicalPath()); //$NON-NLS-1$//$NON-NLS-2$ $NON-NLS-3$ + ProcessBuilder exec = new ProcessBuilder(wget, "--read-timeout=5", "--progress=dot:binary", url, "-O", //$NON-NLS-1$//$NON-NLS-2$ $NON-NLS-3$ + toSave.getCanonicalPath()); + exec.redirectErrorStream(true); + wgetProc = exec.start(); + wgetOutput = new BufferedReader(new InputStreamReader(wgetProc.getInputStream())); + String line; + while ((line = wgetOutput.readLine()) != null) { + log.info("wget output:" + line); //$NON-NLS-1$ + } + int exitValue = wgetProc.waitFor(); + wgetProc = null; + if (exitValue != 0) { + log.error("Wget exited with error code: " + exitValue); //$NON-NLS-1$ + } else { + return toSave; + } + } catch (IOException e) { + log.error("Input/output exception " + toSave.getName() + " downloading from " + url + "using wget: " + wget, e); //$NON-NLS-1$ //$NON-NLS-2$ $NON-NLS-3$ + } catch (InterruptedException e) { + log.error("Interrupted exception " + toSave.getName() + " downloading from " + url + "using wget: " + wget, e); //$NON-NLS-1$ //$NON-NLS-2$ $NON-NLS-3$ + } finally { + safeClose(wgetOutput, ""); //$NON-NLS-1$ + safeClose(wgetInput, ""); //$NON-NLS-1$ + if (wgetProc != null) { + wgetProc.destroy(); + } + } + return null; + } + + private final static int DOWNLOAD_DEBUG = 1 << 20; + private final static int MB = 1 << 20; + private final static int BUFFER_SIZE = 1 << 15; + private File internalDownload(String url, String country, File toSave) { + int count = 0; + int downloaded = 0; + int mbDownloaded = 0; + byte[] buffer = new byte[BUFFER_SIZE]; + OutputStream ostream = null; + InputStream stream = null; + try { + ostream = new FileOutputStream(toSave); + stream = new URL(url).openStream(); + log.info("Downloading country " + country + " from " + url); //$NON-NLS-1$//$NON-NLS-2$ + while ((count = stream.read(buffer)) != -1) { + ostream.write(buffer, 0, count); + downloaded += count; + if(downloaded > DOWNLOAD_DEBUG){ + downloaded -= DOWNLOAD_DEBUG; + mbDownloaded += (DOWNLOAD_DEBUG>>20); + log.info(mbDownloaded +" megabytes downloaded of " + toSave.getName()); + } + } + return toSave; + } catch (IOException e) { + log.error("Input/output exception " + toSave.getName() + " downloading from " + url, e); //$NON-NLS-1$ //$NON-NLS-2$ + } finally { + safeClose(ostream, "Input/output exception " + toSave.getName() + " to close stream "); //$NON-NLS-1$ //$NON-NLS-2$ + safeClose(stream, "Input/output exception " + url + " to close stream "); //$NON-NLS-1$ //$NON-NLS-2$ + } + return null; + } + + private static void safeClose(Closeable ostream, String message) { + if (ostream != null) { + try { + ostream.close(); + } catch (Exception e) { + log.error(message, e); //$NON-NLS-1$ //$NON-NLS-2$ + } + } + } + + protected void generatedIndexes(Set alreadyGeneratedFiles, Set alreadyUploadedFiles) { + for (File f : getSortedFiles(osmDirFiles)) { + if (alreadyGeneratedFiles.contains(f.getName())) { + continue; + } + if (f.getName().endsWith(".osm.bz2") || f.getName().endsWith(".osm") || f.getName().endsWith(".osm.pbf")) { + generateIndex(f, null, null, alreadyGeneratedFiles, alreadyUploadedFiles); + } + } + System.out.println("GENERATING INDEXES FINISHED "); + } + + + + protected void generateIndex(File f, String rName, RegionSpecificData regionSpecificData, Set alreadyGeneratedFiles, Set alreadyUploadedFiles) { + if (!generateIndexes) { + return; + } + try { + // be independent of previous results + RTree.clearCache(); + + String regionName = f.getName(); + int i = f.getName().indexOf('.'); + if (i > -1) { + regionName = Algoritms.capitalizeFirstLetterAndLowercase(f.getName().substring(0, i)); + } + if(Algoritms.isEmpty(rName)){ + rName = regionName; + } else { + rName = Algoritms.capitalizeFirstLetterAndLowercase(rName); + } + + IndexCreator indexCreator = new IndexCreator(indexDirFiles); + indexCreator.setIndexAddress(indexAddress); + indexCreator.setIndexPOI(indexPOI); + indexCreator.setIndexTransport(indexTransport); + indexCreator.setIndexMap(indexMap); + indexCreator.setLastModifiedDate(f.lastModified()); + indexCreator.setNormalizeStreets(true); + indexCreator.setSaveAddressWays(true); + indexCreator.setRegionName(rName); + if (regionSpecificData != null && regionSpecificData.cityAdminLevel != null) { + indexCreator.setCityAdminLevel(regionSpecificData.cityAdminLevel); + } + + String poiFileName = regionName + "_" + IndexConstants.POI_TABLE_VERSION + IndexConstants.POI_INDEX_EXT; + indexCreator.setPoiFileName(poiFileName); + String mapFileName = regionName + "_" + IndexConstants.BINARY_MAP_VERSION + IndexConstants.BINARY_MAP_INDEX_EXT; + indexCreator.setMapFileName(mapFileName); + try { + alreadyGeneratedFiles.add(f.getName()); + indexCreator.generateIndexes(f, new ConsoleProgressImplementation(3), null, mapZooms, types); + if (indexPOI) { + uploadIndex(new File(indexDirFiles, poiFileName), alreadyUploadedFiles); + } + if (indexMap || indexAddress || indexTransport) { + uploadIndex(new File(indexDirFiles, mapFileName), alreadyUploadedFiles); + } + } catch (Exception e) { + log.error("Exception generating indexes for " + f.getName(), e); //$NON-NLS-1$ + } + } catch (OutOfMemoryError e) { + System.gc(); + log.error("OutOfMemory", e); + + } + System.gc(); + } + + protected File[] getSortedFiles(File dir){ + File[] listFiles = dir.listFiles(); + Arrays.sort(listFiles, new Comparator(){ + @Override + public int compare(File o1, File o2) { + return o1.getName().compareTo(o2.getName()); + } + }); + return listFiles; + } + + protected void uploadIndexes(Set alreadyUploadedFiles){ + for(File f : getSortedFiles(indexDirFiles)){ + if(!alreadyUploadedFiles.contains(f.getName())){ + uploadIndex(f, alreadyUploadedFiles); + if(!alreadyUploadedFiles.contains(f.getName())){ + System.out.println("! NOT UPLOADED " + f.getName()); + } + } + } + System.out.println("UPLOADING INDEXES FINISHED "); + + } + + + + protected void uploadIndex(File f, Set alreadyUploadedFiles){ + if(!uploadIndexes){ + return; + } + + String summary; + double mbLengh = (double)f.length() / MB; + boolean zip = true; String regionName = f.getName().substring(0, f.getName().lastIndexOf('_', f.getName().indexOf('.'))); - if(f.getName().endsWith(IndexConstants.POI_INDEX_EXT) || f.getName().endsWith(IndexConstants.POI_INDEX_EXT_ZIP)){ - summary = "POI index for " ; - } else if(f.getName().endsWith(IndexConstants.BINARY_MAP_INDEX_EXT) || f.getName().endsWith(IndexConstants.BINARY_MAP_INDEX_EXT_ZIP)){ - boolean addr = indexAddress; - boolean trans = indexTransport; - boolean map = indexMap; - RandomAccessFile raf = null; + if(f.getName().endsWith(IndexConstants.POI_INDEX_EXT) || f.getName().endsWith(IndexConstants.POI_INDEX_EXT_ZIP)){ + summary = "POI index for " ; + } else if(f.getName().endsWith(IndexConstants.BINARY_MAP_INDEX_EXT) || f.getName().endsWith(IndexConstants.BINARY_MAP_INDEX_EXT_ZIP)){ + boolean addr = indexAddress; + boolean trans = indexTransport; + boolean map = indexMap; + RandomAccessFile raf = null; if (f.getName().endsWith(IndexConstants.BINARY_MAP_INDEX_EXT)) { try { raf = new RandomAccessFile(f, "r"); @@ -504,91 +504,91 @@ public class IndexBatchCreator { raf.close(); } catch (IOException e1) { } - } - } - } - summary = " index for "; - boolean fir = true; - if (addr) { - summary = "Address" + (fir ? "" : ", ") + summary; - fir = false; - } - if (trans) { - summary = "Transport" + (fir ? "" : ", ") + summary; - fir = false; - } - if (map) { - summary = "Map" + (fir ? "" : ", ") + summary; - fir = false; - } - - } else { - return; - } + } + } + } + summary = " index for "; + boolean fir = true; + if (addr) { + summary = "Address" + (fir ? "" : ", ") + summary; + fir = false; + } + if (trans) { + summary = "Transport" + (fir ? "" : ", ") + summary; + fir = false; + } + if (map) { + summary = "Map" + (fir ? "" : ", ") + summary; + fir = false; + } + + } else { + return; + } if(mbLengh < MIN_SIZE_TO_UPLOAD){ - // do not upload small files - return; - } + // do not upload small files + return; + } if(mbLengh > MIN_SIZE_TO_NOT_ZIP && (f.getName().endsWith(".odb") || f.getName().endsWith(".obf")) && zip){ - String n = f.getName(); - if(f.getName().endsWith(".odb")){ - n = f.getName().substring(0, f.getName().length() - 4); - } - String zipFileName = n+".zip"; - File zFile = new File(f.getParentFile(), zipFileName); - log.info("Zipping file " + f.getName()); - try { - ZipOutputStream zout = new ZipOutputStream(new FileOutputStream(zFile)); - zout.setLevel(9); - zout.putNextEntry(new ZipEntry(f.getName())); - FileInputStream is = new FileInputStream(f); - byte[] BUFFER = new byte[8192]; - int read = 0; - while((read = is.read(BUFFER)) != -1){ - zout.write(BUFFER, 0, read); - } - is.close(); - zout.close(); - } catch (IOException e) { - log.error("Exception while zipping file", e); - } - if(f.delete()){ - log.info("Source odb file was deleted"); - } - f = zFile; - - } - + String n = f.getName(); + if(f.getName().endsWith(".odb")){ + n = f.getName().substring(0, f.getName().length() - 4); + } + String zipFileName = n+".zip"; + File zFile = new File(f.getParentFile(), zipFileName); + log.info("Zipping file " + f.getName()); + try { + ZipOutputStream zout = new ZipOutputStream(new FileOutputStream(zFile)); + zout.setLevel(9); + zout.putNextEntry(new ZipEntry(f.getName())); + FileInputStream is = new FileInputStream(f); + byte[] BUFFER = new byte[8192]; + int read = 0; + while((read = is.read(BUFFER)) != -1){ + zout.write(BUFFER, 0, read); + } + is.close(); + zout.close(); + } catch (IOException e) { + log.error("Exception while zipping file", e); + } + if(f.delete()){ + log.info("Source odb file was deleted"); + } + f = zFile; + + } + summary += regionName; summary = summary.replace('_', ' '); - + List splittedFiles = Collections.emptyList(); try { - splittedFiles = splitFiles(f); + splittedFiles = splitFiles(f); boolean uploaded =true; for (File fs : splittedFiles) { uploaded &= uploadFileToServer(fs, f, summary); - } - // remove source file if file was splitted - if (uploaded && deleteFilesAfterUploading && f.exists()) { - f.delete(); + } + // remove source file if file was splitted + if (uploaded && deleteFilesAfterUploading && f.exists()) { + f.delete(); } alreadyUploadedFiles.add(f.getName()); - } catch (IOException e) { + } catch (IOException e) { log.error("Input/output exception uploading " + f.getName(), e); - } finally { - // remove all splitted files - for(File fs : splittedFiles){ - if(!fs.equals(f)){ - fs.delete(); - } - } + } finally { + // remove all splitted files + for(File fs : splittedFiles){ + if(!fs.equals(f)){ + fs.delete(); + } + } } } - private List splitFiles(File f) throws IOException { + private List splitFiles(File f) throws IOException { double mbLengh = (double)f.length() / MB; if(mbLengh < MAX_SIZE_TO_NOT_SPLIT) { return Collections.singletonList(f); @@ -620,26 +620,26 @@ public class IndexBatchCreator { } - private boolean uploadFileToServer(File f, File original, String summary) throws IOException { - if (f.length() / MB > MAX_UPLOAD_SIZE && !uploadToOsmandDownloads) { - System.err.println("ERROR : file " + f.getName() + " exceeded 200 mb!!! Could not be uploaded."); - return false; // restriction for google code - } + private boolean uploadFileToServer(File f, File original, String summary) throws IOException { + if (f.length() / MB > MAX_UPLOAD_SIZE && !uploadToOsmandDownloads) { + System.err.println("ERROR : file " + f.getName() + " exceeded 200 mb!!! Could not be uploaded."); + return false; // restriction for google code + } double originalLength = original.length() / MB; - if (!uploadToOsmandDownloads) { - try { - DownloaderIndexFromGoogleCode.deleteFileFromGoogleDownloads(f.getName(), token, pagegen, cookieHSID, cookieSID); - try { - Thread.sleep(4000); - } catch (InterruptedException e) { - // wait 5 seconds - } - } catch (IOException e) { - log.warn("Deleting file from downloads" + f.getName() + " " + e.getMessage()); - } - } - - + if (!uploadToOsmandDownloads) { + try { + DownloaderIndexFromGoogleCode.deleteFileFromGoogleDownloads(f.getName(), token, pagegen, cookieHSID, cookieSID); + try { + Thread.sleep(4000); + } catch (InterruptedException e) { + // wait 5 seconds + } + } catch (IOException e) { + log.warn("Deleting file from downloads" + f.getName() + " " + e.getMessage()); + } + } + + MessageFormat dateFormat = new MessageFormat("{0,date,dd.MM.yyyy}", Locale.US); MessageFormat numberFormat = new MessageFormat("{0,number,##.#}", Locale.US); String size = numberFormat.format(new Object[] { originalLength }); @@ -662,34 +662,34 @@ public class IndexBatchCreator { } - return true; - } - - @SuppressWarnings("deprecation") - private void uploadToDownloadOsmandNet(File f, String description, String size, String date) throws IOException{ - log.info("Uploading file " + f.getName() + " " + size + " MB " + date + " of " + description); - // Upload to ftp - FTPFileUpload upload = new FTPFileUpload(); - upload.upload("download.osmand.net", user, password, "indexes/" + f.getName(), f, 1 << 15); - - - String url = "http://download.osmand.net/xml_update.php?"; - url += "index="+URLEncoder.encode(f.getName()); - url += "&description="+URLEncoder.encode(description); - url += "&date="+URLEncoder.encode(date); - url += "&size="+URLEncoder.encode(size); - url += "&action=update"; - log.info("Updating index " + url); //$NON-NLS-1$//$NON-NLS-2$ - HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection(); - connection.setRequestMethod("POST"); - connection.connect(); - if(connection.getResponseCode() != HttpURLConnection.HTTP_OK){ - log.error("Error updating indexes " + connection.getResponseMessage()); - } - InputStream is = connection.getInputStream(); - while(is.read() != -1); - connection.disconnect(); - log.info("Finish updating index"); - } - -} + return true; + } + + @SuppressWarnings("deprecation") + private void uploadToDownloadOsmandNet(File f, String description, String size, String date) throws IOException{ + log.info("Uploading file " + f.getName() + " " + size + " MB " + date + " of " + description); + // Upload to ftp + FTPFileUpload upload = new FTPFileUpload(); + upload.upload("download.osmand.net", user, password, "indexes/" + f.getName(), f, 1 << 15); + + + String url = "http://download.osmand.net/xml_update.php?"; + url += "index="+URLEncoder.encode(f.getName()); + url += "&description="+URLEncoder.encode(description); + url += "&date="+URLEncoder.encode(date); + url += "&size="+URLEncoder.encode(size); + url += "&action=update"; + log.info("Updating index " + url); //$NON-NLS-1$//$NON-NLS-2$ + HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection(); + connection.setRequestMethod("POST"); + connection.connect(); + if(connection.getResponseCode() != HttpURLConnection.HTTP_OK){ + log.error("Error updating indexes " + connection.getResponseMessage()); + } + InputStream is = connection.getInputStream(); + while(is.read() != -1); + connection.disconnect(); + log.info("Finish updating index"); + } + +} diff --git a/DataExtractionOSM/src/net/osmand/data/index/IndexConstants.java b/DataExtractionOSM/src/net/osmand/data/index/IndexConstants.java index 171a0d6d26..490294b8b2 100644 --- a/DataExtractionOSM/src/net/osmand/data/index/IndexConstants.java +++ b/DataExtractionOSM/src/net/osmand/data/index/IndexConstants.java @@ -1,27 +1,27 @@ -package net.osmand.data.index; - - -public class IndexConstants { - - // Important : Every time you change schema of db upgrade version!!! - // If you want that new application support old index : put upgrade code in android app ResourceManager - public final static int POI_TABLE_VERSION = 1; - public final static int BINARY_MAP_VERSION = 1; // starts with 1 - public final static int VOICE_VERSION = 0; - - public static final String POI_INDEX_DIR = "POI/"; //$NON-NLS-1$ - public static final String VOICE_INDEX_DIR = "voice/"; //$NON-NLS-1$ - public static final String RENDERERS_DIR = "rendering/"; //$NON-NLS-1$ - - public static final String POI_INDEX_EXT = ".poi.odb"; //$NON-NLS-1$ - public static final String BINARY_MAP_INDEX_EXT = ".obf"; //$NON-NLS-1$ - - public static final String POI_INDEX_EXT_ZIP = ".poi.zip"; //$NON-NLS-1$ - public static final String VOICE_INDEX_EXT_ZIP = ".voice.zip"; //$NON-NLS-1$ - public static final String BINARY_MAP_INDEX_EXT_ZIP = ".obf.zip"; //$NON-NLS-1$ - - public static final String RENDERER_INDEX_EXT = ".render.xml"; //$NON-NLS-1$ - - public final static String POI_TABLE = "poi"; //$NON-NLS-1$ - -} +package net.osmand.data.index; + + +public class IndexConstants { + + // Important : Every time you change schema of db upgrade version!!! + // If you want that new application support old index : put upgrade code in android app ResourceManager + public final static int POI_TABLE_VERSION = 1; + public final static int BINARY_MAP_VERSION = 1; // starts with 1 + public final static int VOICE_VERSION = 0; + + public static final String POI_INDEX_DIR = "POI/"; //$NON-NLS-1$ + public static final String VOICE_INDEX_DIR = "voice/"; //$NON-NLS-1$ + public static final String RENDERERS_DIR = "rendering/"; //$NON-NLS-1$ + + public static final String POI_INDEX_EXT = ".poi.odb"; //$NON-NLS-1$ + public static final String BINARY_MAP_INDEX_EXT = ".obf"; //$NON-NLS-1$ + + public static final String POI_INDEX_EXT_ZIP = ".poi.zip"; //$NON-NLS-1$ + public static final String VOICE_INDEX_EXT_ZIP = ".voice.zip"; //$NON-NLS-1$ + public static final String BINARY_MAP_INDEX_EXT_ZIP = ".obf.zip"; //$NON-NLS-1$ + + public static final String RENDERER_INDEX_EXT = ".render.xml"; //$NON-NLS-1$ + + public final static String POI_TABLE = "poi"; //$NON-NLS-1$ + +} diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java b/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java index 43a8c884d9..4f77f1b749 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java @@ -1,621 +1,621 @@ -package net.osmand.data.preparation; - -import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.io.RandomAccessFile; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; - -import net.osmand.Algoritms; -import net.osmand.IProgress; -import net.osmand.binary.BinaryMapIndexWriter; -import net.osmand.data.index.IndexConstants; -import net.osmand.data.preparation.OsmDbAccessor.OsmDbVisitor; -import net.osmand.impl.ConsoleProgressImplementation; -import net.osmand.osm.Entity; -import net.osmand.osm.MapRenderingTypes; -import net.osmand.osm.Relation; -import net.osmand.osm.Entity.EntityId; -import net.osmand.osm.Entity.EntityType; -import net.osmand.osm.io.IOsmStorageFilter; -import net.osmand.osm.io.OsmBaseStorage; -import net.osmand.swing.DataExtractionSettings; -import net.osmand.swing.Messages; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.tools.bzip2.CBZip2InputStream; -import org.xml.sax.SAXException; - -import rtree.RTreeException; - -/** - * http://wiki.openstreetmap.org/wiki/OSM_tags_for_routing#Is_inside.2Foutside - * http://wiki.openstreetmap.org/wiki/Relations/Proposed/Postal_Addresses - * http://wiki.openstreetmap.org/wiki/Proposed_features/House_numbers/Karlsruhe_Schema#Tags (node, way) - * - * That data extraction has aim, save runtime memory and generate indexes on the fly. It will be longer than load in memory (needed part) - * and save into index. - */ -public class IndexCreator { - private static final Log log = LogFactory.getLog(IndexCreator.class); - - // ONLY derby.jar needed for derby dialect - private final DBDialect dialect = DBDialect.SQLITE; - - public static final int BATCH_SIZE = 5000; - public static final int BATCH_SIZE_OSM = 10000; - public static final String TEMP_NODES_DB = "nodes.tmp.odb"; - - public static final int STEP_MAIN = 4; - - private File workingDir = null; - - private boolean indexMap; - private boolean indexPOI; - private boolean indexTransport; - private boolean indexAddress; - - private boolean normalizeStreets = true; // true by default - private boolean saveAddressWays = true; // true by default - - private String regionName; - private String poiFileName = null; - private String mapFileName = null; - private Long lastModifiedDate = null; - - - private IndexTransportCreator indexTransportCreator; - private IndexPoiCreator indexPoiCreator; - private IndexAddressCreator indexAddressCreator; - private IndexVectorMapCreator indexMapCreator; - private OsmDbAccessor accessor; - // constants to start process from the middle and save temporary results - private boolean recreateOnlyBinaryFile = false; // false; - private boolean deleteOsmDB = false; - private boolean deleteDatabaseIndexes = true; - - private Connection dbConn; - private File dbFile; - - private File mapFile; - private RandomAccessFile mapRAFile; - private Connection mapConnection; - private String cityAdminLevel = "8"; - - - public IndexCreator(File workingDir) { - this.workingDir = workingDir; - } - - public void setIndexAddress(boolean indexAddress) { - this.indexAddress = indexAddress; - } - - public void setIndexMap(boolean indexMap) { - this.indexMap = indexMap; - } - - public void setIndexPOI(boolean indexPOI) { - this.indexPOI = indexPOI; - } - - public void setIndexTransport(boolean indexTransport) { - this.indexTransport = indexTransport; - } - - public void setSaveAddressWays(boolean saveAddressWays) { - this.saveAddressWays = saveAddressWays; - } - - public void setNormalizeStreets(boolean normalizeStreets) { - this.normalizeStreets = normalizeStreets; - } - - public String getRegionName() { - if (regionName == null) { - return "Region"; //$NON-NLS-1$ - } - return regionName; - } - - public void setRegionName(String regionName) { - this.regionName = regionName; - } - - private Connection getDatabaseConnection(String fileName) throws SQLException { - return dialect.getDatabaseConnection(fileName, log); - } - - public void setPoiFileName(String poiFileName) { - this.poiFileName = poiFileName; - } - - public void setNodesDBFile(File file) { - dbFile = file; - } - - public void setMapFileName(String mapFileName) { - this.mapFileName = mapFileName; - } - - public String getMapFileName() { - if (mapFileName == null) { - return getRegionName() + IndexConstants.BINARY_MAP_INDEX_EXT; - } - return mapFileName; - } - - public String getTempMapDBFileName() { - return getMapFileName() + ".tmp"; //$NON-NLS-1$ - } - - public Long getLastModifiedDate() { - return lastModifiedDate; - } - - public void setLastModifiedDate(Long lastModifiedDate) { - this.lastModifiedDate = lastModifiedDate; - } - - public String getPoiFileName() { - if (poiFileName == null) { - return IndexConstants.POI_INDEX_DIR + getRegionName() + IndexConstants.POI_INDEX_EXT; - } - return poiFileName; - } - - public String getCityAdminLevel() { - return cityAdminLevel; - } - - public void setCityAdminLevel(String cityAdminLevel) { - this.cityAdminLevel = cityAdminLevel; - } - - public String getRTreeMapIndexNonPackFileName() { - return mapFile.getAbsolutePath() + ".rtree"; //$NON-NLS-1$ - } - - public String getRTreeTransportStopsFileName() { - return mapFile.getAbsolutePath() + ".trans"; //$NON-NLS-1$ - } - - public String getRTreeTransportStopsPackFileName() { - return mapFile.getAbsolutePath() + ".ptrans"; //$NON-NLS-1$ - } - - public String getRTreeMapIndexPackFileName() { - return mapFile.getAbsolutePath() + ".prtree"; //$NON-NLS-1$ - } - - /* ***** END OF GETTERS/SETTERS ***** */ - public void generateIndexes(File readFile, IProgress progress, IOsmStorageFilter addFilter) throws IOException, SAXException, SQLException{ - generateIndexes(readFile, progress, addFilter, null, null); - } - - private void iterateMainEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException { - if (indexPOI) { - indexPoiCreator.iterateEntity(e, ctx); - } - if (indexTransport) { - indexTransportCreator.visitEntityMainStep(e, ctx); - } - if (indexMap) { - indexMapCreator.iterateMainEntity(e, ctx); - } - if (indexAddress) { - indexAddressCreator.iterateMainEntity(e, ctx); - } - } - - private OsmDbCreator extractOsmToNodesDB(File readFile, IProgress progress, IOsmStorageFilter addFilter) throws FileNotFoundException, - IOException, SQLException, SAXException { - boolean pbfFile = false; - InputStream stream = new BufferedInputStream(new FileInputStream(readFile), 8192 * 4); - InputStream streamFile = stream; - long st = System.currentTimeMillis(); - if (readFile.getName().endsWith(".bz2")) { //$NON-NLS-1$ - if (stream.read() != 'B' || stream.read() != 'Z') { - throw new RuntimeException("The source stream must start with the characters BZ if it is to be read as a BZip2 stream."); //$NON-NLS-1$ - } else { - stream = new CBZip2InputStream(stream); - } - } else if (readFile.getName().endsWith(".pbf")) { //$NON-NLS-1$ - pbfFile = true; - } - - OsmBaseStorage storage = new OsmBaseStorage(); - storage.setSupressWarnings(DataExtractionSettings.getSettings().isSupressWarningsForDuplicatedId()); - if (addFilter != null) { - storage.getFilters().add(addFilter); - } - - storage.getFilters().add(new IOsmStorageFilter() { - - @Override - public boolean acceptEntityToLoad(OsmBaseStorage storage, EntityId entityId, Entity entity) { - indexAddressCreator.registerCityIfNeeded(entity); - // accept to allow db creator parse it - return true; - } - }); - - // 1. Loading osm file - OsmDbCreator dbCreator = new OsmDbCreator(this); - try { - progress.setGeneralProgress("[35 / 100]"); //$NON-NLS-1$ - progress.startTask(Messages.getString("IndexCreator.LOADING_FILE") + readFile.getAbsolutePath(), -1); //$NON-NLS-1$ - // 1 init database to store temporary data - dbCreator.initDatabase(dialect, dbConn); - storage.getFilters().add(dbCreator); - if (pbfFile) { - storage.parseOSMPbf(stream, progress, false); - } else { - storage.parseOSM(stream, progress, streamFile, false); - } - dbCreator.finishLoading(); - dbConn.commit(); - - if (log.isInfoEnabled()) { - log.info("File parsed : " + (System.currentTimeMillis() - st)); //$NON-NLS-1$ - } - progress.finishTask(); - return dbCreator; - } finally { - if (log.isInfoEnabled()) { - log.info("File indexed : " + (System.currentTimeMillis() - st)); //$NON-NLS-1$ - } - } - } - - private boolean createPlainOsmDb(IProgress progress, File readFile, IOsmStorageFilter addFilter) throws SQLException, FileNotFoundException, IOException, SAXException{ - // initialize db file - boolean loadFromExistingFile = dbFile != null && dialect.databaseFileExists(dbFile); - if (dbFile == null) { - dbFile = new File(workingDir, TEMP_NODES_DB); - // to save space - if (dialect.databaseFileExists(dbFile)) { - dialect.removeDatabase(dbFile); - } - } - dbConn = getDatabaseConnection(dbFile.getAbsolutePath()); - int allRelations = 100000; - int allWays = 1000000; - int allNodes = 10000000; - if (!loadFromExistingFile) { - OsmDbCreator dbCreator = extractOsmToNodesDB(readFile, progress, addFilter); - if (dbCreator != null) { - allNodes = dbCreator.getAllNodes(); - allWays = dbCreator.getAllWays(); - allRelations = dbCreator.getAllRelations(); - } - } - accessor.initDatabase(dbConn, allNodes, allWays, allRelations); - return loadFromExistingFile; - } - - private void createDatabaseIndexesStructure() throws SQLException, IOException { - // 2.1 create temporary sqlite database to put temporary results to it - if (indexMap || indexAddress || indexTransport) { - mapFile = new File(workingDir, getMapFileName()); - // to save space - mapFile.getParentFile().mkdirs(); - File tempDBMapFile = new File(workingDir, getTempMapDBFileName()); - dialect.removeDatabase(tempDBMapFile); - mapConnection = getDatabaseConnection(tempDBMapFile.getAbsolutePath()); - mapConnection.setAutoCommit(false); - } - - // 2.2 create rtree map - if (indexMap) { - indexMapCreator.createDatabaseStructure(mapConnection, dialect, getRTreeMapIndexNonPackFileName()); - } - if (indexAddress) { - indexAddressCreator.createDatabaseStructure(mapConnection, dialect); - } - if (indexPOI) { - indexPoiCreator.createDatabaseStructure(new File(workingDir, getPoiFileName())); - } - if (indexTransport) { - indexTransportCreator.createDatabaseStructure(mapConnection, dialect, getRTreeTransportStopsFileName()); - } - } - - - public void generateIndexes(File readFile, IProgress progress, IOsmStorageFilter addFilter, MapZooms mapZooms, - MapRenderingTypes renderingTypes) throws IOException, SAXException, SQLException { - if (renderingTypes == null) { - renderingTypes = MapRenderingTypes.getDefault(); - } - if (mapZooms == null) { - mapZooms = MapZooms.getDefault(); - } - - // clear previous results and setting variables - if (readFile != null && regionName == null) { - int i = readFile.getName().indexOf('.'); - if (i > -1) { - regionName = Algoritms.capitalizeFirstLetterAndLowercase(readFile.getName().substring(0, i)); - } - } - this.indexTransportCreator = new IndexTransportCreator(); - this.indexPoiCreator = new IndexPoiCreator(); - this.indexAddressCreator = new IndexAddressCreator(); - this.indexMapCreator = new IndexVectorMapCreator(); - this.accessor = new OsmDbAccessor(); - - - indexMapCreator.initSettings(mapZooms, renderingTypes); - - // init address - String[] normalizeDefaultSuffixes = null; - String[] normalizeSuffixes = null; - if (normalizeStreets) { - normalizeDefaultSuffixes = DataExtractionSettings.getSettings().getDefaultSuffixesToNormalizeStreets(); - normalizeSuffixes = DataExtractionSettings.getSettings().getSuffixesToNormalizeStreets(); - } - indexAddressCreator.initSettings(normalizeStreets, normalizeDefaultSuffixes, normalizeSuffixes, saveAddressWays, cityAdminLevel); - - // Main generation method - try { - // //////////////////////////////////////////////////////////////////////// - // 1. creating nodes db to fast access for all nodes and simply import all relations, ways, nodes to it - boolean loadFromExistingFile = createPlainOsmDb(progress, readFile, addFilter); - - // do not create temp map file and rtree files - if (recreateOnlyBinaryFile) { - mapFile = new File(workingDir, getMapFileName()); - File tempDBMapFile = new File(workingDir, getTempMapDBFileName()); - mapConnection = getDatabaseConnection(tempDBMapFile.getAbsolutePath()); - mapConnection.setAutoCommit(false); - try { - if (indexMap) { - indexMapCreator.createRTreeFiles(getRTreeMapIndexPackFileName()); - } - if (indexTransport) { - indexTransportCreator.createRTreeFile(getRTreeTransportStopsPackFileName()); - } - } catch (RTreeException e) { - log.error("Error flushing", e); //$NON-NLS-1$ - throw new IOException(e); - } - } else { - - // 2. Create index connections and index structure - createDatabaseIndexesStructure(); - - // 3. Processing all entries - - // 3.1 write all cities - if (indexAddress) { - progress.setGeneralProgress("[40 / 100]"); //$NON-NLS-1$ - progress.startTask(Messages.getString("IndexCreator.INDEX_CITIES"), accessor.getAllNodes()); //$NON-NLS-1$ - if (loadFromExistingFile) { - // load cities names - accessor.iterateOverEntities(progress, EntityType.NODE, new OsmDbVisitor() { - @Override - public void iterateEntity(Entity e, OsmDbAccessorContext ctx) { - indexAddressCreator.registerCityIfNeeded(e); - } - }); - } - indexAddressCreator.writeCitiesIntoDb(); - } - - // 3.2 index address relations - if (indexAddress || indexMap) { - progress.setGeneralProgress("[30 / 100]"); //$NON-NLS-1$ - progress.startTask(Messages.getString("IndexCreator.PREINDEX_ADRESS_MAP"), accessor.getAllRelations()); //$NON-NLS-1$ - accessor.iterateOverEntities(progress, EntityType.RELATION, new OsmDbVisitor() { - @Override - public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException { - if (indexAddress) { - indexAddressCreator.indexAddressRelation((Relation) e, ctx); - indexAddressCreator.indexBoundariesRelation((Relation) e, ctx); - } - if (indexMap) { - indexMapCreator.indexMapRelationsAndMultiPolygons(e, ctx); - } - } - }); - if (indexAddress) { - progress.setGeneralProgress("[40 / 100]"); //$NON-NLS-1$ - progress.startTask(Messages.getString("IndexCreator.PREINDEX_ADRESS_MAP"), accessor.getAllWays()); //$NON-NLS-1$ - accessor.iterateOverEntities(progress, EntityType.WAY, new OsmDbVisitor() { - @Override - public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException { - indexAddressCreator.indexBoundariesRelation(e, ctx); - } - }); - - indexAddressCreator.commitToPutAllCities(); - } - - } - - // 3.3 MAIN iterate over all entities - if (indexPOI || indexAddress || indexMap) { - progress.setGeneralProgress("[50 / 100]"); - progress.startTask(Messages.getString("IndexCreator.PROCESS_OSM_NODES"), accessor.getAllNodes()); - accessor.iterateOverEntities(progress, EntityType.NODE, new OsmDbVisitor() { - @Override - public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException { - iterateMainEntity(e, ctx); - } - }); - progress.setGeneralProgress("[70 / 100]"); - progress.startTask(Messages.getString("IndexCreator.PROCESS_OSM_WAYS"), accessor.getAllWays()); - accessor.iterateOverEntities(progress, EntityType.WAY, new OsmDbVisitor() { - @Override - public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException { - iterateMainEntity(e, ctx); - } - }); - } - progress.setGeneralProgress("[85 / 100]"); - progress.startTask(Messages.getString("IndexCreator.PROCESS_OSM_REL"), accessor.getAllRelations()); - accessor.iterateOverEntities(progress, EntityType.RELATION, new OsmDbVisitor() { - @Override - public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException { - iterateMainEntity(e, ctx); - } - }); - - // 3.4 combine all low level ways and simplify them - if (indexMap) { - progress.setGeneralProgress("[90 / 100]"); - progress.startTask(Messages.getString("IndexCreator.INDEX_LO_LEVEL_WAYS"), indexMapCreator.getLowLevelWays()); - indexMapCreator.processingLowLevelWays(progress); - } - - // 3.5 update all postal codes from relations - if (indexAddress) { - progress.setGeneralProgress("[90 / 100]"); - progress.startTask(Messages.getString("IndexCreator.REGISTER_PCODES"), -1); - indexAddressCreator.processingPostcodes(); - } - - // 4. packing map rtree indexes - if (indexMap) { - progress.setGeneralProgress("[90 / 100]"); //$NON-NLS-1$ - progress.startTask(Messages.getString("IndexCreator.PACK_RTREE_MAP"), -1); //$NON-NLS-1$ - indexMapCreator.packRtreeFiles(getRTreeMapIndexNonPackFileName(), getRTreeMapIndexPackFileName()); - } - - if (indexTransport) { - progress.setGeneralProgress("[90 / 100]"); //$NON-NLS-1$ - progress.startTask(Messages.getString("IndexCreator.PACK_RTREE_TRANSP"), -1); //$NON-NLS-1$ - indexTransportCreator.packRTree(getRTreeTransportStopsFileName(), getRTreeTransportStopsPackFileName()); - } - } - - // 5. Writing binary file - if (indexMap || indexAddress || indexTransport) { - if (mapFile.exists()) { - mapFile.delete(); - } - mapRAFile = new RandomAccessFile(mapFile, "rw"); - BinaryMapIndexWriter writer = new BinaryMapIndexWriter(mapRAFile); - if (indexMap) { - progress.setGeneralProgress("[95 of 100]"); - progress.startTask("Writing map index to binary file...", -1); - indexMapCreator.writeBinaryMapIndex(writer, regionName); - } - - if (indexAddress) { - progress.setGeneralProgress("[95 of 100]"); - progress.startTask("Writing address index to binary file...", -1); - indexAddressCreator.writeBinaryAddressIndex(writer, regionName, progress); - } - - if (indexTransport) { - progress.setGeneralProgress("[95 of 100]"); - progress.startTask("Writing transport index to binary file...", -1); - indexTransportCreator.writeBinaryTransportIndex(writer, regionName, mapConnection); - } - progress.finishTask(); - writer.close(); - mapRAFile.close(); - log.info("Finish writing binary file"); //$NON-NLS-1$ - } - } catch (RuntimeException e) { - log.error("Log exception", e); //$NON-NLS-1$ - throw e; - } catch (SQLException e) { - log.error("Log exception", e); //$NON-NLS-1$ - throw e; - } catch (IOException e) { - log.error("Log exception", e); //$NON-NLS-1$ - throw e; - } catch (SAXException e) { - log.error("Log exception", e); //$NON-NLS-1$ - throw e; - } finally { - try { - accessor.closeReadingConnection(); - - indexPoiCreator.commitAndClosePoiFile(lastModifiedDate); - indexAddressCreator.closeAllPreparedStatements(); - indexTransportCreator.commitAndCloseFiles(getRTreeTransportStopsFileName(), getRTreeTransportStopsPackFileName(), - deleteDatabaseIndexes); - indexMapCreator.commitAndCloseFiles(getRTreeMapIndexNonPackFileName(), getRTreeMapIndexPackFileName(), - deleteDatabaseIndexes); - - if (mapConnection != null) { - mapConnection.commit(); - mapConnection.close(); - mapConnection = null; - File tempDBFile = new File(workingDir, getTempMapDBFileName()); - if (dialect.databaseFileExists(tempDBFile) && deleteDatabaseIndexes) { - // do not delete it for now - dialect.removeDatabase(tempDBFile); - } - } - - // do not delete first db connection - if (dbConn != null) { - if (DBDialect.H2 == dialect) { - dbConn.createStatement().execute("SHUTDOWN COMPACT"); //$NON-NLS-1$ - } - dbConn.close(); - } - if (deleteOsmDB) { - if (DBDialect.DERBY == dialect) { - try { - DriverManager.getConnection("jdbc:derby:;shutdown=true"); //$NON-NLS-1$ - } catch (SQLException e) { - // ignore exception - } - } - dialect.removeDatabase(dbFile); - } - } catch (SQLException e) { - e.printStackTrace(); - } - } - } - - - public static void main(String[] args) throws IOException, SAXException, SQLException { - - long time = System.currentTimeMillis(); - IndexCreator creator = new IndexCreator(new File("/home/victor/projects/OsmAnd/data/osm-gen/")); //$NON-NLS-1$ - creator.setIndexMap(true); - creator.setIndexAddress(true); - creator.setIndexPOI(true); - creator.setIndexTransport(true); - // for NL -// creator.setCityAdminLevel("10"); - - creator.recreateOnlyBinaryFile = false; - creator.deleteDatabaseIndexes = true; - - creator.generateIndexes(new File("/home/victor/projects/OsmAnd/download/406/map.osm"), - new ConsoleProgressImplementation(1), null, MapZooms.getDefault(), null); -// creator.generateIndexes(new File("/home/victor/projects/OsmAnd/data/osm-maps/minsk_around.osm"), -// new ConsoleProgressImplementation(1), null, MapZooms.getDefault(), null); - - - -// creator.setNodesDBFile(new File("e:/Information/OSM maps/osmand/minsk.tmp.odb")); -// creator.generateIndexes(new File("e:/Information/OSM maps/belarus osm/minsk.osm"), new ConsoleProgressImplementation(3), null, MapZooms.getDefault(), null); - - - - System.out.println("WHOLE GENERATION TIME : " + (System.currentTimeMillis() - time)); //$NON-NLS-1$ - System.out.println("COORDINATES_SIZE " + BinaryMapIndexWriter.COORDINATES_SIZE + " count " + BinaryMapIndexWriter.COORDINATES_COUNT); //$NON-NLS-1$ //$NON-NLS-2$ - System.out.println("TYPES_SIZE " + BinaryMapIndexWriter.TYPES_SIZE); //$NON-NLS-1$ - System.out.println("ID_SIZE " + BinaryMapIndexWriter.ID_SIZE); //$NON-NLS-1$ - System.out.println("- COORD_TYPES_ID SIZE " + (BinaryMapIndexWriter.COORDINATES_SIZE + BinaryMapIndexWriter.TYPES_SIZE + BinaryMapIndexWriter.ID_SIZE)); //$NON-NLS-1$ - System.out.println("- MAP_DATA_SIZE " + BinaryMapIndexWriter.MAP_DATA_SIZE); //$NON-NLS-1$ - System.out.println("- STRING_TABLE_SIZE " + BinaryMapIndexWriter.STRING_TABLE_SIZE); //$NON-NLS-1$ - System.out.println("-- MAP_DATA_AND_STRINGS SIZE " + (BinaryMapIndexWriter.MAP_DATA_SIZE + BinaryMapIndexWriter.STRING_TABLE_SIZE)); //$NON-NLS-1$ - - } +package net.osmand.data.preparation; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; + +import net.osmand.Algoritms; +import net.osmand.IProgress; +import net.osmand.binary.BinaryMapIndexWriter; +import net.osmand.data.index.IndexConstants; +import net.osmand.data.preparation.OsmDbAccessor.OsmDbVisitor; +import net.osmand.impl.ConsoleProgressImplementation; +import net.osmand.osm.Entity; +import net.osmand.osm.MapRenderingTypes; +import net.osmand.osm.Relation; +import net.osmand.osm.Entity.EntityId; +import net.osmand.osm.Entity.EntityType; +import net.osmand.osm.io.IOsmStorageFilter; +import net.osmand.osm.io.OsmBaseStorage; +import net.osmand.swing.DataExtractionSettings; +import net.osmand.swing.Messages; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.tools.bzip2.CBZip2InputStream; +import org.xml.sax.SAXException; + +import rtree.RTreeException; + +/** + * http://wiki.openstreetmap.org/wiki/OSM_tags_for_routing#Is_inside.2Foutside + * http://wiki.openstreetmap.org/wiki/Relations/Proposed/Postal_Addresses + * http://wiki.openstreetmap.org/wiki/Proposed_features/House_numbers/Karlsruhe_Schema#Tags (node, way) + * + * That data extraction has aim, save runtime memory and generate indexes on the fly. It will be longer than load in memory (needed part) + * and save into index. + */ +public class IndexCreator { + private static final Log log = LogFactory.getLog(IndexCreator.class); + + // ONLY derby.jar needed for derby dialect + private final DBDialect dialect = DBDialect.SQLITE; + + public static final int BATCH_SIZE = 5000; + public static final int BATCH_SIZE_OSM = 10000; + public static final String TEMP_NODES_DB = "nodes.tmp.odb"; + + public static final int STEP_MAIN = 4; + + private File workingDir = null; + + private boolean indexMap; + private boolean indexPOI; + private boolean indexTransport; + private boolean indexAddress; + + private boolean normalizeStreets = true; // true by default + private boolean saveAddressWays = true; // true by default + + private String regionName; + private String poiFileName = null; + private String mapFileName = null; + private Long lastModifiedDate = null; + + + private IndexTransportCreator indexTransportCreator; + private IndexPoiCreator indexPoiCreator; + private IndexAddressCreator indexAddressCreator; + private IndexVectorMapCreator indexMapCreator; + private OsmDbAccessor accessor; + // constants to start process from the middle and save temporary results + private boolean recreateOnlyBinaryFile = false; // false; + private boolean deleteOsmDB = false; + private boolean deleteDatabaseIndexes = true; + + private Connection dbConn; + private File dbFile; + + private File mapFile; + private RandomAccessFile mapRAFile; + private Connection mapConnection; + private String cityAdminLevel = "8"; + + + public IndexCreator(File workingDir) { + this.workingDir = workingDir; + } + + public void setIndexAddress(boolean indexAddress) { + this.indexAddress = indexAddress; + } + + public void setIndexMap(boolean indexMap) { + this.indexMap = indexMap; + } + + public void setIndexPOI(boolean indexPOI) { + this.indexPOI = indexPOI; + } + + public void setIndexTransport(boolean indexTransport) { + this.indexTransport = indexTransport; + } + + public void setSaveAddressWays(boolean saveAddressWays) { + this.saveAddressWays = saveAddressWays; + } + + public void setNormalizeStreets(boolean normalizeStreets) { + this.normalizeStreets = normalizeStreets; + } + + public String getRegionName() { + if (regionName == null) { + return "Region"; //$NON-NLS-1$ + } + return regionName; + } + + public void setRegionName(String regionName) { + this.regionName = regionName; + } + + private Connection getDatabaseConnection(String fileName) throws SQLException { + return dialect.getDatabaseConnection(fileName, log); + } + + public void setPoiFileName(String poiFileName) { + this.poiFileName = poiFileName; + } + + public void setNodesDBFile(File file) { + dbFile = file; + } + + public void setMapFileName(String mapFileName) { + this.mapFileName = mapFileName; + } + + public String getMapFileName() { + if (mapFileName == null) { + return getRegionName() + IndexConstants.BINARY_MAP_INDEX_EXT; + } + return mapFileName; + } + + public String getTempMapDBFileName() { + return getMapFileName() + ".tmp"; //$NON-NLS-1$ + } + + public Long getLastModifiedDate() { + return lastModifiedDate; + } + + public void setLastModifiedDate(Long lastModifiedDate) { + this.lastModifiedDate = lastModifiedDate; + } + + public String getPoiFileName() { + if (poiFileName == null) { + return IndexConstants.POI_INDEX_DIR + getRegionName() + IndexConstants.POI_INDEX_EXT; + } + return poiFileName; + } + + public String getCityAdminLevel() { + return cityAdminLevel; + } + + public void setCityAdminLevel(String cityAdminLevel) { + this.cityAdminLevel = cityAdminLevel; + } + + public String getRTreeMapIndexNonPackFileName() { + return mapFile.getAbsolutePath() + ".rtree"; //$NON-NLS-1$ + } + + public String getRTreeTransportStopsFileName() { + return mapFile.getAbsolutePath() + ".trans"; //$NON-NLS-1$ + } + + public String getRTreeTransportStopsPackFileName() { + return mapFile.getAbsolutePath() + ".ptrans"; //$NON-NLS-1$ + } + + public String getRTreeMapIndexPackFileName() { + return mapFile.getAbsolutePath() + ".prtree"; //$NON-NLS-1$ + } + + /* ***** END OF GETTERS/SETTERS ***** */ + public void generateIndexes(File readFile, IProgress progress, IOsmStorageFilter addFilter) throws IOException, SAXException, SQLException{ + generateIndexes(readFile, progress, addFilter, null, null); + } + + private void iterateMainEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException { + if (indexPOI) { + indexPoiCreator.iterateEntity(e, ctx); + } + if (indexTransport) { + indexTransportCreator.visitEntityMainStep(e, ctx); + } + if (indexMap) { + indexMapCreator.iterateMainEntity(e, ctx); + } + if (indexAddress) { + indexAddressCreator.iterateMainEntity(e, ctx); + } + } + + private OsmDbCreator extractOsmToNodesDB(File readFile, IProgress progress, IOsmStorageFilter addFilter) throws FileNotFoundException, + IOException, SQLException, SAXException { + boolean pbfFile = false; + InputStream stream = new BufferedInputStream(new FileInputStream(readFile), 8192 * 4); + InputStream streamFile = stream; + long st = System.currentTimeMillis(); + if (readFile.getName().endsWith(".bz2")) { //$NON-NLS-1$ + if (stream.read() != 'B' || stream.read() != 'Z') { + throw new RuntimeException("The source stream must start with the characters BZ if it is to be read as a BZip2 stream."); //$NON-NLS-1$ + } else { + stream = new CBZip2InputStream(stream); + } + } else if (readFile.getName().endsWith(".pbf")) { //$NON-NLS-1$ + pbfFile = true; + } + + OsmBaseStorage storage = new OsmBaseStorage(); + storage.setSupressWarnings(DataExtractionSettings.getSettings().isSupressWarningsForDuplicatedId()); + if (addFilter != null) { + storage.getFilters().add(addFilter); + } + + storage.getFilters().add(new IOsmStorageFilter() { + + @Override + public boolean acceptEntityToLoad(OsmBaseStorage storage, EntityId entityId, Entity entity) { + indexAddressCreator.registerCityIfNeeded(entity); + // accept to allow db creator parse it + return true; + } + }); + + // 1. Loading osm file + OsmDbCreator dbCreator = new OsmDbCreator(this); + try { + progress.setGeneralProgress("[35 / 100]"); //$NON-NLS-1$ + progress.startTask(Messages.getString("IndexCreator.LOADING_FILE") + readFile.getAbsolutePath(), -1); //$NON-NLS-1$ + // 1 init database to store temporary data + dbCreator.initDatabase(dialect, dbConn); + storage.getFilters().add(dbCreator); + if (pbfFile) { + storage.parseOSMPbf(stream, progress, false); + } else { + storage.parseOSM(stream, progress, streamFile, false); + } + dbCreator.finishLoading(); + dbConn.commit(); + + if (log.isInfoEnabled()) { + log.info("File parsed : " + (System.currentTimeMillis() - st)); //$NON-NLS-1$ + } + progress.finishTask(); + return dbCreator; + } finally { + if (log.isInfoEnabled()) { + log.info("File indexed : " + (System.currentTimeMillis() - st)); //$NON-NLS-1$ + } + } + } + + private boolean createPlainOsmDb(IProgress progress, File readFile, IOsmStorageFilter addFilter) throws SQLException, FileNotFoundException, IOException, SAXException{ + // initialize db file + boolean loadFromExistingFile = dbFile != null && dialect.databaseFileExists(dbFile); + if (dbFile == null) { + dbFile = new File(workingDir, TEMP_NODES_DB); + // to save space + if (dialect.databaseFileExists(dbFile)) { + dialect.removeDatabase(dbFile); + } + } + dbConn = getDatabaseConnection(dbFile.getAbsolutePath()); + int allRelations = 100000; + int allWays = 1000000; + int allNodes = 10000000; + if (!loadFromExistingFile) { + OsmDbCreator dbCreator = extractOsmToNodesDB(readFile, progress, addFilter); + if (dbCreator != null) { + allNodes = dbCreator.getAllNodes(); + allWays = dbCreator.getAllWays(); + allRelations = dbCreator.getAllRelations(); + } + } + accessor.initDatabase(dbConn, allNodes, allWays, allRelations); + return loadFromExistingFile; + } + + private void createDatabaseIndexesStructure() throws SQLException, IOException { + // 2.1 create temporary sqlite database to put temporary results to it + if (indexMap || indexAddress || indexTransport) { + mapFile = new File(workingDir, getMapFileName()); + // to save space + mapFile.getParentFile().mkdirs(); + File tempDBMapFile = new File(workingDir, getTempMapDBFileName()); + dialect.removeDatabase(tempDBMapFile); + mapConnection = getDatabaseConnection(tempDBMapFile.getAbsolutePath()); + mapConnection.setAutoCommit(false); + } + + // 2.2 create rtree map + if (indexMap) { + indexMapCreator.createDatabaseStructure(mapConnection, dialect, getRTreeMapIndexNonPackFileName()); + } + if (indexAddress) { + indexAddressCreator.createDatabaseStructure(mapConnection, dialect); + } + if (indexPOI) { + indexPoiCreator.createDatabaseStructure(new File(workingDir, getPoiFileName())); + } + if (indexTransport) { + indexTransportCreator.createDatabaseStructure(mapConnection, dialect, getRTreeTransportStopsFileName()); + } + } + + + public void generateIndexes(File readFile, IProgress progress, IOsmStorageFilter addFilter, MapZooms mapZooms, + MapRenderingTypes renderingTypes) throws IOException, SAXException, SQLException { + if (renderingTypes == null) { + renderingTypes = MapRenderingTypes.getDefault(); + } + if (mapZooms == null) { + mapZooms = MapZooms.getDefault(); + } + + // clear previous results and setting variables + if (readFile != null && regionName == null) { + int i = readFile.getName().indexOf('.'); + if (i > -1) { + regionName = Algoritms.capitalizeFirstLetterAndLowercase(readFile.getName().substring(0, i)); + } + } + this.indexTransportCreator = new IndexTransportCreator(); + this.indexPoiCreator = new IndexPoiCreator(); + this.indexAddressCreator = new IndexAddressCreator(); + this.indexMapCreator = new IndexVectorMapCreator(); + this.accessor = new OsmDbAccessor(); + + + indexMapCreator.initSettings(mapZooms, renderingTypes); + + // init address + String[] normalizeDefaultSuffixes = null; + String[] normalizeSuffixes = null; + if (normalizeStreets) { + normalizeDefaultSuffixes = DataExtractionSettings.getSettings().getDefaultSuffixesToNormalizeStreets(); + normalizeSuffixes = DataExtractionSettings.getSettings().getSuffixesToNormalizeStreets(); + } + indexAddressCreator.initSettings(normalizeStreets, normalizeDefaultSuffixes, normalizeSuffixes, saveAddressWays, cityAdminLevel); + + // Main generation method + try { + // //////////////////////////////////////////////////////////////////////// + // 1. creating nodes db to fast access for all nodes and simply import all relations, ways, nodes to it + boolean loadFromExistingFile = createPlainOsmDb(progress, readFile, addFilter); + + // do not create temp map file and rtree files + if (recreateOnlyBinaryFile) { + mapFile = new File(workingDir, getMapFileName()); + File tempDBMapFile = new File(workingDir, getTempMapDBFileName()); + mapConnection = getDatabaseConnection(tempDBMapFile.getAbsolutePath()); + mapConnection.setAutoCommit(false); + try { + if (indexMap) { + indexMapCreator.createRTreeFiles(getRTreeMapIndexPackFileName()); + } + if (indexTransport) { + indexTransportCreator.createRTreeFile(getRTreeTransportStopsPackFileName()); + } + } catch (RTreeException e) { + log.error("Error flushing", e); //$NON-NLS-1$ + throw new IOException(e); + } + } else { + + // 2. Create index connections and index structure + createDatabaseIndexesStructure(); + + // 3. Processing all entries + + // 3.1 write all cities + if (indexAddress) { + progress.setGeneralProgress("[40 / 100]"); //$NON-NLS-1$ + progress.startTask(Messages.getString("IndexCreator.INDEX_CITIES"), accessor.getAllNodes()); //$NON-NLS-1$ + if (loadFromExistingFile) { + // load cities names + accessor.iterateOverEntities(progress, EntityType.NODE, new OsmDbVisitor() { + @Override + public void iterateEntity(Entity e, OsmDbAccessorContext ctx) { + indexAddressCreator.registerCityIfNeeded(e); + } + }); + } + indexAddressCreator.writeCitiesIntoDb(); + } + + // 3.2 index address relations + if (indexAddress || indexMap) { + progress.setGeneralProgress("[30 / 100]"); //$NON-NLS-1$ + progress.startTask(Messages.getString("IndexCreator.PREINDEX_ADRESS_MAP"), accessor.getAllRelations()); //$NON-NLS-1$ + accessor.iterateOverEntities(progress, EntityType.RELATION, new OsmDbVisitor() { + @Override + public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException { + if (indexAddress) { + indexAddressCreator.indexAddressRelation((Relation) e, ctx); + indexAddressCreator.indexBoundariesRelation((Relation) e, ctx); + } + if (indexMap) { + indexMapCreator.indexMapRelationsAndMultiPolygons(e, ctx); + } + } + }); + if (indexAddress) { + progress.setGeneralProgress("[40 / 100]"); //$NON-NLS-1$ + progress.startTask(Messages.getString("IndexCreator.PREINDEX_ADRESS_MAP"), accessor.getAllWays()); //$NON-NLS-1$ + accessor.iterateOverEntities(progress, EntityType.WAY, new OsmDbVisitor() { + @Override + public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException { + indexAddressCreator.indexBoundariesRelation(e, ctx); + } + }); + + indexAddressCreator.commitToPutAllCities(); + } + + } + + // 3.3 MAIN iterate over all entities + if (indexPOI || indexAddress || indexMap) { + progress.setGeneralProgress("[50 / 100]"); + progress.startTask(Messages.getString("IndexCreator.PROCESS_OSM_NODES"), accessor.getAllNodes()); + accessor.iterateOverEntities(progress, EntityType.NODE, new OsmDbVisitor() { + @Override + public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException { + iterateMainEntity(e, ctx); + } + }); + progress.setGeneralProgress("[70 / 100]"); + progress.startTask(Messages.getString("IndexCreator.PROCESS_OSM_WAYS"), accessor.getAllWays()); + accessor.iterateOverEntities(progress, EntityType.WAY, new OsmDbVisitor() { + @Override + public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException { + iterateMainEntity(e, ctx); + } + }); + } + progress.setGeneralProgress("[85 / 100]"); + progress.startTask(Messages.getString("IndexCreator.PROCESS_OSM_REL"), accessor.getAllRelations()); + accessor.iterateOverEntities(progress, EntityType.RELATION, new OsmDbVisitor() { + @Override + public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException { + iterateMainEntity(e, ctx); + } + }); + + // 3.4 combine all low level ways and simplify them + if (indexMap) { + progress.setGeneralProgress("[90 / 100]"); + progress.startTask(Messages.getString("IndexCreator.INDEX_LO_LEVEL_WAYS"), indexMapCreator.getLowLevelWays()); + indexMapCreator.processingLowLevelWays(progress); + } + + // 3.5 update all postal codes from relations + if (indexAddress) { + progress.setGeneralProgress("[90 / 100]"); + progress.startTask(Messages.getString("IndexCreator.REGISTER_PCODES"), -1); + indexAddressCreator.processingPostcodes(); + } + + // 4. packing map rtree indexes + if (indexMap) { + progress.setGeneralProgress("[90 / 100]"); //$NON-NLS-1$ + progress.startTask(Messages.getString("IndexCreator.PACK_RTREE_MAP"), -1); //$NON-NLS-1$ + indexMapCreator.packRtreeFiles(getRTreeMapIndexNonPackFileName(), getRTreeMapIndexPackFileName()); + } + + if (indexTransport) { + progress.setGeneralProgress("[90 / 100]"); //$NON-NLS-1$ + progress.startTask(Messages.getString("IndexCreator.PACK_RTREE_TRANSP"), -1); //$NON-NLS-1$ + indexTransportCreator.packRTree(getRTreeTransportStopsFileName(), getRTreeTransportStopsPackFileName()); + } + } + + // 5. Writing binary file + if (indexMap || indexAddress || indexTransport) { + if (mapFile.exists()) { + mapFile.delete(); + } + mapRAFile = new RandomAccessFile(mapFile, "rw"); + BinaryMapIndexWriter writer = new BinaryMapIndexWriter(mapRAFile); + if (indexMap) { + progress.setGeneralProgress("[95 of 100]"); + progress.startTask("Writing map index to binary file...", -1); + indexMapCreator.writeBinaryMapIndex(writer, regionName); + } + + if (indexAddress) { + progress.setGeneralProgress("[95 of 100]"); + progress.startTask("Writing address index to binary file...", -1); + indexAddressCreator.writeBinaryAddressIndex(writer, regionName, progress); + } + + if (indexTransport) { + progress.setGeneralProgress("[95 of 100]"); + progress.startTask("Writing transport index to binary file...", -1); + indexTransportCreator.writeBinaryTransportIndex(writer, regionName, mapConnection); + } + progress.finishTask(); + writer.close(); + mapRAFile.close(); + log.info("Finish writing binary file"); //$NON-NLS-1$ + } + } catch (RuntimeException e) { + log.error("Log exception", e); //$NON-NLS-1$ + throw e; + } catch (SQLException e) { + log.error("Log exception", e); //$NON-NLS-1$ + throw e; + } catch (IOException e) { + log.error("Log exception", e); //$NON-NLS-1$ + throw e; + } catch (SAXException e) { + log.error("Log exception", e); //$NON-NLS-1$ + throw e; + } finally { + try { + accessor.closeReadingConnection(); + + indexPoiCreator.commitAndClosePoiFile(lastModifiedDate); + indexAddressCreator.closeAllPreparedStatements(); + indexTransportCreator.commitAndCloseFiles(getRTreeTransportStopsFileName(), getRTreeTransportStopsPackFileName(), + deleteDatabaseIndexes); + indexMapCreator.commitAndCloseFiles(getRTreeMapIndexNonPackFileName(), getRTreeMapIndexPackFileName(), + deleteDatabaseIndexes); + + if (mapConnection != null) { + mapConnection.commit(); + mapConnection.close(); + mapConnection = null; + File tempDBFile = new File(workingDir, getTempMapDBFileName()); + if (dialect.databaseFileExists(tempDBFile) && deleteDatabaseIndexes) { + // do not delete it for now + dialect.removeDatabase(tempDBFile); + } + } + + // do not delete first db connection + if (dbConn != null) { + if (DBDialect.H2 == dialect) { + dbConn.createStatement().execute("SHUTDOWN COMPACT"); //$NON-NLS-1$ + } + dbConn.close(); + } + if (deleteOsmDB) { + if (DBDialect.DERBY == dialect) { + try { + DriverManager.getConnection("jdbc:derby:;shutdown=true"); //$NON-NLS-1$ + } catch (SQLException e) { + // ignore exception + } + } + dialect.removeDatabase(dbFile); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } + } + + + public static void main(String[] args) throws IOException, SAXException, SQLException { + + long time = System.currentTimeMillis(); + IndexCreator creator = new IndexCreator(new File("/home/victor/projects/OsmAnd/data/osm-gen/")); //$NON-NLS-1$ + creator.setIndexMap(true); + creator.setIndexAddress(true); + creator.setIndexPOI(true); + creator.setIndexTransport(true); + // for NL +// creator.setCityAdminLevel("10"); + + creator.recreateOnlyBinaryFile = false; + creator.deleteDatabaseIndexes = true; + + creator.generateIndexes(new File("/home/victor/projects/OsmAnd/download/406/map.osm"), + new ConsoleProgressImplementation(1), null, MapZooms.getDefault(), null); +// creator.generateIndexes(new File("/home/victor/projects/OsmAnd/data/osm-maps/minsk_around.osm"), +// new ConsoleProgressImplementation(1), null, MapZooms.getDefault(), null); + + + +// creator.setNodesDBFile(new File("e:/Information/OSM maps/osmand/minsk.tmp.odb")); +// creator.generateIndexes(new File("e:/Information/OSM maps/belarus osm/minsk.osm"), new ConsoleProgressImplementation(3), null, MapZooms.getDefault(), null); + + + + System.out.println("WHOLE GENERATION TIME : " + (System.currentTimeMillis() - time)); //$NON-NLS-1$ + System.out.println("COORDINATES_SIZE " + BinaryMapIndexWriter.COORDINATES_SIZE + " count " + BinaryMapIndexWriter.COORDINATES_COUNT); //$NON-NLS-1$ //$NON-NLS-2$ + System.out.println("TYPES_SIZE " + BinaryMapIndexWriter.TYPES_SIZE); //$NON-NLS-1$ + System.out.println("ID_SIZE " + BinaryMapIndexWriter.ID_SIZE); //$NON-NLS-1$ + System.out.println("- COORD_TYPES_ID SIZE " + (BinaryMapIndexWriter.COORDINATES_SIZE + BinaryMapIndexWriter.TYPES_SIZE + BinaryMapIndexWriter.ID_SIZE)); //$NON-NLS-1$ + System.out.println("- MAP_DATA_SIZE " + BinaryMapIndexWriter.MAP_DATA_SIZE); //$NON-NLS-1$ + System.out.println("- STRING_TABLE_SIZE " + BinaryMapIndexWriter.STRING_TABLE_SIZE); //$NON-NLS-1$ + System.out.println("-- MAP_DATA_AND_STRINGS SIZE " + (BinaryMapIndexWriter.MAP_DATA_SIZE + BinaryMapIndexWriter.STRING_TABLE_SIZE)); //$NON-NLS-1$ + + } } \ No newline at end of file diff --git a/DataExtractionOSM/src/net/osmand/swing/DataExtractionSettings.java b/DataExtractionOSM/src/net/osmand/swing/DataExtractionSettings.java index 066325000a..5dd17c8f91 100644 --- a/DataExtractionOSM/src/net/osmand/swing/DataExtractionSettings.java +++ b/DataExtractionOSM/src/net/osmand/swing/DataExtractionSettings.java @@ -154,11 +154,11 @@ public class DataExtractionSettings { } public String getCityAdminLevel(){ - return preferences.getString("cityAdminLevel", "8"); + return preferences.get("cityAdminLevel", "8"); } public void setCityAdminLevel(String s){ - preferences.putString("cityAdminLevel", s); + preferences.put("cityAdminLevel", s); } public boolean isSupressWarningsForDuplicatedId(){ diff --git a/DataExtractionOSM/src/net/osmand/swing/OsmExtractionPreferencesDialog.java b/DataExtractionOSM/src/net/osmand/swing/OsmExtractionPreferencesDialog.java index 4229860271..f8681b4345 100644 --- a/DataExtractionOSM/src/net/osmand/swing/OsmExtractionPreferencesDialog.java +++ b/DataExtractionOSM/src/net/osmand/swing/OsmExtractionPreferencesDialog.java @@ -123,9 +123,9 @@ public class OsmExtractionPreferencesDialog extends JDialog { label = new JLabel("City admin level : "); panel.add(label); constr = new GridBagConstraints(); - constr.ipadx = 6; + constr.ipadx = 5; constr.gridx = 0; - constr.gridy = 1; + constr.gridy = 2; constr.anchor = GridBagConstraints.WEST; l.setConstraints(label, constr); @@ -136,9 +136,9 @@ public class OsmExtractionPreferencesDialog extends JDialog { constr = new GridBagConstraints(); constr.weightx = 1; constr.fill = GridBagConstraints.HORIZONTAL; - constr.ipadx = 6; + constr.ipadx = 5; constr.gridx = 1; - constr.gridy = 1; + constr.gridy = 2; l.setConstraints(cityAdminLevel, constr); // supressWarning = new JCheckBox(); diff --git a/OsmAnd/src/net/osmand/GPXUtilities.java b/OsmAnd/src/net/osmand/GPXUtilities.java index b765800c5e..c728735aa0 100644 --- a/OsmAnd/src/net/osmand/GPXUtilities.java +++ b/OsmAnd/src/net/osmand/GPXUtilities.java @@ -1,215 +1,215 @@ -package net.osmand; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.text.DecimalFormat; -import java.text.DecimalFormatSymbols; -import java.text.NumberFormat; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Locale; -import java.util.TimeZone; - -import net.osmand.plus.R; - -import org.apache.commons.logging.Log; -import org.xmlpull.v1.XmlPullParser; -import org.xmlpull.v1.XmlPullParserException; -import org.xmlpull.v1.XmlSerializer; - -import android.content.Context; -import android.location.Location; -import android.util.Xml; - -public class GPXUtilities { - public final static Log log = LogUtil.getLog(GPXUtilities.class); - - - private final static String GPX_TIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ss'Z'"; //$NON-NLS-1$ - - private final static NumberFormat latLonFormat = new DecimalFormat("0.00#####", new DecimalFormatSymbols(Locale.US)); - - public static class TrkPt { - public double lat; - public double lon; - public double ele; - public double speed; - public long time; - } - - public static class WptPt { - public double lat; - public double lon; - public String name; - // by default - public long time = 0; - } - - public static class TrkSegment { - public List points = new ArrayList(); - } - - public static class Track { - public List segments = new ArrayList(); - } - - public static class GPXFile { - public List tracks = new ArrayList(); - public List points = new ArrayList(); - } - - - public static String writeGpxFile(File fout, GPXFile file, Context ctx) { - try { - SimpleDateFormat format = new SimpleDateFormat(GPX_TIME_FORMAT); - format.setTimeZone(TimeZone.getTimeZone("UTC")); - FileOutputStream output = new FileOutputStream(fout); - XmlSerializer serializer = Xml.newSerializer(); - serializer.setOutput(output, "UTF-8"); //$NON-NLS-1$ - serializer.setFeature("http://xmlpull.org/v1/doc/features.html#indent-output", true); //$NON-NLS-1$ - serializer.startDocument("UTF-8", true); //$NON-NLS-1$ - serializer.startTag(null, "gpx"); //$NON-NLS-1$ - serializer.attribute(null, "version", "1.1"); //$NON-NLS-1$ //$NON-NLS-2$ - serializer.attribute(null, "creator", Version.APP_NAME_VERSION); //$NON-NLS-1$ - serializer.attribute(null, "xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance"); - serializer.attribute(null, "xsi:schemaLocation", "http://www.topografix.com/GPX/1/1 http://www.topografix.com/GPX/1/1/gpx.xsd"); - - - for (Track track : file.tracks) { - serializer.startTag(null, "trk"); //$NON-NLS-1$ - for (TrkSegment segment : track.segments) { - serializer.startTag(null, "trkseg"); //$NON-NLS-1$ - for (TrkPt p : segment.points) { - serializer.startTag(null, "trkpt"); //$NON-NLS-1$ - serializer.attribute(null, "lat", latLonFormat.format(p.lat)); //$NON-NLS-1$ //$NON-NLS-2$ - serializer.attribute(null, "lon", latLonFormat.format(p.lon)); //$NON-NLS-1$ //$NON-NLS-2$ - serializer.startTag(null, "ele"); //$NON-NLS-1$ - serializer.text(p.ele + ""); //$NON-NLS-1$ - serializer.endTag(null, "ele"); //$NON-NLS-1$ - serializer.startTag(null, "time"); //$NON-NLS-1$ - serializer.text(format.format(new Date(p.time))); - serializer.endTag(null, "time"); //$NON-NLS-1$ - if (p.speed > 0) { - serializer.startTag(null, "extensions"); - serializer.startTag(null, "speed"); //$NON-NLS-1$ - serializer.text(p.speed + ""); //$NON-NLS-1$ - serializer.endTag(null, "speed"); //$NON-NLS-1$ - serializer.endTag(null, "extensions"); - } - - serializer.endTag(null, "trkpt"); //$NON-NLS-1$ - } - serializer.endTag(null, "trkseg"); //$NON-NLS-1$ - } - serializer.endTag(null, "trk"); //$NON-NLS-1$ - } - - for (WptPt l : file.points) { - serializer.startTag(null, "wpt"); //$NON-NLS-1$ - serializer.attribute(null, "lat", latLonFormat.format(l.lat)); //$NON-NLS-1$ - serializer.attribute(null, "lon", latLonFormat.format(l.lon)); //$NON-NLS-1$ //$NON-NLS-2$ - if (l.time != 0) { - serializer.startTag(null, "time"); //$NON-NLS-1$ - serializer.text(format.format(new Date(l.time))); - serializer.endTag(null, "time"); //$NON-NLS-1$ - } - serializer.startTag(null, "name"); //$NON-NLS-1$ - serializer.text(l.name); - serializer.endTag(null, "name"); //$NON-NLS-1$ - serializer.endTag(null, "wpt"); //$NON-NLS-1$ - } - - serializer.endTag(null, "gpx"); //$NON-NLS-1$ - serializer.flush(); - serializer.endDocument(); - } catch (RuntimeException e) { - log.error("Error saving gpx", e); //$NON-NLS-1$ - return ctx.getString(R.string.error_occurred_saving_gpx); - } catch (IOException e) { - log.error("Error saving gpx", e); //$NON-NLS-1$ - return ctx.getString(R.string.error_occurred_saving_gpx); - } - return null; - } - - - public static class GPXFileResult { - public ArrayList> locations = new ArrayList>(); - public ArrayList wayPoints = new ArrayList(); - // special case for cloudmate gpx : they discourage common schema - // by using waypoint as track points and rtept are not very close to real way - // such as wpt. However they provide additional information into gpx. - public boolean cloudMadeFile; - public String error; - } - - public static GPXFileResult loadGPXFile(Context ctx, File f){ - GPXFileResult res = new GPXFileResult(); - try { - boolean cloudMade = false; - XmlPullParser parser = Xml.newPullParser(); - parser.setInput(new FileInputStream(f), "UTF-8"); //$NON-NLS-1$ - - int tok; - Location current = null; - String currentName = ""; //$NON-NLS-1$ - while ((tok = parser.next()) != XmlPullParser.END_DOCUMENT) { - if (tok == XmlPullParser.START_TAG) { - if (parser.getName().equals("copyright")) { //$NON-NLS-1$ - cloudMade |= "cloudmade".equalsIgnoreCase(parser.getAttributeValue("", "author")); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ - - } else if (parser.getName().equals("trkseg")) { //$NON-NLS-1$ - res.locations.add(new ArrayList()); - } else if (parser.getName().equals("wpt") || parser.getName().equals("trkpt") || //$NON-NLS-1$//$NON-NLS-2$ - (!cloudMade && parser.getName().equals("rtept"))) { //$NON-NLS-1$ - // currently not distinguish different point represents all as a line - try { - currentName = ""; //$NON-NLS-1$ - current = new Location("gpx_file"); //$NON-NLS-1$ - current.setLatitude(Double.parseDouble(parser.getAttributeValue("", "lat"))); //$NON-NLS-1$ //$NON-NLS-2$ - current.setLongitude(Double.parseDouble(parser.getAttributeValue("", "lon"))); //$NON-NLS-1$ //$NON-NLS-2$ - } catch (NumberFormatException e) { - current = null; - - } - } else if (current != null && parser.getName().equals("name")) { //$NON-NLS-1$ - if (parser.next() == XmlPullParser.TEXT) { - currentName = parser.getText(); - } - } - } else if (tok == XmlPullParser.END_TAG) { - if (parser.getName().equals("wpt") || //$NON-NLS-1$ - parser.getName().equals("trkpt") || (!cloudMade && parser.getName().equals("rtept"))) { //$NON-NLS-1$ //$NON-NLS-2$ - if (current != null) { - if (parser.getName().equals("wpt") && !cloudMade) { //$NON-NLS-1$ - WptPt pt = new WptPt(); - pt.lat = current.getLatitude(); - pt.lon = current.getLongitude(); - pt.name = currentName; - res.wayPoints.add(pt); - } else { - if (res.locations.isEmpty()) { - res.locations.add(new ArrayList()); - } - res.locations.get(res.locations.size() - 1).add(current); - } - } - } - } - } - } catch (XmlPullParserException e) { - log.error("Error reading gpx", e); //$NON-NLS-1$ - res.error = ctx.getString(R.string.error_reading_gpx); - } catch (IOException e) { - log.error("Error reading gpx", e); //$NON-NLS-1$ - res.error = ctx.getString(R.string.error_reading_gpx); - } - - return res; - } - -} +package net.osmand; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.text.DecimalFormat; +import java.text.DecimalFormatSymbols; +import java.text.NumberFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Locale; +import java.util.TimeZone; + +import net.osmand.plus.R; + +import org.apache.commons.logging.Log; +import org.xmlpull.v1.XmlPullParser; +import org.xmlpull.v1.XmlPullParserException; +import org.xmlpull.v1.XmlSerializer; + +import android.content.Context; +import android.location.Location; +import android.util.Xml; + +public class GPXUtilities { + public final static Log log = LogUtil.getLog(GPXUtilities.class); + + + private final static String GPX_TIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ss'Z'"; //$NON-NLS-1$ + + private final static NumberFormat latLonFormat = new DecimalFormat("0.00#####", new DecimalFormatSymbols(Locale.US)); + + public static class TrkPt { + public double lat; + public double lon; + public double ele; + public double speed; + public long time; + } + + public static class WptPt { + public double lat; + public double lon; + public String name; + // by default + public long time = 0; + } + + public static class TrkSegment { + public List points = new ArrayList(); + } + + public static class Track { + public List segments = new ArrayList(); + } + + public static class GPXFile { + public List tracks = new ArrayList(); + public List points = new ArrayList(); + } + + + public static String writeGpxFile(File fout, GPXFile file, Context ctx) { + try { + SimpleDateFormat format = new SimpleDateFormat(GPX_TIME_FORMAT); + format.setTimeZone(TimeZone.getTimeZone("UTC")); + FileOutputStream output = new FileOutputStream(fout); + XmlSerializer serializer = Xml.newSerializer(); + serializer.setOutput(output, "UTF-8"); //$NON-NLS-1$ + serializer.setFeature("http://xmlpull.org/v1/doc/features.html#indent-output", true); //$NON-NLS-1$ + serializer.startDocument("UTF-8", true); //$NON-NLS-1$ + serializer.startTag(null, "gpx"); //$NON-NLS-1$ + serializer.attribute(null, "version", "1.1"); //$NON-NLS-1$ //$NON-NLS-2$ + serializer.attribute(null, "creator", Version.APP_NAME_VERSION); //$NON-NLS-1$ + serializer.attribute(null, "xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance"); + serializer.attribute(null, "xsi:schemaLocation", "http://www.topografix.com/GPX/1/1 http://www.topografix.com/GPX/1/1/gpx.xsd"); + + + for (Track track : file.tracks) { + serializer.startTag(null, "trk"); //$NON-NLS-1$ + for (TrkSegment segment : track.segments) { + serializer.startTag(null, "trkseg"); //$NON-NLS-1$ + for (TrkPt p : segment.points) { + serializer.startTag(null, "trkpt"); //$NON-NLS-1$ + serializer.attribute(null, "lat", latLonFormat.format(p.lat)); //$NON-NLS-1$ //$NON-NLS-2$ + serializer.attribute(null, "lon", latLonFormat.format(p.lon)); //$NON-NLS-1$ //$NON-NLS-2$ + serializer.startTag(null, "ele"); //$NON-NLS-1$ + serializer.text(p.ele + ""); //$NON-NLS-1$ + serializer.endTag(null, "ele"); //$NON-NLS-1$ + serializer.startTag(null, "time"); //$NON-NLS-1$ + serializer.text(format.format(new Date(p.time))); + serializer.endTag(null, "time"); //$NON-NLS-1$ + if (p.speed > 0) { + serializer.startTag(null, "extensions"); + serializer.startTag(null, "speed"); //$NON-NLS-1$ + serializer.text(p.speed + ""); //$NON-NLS-1$ + serializer.endTag(null, "speed"); //$NON-NLS-1$ + serializer.endTag(null, "extensions"); + } + + serializer.endTag(null, "trkpt"); //$NON-NLS-1$ + } + serializer.endTag(null, "trkseg"); //$NON-NLS-1$ + } + serializer.endTag(null, "trk"); //$NON-NLS-1$ + } + + for (WptPt l : file.points) { + serializer.startTag(null, "wpt"); //$NON-NLS-1$ + serializer.attribute(null, "lat", latLonFormat.format(l.lat)); //$NON-NLS-1$ + serializer.attribute(null, "lon", latLonFormat.format(l.lon)); //$NON-NLS-1$ //$NON-NLS-2$ + if (l.time != 0) { + serializer.startTag(null, "time"); //$NON-NLS-1$ + serializer.text(format.format(new Date(l.time))); + serializer.endTag(null, "time"); //$NON-NLS-1$ + } + serializer.startTag(null, "name"); //$NON-NLS-1$ + serializer.text(l.name); + serializer.endTag(null, "name"); //$NON-NLS-1$ + serializer.endTag(null, "wpt"); //$NON-NLS-1$ + } + + serializer.endTag(null, "gpx"); //$NON-NLS-1$ + serializer.flush(); + serializer.endDocument(); + } catch (RuntimeException e) { + log.error("Error saving gpx", e); //$NON-NLS-1$ + return ctx.getString(R.string.error_occurred_saving_gpx); + } catch (IOException e) { + log.error("Error saving gpx", e); //$NON-NLS-1$ + return ctx.getString(R.string.error_occurred_saving_gpx); + } + return null; + } + + + public static class GPXFileResult { + public ArrayList> locations = new ArrayList>(); + public ArrayList wayPoints = new ArrayList(); + // special case for cloudmate gpx : they discourage common schema + // by using waypoint as track points and rtept are not very close to real way + // such as wpt. However they provide additional information into gpx. + public boolean cloudMadeFile; + public String error; + } + + public static GPXFileResult loadGPXFile(Context ctx, File f){ + GPXFileResult res = new GPXFileResult(); + try { + boolean cloudMade = false; + XmlPullParser parser = Xml.newPullParser(); + parser.setInput(new FileInputStream(f), "UTF-8"); //$NON-NLS-1$ + + int tok; + Location current = null; + String currentName = ""; //$NON-NLS-1$ + while ((tok = parser.next()) != XmlPullParser.END_DOCUMENT) { + if (tok == XmlPullParser.START_TAG) { + if (parser.getName().equals("copyright")) { //$NON-NLS-1$ + cloudMade |= "cloudmade".equalsIgnoreCase(parser.getAttributeValue("", "author")); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + + } else if (parser.getName().equals("trkseg")) { //$NON-NLS-1$ + res.locations.add(new ArrayList()); + } else if (parser.getName().equals("wpt") || parser.getName().equals("trkpt") || //$NON-NLS-1$//$NON-NLS-2$ + (!cloudMade && parser.getName().equals("rtept"))) { //$NON-NLS-1$ + // currently not distinguish different point represents all as a line + try { + currentName = ""; //$NON-NLS-1$ + current = new Location("gpx_file"); //$NON-NLS-1$ + current.setLatitude(Double.parseDouble(parser.getAttributeValue("", "lat"))); //$NON-NLS-1$ //$NON-NLS-2$ + current.setLongitude(Double.parseDouble(parser.getAttributeValue("", "lon"))); //$NON-NLS-1$ //$NON-NLS-2$ + } catch (NumberFormatException e) { + current = null; + + } + } else if (current != null && parser.getName().equals("name")) { //$NON-NLS-1$ + if (parser.next() == XmlPullParser.TEXT) { + currentName = parser.getText(); + } + } + } else if (tok == XmlPullParser.END_TAG) { + if (parser.getName().equals("wpt") || //$NON-NLS-1$ + parser.getName().equals("trkpt") || (!cloudMade && parser.getName().equals("rtept"))) { //$NON-NLS-1$ //$NON-NLS-2$ + if (current != null) { + if (parser.getName().equals("wpt") && !cloudMade) { //$NON-NLS-1$ + WptPt pt = new WptPt(); + pt.lat = current.getLatitude(); + pt.lon = current.getLongitude(); + pt.name = currentName; + res.wayPoints.add(pt); + } else { + if (res.locations.isEmpty()) { + res.locations.add(new ArrayList()); + } + res.locations.get(res.locations.size() - 1).add(current); + } + } + } + } + } + } catch (XmlPullParserException e) { + log.error("Error reading gpx", e); //$NON-NLS-1$ + res.error = ctx.getString(R.string.error_reading_gpx); + } catch (IOException e) { + log.error("Error reading gpx", e); //$NON-NLS-1$ + res.error = ctx.getString(R.string.error_reading_gpx); + } + + return res; + } + +} diff --git a/OsmAnd/src/net/osmand/plus/ResourceManager.java b/OsmAnd/src/net/osmand/plus/ResourceManager.java index 62aef122ff..049d38052b 100644 --- a/OsmAnd/src/net/osmand/plus/ResourceManager.java +++ b/OsmAnd/src/net/osmand/plus/ResourceManager.java @@ -1,837 +1,837 @@ -package net.osmand.plus; - -import java.io.File; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.text.Collator; -import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Date; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Stack; -import java.util.TreeMap; - -import net.osmand.IProgress; -import net.osmand.LogUtil; -import net.osmand.binary.BinaryMapIndexReader; -import net.osmand.data.Amenity; -import net.osmand.data.TransportStop; -import net.osmand.data.index.IndexConstants; -import net.osmand.data.preparation.MapTileDownloader; -import net.osmand.data.preparation.MapTileDownloader.DownloadRequest; -import net.osmand.data.preparation.MapTileDownloader.IMapDownloaderCallback; -import net.osmand.map.ITileSource; -import net.osmand.osm.LatLon; -import net.osmand.osm.MapUtils; -import net.osmand.plus.activities.OsmandApplication; -import net.osmand.plus.render.BaseOsmandRender; -import net.osmand.plus.render.MapRenderRepositories; -import net.osmand.plus.render.RendererRegistry; -import net.osmand.plus.views.POIMapLayer; - -import org.apache.commons.logging.Log; - -import android.database.sqlite.SQLiteException; -import android.graphics.Bitmap; -import android.graphics.BitmapFactory; - -/** - * Resource manager is responsible to work with all resources - * that could consume memory (especially with file resources). - * Such as indexes, tiles. - * Also it is responsible to create cache for that resources if they - * can't be loaded fully into memory & clear them on request. - *SQLITE - */ -public class ResourceManager { - - public static final String APP_DIR = "osmand/"; //$NON-NLS-1$ - public static final String POI_PATH = APP_DIR + IndexConstants.POI_INDEX_DIR; - public static final String VOICE_PATH = APP_DIR + IndexConstants.VOICE_INDEX_DIR; - public static final String MAPS_PATH = APP_DIR; - public static final String TILES_PATH = APP_DIR+"tiles/"; //$NON-NLS-1$ - public static final String TEMP_SOURCE_TO_LOAD = "temp"; //$NON-NLS-1$ - public static final String VECTOR_MAP = "#vector_map"; //$NON-NLS-1$ - - public static final int LIMIT_TRANSPORT = 200; - - private static final Log log = LogUtil.getLog(ResourceManager.class); - - protected static ResourceManager manager = null; - - // it is not good investigated but no more than 64 (satellite images) - // Only 8 MB (from 16 Mb whole mem) available for images : image 64K * 128 = 8 MB (8 bit), 64 - 16 bit, 32 - 32 bit - protected int maxImgCacheSize = 32; - - protected Map cacheOfImages = new LinkedHashMap(); - protected Map imagesOnFS = new LinkedHashMap() ; - - protected File dirWithTiles ; - - private final OsmandApplication context; - - private BusyIndicator busyIndicator; - - private final MapTileDownloader downloader = MapTileDownloader.getInstance(); - // Indexes - private final Map addressMap = new TreeMap(Collator.getInstance()); - - protected final List amenityRepositories = new ArrayList(); - - protected final List transportRepositories = new ArrayList(); - - protected final Map indexFileNames = new LinkedHashMap(); - - protected final MapRenderRepositories renderer; - - public final AsyncLoadingThread asyncLoadingTiles = new AsyncLoadingThread(); - - protected boolean internetIsNotAccessible = false; - - - public ResourceManager(OsmandApplication context) { - this.context = context; - this.renderer = new MapRenderRepositories(context); - asyncLoadingTiles.start(); - resetStoreDirectory(); - } - - public void resetStoreDirectory() { - dirWithTiles = OsmandSettings.extendOsmandPath(context, TILES_PATH); - dirWithTiles.mkdirs(); - } - - public OsmandApplication getContext() { - return context; - } - - ////////////////////////////////////////////// Working with tiles //////////////////////////////////////////////// - - public void indexingImageTiles(IProgress progress){ - progress.startTask(context.getString(R.string.reading_cached_tiles), -1); //$NON-NLS-1$ - imagesOnFS.clear(); - for(File c : dirWithTiles.listFiles()){ - indexImageTilesFS("", c); //$NON-NLS-1$ - } - } - - private void indexImageTilesFS(String prefix, File f){ - if(f.isDirectory()){ - for(File c : f.listFiles()){ - indexImageTilesFS(prefix +f.getName() +"/" , c); //$NON-NLS-1$ - } - } else if(f.getName().endsWith(".tile")){ //$NON-NLS-1$ - imagesOnFS.put(prefix + f.getName(), Boolean.TRUE); - } else if(f.getName().endsWith(".sqlitedb")){ //$NON-NLS-1$ - // nothing to do here - } - } - - - public Bitmap getTileImageForMapAsync(String file, ITileSource map, int x, int y, int zoom, boolean loadFromInternetIfNeeded) { - return getTileImageForMap(file, map, x, y, zoom, loadFromInternetIfNeeded, false, true); - } - - - public Bitmap getTileImageFromCache(String file){ - return cacheOfImages.get(file); - } - - - public Bitmap getTileImageForMapSync(String file, ITileSource map, int x, int y, int zoom, boolean loadFromInternetIfNeeded) { - return getTileImageForMap(file, map, x, y, zoom, loadFromInternetIfNeeded, true, true); - } - - public void tileDownloaded(DownloadRequest request){ - if(request instanceof TileLoadDownloadRequest){ - TileLoadDownloadRequest req = ((TileLoadDownloadRequest) request); - imagesOnFS.put(req.tileId, Boolean.TRUE); - if(req.fileToSave != null && req.tileSource instanceof SQLiteTileSource){ - try { - ((SQLiteTileSource) req.tileSource).insertImage(req.xTile, req.yTile, req.zoom, req.fileToSave); - } catch (IOException e) { - log.warn("File "+req.fileToSave.getName() + " couldn't be read", e); //$NON-NLS-1$//$NON-NLS-2$ - } - req.fileToSave.delete(); - String[] l = req.fileToSave.getParentFile().list(); - if(l == null || l.length == 0){ - req.fileToSave.getParentFile().delete(); - l = req.fileToSave.getParentFile().getParentFile().list(); - if(l == null || l.length == 0){ - req.fileToSave.getParentFile().getParentFile().delete(); - } - } - } - } - - } - - public synchronized boolean tileExistOnFileSystem(String file, ITileSource map, int x, int y, int zoom){ - if(!imagesOnFS.containsKey(file)){ - boolean ex = false; - if(map instanceof SQLiteTileSource){ - ex = ((SQLiteTileSource) map).exists(x, y, zoom); - } else { - if(file == null){ - file = calculateTileId(map, x, y, zoom); - } - ex = new File(dirWithTiles, file).exists(); - } - if (ex) { - imagesOnFS.put(file, Boolean.TRUE); - } else { - imagesOnFS.put(file, null); - } - } - return imagesOnFS.get(file) != null; - } - - public void clearTileImageForMap(String file, ITileSource map, int x, int y, int zoom){ - getTileImageForMap(file, map, x, y, zoom, true, false, true, true); - } - - /** - * @param file - null could be passed if you do not call very often with that param - */ - protected Bitmap getTileImageForMap(String file, ITileSource map, int x, int y, int zoom, - boolean loadFromInternetIfNeeded, boolean sync, boolean loadFromFs) { - return getTileImageForMap(file, map, x, y, zoom, loadFromInternetIfNeeded, sync, loadFromFs, false); - } - - // introduce cache in order save memory - private int insertString(char[] ar, int offset, String s) { - for (int j = 0; j < s.length(); j++) { - ar[offset++] = s.charAt(j); - } - return offset; - } - protected StringBuilder builder = new StringBuilder(40); - protected char[] tileId = new char[120]; - public synchronized String calculateTileId(ITileSource map, int x, int y, int zoom){ - if(false){ - // performance improve ? - int ind = 0; - String mapName = map == null ? TEMP_SOURCE_TO_LOAD : map.getName(); - ind = insertString(tileId, ind, mapName); - if (map instanceof SQLiteTileSource) { - tileId[ind++] = '@'; - } else { - tileId[ind++] = '/'; - } - ind = insertString(tileId, ind, Integer.toString(zoom)); - tileId[ind++] = '/'; - ind = insertString(tileId, ind, Integer.toString(x)); - tileId[ind++] = '/'; - ind = insertString(tileId, ind, Integer.toString(y)); - ind = insertString(tileId, ind, map == null ? ".jpg" : map.getTileFormat()); //$NON-NLS-1$ - ind = insertString(tileId, ind, ".tile"); //$NON-NLS-1$ - return new String(tileId, 0, ind); - } else { - - builder.setLength(0); - if (map == null) { - builder.append(TEMP_SOURCE_TO_LOAD); - } else { - builder.append(map.getName()); - } - - if (map instanceof SQLiteTileSource) { - builder.append('@'); - } else { - builder.append('/'); - } - builder.append(zoom).append('/').append(x).append('/').append(y) - .append(map == null ? ".jpg" : map.getTileFormat()).append(".tile"); //$NON-NLS-1$ //$NON-NLS-2$ - return builder.toString(); - } - } - - - protected synchronized Bitmap getTileImageForMap(String tileId, ITileSource map, int x, int y, int zoom, - boolean loadFromInternetIfNeeded, boolean sync, boolean loadFromFs, boolean deleteBefore) { - if (tileId == null) { - tileId = calculateTileId(map, x, y, zoom); - if(tileId == null){ - return null; - } - } - - if(deleteBefore){ - cacheOfImages.remove(tileId); - if (map instanceof SQLiteTileSource) { - ((SQLiteTileSource) map).deleteImage(x, y, zoom); - } else { - File f = new File(dirWithTiles, tileId); - if (f.exists()) { - f.delete(); - } - } - imagesOnFS.put(tileId, null); - } - - if (loadFromFs && cacheOfImages.get(tileId) == null && map != null) { - if(!loadFromInternetIfNeeded && !tileExistOnFileSystem(tileId, map, x, y, zoom)){ - return null; - } - String url = loadFromInternetIfNeeded ? map.getUrlToLoad(x, y, zoom) : null; - File toSave = null; - if (url != null) { - if (map instanceof SQLiteTileSource) { - toSave = new File(dirWithTiles, calculateTileId(((SQLiteTileSource) map).getBase(), x, y, zoom)); - } else { - toSave = new File(dirWithTiles, tileId); - } - } - TileLoadDownloadRequest req = new TileLoadDownloadRequest(dirWithTiles, url, toSave, - tileId, map, x, y, zoom); - if(sync){ - return getRequestedImageTile(req); - } else { - asyncLoadingTiles.requestToLoadImage(req); - } - } - return cacheOfImages.get(tileId); - } - - - - private Bitmap getRequestedImageTile(TileLoadDownloadRequest req){ - if(req.tileId == null || req.dirWithTiles == null){ - return null; - } - if (cacheOfImages.size() > maxImgCacheSize) { - clearTiles(); - } - if (req.dirWithTiles.canRead() && !downloader.isFileCurrentlyDownloaded(req.fileToSave)) { - long time = System.currentTimeMillis(); - Bitmap bmp = null; - if (req.tileSource instanceof SQLiteTileSource) { - bmp = ((SQLiteTileSource) req.tileSource).getImage(req.xTile, req.yTile, req.zoom); - } else { - File en = new File(req.dirWithTiles, req.tileId); - if (en.exists()) { - try { - bmp = BitmapFactory.decodeFile(en.getAbsolutePath()); - } catch (OutOfMemoryError e) { - log.error("Out of memory error", e); //$NON-NLS-1$ - clearTiles(); - } - } - } - - if (bmp != null) { - cacheOfImages.put(req.tileId, bmp); - if (log.isDebugEnabled()) { - log.debug("Loaded file : " + req.tileId + " " + -(time - System.currentTimeMillis()) + " ms " + cacheOfImages.size()); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ - } - } - - if (cacheOfImages.get(req.tileId) == null && req.url != null) { - downloader.requestToDownload(req); - } - - } - return cacheOfImages.get(req.tileId); - } - - ////////////////////////////////////////////// Working with indexes //////////////////////////////////////////////// - - public List reloadIndexes(IProgress progress){ - close(); - initRenderers(progress); - // do it lazy - // indexingImageTiles(progress); - List warnings = new ArrayList(); - warnings.addAll(indexingPoi(progress)); - warnings.addAll(indexingMaps(progress)); - return warnings; - } - - private void initRenderers(IProgress progress) { - File file = OsmandSettings.extendOsmandPath(context, APP_DIR + IndexConstants.RENDERERS_DIR); - file.mkdirs(); - Map externalRenderers = new LinkedHashMap(); - if (file.exists() && file.canRead()) { - for (File f : file.listFiles()) { - if (f.getName().endsWith(IndexConstants.RENDERER_INDEX_EXT)) { - String name = f.getName().substring(0, f.getName().length() - IndexConstants.RENDERER_INDEX_EXT.length()); - externalRenderers.put(name, f); - } - } - } - RendererRegistry.getRegistry().setExternalRenderers(externalRenderers); - String r = OsmandSettings.getVectorRenderer(OsmandSettings.getPrefs(context)); - if(r != null){ - BaseOsmandRender obj = RendererRegistry.getRegistry().getRenderer(r); - if(obj != null){ - RendererRegistry.getRegistry().setCurrentSelectedRender(obj); - } - } - } - - public List indexingMaps(final IProgress progress) { - File file = OsmandSettings.extendOsmandPath(context, MAPS_PATH); - file.mkdirs(); - List warnings = new ArrayList(); - renderer.clearAllResources(); - if (file.exists() && file.canRead()) { - for (File f : file.listFiles()) { - if (f.getName().endsWith(IndexConstants.BINARY_MAP_INDEX_EXT)) { - progress.startTask(context.getString(R.string.indexing_map) + " " + f.getName(), -1); //$NON-NLS-1$ - try { - BinaryMapIndexReader index = renderer.initializeNewResource(progress, f); - if (index == null) { - warnings.add(MessageFormat.format(context.getString(R.string.version_index_is_not_supported), f.getName())); //$NON-NLS-1$ - } else { - indexFileNames.put(f.getName(), MessageFormat.format("{0,date,dd.MM.yyyy}", new Date(f.lastModified()))); //$NON-NLS-1$ - for(String rName : index.getRegionNames()) { - // skip duplicate names (don't make collision between getName() and name in the map) - RegionAddressRepositoryBinary rarb = new RegionAddressRepositoryBinary(index, rName); - addressMap.put(rName, rarb); - } - if (index.hasTransportData()) { - try { - RandomAccessFile raf = new RandomAccessFile(f, "r"); //$NON-NLS-1$ - transportRepositories.add(new TransportIndexRepositoryBinary(new BinaryMapIndexReader(raf))); - } catch (IOException e) { - log.error("Exception reading " + f.getAbsolutePath(), e); //$NON-NLS-1$ - warnings.add(MessageFormat.format(context.getString(R.string.version_index_is_not_supported), f.getName())); //$NON-NLS-1$ - } - } - if(index.containsMapData()){ - // that's not fully acceptable - // TODO -// try { -// RandomAccessFile raf = new RandomAccessFile(f, "r"); //$NON-NLS-1$ -// amenityRepositories.add(new AmenityIndexRepositoryBinary(new BinaryMapIndexReader(raf))); -// } catch (IOException e) { -// log.error("Exception reading " + f.getAbsolutePath(), e); //$NON-NLS-1$ -// warnings.add(MessageFormat.format(Messages.getMessage("version_index_is_not_supported"), f.getName())); //$NON-NLS-1$ -// } - } - } - } catch (SQLiteException e) { - log.error("Exception reading " + f.getAbsolutePath(), e); //$NON-NLS-1$ - warnings.add(MessageFormat.format(context.getString(R.string.version_index_is_not_supported), f.getName())); //$NON-NLS-1$ - } catch (OutOfMemoryError oome) { - log.error("Exception reading " + f.getAbsolutePath(), oome); //$NON-NLS-1$ - warnings.add(MessageFormat.format(context.getString(R.string.version_index_is_big_for_memory), f.getName())); - } - } else if(f.getName().endsWith(".map.odb")){ //$NON-NLS-1$ - warnings.add(MessageFormat.format(context.getString(R.string.old_map_index_is_not_supported), f.getName())); //$NON-NLS-1$ - } - } - } - return warnings; - } - - // POI INDEX // - public List indexingPoi(final IProgress progress) { - File file = OsmandSettings.extendOsmandPath(context, POI_PATH); - file.mkdirs(); - List warnings = new ArrayList(); - closeAmenities(); - if (file.exists() && file.canRead()) { - for (File f : file.listFiles()) { - indexingPoi(progress, warnings, f); - } - } - return warnings; - } - - public void indexingPoi(final IProgress progress, List warnings, File f) { - if (f.getName().endsWith(IndexConstants.POI_INDEX_EXT)) { - AmenityIndexRepositoryOdb repository = new AmenityIndexRepositoryOdb(); - - progress.startTask(context.getString(R.string.indexing_poi) + " " + f.getName(), -1); //$NON-NLS-1$ - try { - boolean initialized = repository.initialize(progress, f); - if (initialized) { - amenityRepositories.add(repository); - indexFileNames.put(f.getName(), MessageFormat.format("{0,date,dd.MM.yyyy}", new Date(f.lastModified()))); //$NON-NLS-1$ - } else { - warnings.add(MessageFormat.format(context.getString(R.string.version_index_is_not_supported), f.getName())); //$NON-NLS-1$ - } - } catch (SQLiteException e) { - log.error("Exception reading " + f.getAbsolutePath(), e); //$NON-NLS-1$ - warnings.add(MessageFormat.format(context.getString(R.string.version_index_is_not_supported), f.getName())); //$NON-NLS-1$ - } - } - } - - public void updateIndexLastDateModified(File f){ - if(f != null && f.exists()){ - indexFileNames.put(f.getName(), MessageFormat.format("{0,date,dd.MM.yyyy}", new Date(f.lastModified()))); //$NON-NLS-1$ - } - } - - - ////////////////////////////////////////////// Working with amenities //////////////////////////////////////////////// - public List searchAmenityRepositories(double latitude, double longitude) { - List repos = new ArrayList(); - for (AmenityIndexRepository index : amenityRepositories) { - if (index.checkContains(latitude,longitude)) { - repos.add(index); - } - } - return repos; - } - - public List searchAmenities(PoiFilter filter, double latitude, double longitude, int zoom, int limit) { - double tileNumberX = MapUtils.getTileNumberX(zoom, longitude); - double tileNumberY = MapUtils.getTileNumberY(zoom, latitude); - double topLatitude = MapUtils.getLatitudeFromTile(zoom, tileNumberY - 0.5); - double bottomLatitude = MapUtils.getLatitudeFromTile(zoom, tileNumberY + 0.5); - double leftLongitude = MapUtils.getLongitudeFromTile(zoom, tileNumberX - 0.5); - double rightLongitude = MapUtils.getLongitudeFromTile(zoom, tileNumberX + 0.5); - List amenities = new ArrayList(); - for (AmenityIndexRepository index : amenityRepositories) { - if (index.checkContains(topLatitude, leftLongitude, bottomLatitude, rightLongitude)) { - if (!index.checkCachedAmenities(topLatitude, leftLongitude, bottomLatitude, rightLongitude, zoom, filter.getFilterId(), - amenities, false)) { - index.searchAmenities(topLatitude, leftLongitude, bottomLatitude, rightLongitude, limit, filter, amenities); - } - } - } - - return amenities; - } - - public void searchAmenitiesAsync(double topLatitude, double leftLongitude, double bottomLatitude, double rightLongitude, int zoom, PoiFilter filter, List toFill){ - if(filter instanceof NameFinderPoiFilter){ - List amenities = ((NameFinderPoiFilter) filter).getSearchedAmenities(); - for(Amenity a : amenities){ - LatLon l = a.getLocation(); - if(l != null && l.getLatitude() <= topLatitude && l.getLatitude() >= bottomLatitude && l.getLongitude() >= leftLongitude && l.getLongitude() <= rightLongitude){ - toFill.add(a); - } - } - - } else { - String filterId = filter == null ? null : filter.getFilterId(); - for (AmenityIndexRepository index : amenityRepositories) { - if (index.checkContains(topLatitude, leftLongitude, bottomLatitude, rightLongitude)) { - if (!index.checkCachedAmenities(topLatitude, leftLongitude, bottomLatitude, rightLongitude, zoom, filterId, toFill, - true)) { - asyncLoadingTiles.requestToLoadAmenities(new AmenityLoadRequest(index, topLatitude, leftLongitude, bottomLatitude, - rightLongitude, zoom, filter)); - } - } - } - } - } - - ////////////////////////////////////////////// Working with address /////////////////////////////////////////// - - public RegionAddressRepository getRegionRepository(String name){ - return addressMap.get(name); - } - - public Collection getAddressRepositories(){ - return addressMap.values(); - } - - ////////////////////////////////////////////// Working with transport //////////////////////////////////////////////// - public List searchTransportRepositories(double latitude, double longitude) { - List repos = new ArrayList(); - for (TransportIndexRepository index : transportRepositories) { - if (index.checkContains(latitude,longitude)) { - repos.add(index); - } - } - return repos; - } - - - public void searchTransportAsync(double topLatitude, double leftLongitude, double bottomLatitude, double rightLongitude, int zoom, List toFill){ - for(TransportIndexRepository index : transportRepositories){ - if(index.checkContains(topLatitude, leftLongitude, bottomLatitude, rightLongitude)){ - if(!index.checkCachedObjects(topLatitude, leftLongitude, bottomLatitude, rightLongitude, zoom, toFill, true)){ - asyncLoadingTiles.requestToLoadTransport( - new TransportLoadRequest(index, topLatitude, leftLongitude, bottomLatitude, rightLongitude, zoom)); - } - } - } - } - - ////////////////////////////////////////////// Working with map //////////////////////////////////////////////// - public boolean updateRenderedMapNeeded(RotatedTileBox rotatedTileBox){ - return renderer.updateMapIsNeeded(rotatedTileBox); - } - - public void updateRendererMap(RotatedTileBox rotatedTileBox){ - renderer.interruptLoadingMap(); - asyncLoadingTiles.requestToLoadMap( - new MapLoadRequest(new RotatedTileBox(rotatedTileBox))); - } - - public void interruptRendering(){ - renderer.interruptLoadingMap(); - } - - public MapRenderRepositories getRenderer() { - return renderer; - } - - ////////////////////////////////////////////// Closing methods //////////////////////////////////////////////// - - public void closeAmenities(){ - for(AmenityIndexRepository r : amenityRepositories){ - r.close(); - } - amenityRepositories.clear(); - } - - public void closeAddresses(){ - for(RegionAddressRepository r : addressMap.values()){ - r.close(); - } - addressMap.clear(); - } - - public void closeTransport(){ - for(TransportIndexRepository r : transportRepositories){ - r.close(); - } - transportRepositories.clear(); - } - - public BusyIndicator getBusyIndicator() { - return busyIndicator; - } - - public synchronized void setBusyIndicator(BusyIndicator busyIndicator) { - this.busyIndicator = busyIndicator; - } - - public synchronized void close(){ - imagesOnFS.clear(); - indexFileNames.clear(); - renderer.clearAllResources(); - closeAmenities(); - closeAddresses(); - closeTransport(); - } - - public Map getIndexFileNames() { - return indexFileNames; - } - - public synchronized void reloadTilesFromFS(){ - imagesOnFS.clear(); - } - - /// On low memory method /// - public void onLowMemory() { - log.info("On low memory : cleaning tiles - size = " + cacheOfImages.size()); //$NON-NLS-1$ - clearTiles(); - for(AmenityIndexRepository r : amenityRepositories){ - r.clearCache(); - } - for(RegionAddressRepository r : addressMap.values()){ - r.clearCache(); - } - renderer.clearCache(); - - System.gc(); - } - - - public synchronized void updateMapSource(boolean useVectorMap, ITileSource source){ - log.info("Clear cache with new source " + cacheOfImages.size()); //$NON-NLS-1$ - cacheOfImages.clear(); - renderer.clearCache(); - if(source == null || source.getBitDensity() == 0){ - maxImgCacheSize = 32; - } else { - maxImgCacheSize = Math.max(384 / source.getBitDensity() , 32); - } - - } - - - protected synchronized void clearTiles(){ - log.info("Cleaning tiles - size = " + cacheOfImages.size()); //$NON-NLS-1$ - ArrayList list = new ArrayList(cacheOfImages.keySet()); - // remove first images (as we think they are older) - for (int i = 0; i < list.size() /2; i ++) { - cacheOfImages.remove(list.get(i)); - } - } - - - private static class TileLoadDownloadRequest extends DownloadRequest { - - public final String tileId; - public final File dirWithTiles; - public final ITileSource tileSource; - - public TileLoadDownloadRequest(File dirWithTiles, String url, File fileToSave, - String tileId, ITileSource source, int tileX, int tileY, int zoom) { - super(url, fileToSave, tileX, tileY, zoom); - this.dirWithTiles = dirWithTiles; - this.tileSource = source; - this.tileId = tileId; - } - } - - private static class AmenityLoadRequest { - public final AmenityIndexRepository repository; - public final double topLatitude; - public final double bottomLatitude; - public final double leftLongitude; - public final double rightLongitude; - public final PoiFilter filter; - public final int zoom; - - public AmenityLoadRequest(AmenityIndexRepository repository, double topLatitude, double leftLongitude, - double bottomLatitude, double rightLongitude, int zoom, PoiFilter filter) { - super(); - this.bottomLatitude = bottomLatitude; - this.leftLongitude = leftLongitude; - this.repository = repository; - this.rightLongitude = rightLongitude; - this.topLatitude = topLatitude; - this.zoom = zoom; - this.filter = filter; - } - } - - - - private static class TransportLoadRequest { - public final TransportIndexRepository repository; - public final double topLatitude; - public final double bottomLatitude; - public final double leftLongitude; - public final double rightLongitude; - public final int zoom; - - public TransportLoadRequest(TransportIndexRepository repository, double topLatitude, double leftLongitude, - double bottomLatitude, double rightLongitude, int zoom) { - super(); - this.bottomLatitude = bottomLatitude; - this.leftLongitude = leftLongitude; - this.repository = repository; - this.rightLongitude = rightLongitude; - this.topLatitude = topLatitude; - this.zoom = zoom; - } - } - - private static class MapLoadRequest { - public final RotatedTileBox tileBox; - - public MapLoadRequest(RotatedTileBox tileBox) { - super(); - this.tileBox = tileBox; - } - } - - public class AsyncLoadingThread extends Thread { - Stack requests = new Stack(); - - public AsyncLoadingThread(){ - super("Loader map objects (tiles, poi)"); //$NON-NLS-1$ - } - - @Override - public void run() { - while(true){ - try { - boolean update = false; - boolean amenityLoaded = false; - boolean transportLoaded = false; - boolean mapLoaded = false; - int progress = 0; - if(downloader.isSomethingBeingDownloaded()){ - progress = BusyIndicator.STATUS_GREEN; - } - synchronized(ResourceManager.this){ - if(busyIndicator != null){ - if(context.getRoutingHelper().isRouteBeingCalculated()){ - progress = BusyIndicator.STATUS_BLUE; - } else if(!requests.isEmpty()){ - progress = BusyIndicator.STATUS_BLACK;; - } - busyIndicator.updateStatus(progress); - } - } - while(!requests.isEmpty()){ - Object req = requests.pop(); - if (req instanceof TileLoadDownloadRequest) { - TileLoadDownloadRequest r = (TileLoadDownloadRequest) req; - if (cacheOfImages.get(r.tileId) == null) { - update |= getRequestedImageTile(r) != null; - } - } else if(req instanceof AmenityLoadRequest){ - if(!amenityLoaded){ - AmenityLoadRequest r = (AmenityLoadRequest) req; - r.repository.evaluateCachedAmenities(r.topLatitude, r.leftLongitude, - r.bottomLatitude, r.rightLongitude, r.zoom, POIMapLayer.LIMIT_POI, r.filter, null); - amenityLoaded = true; - } - } else if(req instanceof TransportLoadRequest){ - if(!transportLoaded){ - TransportLoadRequest r = (TransportLoadRequest) req; - r.repository.evaluateCachedTransportStops(r.topLatitude, r.leftLongitude, - r.bottomLatitude, r.rightLongitude, r.zoom, LIMIT_TRANSPORT, null); - transportLoaded = true; - } - } else if(req instanceof MapLoadRequest){ - if(!mapLoaded){ - MapLoadRequest r = (MapLoadRequest) req; - renderer.loadMap(r.tileBox, downloader.getDownloaderCallbacks()); - mapLoaded = true; - } - } - } - if(update || amenityLoaded || transportLoaded || mapLoaded){ - // use downloader callback - for(IMapDownloaderCallback c : downloader.getDownloaderCallbacks()){ - c.tileDownloaded(null); - } - } - boolean routeBeingCalculated = context.getRoutingHelper().isRouteBeingCalculated(); - if (progress != 0 || routeBeingCalculated || downloader.isSomethingBeingDownloaded()) { - synchronized (ResourceManager.this) { - if (busyIndicator != null) { - if(routeBeingCalculated){ - progress = BusyIndicator.STATUS_BLUE; - } else if(downloader.isSomethingBeingDownloaded()){ - progress = BusyIndicator.STATUS_GREEN; - } else { - progress = 0; - } - busyIndicator.updateStatus(progress); - } - } - } - sleep(750); - } catch (InterruptedException e) { - log.error(e, e); - } catch (RuntimeException e){ - log.error(e, e); - } - } - } - - public void requestToLoadImage(TileLoadDownloadRequest req){ - requests.push(req); - } - public void requestToLoadAmenities(AmenityLoadRequest req){ - requests.push(req); - } - - public void requestToLoadMap(MapLoadRequest req){ - requests.push(req); - } - - public void requestToLoadTransport(TransportLoadRequest req){ - requests.push(req); - } - }; -} +package net.osmand.plus; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.text.Collator; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Stack; +import java.util.TreeMap; + +import net.osmand.IProgress; +import net.osmand.LogUtil; +import net.osmand.binary.BinaryMapIndexReader; +import net.osmand.data.Amenity; +import net.osmand.data.TransportStop; +import net.osmand.data.index.IndexConstants; +import net.osmand.data.preparation.MapTileDownloader; +import net.osmand.data.preparation.MapTileDownloader.DownloadRequest; +import net.osmand.data.preparation.MapTileDownloader.IMapDownloaderCallback; +import net.osmand.map.ITileSource; +import net.osmand.osm.LatLon; +import net.osmand.osm.MapUtils; +import net.osmand.plus.activities.OsmandApplication; +import net.osmand.plus.render.BaseOsmandRender; +import net.osmand.plus.render.MapRenderRepositories; +import net.osmand.plus.render.RendererRegistry; +import net.osmand.plus.views.POIMapLayer; + +import org.apache.commons.logging.Log; + +import android.database.sqlite.SQLiteException; +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; + +/** + * Resource manager is responsible to work with all resources + * that could consume memory (especially with file resources). + * Such as indexes, tiles. + * Also it is responsible to create cache for that resources if they + * can't be loaded fully into memory & clear them on request. + *SQLITE + */ +public class ResourceManager { + + public static final String APP_DIR = "osmand/"; //$NON-NLS-1$ + public static final String POI_PATH = APP_DIR + IndexConstants.POI_INDEX_DIR; + public static final String VOICE_PATH = APP_DIR + IndexConstants.VOICE_INDEX_DIR; + public static final String MAPS_PATH = APP_DIR; + public static final String TILES_PATH = APP_DIR+"tiles/"; //$NON-NLS-1$ + public static final String TEMP_SOURCE_TO_LOAD = "temp"; //$NON-NLS-1$ + public static final String VECTOR_MAP = "#vector_map"; //$NON-NLS-1$ + + public static final int LIMIT_TRANSPORT = 200; + + private static final Log log = LogUtil.getLog(ResourceManager.class); + + protected static ResourceManager manager = null; + + // it is not good investigated but no more than 64 (satellite images) + // Only 8 MB (from 16 Mb whole mem) available for images : image 64K * 128 = 8 MB (8 bit), 64 - 16 bit, 32 - 32 bit + protected int maxImgCacheSize = 32; + + protected Map cacheOfImages = new LinkedHashMap(); + protected Map imagesOnFS = new LinkedHashMap() ; + + protected File dirWithTiles ; + + private final OsmandApplication context; + + private BusyIndicator busyIndicator; + + private final MapTileDownloader downloader = MapTileDownloader.getInstance(); + // Indexes + private final Map addressMap = new TreeMap(Collator.getInstance()); + + protected final List amenityRepositories = new ArrayList(); + + protected final List transportRepositories = new ArrayList(); + + protected final Map indexFileNames = new LinkedHashMap(); + + protected final MapRenderRepositories renderer; + + public final AsyncLoadingThread asyncLoadingTiles = new AsyncLoadingThread(); + + protected boolean internetIsNotAccessible = false; + + + public ResourceManager(OsmandApplication context) { + this.context = context; + this.renderer = new MapRenderRepositories(context); + asyncLoadingTiles.start(); + resetStoreDirectory(); + } + + public void resetStoreDirectory() { + dirWithTiles = OsmandSettings.extendOsmandPath(context, TILES_PATH); + dirWithTiles.mkdirs(); + } + + public OsmandApplication getContext() { + return context; + } + + ////////////////////////////////////////////// Working with tiles //////////////////////////////////////////////// + + public void indexingImageTiles(IProgress progress){ + progress.startTask(context.getString(R.string.reading_cached_tiles), -1); //$NON-NLS-1$ + imagesOnFS.clear(); + for(File c : dirWithTiles.listFiles()){ + indexImageTilesFS("", c); //$NON-NLS-1$ + } + } + + private void indexImageTilesFS(String prefix, File f){ + if(f.isDirectory()){ + for(File c : f.listFiles()){ + indexImageTilesFS(prefix +f.getName() +"/" , c); //$NON-NLS-1$ + } + } else if(f.getName().endsWith(".tile")){ //$NON-NLS-1$ + imagesOnFS.put(prefix + f.getName(), Boolean.TRUE); + } else if(f.getName().endsWith(".sqlitedb")){ //$NON-NLS-1$ + // nothing to do here + } + } + + + public Bitmap getTileImageForMapAsync(String file, ITileSource map, int x, int y, int zoom, boolean loadFromInternetIfNeeded) { + return getTileImageForMap(file, map, x, y, zoom, loadFromInternetIfNeeded, false, true); + } + + + public Bitmap getTileImageFromCache(String file){ + return cacheOfImages.get(file); + } + + + public Bitmap getTileImageForMapSync(String file, ITileSource map, int x, int y, int zoom, boolean loadFromInternetIfNeeded) { + return getTileImageForMap(file, map, x, y, zoom, loadFromInternetIfNeeded, true, true); + } + + public void tileDownloaded(DownloadRequest request){ + if(request instanceof TileLoadDownloadRequest){ + TileLoadDownloadRequest req = ((TileLoadDownloadRequest) request); + imagesOnFS.put(req.tileId, Boolean.TRUE); + if(req.fileToSave != null && req.tileSource instanceof SQLiteTileSource){ + try { + ((SQLiteTileSource) req.tileSource).insertImage(req.xTile, req.yTile, req.zoom, req.fileToSave); + } catch (IOException e) { + log.warn("File "+req.fileToSave.getName() + " couldn't be read", e); //$NON-NLS-1$//$NON-NLS-2$ + } + req.fileToSave.delete(); + String[] l = req.fileToSave.getParentFile().list(); + if(l == null || l.length == 0){ + req.fileToSave.getParentFile().delete(); + l = req.fileToSave.getParentFile().getParentFile().list(); + if(l == null || l.length == 0){ + req.fileToSave.getParentFile().getParentFile().delete(); + } + } + } + } + + } + + public synchronized boolean tileExistOnFileSystem(String file, ITileSource map, int x, int y, int zoom){ + if(!imagesOnFS.containsKey(file)){ + boolean ex = false; + if(map instanceof SQLiteTileSource){ + ex = ((SQLiteTileSource) map).exists(x, y, zoom); + } else { + if(file == null){ + file = calculateTileId(map, x, y, zoom); + } + ex = new File(dirWithTiles, file).exists(); + } + if (ex) { + imagesOnFS.put(file, Boolean.TRUE); + } else { + imagesOnFS.put(file, null); + } + } + return imagesOnFS.get(file) != null; + } + + public void clearTileImageForMap(String file, ITileSource map, int x, int y, int zoom){ + getTileImageForMap(file, map, x, y, zoom, true, false, true, true); + } + + /** + * @param file - null could be passed if you do not call very often with that param + */ + protected Bitmap getTileImageForMap(String file, ITileSource map, int x, int y, int zoom, + boolean loadFromInternetIfNeeded, boolean sync, boolean loadFromFs) { + return getTileImageForMap(file, map, x, y, zoom, loadFromInternetIfNeeded, sync, loadFromFs, false); + } + + // introduce cache in order save memory + private int insertString(char[] ar, int offset, String s) { + for (int j = 0; j < s.length(); j++) { + ar[offset++] = s.charAt(j); + } + return offset; + } + protected StringBuilder builder = new StringBuilder(40); + protected char[] tileId = new char[120]; + public synchronized String calculateTileId(ITileSource map, int x, int y, int zoom){ + if(false){ + // performance improve ? + int ind = 0; + String mapName = map == null ? TEMP_SOURCE_TO_LOAD : map.getName(); + ind = insertString(tileId, ind, mapName); + if (map instanceof SQLiteTileSource) { + tileId[ind++] = '@'; + } else { + tileId[ind++] = '/'; + } + ind = insertString(tileId, ind, Integer.toString(zoom)); + tileId[ind++] = '/'; + ind = insertString(tileId, ind, Integer.toString(x)); + tileId[ind++] = '/'; + ind = insertString(tileId, ind, Integer.toString(y)); + ind = insertString(tileId, ind, map == null ? ".jpg" : map.getTileFormat()); //$NON-NLS-1$ + ind = insertString(tileId, ind, ".tile"); //$NON-NLS-1$ + return new String(tileId, 0, ind); + } else { + + builder.setLength(0); + if (map == null) { + builder.append(TEMP_SOURCE_TO_LOAD); + } else { + builder.append(map.getName()); + } + + if (map instanceof SQLiteTileSource) { + builder.append('@'); + } else { + builder.append('/'); + } + builder.append(zoom).append('/').append(x).append('/').append(y) + .append(map == null ? ".jpg" : map.getTileFormat()).append(".tile"); //$NON-NLS-1$ //$NON-NLS-2$ + return builder.toString(); + } + } + + + protected synchronized Bitmap getTileImageForMap(String tileId, ITileSource map, int x, int y, int zoom, + boolean loadFromInternetIfNeeded, boolean sync, boolean loadFromFs, boolean deleteBefore) { + if (tileId == null) { + tileId = calculateTileId(map, x, y, zoom); + if(tileId == null){ + return null; + } + } + + if(deleteBefore){ + cacheOfImages.remove(tileId); + if (map instanceof SQLiteTileSource) { + ((SQLiteTileSource) map).deleteImage(x, y, zoom); + } else { + File f = new File(dirWithTiles, tileId); + if (f.exists()) { + f.delete(); + } + } + imagesOnFS.put(tileId, null); + } + + if (loadFromFs && cacheOfImages.get(tileId) == null && map != null) { + if(!loadFromInternetIfNeeded && !tileExistOnFileSystem(tileId, map, x, y, zoom)){ + return null; + } + String url = loadFromInternetIfNeeded ? map.getUrlToLoad(x, y, zoom) : null; + File toSave = null; + if (url != null) { + if (map instanceof SQLiteTileSource) { + toSave = new File(dirWithTiles, calculateTileId(((SQLiteTileSource) map).getBase(), x, y, zoom)); + } else { + toSave = new File(dirWithTiles, tileId); + } + } + TileLoadDownloadRequest req = new TileLoadDownloadRequest(dirWithTiles, url, toSave, + tileId, map, x, y, zoom); + if(sync){ + return getRequestedImageTile(req); + } else { + asyncLoadingTiles.requestToLoadImage(req); + } + } + return cacheOfImages.get(tileId); + } + + + + private Bitmap getRequestedImageTile(TileLoadDownloadRequest req){ + if(req.tileId == null || req.dirWithTiles == null){ + return null; + } + if (cacheOfImages.size() > maxImgCacheSize) { + clearTiles(); + } + if (req.dirWithTiles.canRead() && !downloader.isFileCurrentlyDownloaded(req.fileToSave)) { + long time = System.currentTimeMillis(); + Bitmap bmp = null; + if (req.tileSource instanceof SQLiteTileSource) { + bmp = ((SQLiteTileSource) req.tileSource).getImage(req.xTile, req.yTile, req.zoom); + } else { + File en = new File(req.dirWithTiles, req.tileId); + if (en.exists()) { + try { + bmp = BitmapFactory.decodeFile(en.getAbsolutePath()); + } catch (OutOfMemoryError e) { + log.error("Out of memory error", e); //$NON-NLS-1$ + clearTiles(); + } + } + } + + if (bmp != null) { + cacheOfImages.put(req.tileId, bmp); + if (log.isDebugEnabled()) { + log.debug("Loaded file : " + req.tileId + " " + -(time - System.currentTimeMillis()) + " ms " + cacheOfImages.size()); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + } + } + + if (cacheOfImages.get(req.tileId) == null && req.url != null) { + downloader.requestToDownload(req); + } + + } + return cacheOfImages.get(req.tileId); + } + + ////////////////////////////////////////////// Working with indexes //////////////////////////////////////////////// + + public List reloadIndexes(IProgress progress){ + close(); + initRenderers(progress); + // do it lazy + // indexingImageTiles(progress); + List warnings = new ArrayList(); + warnings.addAll(indexingPoi(progress)); + warnings.addAll(indexingMaps(progress)); + return warnings; + } + + private void initRenderers(IProgress progress) { + File file = OsmandSettings.extendOsmandPath(context, APP_DIR + IndexConstants.RENDERERS_DIR); + file.mkdirs(); + Map externalRenderers = new LinkedHashMap(); + if (file.exists() && file.canRead()) { + for (File f : file.listFiles()) { + if (f.getName().endsWith(IndexConstants.RENDERER_INDEX_EXT)) { + String name = f.getName().substring(0, f.getName().length() - IndexConstants.RENDERER_INDEX_EXT.length()); + externalRenderers.put(name, f); + } + } + } + RendererRegistry.getRegistry().setExternalRenderers(externalRenderers); + String r = OsmandSettings.getVectorRenderer(OsmandSettings.getPrefs(context)); + if(r != null){ + BaseOsmandRender obj = RendererRegistry.getRegistry().getRenderer(r); + if(obj != null){ + RendererRegistry.getRegistry().setCurrentSelectedRender(obj); + } + } + } + + public List indexingMaps(final IProgress progress) { + File file = OsmandSettings.extendOsmandPath(context, MAPS_PATH); + file.mkdirs(); + List warnings = new ArrayList(); + renderer.clearAllResources(); + if (file.exists() && file.canRead()) { + for (File f : file.listFiles()) { + if (f.getName().endsWith(IndexConstants.BINARY_MAP_INDEX_EXT)) { + progress.startTask(context.getString(R.string.indexing_map) + " " + f.getName(), -1); //$NON-NLS-1$ + try { + BinaryMapIndexReader index = renderer.initializeNewResource(progress, f); + if (index == null) { + warnings.add(MessageFormat.format(context.getString(R.string.version_index_is_not_supported), f.getName())); //$NON-NLS-1$ + } else { + indexFileNames.put(f.getName(), MessageFormat.format("{0,date,dd.MM.yyyy}", new Date(f.lastModified()))); //$NON-NLS-1$ + for(String rName : index.getRegionNames()) { + // skip duplicate names (don't make collision between getName() and name in the map) + RegionAddressRepositoryBinary rarb = new RegionAddressRepositoryBinary(index, rName); + addressMap.put(rName, rarb); + } + if (index.hasTransportData()) { + try { + RandomAccessFile raf = new RandomAccessFile(f, "r"); //$NON-NLS-1$ + transportRepositories.add(new TransportIndexRepositoryBinary(new BinaryMapIndexReader(raf))); + } catch (IOException e) { + log.error("Exception reading " + f.getAbsolutePath(), e); //$NON-NLS-1$ + warnings.add(MessageFormat.format(context.getString(R.string.version_index_is_not_supported), f.getName())); //$NON-NLS-1$ + } + } + if(index.containsMapData()){ + // that's not fully acceptable + // TODO +// try { +// RandomAccessFile raf = new RandomAccessFile(f, "r"); //$NON-NLS-1$ +// amenityRepositories.add(new AmenityIndexRepositoryBinary(new BinaryMapIndexReader(raf))); +// } catch (IOException e) { +// log.error("Exception reading " + f.getAbsolutePath(), e); //$NON-NLS-1$ +// warnings.add(MessageFormat.format(Messages.getMessage("version_index_is_not_supported"), f.getName())); //$NON-NLS-1$ +// } + } + } + } catch (SQLiteException e) { + log.error("Exception reading " + f.getAbsolutePath(), e); //$NON-NLS-1$ + warnings.add(MessageFormat.format(context.getString(R.string.version_index_is_not_supported), f.getName())); //$NON-NLS-1$ + } catch (OutOfMemoryError oome) { + log.error("Exception reading " + f.getAbsolutePath(), oome); //$NON-NLS-1$ + warnings.add(MessageFormat.format(context.getString(R.string.version_index_is_big_for_memory), f.getName())); + } + } else if(f.getName().endsWith(".map.odb")){ //$NON-NLS-1$ + warnings.add(MessageFormat.format(context.getString(R.string.old_map_index_is_not_supported), f.getName())); //$NON-NLS-1$ + } + } + } + return warnings; + } + + // POI INDEX // + public List indexingPoi(final IProgress progress) { + File file = OsmandSettings.extendOsmandPath(context, POI_PATH); + file.mkdirs(); + List warnings = new ArrayList(); + closeAmenities(); + if (file.exists() && file.canRead()) { + for (File f : file.listFiles()) { + indexingPoi(progress, warnings, f); + } + } + return warnings; + } + + public void indexingPoi(final IProgress progress, List warnings, File f) { + if (f.getName().endsWith(IndexConstants.POI_INDEX_EXT)) { + AmenityIndexRepositoryOdb repository = new AmenityIndexRepositoryOdb(); + + progress.startTask(context.getString(R.string.indexing_poi) + " " + f.getName(), -1); //$NON-NLS-1$ + try { + boolean initialized = repository.initialize(progress, f); + if (initialized) { + amenityRepositories.add(repository); + indexFileNames.put(f.getName(), MessageFormat.format("{0,date,dd.MM.yyyy}", new Date(f.lastModified()))); //$NON-NLS-1$ + } else { + warnings.add(MessageFormat.format(context.getString(R.string.version_index_is_not_supported), f.getName())); //$NON-NLS-1$ + } + } catch (SQLiteException e) { + log.error("Exception reading " + f.getAbsolutePath(), e); //$NON-NLS-1$ + warnings.add(MessageFormat.format(context.getString(R.string.version_index_is_not_supported), f.getName())); //$NON-NLS-1$ + } + } + } + + public void updateIndexLastDateModified(File f){ + if(f != null && f.exists()){ + indexFileNames.put(f.getName(), MessageFormat.format("{0,date,dd.MM.yyyy}", new Date(f.lastModified()))); //$NON-NLS-1$ + } + } + + + ////////////////////////////////////////////// Working with amenities //////////////////////////////////////////////// + public List searchAmenityRepositories(double latitude, double longitude) { + List repos = new ArrayList(); + for (AmenityIndexRepository index : amenityRepositories) { + if (index.checkContains(latitude,longitude)) { + repos.add(index); + } + } + return repos; + } + + public List searchAmenities(PoiFilter filter, double latitude, double longitude, int zoom, int limit) { + double tileNumberX = MapUtils.getTileNumberX(zoom, longitude); + double tileNumberY = MapUtils.getTileNumberY(zoom, latitude); + double topLatitude = MapUtils.getLatitudeFromTile(zoom, tileNumberY - 0.5); + double bottomLatitude = MapUtils.getLatitudeFromTile(zoom, tileNumberY + 0.5); + double leftLongitude = MapUtils.getLongitudeFromTile(zoom, tileNumberX - 0.5); + double rightLongitude = MapUtils.getLongitudeFromTile(zoom, tileNumberX + 0.5); + List amenities = new ArrayList(); + for (AmenityIndexRepository index : amenityRepositories) { + if (index.checkContains(topLatitude, leftLongitude, bottomLatitude, rightLongitude)) { + if (!index.checkCachedAmenities(topLatitude, leftLongitude, bottomLatitude, rightLongitude, zoom, filter.getFilterId(), + amenities, false)) { + index.searchAmenities(topLatitude, leftLongitude, bottomLatitude, rightLongitude, limit, filter, amenities); + } + } + } + + return amenities; + } + + public void searchAmenitiesAsync(double topLatitude, double leftLongitude, double bottomLatitude, double rightLongitude, int zoom, PoiFilter filter, List toFill){ + if(filter instanceof NameFinderPoiFilter){ + List amenities = ((NameFinderPoiFilter) filter).getSearchedAmenities(); + for(Amenity a : amenities){ + LatLon l = a.getLocation(); + if(l != null && l.getLatitude() <= topLatitude && l.getLatitude() >= bottomLatitude && l.getLongitude() >= leftLongitude && l.getLongitude() <= rightLongitude){ + toFill.add(a); + } + } + + } else { + String filterId = filter == null ? null : filter.getFilterId(); + for (AmenityIndexRepository index : amenityRepositories) { + if (index.checkContains(topLatitude, leftLongitude, bottomLatitude, rightLongitude)) { + if (!index.checkCachedAmenities(topLatitude, leftLongitude, bottomLatitude, rightLongitude, zoom, filterId, toFill, + true)) { + asyncLoadingTiles.requestToLoadAmenities(new AmenityLoadRequest(index, topLatitude, leftLongitude, bottomLatitude, + rightLongitude, zoom, filter)); + } + } + } + } + } + + ////////////////////////////////////////////// Working with address /////////////////////////////////////////// + + public RegionAddressRepository getRegionRepository(String name){ + return addressMap.get(name); + } + + public Collection getAddressRepositories(){ + return addressMap.values(); + } + + ////////////////////////////////////////////// Working with transport //////////////////////////////////////////////// + public List searchTransportRepositories(double latitude, double longitude) { + List repos = new ArrayList(); + for (TransportIndexRepository index : transportRepositories) { + if (index.checkContains(latitude,longitude)) { + repos.add(index); + } + } + return repos; + } + + + public void searchTransportAsync(double topLatitude, double leftLongitude, double bottomLatitude, double rightLongitude, int zoom, List toFill){ + for(TransportIndexRepository index : transportRepositories){ + if(index.checkContains(topLatitude, leftLongitude, bottomLatitude, rightLongitude)){ + if(!index.checkCachedObjects(topLatitude, leftLongitude, bottomLatitude, rightLongitude, zoom, toFill, true)){ + asyncLoadingTiles.requestToLoadTransport( + new TransportLoadRequest(index, topLatitude, leftLongitude, bottomLatitude, rightLongitude, zoom)); + } + } + } + } + + ////////////////////////////////////////////// Working with map //////////////////////////////////////////////// + public boolean updateRenderedMapNeeded(RotatedTileBox rotatedTileBox){ + return renderer.updateMapIsNeeded(rotatedTileBox); + } + + public void updateRendererMap(RotatedTileBox rotatedTileBox){ + renderer.interruptLoadingMap(); + asyncLoadingTiles.requestToLoadMap( + new MapLoadRequest(new RotatedTileBox(rotatedTileBox))); + } + + public void interruptRendering(){ + renderer.interruptLoadingMap(); + } + + public MapRenderRepositories getRenderer() { + return renderer; + } + + ////////////////////////////////////////////// Closing methods //////////////////////////////////////////////// + + public void closeAmenities(){ + for(AmenityIndexRepository r : amenityRepositories){ + r.close(); + } + amenityRepositories.clear(); + } + + public void closeAddresses(){ + for(RegionAddressRepository r : addressMap.values()){ + r.close(); + } + addressMap.clear(); + } + + public void closeTransport(){ + for(TransportIndexRepository r : transportRepositories){ + r.close(); + } + transportRepositories.clear(); + } + + public BusyIndicator getBusyIndicator() { + return busyIndicator; + } + + public synchronized void setBusyIndicator(BusyIndicator busyIndicator) { + this.busyIndicator = busyIndicator; + } + + public synchronized void close(){ + imagesOnFS.clear(); + indexFileNames.clear(); + renderer.clearAllResources(); + closeAmenities(); + closeAddresses(); + closeTransport(); + } + + public Map getIndexFileNames() { + return indexFileNames; + } + + public synchronized void reloadTilesFromFS(){ + imagesOnFS.clear(); + } + + /// On low memory method /// + public void onLowMemory() { + log.info("On low memory : cleaning tiles - size = " + cacheOfImages.size()); //$NON-NLS-1$ + clearTiles(); + for(AmenityIndexRepository r : amenityRepositories){ + r.clearCache(); + } + for(RegionAddressRepository r : addressMap.values()){ + r.clearCache(); + } + renderer.clearCache(); + + System.gc(); + } + + + public synchronized void updateMapSource(boolean useVectorMap, ITileSource source){ + log.info("Clear cache with new source " + cacheOfImages.size()); //$NON-NLS-1$ + cacheOfImages.clear(); + renderer.clearCache(); + if(source == null || source.getBitDensity() == 0){ + maxImgCacheSize = 32; + } else { + maxImgCacheSize = Math.max(384 / source.getBitDensity() , 32); + } + + } + + + protected synchronized void clearTiles(){ + log.info("Cleaning tiles - size = " + cacheOfImages.size()); //$NON-NLS-1$ + ArrayList list = new ArrayList(cacheOfImages.keySet()); + // remove first images (as we think they are older) + for (int i = 0; i < list.size() /2; i ++) { + cacheOfImages.remove(list.get(i)); + } + } + + + private static class TileLoadDownloadRequest extends DownloadRequest { + + public final String tileId; + public final File dirWithTiles; + public final ITileSource tileSource; + + public TileLoadDownloadRequest(File dirWithTiles, String url, File fileToSave, + String tileId, ITileSource source, int tileX, int tileY, int zoom) { + super(url, fileToSave, tileX, tileY, zoom); + this.dirWithTiles = dirWithTiles; + this.tileSource = source; + this.tileId = tileId; + } + } + + private static class AmenityLoadRequest { + public final AmenityIndexRepository repository; + public final double topLatitude; + public final double bottomLatitude; + public final double leftLongitude; + public final double rightLongitude; + public final PoiFilter filter; + public final int zoom; + + public AmenityLoadRequest(AmenityIndexRepository repository, double topLatitude, double leftLongitude, + double bottomLatitude, double rightLongitude, int zoom, PoiFilter filter) { + super(); + this.bottomLatitude = bottomLatitude; + this.leftLongitude = leftLongitude; + this.repository = repository; + this.rightLongitude = rightLongitude; + this.topLatitude = topLatitude; + this.zoom = zoom; + this.filter = filter; + } + } + + + + private static class TransportLoadRequest { + public final TransportIndexRepository repository; + public final double topLatitude; + public final double bottomLatitude; + public final double leftLongitude; + public final double rightLongitude; + public final int zoom; + + public TransportLoadRequest(TransportIndexRepository repository, double topLatitude, double leftLongitude, + double bottomLatitude, double rightLongitude, int zoom) { + super(); + this.bottomLatitude = bottomLatitude; + this.leftLongitude = leftLongitude; + this.repository = repository; + this.rightLongitude = rightLongitude; + this.topLatitude = topLatitude; + this.zoom = zoom; + } + } + + private static class MapLoadRequest { + public final RotatedTileBox tileBox; + + public MapLoadRequest(RotatedTileBox tileBox) { + super(); + this.tileBox = tileBox; + } + } + + public class AsyncLoadingThread extends Thread { + Stack requests = new Stack(); + + public AsyncLoadingThread(){ + super("Loader map objects (tiles, poi)"); //$NON-NLS-1$ + } + + @Override + public void run() { + while(true){ + try { + boolean update = false; + boolean amenityLoaded = false; + boolean transportLoaded = false; + boolean mapLoaded = false; + int progress = 0; + if(downloader.isSomethingBeingDownloaded()){ + progress = BusyIndicator.STATUS_GREEN; + } + synchronized(ResourceManager.this){ + if(busyIndicator != null){ + if(context.getRoutingHelper().isRouteBeingCalculated()){ + progress = BusyIndicator.STATUS_BLUE; + } else if(!requests.isEmpty()){ + progress = BusyIndicator.STATUS_BLACK;; + } + busyIndicator.updateStatus(progress); + } + } + while(!requests.isEmpty()){ + Object req = requests.pop(); + if (req instanceof TileLoadDownloadRequest) { + TileLoadDownloadRequest r = (TileLoadDownloadRequest) req; + if (cacheOfImages.get(r.tileId) == null) { + update |= getRequestedImageTile(r) != null; + } + } else if(req instanceof AmenityLoadRequest){ + if(!amenityLoaded){ + AmenityLoadRequest r = (AmenityLoadRequest) req; + r.repository.evaluateCachedAmenities(r.topLatitude, r.leftLongitude, + r.bottomLatitude, r.rightLongitude, r.zoom, POIMapLayer.LIMIT_POI, r.filter, null); + amenityLoaded = true; + } + } else if(req instanceof TransportLoadRequest){ + if(!transportLoaded){ + TransportLoadRequest r = (TransportLoadRequest) req; + r.repository.evaluateCachedTransportStops(r.topLatitude, r.leftLongitude, + r.bottomLatitude, r.rightLongitude, r.zoom, LIMIT_TRANSPORT, null); + transportLoaded = true; + } + } else if(req instanceof MapLoadRequest){ + if(!mapLoaded){ + MapLoadRequest r = (MapLoadRequest) req; + renderer.loadMap(r.tileBox, downloader.getDownloaderCallbacks()); + mapLoaded = true; + } + } + } + if(update || amenityLoaded || transportLoaded || mapLoaded){ + // use downloader callback + for(IMapDownloaderCallback c : downloader.getDownloaderCallbacks()){ + c.tileDownloaded(null); + } + } + boolean routeBeingCalculated = context.getRoutingHelper().isRouteBeingCalculated(); + if (progress != 0 || routeBeingCalculated || downloader.isSomethingBeingDownloaded()) { + synchronized (ResourceManager.this) { + if (busyIndicator != null) { + if(routeBeingCalculated){ + progress = BusyIndicator.STATUS_BLUE; + } else if(downloader.isSomethingBeingDownloaded()){ + progress = BusyIndicator.STATUS_GREEN; + } else { + progress = 0; + } + busyIndicator.updateStatus(progress); + } + } + } + sleep(750); + } catch (InterruptedException e) { + log.error(e, e); + } catch (RuntimeException e){ + log.error(e, e); + } + } + } + + public void requestToLoadImage(TileLoadDownloadRequest req){ + requests.push(req); + } + public void requestToLoadAmenities(AmenityLoadRequest req){ + requests.push(req); + } + + public void requestToLoadMap(MapLoadRequest req){ + requests.push(req); + } + + public void requestToLoadTransport(TransportLoadRequest req){ + requests.push(req); + } + }; +} diff --git a/OsmAnd/src/net/osmand/plus/activities/DownloadIndexActivity.java b/OsmAnd/src/net/osmand/plus/activities/DownloadIndexActivity.java index ce9ebe409f..f38b645e2a 100644 --- a/OsmAnd/src/net/osmand/plus/activities/DownloadIndexActivity.java +++ b/OsmAnd/src/net/osmand/plus/activities/DownloadIndexActivity.java @@ -1,390 +1,390 @@ -package net.osmand.plus.activities; - -import static net.osmand.data.index.IndexConstants.BINARY_MAP_INDEX_EXT; -import static net.osmand.data.index.IndexConstants.BINARY_MAP_INDEX_EXT_ZIP; -import static net.osmand.data.index.IndexConstants.BINARY_MAP_VERSION; -import static net.osmand.data.index.IndexConstants.POI_INDEX_EXT; -import static net.osmand.data.index.IndexConstants.POI_INDEX_EXT_ZIP; -import static net.osmand.data.index.IndexConstants.POI_TABLE_VERSION; -import static net.osmand.data.index.IndexConstants.VOICE_INDEX_EXT_ZIP; -import static net.osmand.data.index.IndexConstants.VOICE_VERSION; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.FilenameFilter; -import java.io.IOException; -import java.io.InputStream; -import java.net.HttpURLConnection; -import java.net.URL; -import java.text.MessageFormat; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Date; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Map.Entry; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; - -import net.osmand.IProgress; -import net.osmand.LogUtil; -import net.osmand.data.index.IndexConstants; -import net.osmand.plus.DownloadOsmandIndexesHelper; -import net.osmand.plus.OsmandSettings; -import net.osmand.plus.ProgressDialogImplementation; -import net.osmand.plus.R; -import net.osmand.plus.ResourceManager; -import net.osmand.plus.DownloadOsmandIndexesHelper.IndexItem; - -import org.apache.commons.logging.Log; - -import android.app.AlertDialog; -import android.app.ListActivity; -import android.app.ProgressDialog; -import android.app.AlertDialog.Builder; -import android.content.DialogInterface; -import android.graphics.Color; -import android.os.Bundle; -import android.text.Editable; -import android.text.TextWatcher; -import android.view.LayoutInflater; -import android.view.Menu; -import android.view.MenuItem; -import android.view.View; -import android.view.ViewGroup; -import android.widget.ArrayAdapter; -import android.widget.CheckBox; -import android.widget.EditText; -import android.widget.Filter; -import android.widget.Filterable; -import android.widget.ListView; -import android.widget.TextView; -import android.widget.Toast; - -public class DownloadIndexActivity extends ListActivity { - - private final static Log log = LogUtil.getLog(DownloadIndexActivity.class); - private static final int RELOAD_ID = 0; - private static final int SELECT_ALL_ID = 1; - private static final int DESELECT_ALL_ID = 2; - private static final int FILTER_EXISTING_REGIONS = 3; - - private static DownloadIndexListThread downloadListIndexThread = new DownloadIndexListThread(); - - private ProgressDialog progressFileDlg = null; - private ProgressDialog progressListDlg = null; - private Map indexFileNames = null; - private LinkedHashMap entriesToDownload = new LinkedHashMap(); - private TextWatcher textWatcher = new TextWatcher() { - - public void afterTextChanged(Editable s) { - } - - public void beforeTextChanged(CharSequence s, int start, int count, - int after) { - } - - public void onTextChanged(CharSequence s, int start, int before, - int count) { - DownloadIndexAdapter adapter = ((DownloadIndexAdapter)getListAdapter()); - if(adapter != null){ - adapter.getFilter().filter(s); - } - } - - }; - - - @Override - protected void onCreate(Bundle savedInstanceState) { - super.onCreate(savedInstanceState); - // recreation upon rotation is prevented in manifest file - setContentView(R.layout.download_index); - findViewById(R.id.DownloadButton).setOnClickListener(new View.OnClickListener(){ - - @Override - public void onClick(View v) { - downloadFiles(); - } - - }); - - indexFileNames = ((OsmandApplication)getApplication()).getResourceManager().getIndexFileNames(); - - EditText filterText = (EditText) findViewById(R.id.search_box); - filterText.addTextChangedListener(textWatcher); - - if(downloadListIndexThread.getCachedIndexFiles() != null){ - setListAdapter(new DownloadIndexAdapter(downloadListIndexThread.getCachedIndexFiles())); - } else { - downloadIndexList(); - } - } - - private void downloadIndexList() { - progressListDlg = ProgressDialog.show(this, getString(R.string.downloading), getString(R.string.downloading_list_indexes)); - progressListDlg.setCancelable(true); - downloadListIndexThread.setUiActivity(this); - if(downloadListIndexThread.getState() == Thread.State.NEW){ - downloadListIndexThread.start(); - } else if(downloadListIndexThread.getState() == Thread.State.TERMINATED){ - // possibly exception occurred we don't have cache of files - downloadListIndexThread = new DownloadIndexListThread(); - downloadListIndexThread.setUiActivity(this); - downloadListIndexThread.start(); - } - } - - @Override - - public boolean onPrepareOptionsMenu(Menu menu) { - super.onPrepareOptionsMenu(menu); - menu.clear(); - menu.add(0, RELOAD_ID, 0, R.string.reload); - if (getListAdapter() != null) { - // item.setIcon(R.drawable.ic_menu_refresh); - menu.add(0, SELECT_ALL_ID, 0, R.string.select_all); - menu.add(0, DESELECT_ALL_ID, 0, R.string.deselect_all); - menu.add(0, FILTER_EXISTING_REGIONS, 0, R.string.filter_existing_indexes); - } - return true; - } - - @Override - public boolean onOptionsItemSelected(MenuItem item) { - if(item.getItemId() == RELOAD_ID){ - //re-create the thread - downloadListIndexThread = new DownloadIndexListThread(); - downloadIndexList(); - } else { - final DownloadIndexAdapter listAdapter = (DownloadIndexAdapter)getListAdapter(); - if(item.getItemId() == SELECT_ALL_ID){ - int selected = 0; - for (int i = 0; i < listAdapter.getCount(); i++) { - IndexItem es = listAdapter.getItem(i); - if(!entriesToDownload.containsKey(es.getFileName())){ - selected++; - entriesToDownload.put(es.getFileName(), createDownloadEntry(es)); - } - } - Toast.makeText(this, MessageFormat.format(getString(R.string.items_were_selected), selected), Toast.LENGTH_SHORT).show(); - listAdapter.notifyDataSetInvalidated(); - if(selected > 0){ - findViewById(R.id.DownloadButton).setVisibility(View.VISIBLE); - } - } else if(item.getItemId() == FILTER_EXISTING_REGIONS){ - final List listAlreadyDownloaded = listAlreadyDownloadedWithAlternatives(); - final List filtered = new ArrayList(); - for(String file : listAlreadyDownloaded) { - IndexItem fileItem = listAdapter.getIndexFiles().get(file); - if (fileItem != null) { - filtered.add(fileItem); - } - } - listAdapter.clear(); - for (IndexItem fileItem : filtered) { - listAdapter.add(fileItem); - } - } else if(item.getItemId() == DESELECT_ALL_ID){ - entriesToDownload.clear(); - listAdapter.notifyDataSetInvalidated(); - findViewById(R.id.DownloadButton).setVisibility(View.GONE); - } else { - return false; - } - } - return true; - } - - private static class DownloadIndexListThread extends Thread { - private DownloadIndexActivity uiActivity = null; - private Map indexFiles = null; - - public DownloadIndexListThread(){ - super("DownloadIndexes"); //$NON-NLS-1$ - } - public void setUiActivity(DownloadIndexActivity uiActivity) { - this.uiActivity = uiActivity; - } - - public Map getCachedIndexFiles() { - return indexFiles; - } - - @Override - public void run() { - indexFiles = DownloadOsmandIndexesHelper.downloadIndexesListFromInternet(); - if(uiActivity != null && uiActivity.progressListDlg != null){ - uiActivity.progressListDlg.dismiss(); - uiActivity.progressListDlg = null; - uiActivity.runOnUiThread(new Runnable() { - @Override - public void run() { - if (indexFiles != null) { - uiActivity.setListAdapter(uiActivity.new DownloadIndexAdapter(indexFiles)); - } else { - Toast.makeText(uiActivity, R.string.list_index_files_was_not_loaded, Toast.LENGTH_LONG).show(); - } - } - }); - } - } - } - - - - +package net.osmand.plus.activities; + +import static net.osmand.data.index.IndexConstants.BINARY_MAP_INDEX_EXT; +import static net.osmand.data.index.IndexConstants.BINARY_MAP_INDEX_EXT_ZIP; +import static net.osmand.data.index.IndexConstants.BINARY_MAP_VERSION; +import static net.osmand.data.index.IndexConstants.POI_INDEX_EXT; +import static net.osmand.data.index.IndexConstants.POI_INDEX_EXT_ZIP; +import static net.osmand.data.index.IndexConstants.POI_TABLE_VERSION; +import static net.osmand.data.index.IndexConstants.VOICE_INDEX_EXT_ZIP; +import static net.osmand.data.index.IndexConstants.VOICE_VERSION; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.FilenameFilter; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.text.MessageFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Map.Entry; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import net.osmand.IProgress; +import net.osmand.LogUtil; +import net.osmand.data.index.IndexConstants; +import net.osmand.plus.DownloadOsmandIndexesHelper; +import net.osmand.plus.OsmandSettings; +import net.osmand.plus.ProgressDialogImplementation; +import net.osmand.plus.R; +import net.osmand.plus.ResourceManager; +import net.osmand.plus.DownloadOsmandIndexesHelper.IndexItem; + +import org.apache.commons.logging.Log; + +import android.app.AlertDialog; +import android.app.ListActivity; +import android.app.ProgressDialog; +import android.app.AlertDialog.Builder; +import android.content.DialogInterface; +import android.graphics.Color; +import android.os.Bundle; +import android.text.Editable; +import android.text.TextWatcher; +import android.view.LayoutInflater; +import android.view.Menu; +import android.view.MenuItem; +import android.view.View; +import android.view.ViewGroup; +import android.widget.ArrayAdapter; +import android.widget.CheckBox; +import android.widget.EditText; +import android.widget.Filter; +import android.widget.Filterable; +import android.widget.ListView; +import android.widget.TextView; +import android.widget.Toast; + +public class DownloadIndexActivity extends ListActivity { - - private final static int MB = 1 << 20; - - @Override - protected void onListItemClick(ListView l, View v, int position, long id) { - super.onListItemClick(l, v, position, id); - final IndexItem e = ((DownloadIndexAdapter)getListAdapter()).getItem(position); - String key = e.getFileName(); - final CheckBox ch = (CheckBox) v.findViewById(R.id.check_download_item); - - if(ch.isChecked()){ - ch.setChecked(!ch.isChecked()); - entriesToDownload.remove(key); - if(entriesToDownload.isEmpty()){ - int x = getListView().getScrollX(); - int y = getListView().getScrollY(); - findViewById(R.id.DownloadButton).setVisibility(View.GONE); - getListView().scrollTo(x, y); - } - return; - } - + private final static Log log = LogUtil.getLog(DownloadIndexActivity.class); + private static final int RELOAD_ID = 0; + private static final int SELECT_ALL_ID = 1; + private static final int DESELECT_ALL_ID = 2; + private static final int FILTER_EXISTING_REGIONS = 3; + + private static DownloadIndexListThread downloadListIndexThread = new DownloadIndexListThread(); + + private ProgressDialog progressFileDlg = null; + private ProgressDialog progressListDlg = null; + private Map indexFileNames = null; + private LinkedHashMap entriesToDownload = new LinkedHashMap(); + private TextWatcher textWatcher = new TextWatcher() { + + public void afterTextChanged(Editable s) { + } + + public void beforeTextChanged(CharSequence s, int start, int count, + int after) { + } + + public void onTextChanged(CharSequence s, int start, int before, + int count) { + DownloadIndexAdapter adapter = ((DownloadIndexAdapter)getListAdapter()); + if(adapter != null){ + adapter.getFilter().filter(s); + } + } + + }; + + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + // recreation upon rotation is prevented in manifest file + setContentView(R.layout.download_index); + findViewById(R.id.DownloadButton).setOnClickListener(new View.OnClickListener(){ + + @Override + public void onClick(View v) { + downloadFiles(); + } + + }); + + indexFileNames = ((OsmandApplication)getApplication()).getResourceManager().getIndexFileNames(); + + EditText filterText = (EditText) findViewById(R.id.search_box); + filterText.addTextChangedListener(textWatcher); + + if(downloadListIndexThread.getCachedIndexFiles() != null){ + setListAdapter(new DownloadIndexAdapter(downloadListIndexThread.getCachedIndexFiles())); + } else { + downloadIndexList(); + } + } + + private void downloadIndexList() { + progressListDlg = ProgressDialog.show(this, getString(R.string.downloading), getString(R.string.downloading_list_indexes)); + progressListDlg.setCancelable(true); + downloadListIndexThread.setUiActivity(this); + if(downloadListIndexThread.getState() == Thread.State.NEW){ + downloadListIndexThread.start(); + } else if(downloadListIndexThread.getState() == Thread.State.TERMINATED){ + // possibly exception occurred we don't have cache of files + downloadListIndexThread = new DownloadIndexListThread(); + downloadListIndexThread.setUiActivity(this); + downloadListIndexThread.start(); + } + } + + @Override + + public boolean onPrepareOptionsMenu(Menu menu) { + super.onPrepareOptionsMenu(menu); + menu.clear(); + menu.add(0, RELOAD_ID, 0, R.string.reload); + if (getListAdapter() != null) { + // item.setIcon(R.drawable.ic_menu_refresh); + menu.add(0, SELECT_ALL_ID, 0, R.string.select_all); + menu.add(0, DESELECT_ALL_ID, 0, R.string.deselect_all); + menu.add(0, FILTER_EXISTING_REGIONS, 0, R.string.filter_existing_indexes); + } + return true; + } + + @Override + public boolean onOptionsItemSelected(MenuItem item) { + if(item.getItemId() == RELOAD_ID){ + //re-create the thread + downloadListIndexThread = new DownloadIndexListThread(); + downloadIndexList(); + } else { + final DownloadIndexAdapter listAdapter = (DownloadIndexAdapter)getListAdapter(); + if(item.getItemId() == SELECT_ALL_ID){ + int selected = 0; + for (int i = 0; i < listAdapter.getCount(); i++) { + IndexItem es = listAdapter.getItem(i); + if(!entriesToDownload.containsKey(es.getFileName())){ + selected++; + entriesToDownload.put(es.getFileName(), createDownloadEntry(es)); + } + } + Toast.makeText(this, MessageFormat.format(getString(R.string.items_were_selected), selected), Toast.LENGTH_SHORT).show(); + listAdapter.notifyDataSetInvalidated(); + if(selected > 0){ + findViewById(R.id.DownloadButton).setVisibility(View.VISIBLE); + } + } else if(item.getItemId() == FILTER_EXISTING_REGIONS){ + final List listAlreadyDownloaded = listAlreadyDownloadedWithAlternatives(); + final List filtered = new ArrayList(); + for(String file : listAlreadyDownloaded) { + IndexItem fileItem = listAdapter.getIndexFiles().get(file); + if (fileItem != null) { + filtered.add(fileItem); + } + } + listAdapter.clear(); + for (IndexItem fileItem : filtered) { + listAdapter.add(fileItem); + } + } else if(item.getItemId() == DESELECT_ALL_ID){ + entriesToDownload.clear(); + listAdapter.notifyDataSetInvalidated(); + findViewById(R.id.DownloadButton).setVisibility(View.GONE); + } else { + return false; + } + } + return true; + } + + private static class DownloadIndexListThread extends Thread { + private DownloadIndexActivity uiActivity = null; + private Map indexFiles = null; + + public DownloadIndexListThread(){ + super("DownloadIndexes"); //$NON-NLS-1$ + } + public void setUiActivity(DownloadIndexActivity uiActivity) { + this.uiActivity = uiActivity; + } + + public Map getCachedIndexFiles() { + return indexFiles; + } + + @Override + public void run() { + indexFiles = DownloadOsmandIndexesHelper.downloadIndexesListFromInternet(); + if(uiActivity != null && uiActivity.progressListDlg != null){ + uiActivity.progressListDlg.dismiss(); + uiActivity.progressListDlg = null; + uiActivity.runOnUiThread(new Runnable() { + @Override + public void run() { + if (indexFiles != null) { + uiActivity.setListAdapter(uiActivity.new DownloadIndexAdapter(indexFiles)); + } else { + Toast.makeText(uiActivity, R.string.list_index_files_was_not_loaded, Toast.LENGTH_LONG).show(); + } + } + }); + } + } + } + + + + + + + private final static int MB = 1 << 20; + + @Override + protected void onListItemClick(ListView l, View v, int position, long id) { + super.onListItemClick(l, v, position, id); + final IndexItem e = ((DownloadIndexAdapter)getListAdapter()).getItem(position); + String key = e.getFileName(); + final CheckBox ch = (CheckBox) v.findViewById(R.id.check_download_item); + + if(ch.isChecked()){ + ch.setChecked(!ch.isChecked()); + entriesToDownload.remove(key); + if(entriesToDownload.isEmpty()){ + int x = getListView().getScrollX(); + int y = getListView().getScrollY(); + findViewById(R.id.DownloadButton).setVisibility(View.GONE); + getListView().scrollTo(x, y); + } + return; + } + final DownloadEntry entry = createDownloadEntry(e); - if (entry != null) { - // if(!fileToUnzip.exists()){ - // builder.setMessage(MessageFormat.format(getString(R.string.download_question), baseName, extractDateAndSize(e.getValue()))); - if (entry.fileToUnzip.exists()) { - Builder builder = new AlertDialog.Builder(this); - MessageFormat format; - if (entry.fileToUnzip.isDirectory()) { - format = new MessageFormat("{0,date,dd.MM.yyyy}", Locale.US); //$NON-NLS-1$ - } else { - format = new MessageFormat("{0,date,dd.MM.yyyy} : {1, number,##.#} MB", Locale.US); //$NON-NLS-1$ - } - String description = format.format(new Object[] { new Date(entry.fileToUnzip.lastModified()), - ((float) entry.fileToUnzip.length() / MB) }); + if (entry != null) { + // if(!fileToUnzip.exists()){ + // builder.setMessage(MessageFormat.format(getString(R.string.download_question), baseName, extractDateAndSize(e.getValue()))); + if (entry.fileToUnzip.exists()) { + Builder builder = new AlertDialog.Builder(this); + MessageFormat format; + if (entry.fileToUnzip.isDirectory()) { + format = new MessageFormat("{0,date,dd.MM.yyyy}", Locale.US); //$NON-NLS-1$ + } else { + format = new MessageFormat("{0,date,dd.MM.yyyy} : {1, number,##.#} MB", Locale.US); //$NON-NLS-1$ + } + String description = format.format(new Object[] { new Date(entry.fileToUnzip.lastModified()), + ((float) entry.fileToUnzip.length() / MB) }); String descriptionEx = e.getDate() + " : " +e.getSize(); - builder.setMessage(MessageFormat.format(getString(R.string.download_question_exist), entry.baseName, description, + builder.setMessage(MessageFormat.format(getString(R.string.download_question_exist), entry.baseName, description, descriptionEx)); - - builder.setPositiveButton(R.string.default_buttons_yes, new DialogInterface.OnClickListener() { - @Override - public void onClick(DialogInterface dialog, int which) { - entriesToDownload.put(e.getFileName(), entry); - int x = getListView().getScrollX(); - int y = getListView().getScrollY(); - findViewById(R.id.DownloadButton).setVisibility(View.VISIBLE); - getListView().scrollTo(x, y); - ch.setChecked(!ch.isChecked()); - } - }); - builder.setNegativeButton(R.string.default_buttons_no, null); - builder.show(); - } else { - entriesToDownload.put(e.getFileName(), entry); - int x = getListView().getScrollX(); - int y = getListView().getScrollY(); - findViewById(R.id.DownloadButton).setVisibility(View.VISIBLE); - getListView().scrollTo(x, y); - ch.setChecked(!ch.isChecked()); - } - - } - - } - - private List listAlreadyDownloadedWithAlternatives() { - List files = new ArrayList(); - File externalStorageDirectory = OsmandSettings.getExternalStorageDirectory(getApplicationContext()); - files.addAll(listWithAlternatives(new File(externalStorageDirectory, ResourceManager.POI_PATH),POI_INDEX_EXT,POI_INDEX_EXT_ZIP,POI_TABLE_VERSION)); - files.addAll(listWithAlternatives(new File(externalStorageDirectory, ResourceManager.APP_DIR),BINARY_MAP_INDEX_EXT,BINARY_MAP_INDEX_EXT_ZIP,BINARY_MAP_VERSION)); - files.addAll(listWithAlternatives(new File(externalStorageDirectory, ResourceManager.VOICE_PATH),"",VOICE_INDEX_EXT_ZIP, VOICE_VERSION)); - return files; - } - - private Collection listWithAlternatives(File file, final String ext, final String secondaryExt, final int version) { - final List files = new ArrayList(); - if (file.isDirectory()) { - file.list(new FilenameFilter() { - @Override - public boolean accept(File dir, String filename) { - if (filename.endsWith(ext)) { - files.add(filename); - files.add(filename.substring(0,filename.length()-ext.length())+"_"+version+ext); - if (secondaryExt != null) { - files.add(filename.substring(0,filename.length()-ext.length())+"_"+version+secondaryExt); - } - } - return filename.endsWith(ext); - } - }); - - } - return files; - } - + + builder.setPositiveButton(R.string.default_buttons_yes, new DialogInterface.OnClickListener() { + @Override + public void onClick(DialogInterface dialog, int which) { + entriesToDownload.put(e.getFileName(), entry); + int x = getListView().getScrollX(); + int y = getListView().getScrollY(); + findViewById(R.id.DownloadButton).setVisibility(View.VISIBLE); + getListView().scrollTo(x, y); + ch.setChecked(!ch.isChecked()); + } + }); + builder.setNegativeButton(R.string.default_buttons_no, null); + builder.show(); + } else { + entriesToDownload.put(e.getFileName(), entry); + int x = getListView().getScrollX(); + int y = getListView().getScrollY(); + findViewById(R.id.DownloadButton).setVisibility(View.VISIBLE); + getListView().scrollTo(x, y); + ch.setChecked(!ch.isChecked()); + } + + } + + } + + private List listAlreadyDownloadedWithAlternatives() { + List files = new ArrayList(); + File externalStorageDirectory = OsmandSettings.getExternalStorageDirectory(getApplicationContext()); + files.addAll(listWithAlternatives(new File(externalStorageDirectory, ResourceManager.POI_PATH),POI_INDEX_EXT,POI_INDEX_EXT_ZIP,POI_TABLE_VERSION)); + files.addAll(listWithAlternatives(new File(externalStorageDirectory, ResourceManager.APP_DIR),BINARY_MAP_INDEX_EXT,BINARY_MAP_INDEX_EXT_ZIP,BINARY_MAP_VERSION)); + files.addAll(listWithAlternatives(new File(externalStorageDirectory, ResourceManager.VOICE_PATH),"",VOICE_INDEX_EXT_ZIP, VOICE_VERSION)); + return files; + } + + private Collection listWithAlternatives(File file, final String ext, final String secondaryExt, final int version) { + final List files = new ArrayList(); + if (file.isDirectory()) { + file.list(new FilenameFilter() { + @Override + public boolean accept(File dir, String filename) { + if (filename.endsWith(ext)) { + files.add(filename); + files.add(filename.substring(0,filename.length()-ext.length())+"_"+version+ext); + if (secondaryExt != null) { + files.add(filename.substring(0,filename.length()-ext.length())+"_"+version+secondaryExt); + } + } + return filename.endsWith(ext); + } + }); + + } + return files; + } + private DownloadEntry createDownloadEntry(IndexItem item) { String fileName = item.getFileName(); - File parent = null; - String toSavePostfix = null; - String toCheckPostfix = null; - boolean unzipDir = false; + File parent = null; + String toSavePostfix = null; + String toCheckPostfix = null; + boolean unzipDir = false; - File externalStorageDirectory = OsmandSettings.getExternalStorageDirectory(getApplicationContext()); + File externalStorageDirectory = OsmandSettings.getExternalStorageDirectory(getApplicationContext()); if(fileName.endsWith(IndexConstants.POI_INDEX_EXT)){ - parent = new File(externalStorageDirectory, ResourceManager.POI_PATH); - toSavePostfix = POI_INDEX_EXT; - toCheckPostfix = POI_INDEX_EXT; + parent = new File(externalStorageDirectory, ResourceManager.POI_PATH); + toSavePostfix = POI_INDEX_EXT; + toCheckPostfix = POI_INDEX_EXT; } else if(fileName.endsWith(IndexConstants.POI_INDEX_EXT_ZIP)){ - parent = new File(externalStorageDirectory, ResourceManager.POI_PATH); - toSavePostfix = POI_INDEX_EXT_ZIP; - toCheckPostfix = POI_INDEX_EXT; + parent = new File(externalStorageDirectory, ResourceManager.POI_PATH); + toSavePostfix = POI_INDEX_EXT_ZIP; + toCheckPostfix = POI_INDEX_EXT; } else if(fileName.endsWith(IndexConstants.BINARY_MAP_INDEX_EXT)){ - parent = new File(externalStorageDirectory, ResourceManager.APP_DIR); - toSavePostfix = BINARY_MAP_INDEX_EXT; - toCheckPostfix = BINARY_MAP_INDEX_EXT; + parent = new File(externalStorageDirectory, ResourceManager.APP_DIR); + toSavePostfix = BINARY_MAP_INDEX_EXT; + toCheckPostfix = BINARY_MAP_INDEX_EXT; } else if(fileName.endsWith(IndexConstants.BINARY_MAP_INDEX_EXT_ZIP)){ - parent = new File(externalStorageDirectory, ResourceManager.APP_DIR); - toSavePostfix = BINARY_MAP_INDEX_EXT_ZIP; - toCheckPostfix = BINARY_MAP_INDEX_EXT; + parent = new File(externalStorageDirectory, ResourceManager.APP_DIR); + toSavePostfix = BINARY_MAP_INDEX_EXT_ZIP; + toCheckPostfix = BINARY_MAP_INDEX_EXT; } else if(fileName.endsWith(IndexConstants.VOICE_INDEX_EXT_ZIP)){ - parent = new File(externalStorageDirectory, ResourceManager.VOICE_PATH); - toSavePostfix = VOICE_INDEX_EXT_ZIP; - toCheckPostfix = ""; //$NON-NLS-1$ - unzipDir = true; - } - if(parent != null){ - parent.mkdirs(); - } - final DownloadEntry entry; - if(parent == null || !parent.exists()){ - Toast.makeText(DownloadIndexActivity.this, getString(R.string.sd_dir_not_accessible), Toast.LENGTH_LONG).show(); - entry = null; - } else { - entry = new DownloadEntry(); + parent = new File(externalStorageDirectory, ResourceManager.VOICE_PATH); + toSavePostfix = VOICE_INDEX_EXT_ZIP; + toCheckPostfix = ""; //$NON-NLS-1$ + unzipDir = true; + } + if(parent != null){ + parent.mkdirs(); + } + final DownloadEntry entry; + if(parent == null || !parent.exists()){ + Toast.makeText(DownloadIndexActivity.this, getString(R.string.sd_dir_not_accessible), Toast.LENGTH_LONG).show(); + entry = null; + } else { + entry = new DownloadEntry(); int ls = fileName.lastIndexOf('_'); entry.baseName = fileName.substring(0, ls); - entry.fileToSave = new File(parent, entry.baseName + toSavePostfix); - entry.unzip = unzipDir; + entry.fileToSave = new File(parent, entry.baseName + toSavePostfix); + entry.unzip = unzipDir; SimpleDateFormat format = new SimpleDateFormat("dd.MM.yyyy"); //$NON-NLS-1$ try { Date d = format.parse(item.getDate()); entry.dateModified = d.getTime(); } catch (ParseException e1) { - } + } try { entry.sizeMB = Double.parseDouble(item.getSize()); } catch (NumberFormatException e1) { @@ -392,188 +392,188 @@ public class DownloadIndexActivity extends ListActivity { entry.parts = 1; if(item.getParts() != null){ entry.parts = Integer.parseInt(item.getParts()); - } - entry.fileToUnzip = new File(parent, entry.baseName + toCheckPostfix); - } - return entry; - } - - protected void downloadFiles() { - Builder builder = new AlertDialog.Builder(this); - double sz = 0; - for(DownloadEntry es : entriesToDownload.values()){ - sz += es.sizeMB; - } - builder.setMessage(MessageFormat.format(getString(R.string.download_files_question), entriesToDownload.size(), sz)); - builder.setPositiveButton(R.string.default_buttons_yes, new DialogInterface.OnClickListener() { - @Override - public void onClick(DialogInterface dialog, int which) { - progressFileDlg = ProgressDialog.show(DownloadIndexActivity.this, getString(R.string.downloading), - getString(R.string.downloading_file), true, true); - interruptDownloading = false; - progressFileDlg.show(); - final ProgressDialogImplementation impl = new ProgressDialogImplementation(progressFileDlg, true); - progressFileDlg.setOnCancelListener(new DialogInterface.OnCancelListener() { - - @Override - public void onCancel(DialogInterface dialog) { - interruptDownloading = true; - } - }); - - impl.setRunnable("DownloadIndex", new Runnable() { //$NON-NLS-1$ - @Override - public void run() { - try { - for (String s : new ArrayList(entriesToDownload.keySet())) { - DownloadEntry entry = entriesToDownload.get(s); - if (entry != null) { + } + entry.fileToUnzip = new File(parent, entry.baseName + toCheckPostfix); + } + return entry; + } + + protected void downloadFiles() { + Builder builder = new AlertDialog.Builder(this); + double sz = 0; + for(DownloadEntry es : entriesToDownload.values()){ + sz += es.sizeMB; + } + builder.setMessage(MessageFormat.format(getString(R.string.download_files_question), entriesToDownload.size(), sz)); + builder.setPositiveButton(R.string.default_buttons_yes, new DialogInterface.OnClickListener() { + @Override + public void onClick(DialogInterface dialog, int which) { + progressFileDlg = ProgressDialog.show(DownloadIndexActivity.this, getString(R.string.downloading), + getString(R.string.downloading_file), true, true); + interruptDownloading = false; + progressFileDlg.show(); + final ProgressDialogImplementation impl = new ProgressDialogImplementation(progressFileDlg, true); + progressFileDlg.setOnCancelListener(new DialogInterface.OnCancelListener() { + + @Override + public void onCancel(DialogInterface dialog) { + interruptDownloading = true; + } + }); + + impl.setRunnable("DownloadIndex", new Runnable() { //$NON-NLS-1$ + @Override + public void run() { + try { + for (String s : new ArrayList(entriesToDownload.keySet())) { + DownloadEntry entry = entriesToDownload.get(s); + if (entry != null) { if (downloadFile(s, entry.fileToSave, entry.fileToUnzip, entry.unzip, impl, entry.dateModified, entry.parts)) { - entriesToDownload.remove(s); - runOnUiThread(new Runnable() { - @Override - public void run() { - ((DownloadIndexAdapter) getListAdapter()).notifyDataSetChanged(); - findViewById(R.id.DownloadButton).setVisibility(entriesToDownload.isEmpty() ? View.GONE : View.VISIBLE); - } - }); - } - } - } - } catch (InterruptedException e) { - // do not dismiss dialog - progressFileDlg = null; - } finally { - if (progressFileDlg != null) { - progressFileDlg.dismiss(); - progressFileDlg = null; - } - } - } - }); - impl.run(); - } - }); - builder.setNegativeButton(R.string.default_buttons_no, null); - builder.show(); - } - - private static final int BUFFER_SIZE = 32256; - - private static class DownloadEntry { - public File fileToSave; - public File fileToUnzip; - public boolean unzip; - public Long dateModified; - public double sizeMB; - public String baseName; + entriesToDownload.remove(s); + runOnUiThread(new Runnable() { + @Override + public void run() { + ((DownloadIndexAdapter) getListAdapter()).notifyDataSetChanged(); + findViewById(R.id.DownloadButton).setVisibility(entriesToDownload.isEmpty() ? View.GONE : View.VISIBLE); + } + }); + } + } + } + } catch (InterruptedException e) { + // do not dismiss dialog + progressFileDlg = null; + } finally { + if (progressFileDlg != null) { + progressFileDlg.dismiss(); + progressFileDlg = null; + } + } + } + }); + impl.run(); + } + }); + builder.setNegativeButton(R.string.default_buttons_no, null); + builder.show(); + } + + private static final int BUFFER_SIZE = 32256; + + private static class DownloadEntry { + public File fileToSave; + public File fileToUnzip; + public boolean unzip; + public Long dateModified; + public double sizeMB; + public String baseName; public int parts; - } - - protected final int TRIES_TO_DOWNLOAD = 15; - protected final long TIMEOUT_BETWEEN_DOWNLOADS = 8000; - private boolean interruptDownloading = false; - + } + + protected final int TRIES_TO_DOWNLOAD = 15; + protected final long TIMEOUT_BETWEEN_DOWNLOADS = 8000; + private boolean interruptDownloading = false; + protected void downloadFile(String fileName, FileOutputStream out, URL url, String part, IProgress progress) throws IOException, InterruptedException { - InputStream is = null; - - byte[] buffer = new byte[BUFFER_SIZE]; - int read = 0; - int length = 0; - int fileread = 0; - int triesDownload = TRIES_TO_DOWNLOAD; - boolean first = true; - try { - while (triesDownload > 0) { - try { - if (!first) { - log.info("Reconnecting"); //$NON-NLS-1$ - try { - Thread.sleep(TIMEOUT_BETWEEN_DOWNLOADS); - } catch (InterruptedException e) { - } - } - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - conn.setReadTimeout(30000); - if (fileread > 0) { - String range = "bytes="+fileread + "-" + (length -1); //$NON-NLS-1$ //$NON-NLS-2$ - conn.setRequestProperty("Range", range); //$NON-NLS-1$ - } - conn.setConnectTimeout(30000); - log.info(conn.getResponseMessage() + " " + conn.getResponseCode()); //$NON-NLS-1$ - if (conn.getResponseCode() != HttpURLConnection.HTTP_PARTIAL && - conn.getResponseCode() != HttpURLConnection.HTTP_OK) { - conn.disconnect(); - triesDownload--; - continue; - } - is = conn.getInputStream(); -// long skipped = 0; -// while (skipped < fileread) { -// skipped += is.skip(fileread - skipped); -// } - if (first) { - length = conn.getContentLength(); + InputStream is = null; + + byte[] buffer = new byte[BUFFER_SIZE]; + int read = 0; + int length = 0; + int fileread = 0; + int triesDownload = TRIES_TO_DOWNLOAD; + boolean first = true; + try { + while (triesDownload > 0) { + try { + if (!first) { + log.info("Reconnecting"); //$NON-NLS-1$ + try { + Thread.sleep(TIMEOUT_BETWEEN_DOWNLOADS); + } catch (InterruptedException e) { + } + } + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setReadTimeout(30000); + if (fileread > 0) { + String range = "bytes="+fileread + "-" + (length -1); //$NON-NLS-1$ //$NON-NLS-2$ + conn.setRequestProperty("Range", range); //$NON-NLS-1$ + } + conn.setConnectTimeout(30000); + log.info(conn.getResponseMessage() + " " + conn.getResponseCode()); //$NON-NLS-1$ + if (conn.getResponseCode() != HttpURLConnection.HTTP_PARTIAL && + conn.getResponseCode() != HttpURLConnection.HTTP_OK) { + conn.disconnect(); + triesDownload--; + continue; + } + is = conn.getInputStream(); +// long skipped = 0; +// while (skipped < fileread) { +// skipped += is.skip(fileread - skipped); +// } + if (first) { + length = conn.getContentLength(); String taskName = getString(R.string.downloading_file) + " " + fileName; if(part != null){ taskName += part; } progress.startTask(taskName, length); //$NON-NLS-1$ - } - - first = false; - while ((read = is.read(buffer)) != -1) { - if(interruptDownloading){ - throw new InterruptedException(); - } - out.write(buffer, 0, read); - progress.progress(read); - fileread += read; - } - if(length <= fileread){ - triesDownload = 0; - } - } catch (IOException e) { - log.error("IOException", e); //$NON-NLS-1$ - triesDownload--; - } - - } - } finally { - if (is != null) { - is.close(); - } - } - if(length != fileread){ - throw new IOException("File was not fully read"); //$NON-NLS-1$ - } - - } - - - @Override - protected void onDestroy() { - super.onDestroy(); - if(isFinishing()){ - interruptDownloading = true; - } - if (textWatcher != null) { - EditText filterText = (EditText) findViewById(R.id.search_box); - filterText.removeTextChangedListener(textWatcher); - } - downloadListIndexThread.setUiActivity(null); - progressFileDlg = null; - } - - protected boolean downloadFile(final String key, final File fileToDownload, final File fileToUnZip, final boolean unzipToDir, + } + + first = false; + while ((read = is.read(buffer)) != -1) { + if(interruptDownloading){ + throw new InterruptedException(); + } + out.write(buffer, 0, read); + progress.progress(read); + fileread += read; + } + if(length <= fileread){ + triesDownload = 0; + } + } catch (IOException e) { + log.error("IOException", e); //$NON-NLS-1$ + triesDownload--; + } + + } + } finally { + if (is != null) { + is.close(); + } + } + if(length != fileread){ + throw new IOException("File was not fully read"); //$NON-NLS-1$ + } + + } + + + @Override + protected void onDestroy() { + super.onDestroy(); + if(isFinishing()){ + interruptDownloading = true; + } + if (textWatcher != null) { + EditText filterText = (EditText) findViewById(R.id.search_box); + filterText.removeTextChangedListener(textWatcher); + } + downloadListIndexThread.setUiActivity(null); + progressFileDlg = null; + } + + protected boolean downloadFile(final String key, final File fileToDownload, final File fileToUnZip, final boolean unzipToDir, IProgress progress, Long dateModified, int parts) throws InterruptedException { - FileOutputStream out = null; - try { - - out = new FileOutputStream(fileToDownload); + FileOutputStream out = null; + try { + + out = new FileOutputStream(fileToDownload); - try { + try { if(parts == 1){ URL url = new URL("http://download.osmand.net/download?file="+key); //$NON-NLS-1$ downloadFile(key, out, url, null, progress); @@ -583,223 +583,223 @@ public class DownloadIndexActivity extends ListActivity { downloadFile(key, out, url, " ["+i+"/"+parts+"]", progress); } } - } finally { - out.close(); - out = null; - } - - File toIndex = fileToDownload; - if (fileToDownload.getName().endsWith(".zip")) { //$NON-NLS-1$ - progress.startTask(getString(R.string.unzipping_file), -1); - if (!unzipToDir) { - toIndex = fileToUnZip; - } else { - fileToUnZip.mkdirs(); - } - ZipInputStream zipIn = new ZipInputStream(new FileInputStream(fileToDownload)); - ZipEntry entry = null; - boolean first = true; - while ((entry = zipIn.getNextEntry()) != null) { - File fs; - if (!unzipToDir) { - if (first) { - fs = toIndex; - first = false; - } else { - String name = entry.getName(); - // small simplification - int ind = name.lastIndexOf('_'); - if (ind > 0) { - // cut version - int i = name.indexOf('.', ind); - if (i > 0) { - name = name.substring(0, ind) + name.substring(i, name.length()); - } - } - fs = new File(fileToUnZip.getParent(), name); - toIndex = fs; - } - } else { - fs = new File(fileToUnZip, entry.getName()); - } - out = new FileOutputStream(fs); - int read; - byte[] buffer = new byte[BUFFER_SIZE]; - while ((read = zipIn.read(buffer)) != -1) { - out.write(buffer, 0, read); - } - out.close(); - } - zipIn.close(); - fileToDownload.delete(); // zip is no needed more - } - - ArrayList warnings = new ArrayList(); - ResourceManager manager = ((OsmandApplication) getApplication()).getResourceManager(); - if(dateModified != null){ - toIndex.setLastModified(dateModified); - } - if (toIndex.getName().endsWith(IndexConstants.POI_INDEX_EXT)) { - manager.indexingPoi(progress, warnings, toIndex); - } else if (toIndex.getName().endsWith(IndexConstants.BINARY_MAP_INDEX_EXT)) { - warnings.addAll(manager.indexingMaps(progress)); - } else if (toIndex.getName().endsWith(IndexConstants.VOICE_INDEX_EXT_ZIP)) { - } - if(dateModified != null){ - toIndex.setLastModified(dateModified); - manager.updateIndexLastDateModified(toIndex); - } - if (warnings.isEmpty()) { - showWarning(getString(R.string.download_index_success)); - } else { - showWarning(warnings.get(0)); - } - return true; - } catch (IOException e) { - log.error("Exception ocurred", e); //$NON-NLS-1$ - showWarning(getString(R.string.error_io_error)); - if(out != null){ - try { - out.close(); - } catch (IOException e1) { - } - } - // Possibly file is corrupted - fileToDownload.delete(); - return false; - } catch (InterruptedException e) { - // Possibly file is corrupted - fileToDownload.delete(); - throw e; - } - } - - public void showWarning(final String messages){ - runOnUiThread(new Runnable(){ - @Override - public void run() { - Toast.makeText(DownloadIndexActivity.this, messages, Toast.LENGTH_LONG).show(); - } - - }); - } - - private String convertServerFileNameToLocal(String name){ - int l = name.lastIndexOf('_'); - String s; - if(name.endsWith(IndexConstants.POI_INDEX_EXT) || name.endsWith(IndexConstants.POI_INDEX_EXT_ZIP)){ - s = IndexConstants.POI_INDEX_EXT; - } else if(name.endsWith(IndexConstants.BINARY_MAP_INDEX_EXT) || name.endsWith(IndexConstants.BINARY_MAP_INDEX_EXT_ZIP)){ - s = IndexConstants.BINARY_MAP_INDEX_EXT; - } else { - s = ""; //$NON-NLS-1$ - } - return name.substring(0, l) + s; - } - + } finally { + out.close(); + out = null; + } + + File toIndex = fileToDownload; + if (fileToDownload.getName().endsWith(".zip")) { //$NON-NLS-1$ + progress.startTask(getString(R.string.unzipping_file), -1); + if (!unzipToDir) { + toIndex = fileToUnZip; + } else { + fileToUnZip.mkdirs(); + } + ZipInputStream zipIn = new ZipInputStream(new FileInputStream(fileToDownload)); + ZipEntry entry = null; + boolean first = true; + while ((entry = zipIn.getNextEntry()) != null) { + File fs; + if (!unzipToDir) { + if (first) { + fs = toIndex; + first = false; + } else { + String name = entry.getName(); + // small simplification + int ind = name.lastIndexOf('_'); + if (ind > 0) { + // cut version + int i = name.indexOf('.', ind); + if (i > 0) { + name = name.substring(0, ind) + name.substring(i, name.length()); + } + } + fs = new File(fileToUnZip.getParent(), name); + toIndex = fs; + } + } else { + fs = new File(fileToUnZip, entry.getName()); + } + out = new FileOutputStream(fs); + int read; + byte[] buffer = new byte[BUFFER_SIZE]; + while ((read = zipIn.read(buffer)) != -1) { + out.write(buffer, 0, read); + } + out.close(); + } + zipIn.close(); + fileToDownload.delete(); // zip is no needed more + } + + ArrayList warnings = new ArrayList(); + ResourceManager manager = ((OsmandApplication) getApplication()).getResourceManager(); + if(dateModified != null){ + toIndex.setLastModified(dateModified); + } + if (toIndex.getName().endsWith(IndexConstants.POI_INDEX_EXT)) { + manager.indexingPoi(progress, warnings, toIndex); + } else if (toIndex.getName().endsWith(IndexConstants.BINARY_MAP_INDEX_EXT)) { + warnings.addAll(manager.indexingMaps(progress)); + } else if (toIndex.getName().endsWith(IndexConstants.VOICE_INDEX_EXT_ZIP)) { + } + if(dateModified != null){ + toIndex.setLastModified(dateModified); + manager.updateIndexLastDateModified(toIndex); + } + if (warnings.isEmpty()) { + showWarning(getString(R.string.download_index_success)); + } else { + showWarning(warnings.get(0)); + } + return true; + } catch (IOException e) { + log.error("Exception ocurred", e); //$NON-NLS-1$ + showWarning(getString(R.string.error_io_error)); + if(out != null){ + try { + out.close(); + } catch (IOException e1) { + } + } + // Possibly file is corrupted + fileToDownload.delete(); + return false; + } catch (InterruptedException e) { + // Possibly file is corrupted + fileToDownload.delete(); + throw e; + } + } + + public void showWarning(final String messages){ + runOnUiThread(new Runnable(){ + @Override + public void run() { + Toast.makeText(DownloadIndexActivity.this, messages, Toast.LENGTH_LONG).show(); + } + + }); + } + + private String convertServerFileNameToLocal(String name){ + int l = name.lastIndexOf('_'); + String s; + if(name.endsWith(IndexConstants.POI_INDEX_EXT) || name.endsWith(IndexConstants.POI_INDEX_EXT_ZIP)){ + s = IndexConstants.POI_INDEX_EXT; + } else if(name.endsWith(IndexConstants.BINARY_MAP_INDEX_EXT) || name.endsWith(IndexConstants.BINARY_MAP_INDEX_EXT_ZIP)){ + s = IndexConstants.BINARY_MAP_INDEX_EXT; + } else { + s = ""; //$NON-NLS-1$ + } + return name.substring(0, l) + s; + } + - protected class DownloadIndexAdapter extends ArrayAdapter implements Filterable { - - private DownloadIndexFilter myFilter; - private final Map indexFiles; - + protected class DownloadIndexAdapter extends ArrayAdapter implements Filterable { + + private DownloadIndexFilter myFilter; + private final Map indexFiles; + public DownloadIndexAdapter(Map indexFiles) { - super(DownloadIndexActivity.this, net.osmand.plus.R.layout.download_index_list_item); + super(DownloadIndexActivity.this, net.osmand.plus.R.layout.download_index_list_item); this.indexFiles = new LinkedHashMap(indexFiles); for (Entry entry : indexFiles.entrySet()) { add(entry.getValue()); - } - } - - public Map getIndexFiles() { - return indexFiles; - } - - @Override - public View getView(final int position, View convertView, ViewGroup parent) { - View v = convertView; - if (v == null) { - LayoutInflater inflater = getLayoutInflater(); - v = inflater.inflate(net.osmand.plus.R.layout.download_index_list_item, parent, false); - } - final View row = v; - TextView item = (TextView) row.findViewById(R.id.download_item); - TextView description = (TextView) row.findViewById(R.id.download_descr); - IndexItem e = getItem(position); + } + } + + public Map getIndexFiles() { + return indexFiles; + } + + @Override + public View getView(final int position, View convertView, ViewGroup parent) { + View v = convertView; + if (v == null) { + LayoutInflater inflater = getLayoutInflater(); + v = inflater.inflate(net.osmand.plus.R.layout.download_index_list_item, parent, false); + } + final View row = v; + TextView item = (TextView) row.findViewById(R.id.download_item); + TextView description = (TextView) row.findViewById(R.id.download_descr); + IndexItem e = getItem(position); item.setText(e.getVisibleDescription(DownloadIndexActivity.this) + "\n" + e.getVisibleName()); //$NON-NLS-1$ description.setText(e.getDate() + "\n" + e.getSize() + " MB"); - - CheckBox ch = (CheckBox) row.findViewById(R.id.check_download_item); - ch.setChecked(entriesToDownload.containsKey(e.getFileName())); - ch.setOnClickListener(new View.OnClickListener() { - @Override - public void onClick(View v) { - final CheckBox ch = (CheckBox) v.findViewById(R.id.check_download_item); - ch.setChecked(!ch.isChecked()); - DownloadIndexActivity.this.onListItemClick(getListView(), row, position, getItemId(position)); - } - }); - - if(indexFileNames != null){ - String sfName = convertServerFileNameToLocal(e.getFileName()); - if(!indexFileNames.containsKey(sfName)){ - item.setTextColor(Color.WHITE); - } else { - if(e.getDate() != null){ - if(e.getDate().equals(indexFileNames.get(sfName))){ - item.setTextColor(Color.GREEN); - } else { - item.setTextColor(Color.BLUE); - } - } else { - item.setTextColor(Color.GREEN); - } - } - } - return row; - } - - @Override - public Filter getFilter() { - if (myFilter == null) { - myFilter = new DownloadIndexFilter(); - } - return myFilter; - } - - private final class DownloadIndexFilter extends Filter { - @Override - protected FilterResults performFiltering(CharSequence constraint) { - FilterResults results = new FilterResults(); - if (constraint == null || constraint.length() == 0) { - results.values = indexFiles.values(); - results.count = indexFiles.size(); - } else { - CharSequence lowerCase = constraint.toString() - .toLowerCase(); - List filter = new ArrayList(); - for (IndexItem item : indexFiles.values()) { + + CheckBox ch = (CheckBox) row.findViewById(R.id.check_download_item); + ch.setChecked(entriesToDownload.containsKey(e.getFileName())); + ch.setOnClickListener(new View.OnClickListener() { + @Override + public void onClick(View v) { + final CheckBox ch = (CheckBox) v.findViewById(R.id.check_download_item); + ch.setChecked(!ch.isChecked()); + DownloadIndexActivity.this.onListItemClick(getListView(), row, position, getItemId(position)); + } + }); + + if(indexFileNames != null){ + String sfName = convertServerFileNameToLocal(e.getFileName()); + if(!indexFileNames.containsKey(sfName)){ + item.setTextColor(Color.WHITE); + } else { + if(e.getDate() != null){ + if(e.getDate().equals(indexFileNames.get(sfName))){ + item.setTextColor(Color.GREEN); + } else { + item.setTextColor(Color.BLUE); + } + } else { + item.setTextColor(Color.GREEN); + } + } + } + return row; + } + + @Override + public Filter getFilter() { + if (myFilter == null) { + myFilter = new DownloadIndexFilter(); + } + return myFilter; + } + + private final class DownloadIndexFilter extends Filter { + @Override + protected FilterResults performFiltering(CharSequence constraint) { + FilterResults results = new FilterResults(); + if (constraint == null || constraint.length() == 0) { + results.values = indexFiles.values(); + results.count = indexFiles.size(); + } else { + CharSequence lowerCase = constraint.toString() + .toLowerCase(); + List filter = new ArrayList(); + for (IndexItem item : indexFiles.values()) { if (item.getVisibleName().toLowerCase().contains(lowerCase)) { - filter.add(item); - } - } - results.values = filter; - results.count = filter.size(); - } - return results; - } - - @SuppressWarnings("unchecked") - @Override - protected void publishResults(CharSequence constraint, FilterResults results) { - clear(); - for (IndexItem item : (Collection) results.values) { - add(item); - } - } - } - } - - -} + filter.add(item); + } + } + results.values = filter; + results.count = filter.size(); + } + return results; + } + + @SuppressWarnings("unchecked") + @Override + protected void publishResults(CharSequence constraint, FilterResults results) { + clear(); + for (IndexItem item : (Collection) results.values) { + add(item); + } + } + } + } + + +}