From fa091e92d7fc6e6ef0ffe22e8f186dc20968b6f0 Mon Sep 17 00:00:00 2001 From: Victor Shcherb Date: Sat, 14 Aug 2010 10:18:30 +0000 Subject: [PATCH] implement new indexing mechanism git-svn-id: https://osmand.googlecode.com/svn/trunk@452 e29c36b1-1cfa-d876-8d93-3434fc2bb7b8 --- .../src/com/osmand/data/City.java | 2 +- .../osmand/data/index/IndexBatchCreator.java | 53 ++++++++++++++++++- .../osmand/data/preparation/IndexCreator.java | 37 ++++++++++--- 3 files changed, 81 insertions(+), 11 deletions(-) diff --git a/DataExtractionOSM/src/com/osmand/data/City.java b/DataExtractionOSM/src/com/osmand/data/City.java index 0358f00be9..c7f3f5105c 100644 --- a/DataExtractionOSM/src/com/osmand/data/City.java +++ b/DataExtractionOSM/src/com/osmand/data/City.java @@ -15,7 +15,7 @@ public class City extends MapObject { public enum CityType { // that's tricky way to play with that numbers (to avoid including suburbs in city & vice verse) - CITY(10000), TOWN(5000), VILLAGE(1300), HAMLET(1000), SUBURB(300); + CITY(10000), TOWN(5000), VILLAGE(1300), HAMLET(1000), SUBURB(400); private double radius; diff --git a/DataExtractionOSM/src/com/osmand/data/index/IndexBatchCreator.java b/DataExtractionOSM/src/com/osmand/data/index/IndexBatchCreator.java index 9b5725f18f..a9bb50acbc 100644 --- a/DataExtractionOSM/src/com/osmand/data/index/IndexBatchCreator.java +++ b/DataExtractionOSM/src/com/osmand/data/index/IndexBatchCreator.java @@ -18,9 +18,11 @@ import java.util.zip.ZipOutputStream; import org.apache.commons.logging.Log; +import com.osmand.Algoritms; import com.osmand.LogUtil; import com.osmand.data.Region; import com.osmand.data.preparation.DataExtraction; +import com.osmand.data.preparation.IndexCreator; import com.osmand.impl.ConsoleProgressImplementation; public class IndexBatchCreator { @@ -99,7 +101,8 @@ public class IndexBatchCreator { // TODO only australia, new zealand created // TODO australia out of memory address protected static final String[] oceania = new String[] { -// "Australia","New_Zealand", +// "Australia", +// "New_Zealand", // "American_Samoa","Baker_Island","Cocos_Keeling_Islands","Cook_Islands", // "Federated_States_of_Micronesia","Fiji", "French_Polynesia","Guam","Howland_Island", // "Independent_State_of_Samoa","Jarvis_Island","Johnston_Atoll","Kiribati", @@ -301,7 +304,7 @@ public class IndexBatchCreator { } System.out.println("GENERATING INDEXES FINISHED "); } - protected void generateIndex(File f, Set alreadyGeneratedFiles, Set alreadyUploadedFiles) { + protected void generateIndexOld(File f, Set alreadyGeneratedFiles, Set alreadyUploadedFiles) { if (!generateIndexes) { return; } @@ -338,6 +341,52 @@ public class IndexBatchCreator { System.gc(); } + protected void generateIndex(File f, Set alreadyGeneratedFiles, Set alreadyUploadedFiles) { + if (!generateIndexes) { + return; + } + try { + IndexCreator indexCreator = new IndexCreator(indexDirFiles); + indexCreator.setIndexAddress(indexAddress); + indexCreator.setIndexAddress(indexPOI); + indexCreator.setIndexAddress(indexTransport); + indexCreator.setNormalizeStreets(true); + indexCreator.setSaveAddressWays(writeWayNodes); + + String regionName = f.getName(); + int i = f.getName().indexOf('.'); + if (i > -1) { + regionName = Algoritms.capitalizeFirstLetterAndLowercase(f.getName().substring(0, i)); + } + String addressFileName = regionName + "_" + IndexConstants.ADDRESS_TABLE_VERSION + IndexConstants.ADDRESS_INDEX_EXT; + indexCreator.setAddressFileName(addressFileName); + String transportFileName = regionName + "_" + IndexConstants.TRANSPORT_TABLE_VERSION + IndexConstants.TRANSPORT_INDEX_EXT; + indexCreator.setTransportFileName(transportFileName); + String poiFileName = regionName + "_" + IndexConstants.POI_TABLE_VERSION + IndexConstants.POI_INDEX_EXT; + indexCreator.setPoiFileName(poiFileName); + try { + alreadyGeneratedFiles.add(f.getName()); + indexCreator.generateIndexes(f, new ConsoleProgressImplementation(3), null); + if (indexAddress) { + uploadIndex(new File(indexDirFiles, addressFileName), alreadyUploadedFiles); + } + if (indexPOI) { + uploadIndex(new File(indexDirFiles, poiFileName), alreadyUploadedFiles); + } + if (indexTransport) { + uploadIndex(new File(indexDirFiles, transportFileName), alreadyUploadedFiles); + } + } catch (Exception e) { + log.error("Exception generating indexes for " + f.getName(), e); //$NON-NLS-1$ + } + } catch (OutOfMemoryError e) { + System.gc(); + log.error("OutOfMemory", e); + + } + System.gc(); + } + protected File[] getSortedFiles(File dir){ File[] listFiles = dir.listFiles(); Arrays.sort(listFiles, new Comparator(){ diff --git a/DataExtractionOSM/src/com/osmand/data/preparation/IndexCreator.java b/DataExtractionOSM/src/com/osmand/data/preparation/IndexCreator.java index bf643e6a2a..8cf35d71a5 100644 --- a/DataExtractionOSM/src/com/osmand/data/preparation/IndexCreator.java +++ b/DataExtractionOSM/src/com/osmand/data/preparation/IndexCreator.java @@ -38,6 +38,7 @@ import com.osmand.data.DataTileManager; import com.osmand.data.MapObject; import com.osmand.data.TransportRoute; import com.osmand.data.TransportStop; +import com.osmand.data.City.CityType; import com.osmand.data.index.DataIndexWriter; import com.osmand.data.index.IndexConstants; import com.osmand.data.index.IndexConstants.IndexBuildingTable; @@ -137,6 +138,7 @@ public class IndexCreator { // address structure // load it in memory private Map cities = new LinkedHashMap(); + private DataTileManager cityVillageManager = new DataTileManager(13); private DataTileManager cityManager = new DataTileManager(10); private List postalCodeRelations = new ArrayList(); @@ -755,7 +757,20 @@ public class IndexCreator { } City closest = null; double relDist = Double.POSITIVE_INFINITY; - for (City c : cityManager.getClosestObjects(point.getLatitude(), point.getLongitude())) { + for (City c : cityManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) { + double rel = MapUtils.getDistance(c.getLocation(), point) / c.getType().getRadius(); + if (rel < relDist) { + closest = c; + relDist = rel; + if(relDist < 0.2d){ + break; + } + } + } + if(relDist < 0.2d){ + return closest; + } + for (City c : cityVillageManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) { double rel = MapUtils.getDistance(c.getLocation(), point) / c.getType().getRadius(); if (rel < relDist) { closest = c; @@ -912,12 +927,13 @@ public class IndexCreator { // check that building is not registered already boolean exist = false; if(loadInMemory){ + exist = addressBuildingLocalSet.contains(e.getId()); + } else { addressSearchBuildingStat.setLong(1, e.getId()); ResultSet rs = addressSearchBuildingStat.executeQuery(); exist = rs.next(); rs.close(); - } else { - exist = addressBuildingLocalSet.contains(e.getId()); + } if (!exist) { loadEntityData(e, false); @@ -937,12 +953,13 @@ public class IndexCreator { && e.getTag(OSMTagKey.HIGHWAY) != null && e.getTag(OSMTagKey.NAME) != null) { boolean exist = false; if(loadInMemory){ + exist = addressStreetNodeLocalSet.contains(e.getId()); + } else { addressSearchStreetNodeStat.setLong(1, e.getId()); ResultSet rs = addressSearchStreetNodeStat.executeQuery(); exist = rs.next(); rs.close(); - } else { - exist = addressStreetNodeLocalSet.contains(e.getId()); + } // check that building is not registered already if (!exist) { @@ -978,7 +995,11 @@ public class IndexCreator { City city = new City((Node) e); if(city.getType() != null && !Algoritms.isEmpty(city.getName())){ convertEnglishName(city); - cityManager.registerObject(((Node) e).getLatitude(), ((Node) e).getLongitude(), city); + if(city.getType() == CityType.CITY || city.getType() == CityType.TOWN){ + cityManager.registerObject(((Node) e).getLatitude(), ((Node) e).getLongitude(), city); + } else { + cityVillageManager.registerObject(((Node) e).getLatitude(), ((Node) e).getLongitude(), city); + } cities.put(city.getEntityId(), city); } } @@ -1291,8 +1312,8 @@ public class IndexCreator { public static void main(String[] args) throws IOException, SAXException, SQLException { File workDir = new File("e:/Information/OSM maps/osmand/"); IndexCreator extr = new IndexCreator(workDir); - extr.setIndexPOI(true); - extr.setIndexTransport(true); +// extr.setIndexPOI(true); +// extr.setIndexTransport(true); extr.setIndexAddress(true); extr.setNormalizeStreets(true); extr.setSaveAddressWays(true);