implement new indexing mechanism

git-svn-id: https://osmand.googlecode.com/svn/trunk@452 e29c36b1-1cfa-d876-8d93-3434fc2bb7b8

DataExtractionOSM/src/com/osmand/data/City.java

@@ -15,7 +15,7 @@ public class City extends MapObject {
 	
 	public enum CityType {
 		// that's tricky way to play with that numbers (to avoid including suburbs in city & vice verse)
-		CITY(10000), TOWN(5000), VILLAGE(1300), HAMLET(1000), SUBURB(300);
+		CITY(10000), TOWN(5000), VILLAGE(1300), HAMLET(1000), SUBURB(400);
 
 		private double radius;
 

DataExtractionOSM/src/com/osmand/data/index/IndexBatchCreator.java

@@ -18,9 +18,11 @@ import java.util.zip.ZipOutputStream;
 
 import org.apache.commons.logging.Log;
 
+import com.osmand.Algoritms;
 import com.osmand.LogUtil;
 import com.osmand.data.Region;
 import com.osmand.data.preparation.DataExtraction;
+import com.osmand.data.preparation.IndexCreator;
 import com.osmand.impl.ConsoleProgressImplementation;
 
 public class IndexBatchCreator {
@@ -99,7 +101,8 @@ public class IndexBatchCreator {
 	// TODO only australia, new zealand created
 	// TODO australia out of memory address
 	protected static final String[] oceania = new String[] {
-//		"Australia","New_Zealand",
+//		"Australia",
+//		"New_Zealand",
 //		"American_Samoa","Baker_Island","Cocos_Keeling_Islands","Cook_Islands",
 //		"Federated_States_of_Micronesia","Fiji", "French_Polynesia","Guam","Howland_Island",
 //		"Independent_State_of_Samoa","Jarvis_Island","Johnston_Atoll","Kiribati",
@@ -301,7 +304,7 @@ public class IndexBatchCreator {
 		}
 		System.out.println("GENERATING INDEXES FINISHED ");
 	}
-	protected void generateIndex(File f, Set<String> alreadyGeneratedFiles, Set<String> alreadyUploadedFiles) {
+	protected void generateIndexOld(File f, Set<String> alreadyGeneratedFiles, Set<String> alreadyUploadedFiles) {
 		if (!generateIndexes) {
 			return;
 		}
@@ -338,6 +341,52 @@ public class IndexBatchCreator {
 		System.gc();
 	}
 	
+	protected void generateIndex(File f, Set<String> alreadyGeneratedFiles, Set<String> alreadyUploadedFiles) {
+		if (!generateIndexes) {
+			return;
+		}
+		try {
+			IndexCreator indexCreator = new IndexCreator(indexDirFiles);
+			indexCreator.setIndexAddress(indexAddress);
+			indexCreator.setIndexAddress(indexPOI);
+			indexCreator.setIndexAddress(indexTransport);
+			indexCreator.setNormalizeStreets(true);
+			indexCreator.setSaveAddressWays(writeWayNodes);
+
+			String regionName = f.getName();
+			int i = f.getName().indexOf('.');
+			if (i > -1) {
+				regionName = Algoritms.capitalizeFirstLetterAndLowercase(f.getName().substring(0, i));
+			}
+			String addressFileName = regionName + "_" + IndexConstants.ADDRESS_TABLE_VERSION + IndexConstants.ADDRESS_INDEX_EXT;
+			indexCreator.setAddressFileName(addressFileName);
+			String transportFileName = regionName + "_" + IndexConstants.TRANSPORT_TABLE_VERSION + IndexConstants.TRANSPORT_INDEX_EXT;
+			indexCreator.setTransportFileName(transportFileName);
+			String poiFileName = regionName + "_" + IndexConstants.POI_TABLE_VERSION + IndexConstants.POI_INDEX_EXT;
+			indexCreator.setPoiFileName(poiFileName);
+			try {
+				alreadyGeneratedFiles.add(f.getName());
+				indexCreator.generateIndexes(f, new ConsoleProgressImplementation(3),  null);
+				if (indexAddress) {
+					uploadIndex(new File(indexDirFiles, addressFileName), alreadyUploadedFiles);
+				}
+				if (indexPOI) {
+					uploadIndex(new File(indexDirFiles, poiFileName), alreadyUploadedFiles);
+				}
+				if (indexTransport) {
+					uploadIndex(new File(indexDirFiles, transportFileName), alreadyUploadedFiles);
+				}
+			} catch (Exception e) {
+				log.error("Exception generating indexes for " + f.getName(), e); //$NON-NLS-1$ 
+			}
+		} catch (OutOfMemoryError e) {
+			System.gc();
+			log.error("OutOfMemory", e);
+
+		}
+		System.gc();
+	}
+	
 	protected File[] getSortedFiles(File dir){
 		File[] listFiles = dir.listFiles();
 		Arrays.sort(listFiles, new Comparator<File>(){

DataExtractionOSM/src/com/osmand/data/preparation/IndexCreator.java

@@ -38,6 +38,7 @@ import com.osmand.data.DataTileManager;
 import com.osmand.data.MapObject;
 import com.osmand.data.TransportRoute;
 import com.osmand.data.TransportStop;
+import com.osmand.data.City.CityType;
 import com.osmand.data.index.DataIndexWriter;
 import com.osmand.data.index.IndexConstants;
 import com.osmand.data.index.IndexConstants.IndexBuildingTable;
@@ -137,6 +138,7 @@ public class IndexCreator {
 	// address structure
 	// load it in memory
 	private Map<EntityId, City> cities = new LinkedHashMap<EntityId, City>();
+	private DataTileManager<City> cityVillageManager = new DataTileManager<City>(13);
 	private DataTileManager<City> cityManager = new DataTileManager<City>(10);
 	private List<Relation> postalCodeRelations = new ArrayList<Relation>(); 
 
@@ -755,7 +757,20 @@ public class IndexCreator {
 		}
 		City closest = null;
 		double relDist = Double.POSITIVE_INFINITY;
-		for (City c : cityManager.getClosestObjects(point.getLatitude(), point.getLongitude())) {
+		for (City c : cityManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) {
+			double rel = MapUtils.getDistance(c.getLocation(), point) / c.getType().getRadius();
+			if (rel < relDist) {
+				closest = c;
+				relDist = rel;
+				if(relDist < 0.2d){
+					break;
+				}
+			}
+		}
+		if(relDist < 0.2d){
+			return closest;
+		}
+		for (City c : cityVillageManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) {
 			double rel = MapUtils.getDistance(c.getLocation(), point) / c.getType().getRadius();
 			if (rel < relDist) {
 				closest = c;
@@ -912,12 +927,13 @@ public class IndexCreator {
 					// check that building is not registered already
 					boolean exist = false;
 					if(loadInMemory){
+						exist = addressBuildingLocalSet.contains(e.getId());
+					} else {
 						addressSearchBuildingStat.setLong(1, e.getId());
 						ResultSet rs = addressSearchBuildingStat.executeQuery();
 						exist = rs.next();
 						rs.close();
-					} else {
+						
-						exist = addressBuildingLocalSet.contains(e.getId());
 					}
 					if (!exist) {
 						loadEntityData(e, false);
@@ -937,12 +953,13 @@ public class IndexCreator {
 						&& e.getTag(OSMTagKey.HIGHWAY) != null && e.getTag(OSMTagKey.NAME) != null) {
 					boolean exist = false;
 					if(loadInMemory){
+						exist = addressStreetNodeLocalSet.contains(e.getId());
+					} else {
 						addressSearchStreetNodeStat.setLong(1, e.getId());
 						ResultSet rs = addressSearchStreetNodeStat.executeQuery();
 						exist = rs.next();
 						rs.close();
-					} else {
+						
-						exist = addressStreetNodeLocalSet.contains(e.getId());
 					}
 					// check that building is not registered already 
 					if (!exist) {
@@ -978,7 +995,11 @@ public class IndexCreator {
 			City city = new City((Node) e);
 			if(city.getType() != null && !Algoritms.isEmpty(city.getName())){
 				convertEnglishName(city);
-				cityManager.registerObject(((Node) e).getLatitude(), ((Node) e).getLongitude(), city);
+				if(city.getType() == CityType.CITY || city.getType() == CityType.TOWN){
+					cityManager.registerObject(((Node) e).getLatitude(), ((Node) e).getLongitude(), city);
+				} else {
+					cityVillageManager.registerObject(((Node) e).getLatitude(), ((Node) e).getLongitude(), city);
+				}
 				cities.put(city.getEntityId(), city);
 			}
 		}
@@ -1291,8 +1312,8 @@ public class IndexCreator {
 	public static void main(String[] args) throws IOException, SAXException, SQLException {
 		File workDir = new File("e:/Information/OSM maps/osmand/");
 		IndexCreator extr = new IndexCreator(workDir);
-		extr.setIndexPOI(true);
+//		extr.setIndexPOI(true);
-		extr.setIndexTransport(true);
+//		extr.setIndexTransport(true);
 		extr.setIndexAddress(true);
 		extr.setNormalizeStreets(true);
 		extr.setSaveAddressWays(true);