Android-builders-club/OsmAnd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

implement new indexing mechanism

Browse source 
git-svn-id: https://osmand.googlecode.com/svn/trunk@452 e29c36b1-1cfa-d876-8d93-3434fc2bb7b8
This commit is contained in:
Victor Shcherb 2010-08-14 10:18:30 +00:00
parent b0a5076613
commit fa091e92d7
3 changed files with 81 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
DataExtractionOSM/src/com/osmand/data/City.java
View file

@ -15,7 +15,7 @@ public class City extends MapObject {
public enum CityType {
// that's tricky way to play with that numbers (to avoid including suburbs in city & vice verse)
CITY(10000), TOWN(5000), VILLAGE(1300), HAMLET(1000), SUBURB(300);
CITY(10000), TOWN(5000), VILLAGE(1300), HAMLET(1000), SUBURB(400);
private double radius;

53
DataExtractionOSM/src/com/osmand/data/index/IndexBatchCreator.java
View file

@ -18,9 +18,11 @@ import java.util.zip.ZipOutputStream;
import org.apache.commons.logging.Log;
import com.osmand.Algoritms;
import com.osmand.LogUtil;
import com.osmand.data.Region;
import com.osmand.data.preparation.DataExtraction;
import com.osmand.data.preparation.IndexCreator;
import com.osmand.impl.ConsoleProgressImplementation;
public class IndexBatchCreator {
@ -99,7 +101,8 @@ public class IndexBatchCreator {
// TODO only australia, new zealand created
// TODO australia out of memory address
protected static final String[] oceania = new String[] {
// "Australia","New_Zealand",
// "Australia",
// "New_Zealand",
// "American_Samoa","Baker_Island","Cocos_Keeling_Islands","Cook_Islands",
// "Federated_States_of_Micronesia","Fiji", "French_Polynesia","Guam","Howland_Island",
// "Independent_State_of_Samoa","Jarvis_Island","Johnston_Atoll","Kiribati",
@ -301,7 +304,7 @@ public class IndexBatchCreator {
}
System.out.println("GENERATING INDEXES FINISHED ");
}
protected void generateIndex(File f, Set<String> alreadyGeneratedFiles, Set<String> alreadyUploadedFiles) {
protected void generateIndexOld(File f, Set<String> alreadyGeneratedFiles, Set<String> alreadyUploadedFiles) {
if (!generateIndexes) {
return;
}
@ -338,6 +341,52 @@ public class IndexBatchCreator {
System.gc();
}
protected void generateIndex(File f, Set<String> alreadyGeneratedFiles, Set<String> alreadyUploadedFiles) {
if (!generateIndexes) {
return;
}
try {
IndexCreator indexCreator = new IndexCreator(indexDirFiles);
indexCreator.setIndexAddress(indexAddress);
indexCreator.setIndexAddress(indexPOI);
indexCreator.setIndexAddress(indexTransport);
indexCreator.setNormalizeStreets(true);
indexCreator.setSaveAddressWays(writeWayNodes);
String regionName = f.getName();
int i = f.getName().indexOf('.');
if (i > -1) {
regionName = Algoritms.capitalizeFirstLetterAndLowercase(f.getName().substring(0, i));
}
String addressFileName = regionName + "_" + IndexConstants.ADDRESS_TABLE_VERSION + IndexConstants.ADDRESS_INDEX_EXT;
indexCreator.setAddressFileName(addressFileName);
String transportFileName = regionName + "_" + IndexConstants.TRANSPORT_TABLE_VERSION + IndexConstants.TRANSPORT_INDEX_EXT;
indexCreator.setTransportFileName(transportFileName);
String poiFileName = regionName + "_" + IndexConstants.POI_TABLE_VERSION + IndexConstants.POI_INDEX_EXT;
indexCreator.setPoiFileName(poiFileName);
try {
alreadyGeneratedFiles.add(f.getName());
indexCreator.generateIndexes(f, new ConsoleProgressImplementation(3), null);
if (indexAddress) {
uploadIndex(new File(indexDirFiles, addressFileName), alreadyUploadedFiles);
}
if (indexPOI) {
uploadIndex(new File(indexDirFiles, poiFileName), alreadyUploadedFiles);
}
if (indexTransport) {
uploadIndex(new File(indexDirFiles, transportFileName), alreadyUploadedFiles);
}
} catch (Exception e) {
log.error("Exception generating indexes for " + f.getName(), e); //$NON-NLS-1$
}
} catch (OutOfMemoryError e) {
System.gc();
log.error("OutOfMemory", e);
}
System.gc();
}
protected File[] getSortedFiles(File dir){
File[] listFiles = dir.listFiles();
Arrays.sort(listFiles, new Comparator<File>(){

37
DataExtractionOSM/src/com/osmand/data/preparation/IndexCreator.java
View file

@ -38,6 +38,7 @@ import com.osmand.data.DataTileManager;
import com.osmand.data.MapObject;
import com.osmand.data.TransportRoute;
import com.osmand.data.TransportStop;
import com.osmand.data.City.CityType;
import com.osmand.data.index.DataIndexWriter;
import com.osmand.data.index.IndexConstants;
import com.osmand.data.index.IndexConstants.IndexBuildingTable;
@ -137,6 +138,7 @@ public class IndexCreator {
// address structure
// load it in memory
private Map<EntityId, City> cities = new LinkedHashMap<EntityId, City>();
private DataTileManager<City> cityVillageManager = new DataTileManager<City>(13);
private DataTileManager<City> cityManager = new DataTileManager<City>(10);
private List<Relation> postalCodeRelations = new ArrayList<Relation>();
@ -755,7 +757,20 @@ public class IndexCreator {
}
City closest = null;
double relDist = Double.POSITIVE_INFINITY;
for (City c : cityManager.getClosestObjects(point.getLatitude(), point.getLongitude())) {
for (City c : cityManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) {
double rel = MapUtils.getDistance(c.getLocation(), point) / c.getType().getRadius();
if (rel < relDist) {
closest = c;
relDist = rel;
if(relDist < 0.2d){
break;
}
}
}
if(relDist < 0.2d){
return closest;
}
for (City c : cityVillageManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) {
double rel = MapUtils.getDistance(c.getLocation(), point) / c.getType().getRadius();
if (rel < relDist) {
closest = c;
@ -912,12 +927,13 @@ public class IndexCreator {
// check that building is not registered already
boolean exist = false;
if(loadInMemory){
exist = addressBuildingLocalSet.contains(e.getId());
} else {
addressSearchBuildingStat.setLong(1, e.getId());
ResultSet rs = addressSearchBuildingStat.executeQuery();
exist = rs.next();
rs.close();
} else {
exist = addressBuildingLocalSet.contains(e.getId());
}
if (!exist) {
loadEntityData(e, false);
@ -937,12 +953,13 @@ public class IndexCreator {
&& e.getTag(OSMTagKey.HIGHWAY) != null && e.getTag(OSMTagKey.NAME) != null) {
boolean exist = false;
if(loadInMemory){
exist = addressStreetNodeLocalSet.contains(e.getId());
} else {
addressSearchStreetNodeStat.setLong(1, e.getId());
ResultSet rs = addressSearchStreetNodeStat.executeQuery();
exist = rs.next();
rs.close();
} else {
exist = addressStreetNodeLocalSet.contains(e.getId());
}
// check that building is not registered already
if (!exist) {
@ -978,7 +995,11 @@ public class IndexCreator {
City city = new City((Node) e);
if(city.getType() != null && !Algoritms.isEmpty(city.getName())){
convertEnglishName(city);
cityManager.registerObject(((Node) e).getLatitude(), ((Node) e).getLongitude(), city);
if(city.getType() == CityType.CITY || city.getType() == CityType.TOWN){
cityManager.registerObject(((Node) e).getLatitude(), ((Node) e).getLongitude(), city);
} else {
cityVillageManager.registerObject(((Node) e).getLatitude(), ((Node) e).getLongitude(), city);
}
cities.put(city.getEntityId(), city);
}
}
@ -1291,8 +1312,8 @@ public class IndexCreator {
public static void main(String[] args) throws IOException, SAXException, SQLException {
File workDir = new File("e:/Information/OSM maps/osmand/");
IndexCreator extr = new IndexCreator(workDir);
extr.setIndexPOI(true);
extr.setIndexTransport(true);
// extr.setIndexPOI(true);
// extr.setIndexTransport(true);
extr.setIndexAddress(true);
extr.setNormalizeStreets(true);
extr.setSaveAddressWays(true);