Android-builders-club/OsmAnd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

redesign swing UI to support new index creator mechanism

Browse source 
git-svn-id: https://osmand.googlecode.com/svn/trunk@509 e29c36b1-1cfa-d876-8d93-3434fc2bb7b8
This commit is contained in:
Victor Shcherb 2010-09-19 10:42:16 +00:00
parent 0afdfa96dc
commit ce36617681
8 changed files with 358 additions and 1929 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
DataExtractionOSM/src/net/osmand/data/CityComparator.java Normal file
View file

@ -0,0 +1,23 @@
package net.osmand.data;
import java.text.Collator;
import java.util.Comparator;
public class CityComparator implements Comparator<City>{
private final boolean en;
Collator collator = Collator.getInstance();
public CityComparator(boolean en){
this.en = en;
}
@Override
public int compare(City o1, City o2) {
if(en){
return collator.compare(o1.getEnName(), o2.getEnName());
} else {
return collator.compare(o1.getName(), o2.getName());
}
}
}

188
DataExtractionOSM/src/net/osmand/data/Region.java
View file

@ -1,188 +0,0 @@
package net.osmand.data;
import java.text.Collator;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import net.osmand.Algoritms;
import net.osmand.data.City.CityType;
import net.osmand.osm.LatLon;
import net.osmand.osm.MapUtils;
import net.osmand.osm.Node;
import net.osmand.osm.io.OsmBaseStorage;
public class Region extends MapObject {
private DataTileManager<Amenity> amenities = new DataTileManager<Amenity>();
private OsmBaseStorage storage;
private DataTileManager<City> cityManager = new DataTileManager<City>();
private Map<String, List<TransportRoute>> routes = new LinkedHashMap<String, List<TransportRoute>>();
private Map<CityType, List<City>> cities = new HashMap<CityType, List<City>>();
{
cityManager.setZoom(10);
for(CityType type : CityType.values()){
cities.put(type, new ArrayList<City>());
}
}
public static class CityComparator implements Comparator<City>{
private final boolean en;
public CityComparator(boolean en){
this.en = en;
}
Collator collator = Collator.getInstance();
@Override
public int compare(City o1, City o2) {
if(en){
return collator.compare(o1.getEnName(), o2.getEnName());
} else {
return collator.compare(o1.getName(), o2.getName());
}
}
}
public Region(){
name = "Region"; //$NON-NLS-1$
}
public OsmBaseStorage getStorage() {
return storage;
}
public void setStorage(OsmBaseStorage storage) {
this.storage = storage;
}
public Collection<City> getCitiesByType(CityType type){
return cities.get(type);
}
public int getCitiesCount(CityType type) {
if (type == null) {
int am = 0;
for (CityType t : cities.keySet()) {
am += cities.get(t).size();
}
return am;
} else if (!cities.containsKey(type)) {
return 0;
} else {
return cities.get(type).size();
}
}
public Collection<City> getCitiesByName(String name){
return getCityByName(name, true, Integer.MAX_VALUE);
}
public Collection<City> getSuggestedCities(String name, int number){
return getCityByName(name, false, number);
}
protected Collection<City> getCityByName(String name, boolean exactMatch, int number){
List<City> l = new ArrayList<City>();
for(CityType type : CityType.values()){
for(City c : cities.get(type)){
if( (exactMatch && c.getName().equalsIgnoreCase(name)) ||
(!exactMatch && c.getName().toLowerCase().startsWith(name.toLowerCase())
)){
l.add(c);
if(l.size() >= number){
break;
}
}
}
}
return l;
}
public City getClosestCity(LatLon point) {
City closest = null;
double relDist = Double.POSITIVE_INFINITY;
for (City c : cityManager.getClosestObjects(point.getLatitude(), point.getLongitude())) {
double rel = MapUtils.getDistance(c.getLocation(), point) / c.getType().getRadius();
if (rel < relDist) {
closest = c;
relDist = rel;
if(relDist < 0.2d){
break;
}
}
}
return closest;
}
public DataTileManager<Amenity> getAmenityManager(){
return amenities;
}
public void registerAmenity(Amenity a){
LatLon location = a.getLocation();
if(location != null){
amenities.registerObject(location.getLatitude(), location.getLongitude(), a);
}
}
public void registerCity(City city){
if(city.getType() != null && !Algoritms.isEmpty(city.getName()) && city.getLocation() != null){
LatLon l = city.getLocation();
cityManager.registerObject(l.getLatitude(), l.getLongitude(), city);
cities.get(city.getType()).add(city);
}
}
public void unregisterCity(City city){
if(city != null && city.getType() != null){
LatLon l = city.getLocation();
cityManager.unregisterObject(l.getLatitude(), l.getLongitude(), city);
cities.get(city.getType()).remove(city);
}
}
public City registerCity(Node c){
City city = new City(c);
if(city.getType() != null && !Algoritms.isEmpty(city.getName())){
cityManager.registerObject(c.getLatitude(), c.getLongitude(), city);
cities.get(city.getType()).add(city);
return city;
}
return null;
}
public Map<String, List<TransportRoute>> getTransportRoutes() {
return routes;
}
public void doDataPreparation(){
CityComparator comp = new CityComparator(false);
for(CityType t : cities.keySet()){
Collections.sort(cities.get(t), comp);
for(City c : cities.get(t)){
c.doDataPreparation();
}
}
for(String s : routes.keySet()){
List<TransportRoute> trans = routes.get(s);
Collections.sort(trans, new Comparator<TransportRoute>(){
@Override
public int compare(TransportRoute o1, TransportRoute o2) {
int i1 = Algoritms.extractFirstIntegerNumber(o1.getRef());
int i2 = Algoritms.extractFirstIntegerNumber(o2.getRef());
return i1 - i2;
}
});
}
}
}

216
DataExtractionOSM/src/net/osmand/data/index/DataIndexWriter.java
View file

@ -1,30 +1,21 @@
package net.osmand.data.index; package net.osmand.data.index;
import java.io.File;
import java.io.IOException;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.SQLException; import java.sql.SQLException;
import java.sql.Statement; import java.sql.Statement;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import net.osmand.Algoritms; import net.osmand.Algoritms;
import net.osmand.LogUtil;
import net.osmand.data.Amenity; import net.osmand.data.Amenity;
import net.osmand.data.AmenityType; import net.osmand.data.AmenityType;
import net.osmand.data.Building; import net.osmand.data.Building;
import net.osmand.data.City; import net.osmand.data.City;
import net.osmand.data.Region;
import net.osmand.data.Street;
import net.osmand.data.TransportRoute; import net.osmand.data.TransportRoute;
import net.osmand.data.TransportStop; import net.osmand.data.TransportStop;
import net.osmand.data.City.CityType; import net.osmand.data.City.CityType;
@ -44,75 +35,13 @@ import net.osmand.osm.Node;
import net.osmand.osm.Relation; import net.osmand.osm.Relation;
import net.osmand.osm.Way; import net.osmand.osm.Way;
import org.apache.commons.logging.Log;
public class DataIndexWriter { public class DataIndexWriter {
private final File workingDir;
private final Region region;
private static final Log log = LogUtil.getLog(DataIndexWriter.class);
private static final int BATCH_SIZE = 1000; private static final int BATCH_SIZE = 1000;
public DataIndexWriter(File workingDir, Region region){
this.workingDir = workingDir;
this.region = region;
}
protected File checkFile(String name) throws IOException {
String fileName = name;
File f = new File(workingDir, fileName);
f.getParentFile().mkdirs();
// remove existing file
if (f.exists()) {
log.warn("Remove existing index : " + f.getAbsolutePath()); //$NON-NLS-1$
f.delete();
}
return f;
}
public DataIndexWriter writePOI() throws IOException, SQLException {
return writePOI(IndexConstants.POI_INDEX_DIR+region.getName()+IndexConstants.POI_INDEX_EXT, null);
}
public DataIndexWriter writePOI(String fileName, Long date) throws IOException, SQLException {
File file = checkFile(fileName);
long now = System.currentTimeMillis();
try {
Class.forName("org.sqlite.JDBC"); //$NON-NLS-1$
} catch (ClassNotFoundException e) {
log.error("Illegal configuration", e); //$NON-NLS-1$
throw new IllegalStateException(e);
}
Connection conn = DriverManager.getConnection("jdbc:sqlite:"+file.getAbsolutePath()); //$NON-NLS-1$
try {
createPoiIndexStructure(conn);
PreparedStatement prep = createStatementAmenityInsert(conn);
Map<PreparedStatement, Integer> map = new LinkedHashMap<PreparedStatement, Integer>();
map.put(prep, 0);
conn.setAutoCommit(false);
for (Amenity a : region.getAmenityManager().getAllObjects()) {
insertAmenityIntoPoi(prep, map, a, BATCH_SIZE);
}
if(map.get(prep) > 0){
prep.executeBatch();
}
prep.close();
conn.setAutoCommit(true);
} finally {
conn.close();
log.info(String.format("Indexing poi done in %s ms.", System.currentTimeMillis() - now)); //$NON-NLS-1$
}
if(date != null){
file.setLastModified(date);
}
return this;
}
public static void insertAmenityIntoPoi(PreparedStatement prep, Map<PreparedStatement, Integer> map, Amenity amenity, int batchSize) throws SQLException { public static void insertAmenityIntoPoi(PreparedStatement prep, Map<PreparedStatement, Integer> map, Amenity amenity, int batchSize) throws SQLException {
prep.setLong(IndexPoiTable.ID.ordinal() + 1, amenity.getId()); prep.setLong(IndexPoiTable.ID.ordinal() + 1, amenity.getId());
@ -140,91 +69,6 @@ public class DataIndexWriter {
stat.close(); stat.close();
} }
public DataIndexWriter writeAddress() throws IOException, SQLException{
return writeAddress(IndexConstants.ADDRESS_INDEX_DIR+region.getName()+IndexConstants.ADDRESS_INDEX_EXT, null, true);
}
public DataIndexWriter writeAddress(String fileName, Long date, boolean writeWayNodes) throws IOException, SQLException{
File file = checkFile(fileName);
long now = System.currentTimeMillis();
try {
Class.forName("org.sqlite.JDBC"); //$NON-NLS-1$
} catch (ClassNotFoundException e) {
log.error("Illegal configuration", e); //$NON-NLS-1$
throw new IllegalStateException(e);
}
Connection conn = DriverManager.getConnection("jdbc:sqlite:"+file.getAbsolutePath()); //$NON-NLS-1$
try {
createAddressIndexStructure(conn);
PreparedStatement prepCity = conn.prepareStatement(
IndexConstants.generatePrepareStatementToInsert(IndexCityTable.getTable(), IndexCityTable.values().length));
PreparedStatement prepStreet = conn.prepareStatement(
IndexConstants.generatePrepareStatementToInsert(IndexStreetTable.getTable(), IndexStreetTable.values().length));
PreparedStatement prepBuilding = conn.prepareStatement(
IndexConstants.generatePrepareStatementToInsert(IndexBuildingTable.getTable(), IndexBuildingTable.values().length));
PreparedStatement prepStreetNode = conn.prepareStatement(
IndexConstants.generatePrepareStatementToInsert(IndexStreetNodeTable.getTable(), IndexStreetNodeTable.values().length));
Map<PreparedStatement, Integer> count = new HashMap<PreparedStatement, Integer>();
count.put(prepStreet, 0);
count.put(prepCity, 0);
count.put(prepStreetNode, 0);
count.put(prepBuilding, 0);
conn.setAutoCommit(false);
for(CityType t : CityType.values()){
for(City city : region.getCitiesByType(t)) {
if(city.getId() == null || city.getLocation() == null){
continue;
}
writeCity(prepCity, count, city, BATCH_SIZE);
for(Street street : city.getStreets()){
if(street.getId() == null || street.getLocation() == null){
continue;
}
assert IndexStreetTable.values().length == 6;
prepStreet.setLong(IndexStreetTable.ID.ordinal() + 1, street.getId());
prepStreet.setString(IndexStreetTable.NAME_EN.ordinal() + 1, street.getEnName());
prepStreet.setDouble(IndexStreetTable.LATITUDE.ordinal() + 1, street.getLocation().getLatitude());
prepStreet.setDouble(IndexStreetTable.LONGITUDE.ordinal() + 1, street.getLocation().getLongitude());
prepStreet.setString(IndexStreetTable.NAME.ordinal() + 1, street.getName());
prepStreet.setLong(IndexStreetTable.CITY.ordinal() + 1, city.getId());
addBatch(count, prepStreet);
if (writeWayNodes) {
for (Way way : street.getWayNodes()) {
writeStreetWayNodes(prepStreetNode, count, street.getId(), way, BATCH_SIZE);
}
}
for(Building building : street.getBuildings()){
if(building.getId() == null || building.getLocation() == null){
continue;
}
writeBuilding(prepBuilding, count, street.getId(), building, BATCH_SIZE);
}
}
}
}
for(PreparedStatement p : count.keySet()){
if(count.get(p) > 0){
p.executeBatch();
}
p.close();
}
conn.setAutoCommit(true);
} finally {
conn.close();
log.info(String.format("Indexing address done in %s ms.", System.currentTimeMillis() - now)); //$NON-NLS-1$
}
if(date != null){
file.setLastModified(date);
}
return this;
}
public static void writeStreetWayNodes(PreparedStatement prepStreetNode, Map<PreparedStatement, Integer> count, Long streetId, Way way, int batchSize) public static void writeStreetWayNodes(PreparedStatement prepStreetNode, Map<PreparedStatement, Integer> count, Long streetId, Way way, int batchSize)
throws SQLException { throws SQLException {
for (Node n : way.getNodes()) { for (Node n : way.getNodes()) {
@ -283,60 +127,6 @@ public class DataIndexWriter {
} }
public DataIndexWriter writeTransport() throws IOException, SQLException{
return writeTransport(IndexConstants.TRANSPORT_INDEX_DIR+region.getName()+IndexConstants.TRANSPORT_INDEX_EXT, null);
}
public DataIndexWriter writeTransport(String fileName, Long date) throws IOException, SQLException{
File file = checkFile(fileName);
long now = System.currentTimeMillis();
try {
Class.forName("org.sqlite.JDBC"); //$NON-NLS-1$
} catch (ClassNotFoundException e) {
log.error("Illegal configuration", e); //$NON-NLS-1$
throw new IllegalStateException(e);
}
Connection conn = DriverManager.getConnection("jdbc:sqlite:"+file.getAbsolutePath()); //$NON-NLS-1$
try {
createTransportIndexStructure(conn);
PreparedStatement prepRoute = conn.prepareStatement(
IndexConstants.generatePrepareStatementToInsert(IndexTransportRoute.getTable(), IndexTransportRoute.values().length));
PreparedStatement prepRouteStops = conn.prepareStatement(
IndexConstants.generatePrepareStatementToInsert(IndexTransportRouteStop.getTable(), IndexTransportRouteStop.values().length));
PreparedStatement prepStops = conn.prepareStatement(
IndexConstants.generatePrepareStatementToInsert(IndexTransportStop.getTable(), IndexTransportStop.values().length));
Map<PreparedStatement, Integer> count = new HashMap<PreparedStatement, Integer>();
count.put(prepRouteStops, 0);
count.put(prepRoute, 0);
count.put(prepStops, 0);
conn.setAutoCommit(false);
Set<Long> writtenStops = new LinkedHashSet<Long>();
for(String t : region.getTransportRoutes().keySet()){
for(TransportRoute r : region.getTransportRoutes().get(t)) {
insertTransportIntoIndex(prepRoute, prepRouteStops, prepStops, writtenStops, r, count, BATCH_SIZE);
}
}
for(PreparedStatement p : count.keySet()){
if(count.get(p) > 0){
p.executeBatch();
}
p.close();
}
conn.setAutoCommit(true);
} finally {
conn.close();
log.info(String.format("Indexing transport done in %s ms.", System.currentTimeMillis() - now)); //$NON-NLS-1$
}
if(date != null){
file.setLastModified(date);
}
return this;
}
private static void writeRouteStops(PreparedStatement prepRouteStops, PreparedStatement prepStops, Map<PreparedStatement, Integer> count, private static void writeRouteStops(PreparedStatement prepRouteStops, PreparedStatement prepStops, Map<PreparedStatement, Integer> count,
Set<Long> writtenStops, TransportRoute r, List<TransportStop> stops, boolean direction) throws SQLException { Set<Long> writtenStops, TransportRoute r, List<TransportStop> stops, boolean direction) throws SQLException {
int i = 0; int i = 0;
@ -395,9 +185,9 @@ public class DataIndexWriter {
Statement stat = conn.createStatement(); Statement stat = conn.createStatement();
stat.execute(IndexConstants.generateCreateSQL(IndexMapRenderObject.values())); stat.execute(IndexConstants.generateCreateSQL(IndexMapRenderObject.values()));
stat.execute(IndexConstants.generateCreateIndexSQL(IndexMapRenderObject.values())); stat.execute(IndexConstants.generateCreateIndexSQL(IndexMapRenderObject.values()));
stat.execute("CREATE VIRTUAL TABLE "+IndexConstants.indexMapLocationsTable+" USING rtree (id, minLon, maxLon, minLat, maxLat);"); stat.execute("CREATE VIRTUAL TABLE "+IndexConstants.indexMapLocationsTable+" USING rtree (id, minLon, maxLon, minLat, maxLat);"); //$NON-NLS-1$ //$NON-NLS-2$
stat.execute("CREATE VIRTUAL TABLE "+IndexConstants.indexMapLocationsTable2+" USING rtree (id, minLon, maxLon, minLat, maxLat);"); stat.execute("CREATE VIRTUAL TABLE "+IndexConstants.indexMapLocationsTable2+" USING rtree (id, minLon, maxLon, minLat, maxLat);"); //$NON-NLS-1$ //$NON-NLS-2$
stat.execute("CREATE VIRTUAL TABLE "+IndexConstants.indexMapLocationsTable3+" USING rtree (id, minLon, maxLon, minLat, maxLat);"); stat.execute("CREATE VIRTUAL TABLE "+IndexConstants.indexMapLocationsTable3+" USING rtree (id, minLon, maxLon, minLat, maxLat);"); //$NON-NLS-1$ //$NON-NLS-2$
stat.execute("PRAGMA user_version = " + IndexConstants.MAP_TABLE_VERSION); //$NON-NLS-1$ stat.execute("PRAGMA user_version = " + IndexConstants.MAP_TABLE_VERSION); //$NON-NLS-1$
stat.close(); stat.close();
} }

8
DataExtractionOSM/src/net/osmand/data/index/IndexBatchCreator.java
View file

@ -18,8 +18,6 @@ import java.util.zip.ZipOutputStream;
import net.osmand.Algoritms; import net.osmand.Algoritms;
import net.osmand.LogUtil; import net.osmand.LogUtil;
import net.osmand.data.Region;
import net.osmand.data.preparation.DataExtraction;
import net.osmand.data.preparation.IndexCreator; import net.osmand.data.preparation.IndexCreator;
import net.osmand.impl.ConsoleProgressImplementation; import net.osmand.impl.ConsoleProgressImplementation;
@ -375,7 +373,9 @@ public class IndexBatchCreator {
} }
System.out.println("GENERATING INDEXES FINISHED "); System.out.println("GENERATING INDEXES FINISHED ");
} }
protected void generateIndexOld(File f, Set<String> alreadyGeneratedFiles, Set<String> alreadyUploadedFiles) {
/* protected void generateIndexOld(File f, Set<String> alreadyGeneratedFiles, Set<String> alreadyUploadedFiles) {
if (!generateIndexes) { if (!generateIndexes) {
return; return;
} }
@ -410,7 +410,7 @@ public class IndexBatchCreator {
} }
System.gc(); System.gc();
} }*/
protected void generateIndex(File f, Set<String> alreadyGeneratedFiles, Set<String> alreadyUploadedFiles) { protected void generateIndex(File f, Set<String> alreadyGeneratedFiles, Set<String> alreadyUploadedFiles) {
if (!generateIndexes) { if (!generateIndexes) {

1065
DataExtractionOSM/src/net/osmand/data/preparation/DataExtraction.java
View file

File diff suppressed because it is too large Load diff

8
DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java
View file

@ -64,12 +64,16 @@ import org.xml.sax.SAXException;
/** /**
* http://wiki.openstreetmap.org/wiki/OSM_tags_for_routing#Is_inside.2Foutside
* http://wiki.openstreetmap.org/wiki/Relations/Proposed/Postal_Addresses
* http://wiki.openstreetmap.org/wiki/Proposed_features/House_numbers/Karlsruhe_Schema#Tags (node, way)
* That data extraction has aim, * That data extraction has aim,
* save runtime memory and generate indexes on the fly. * save runtime memory and generate indexes on the fly.
* It will be longer than load in memory (needed part) and save into index. * It will be longer than load in memory (needed part) and save into index.
*/ */
public class IndexCreator { public class IndexCreator {
private static final Log log = LogFactory.getLog(DataExtraction.class); private static final Log log = LogFactory.getLog(IndexCreator.class);
public static final int BATCH_SIZE = 5000; public static final int BATCH_SIZE = 5000;
@ -937,7 +941,7 @@ public class IndexCreator {
} }
} }
if(indexMap && (e instanceof Way) || (e instanceof Node)){ if(indexMap && (e instanceof Way || e instanceof Node)){
// manipulate what kind of way to load // manipulate what kind of way to load
loadEntityData(e, true); loadEntityData(e, true);
boolean inverse = "-1".equals(e.getTag(OSMTagKey.ONEWAY)); boolean inverse = "-1".equals(e.getTag(OSMTagKey.ONEWAY));

1293
DataExtractionOSM/src/net/osmand/swing/OsmExtractionUI.java
View file

File diff suppressed because it is too large Load diff

4
OsmAnd/src/net/osmand/RegionAddressRepository.java
View file

@ -17,9 +17,9 @@ import java.util.TreeSet;
import net.osmand.data.Building; import net.osmand.data.Building;
import net.osmand.data.City; import net.osmand.data.City;
import net.osmand.data.CityComparator;
import net.osmand.data.MapObject; import net.osmand.data.MapObject;
import net.osmand.data.PostCode; import net.osmand.data.PostCode;
import net.osmand.data.Region;
import net.osmand.data.Street; import net.osmand.data.Street;
import net.osmand.data.City.CityType; import net.osmand.data.City.CityType;
import net.osmand.data.index.IndexConstants; import net.osmand.data.index.IndexConstants;
@ -183,7 +183,7 @@ public class RegionAddressRepository {
} }
// sort cities // sort cities
ArrayList<City> list = new ArrayList<City>(cities.values()); ArrayList<City> list = new ArrayList<City>(cities.values());
Collections.sort(list, new Region.CityComparator(useEnglishNames)); Collections.sort(list, new CityComparator(useEnglishNames));
cities.clear(); cities.clear();
cityTypes.clear(); cityTypes.clear();
for(City c : list){ for(City c : list){