From ce3661768119c83461f0a394071831607d9b88c2 Mon Sep 17 00:00:00 2001 From: Victor Shcherb Date: Sun, 19 Sep 2010 10:42:16 +0000 Subject: [PATCH] redesign swing UI to support new index creator mechanism git-svn-id: https://osmand.googlecode.com/svn/trunk@509 e29c36b1-1cfa-d876-8d93-3434fc2bb7b8 --- .../src/net/osmand/data/CityComparator.java | 23 + .../src/net/osmand/data/Region.java | 188 --- .../osmand/data/index/DataIndexWriter.java | 216 +--- .../osmand/data/index/IndexBatchCreator.java | 8 +- .../data/preparation/DataExtraction.java | 1065 ----------------- .../osmand/data/preparation/IndexCreator.java | 8 +- .../src/net/osmand/swing/OsmExtractionUI.java | 775 +++++------- .../net/osmand/RegionAddressRepository.java | 4 +- 8 files changed, 358 insertions(+), 1929 deletions(-) create mode 100644 DataExtractionOSM/src/net/osmand/data/CityComparator.java delete mode 100644 DataExtractionOSM/src/net/osmand/data/Region.java delete mode 100644 DataExtractionOSM/src/net/osmand/data/preparation/DataExtraction.java diff --git a/DataExtractionOSM/src/net/osmand/data/CityComparator.java b/DataExtractionOSM/src/net/osmand/data/CityComparator.java new file mode 100644 index 0000000000..31016a1901 --- /dev/null +++ b/DataExtractionOSM/src/net/osmand/data/CityComparator.java @@ -0,0 +1,23 @@ +package net.osmand.data; + +import java.text.Collator; +import java.util.Comparator; + +public class CityComparator implements Comparator{ + private final boolean en; + Collator collator = Collator.getInstance(); + public CityComparator(boolean en){ + this.en = en; + } + + + + @Override + public int compare(City o1, City o2) { + if(en){ + return collator.compare(o1.getEnName(), o2.getEnName()); + } else { + return collator.compare(o1.getName(), o2.getName()); + } + } +} \ No newline at end of file diff --git a/DataExtractionOSM/src/net/osmand/data/Region.java b/DataExtractionOSM/src/net/osmand/data/Region.java deleted file mode 100644 index 9fa0e186f8..0000000000 --- a/DataExtractionOSM/src/net/osmand/data/Region.java +++ /dev/null @@ -1,188 +0,0 @@ -package net.osmand.data; - -import java.text.Collator; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -import net.osmand.Algoritms; -import net.osmand.data.City.CityType; -import net.osmand.osm.LatLon; -import net.osmand.osm.MapUtils; -import net.osmand.osm.Node; -import net.osmand.osm.io.OsmBaseStorage; - - -public class Region extends MapObject { - private DataTileManager amenities = new DataTileManager(); - private OsmBaseStorage storage; - private DataTileManager cityManager = new DataTileManager(); - private Map> routes = new LinkedHashMap>(); - private Map> cities = new HashMap>(); - { - cityManager.setZoom(10); - for(CityType type : CityType.values()){ - cities.put(type, new ArrayList()); - } - } - - public static class CityComparator implements Comparator{ - private final boolean en; - public CityComparator(boolean en){ - this.en = en; - } - Collator collator = Collator.getInstance(); - @Override - public int compare(City o1, City o2) { - if(en){ - return collator.compare(o1.getEnName(), o2.getEnName()); - } else { - return collator.compare(o1.getName(), o2.getName()); - } - } - } - - public Region(){ - name = "Region"; //$NON-NLS-1$ - } - - public OsmBaseStorage getStorage() { - return storage; - } - - public void setStorage(OsmBaseStorage storage) { - this.storage = storage; - } - - - public Collection getCitiesByType(CityType type){ - return cities.get(type); - } - - public int getCitiesCount(CityType type) { - if (type == null) { - int am = 0; - for (CityType t : cities.keySet()) { - am += cities.get(t).size(); - } - return am; - } else if (!cities.containsKey(type)) { - return 0; - } else { - return cities.get(type).size(); - } - - } - - public Collection getCitiesByName(String name){ - return getCityByName(name, true, Integer.MAX_VALUE); - } - - public Collection getSuggestedCities(String name, int number){ - return getCityByName(name, false, number); - } - - protected Collection getCityByName(String name, boolean exactMatch, int number){ - List l = new ArrayList(); - for(CityType type : CityType.values()){ - for(City c : cities.get(type)){ - if( (exactMatch && c.getName().equalsIgnoreCase(name)) || - (!exactMatch && c.getName().toLowerCase().startsWith(name.toLowerCase()) - )){ - l.add(c); - if(l.size() >= number){ - break; - } - } - } - } - return l; - } - - public City getClosestCity(LatLon point) { - City closest = null; - double relDist = Double.POSITIVE_INFINITY; - for (City c : cityManager.getClosestObjects(point.getLatitude(), point.getLongitude())) { - double rel = MapUtils.getDistance(c.getLocation(), point) / c.getType().getRadius(); - if (rel < relDist) { - closest = c; - relDist = rel; - if(relDist < 0.2d){ - break; - } - } - } - return closest; - } - - - public DataTileManager getAmenityManager(){ - return amenities; - } - - public void registerAmenity(Amenity a){ - LatLon location = a.getLocation(); - if(location != null){ - amenities.registerObject(location.getLatitude(), location.getLongitude(), a); - } - } - - public void registerCity(City city){ - if(city.getType() != null && !Algoritms.isEmpty(city.getName()) && city.getLocation() != null){ - LatLon l = city.getLocation(); - cityManager.registerObject(l.getLatitude(), l.getLongitude(), city); - cities.get(city.getType()).add(city); - } - } - - public void unregisterCity(City city){ - if(city != null && city.getType() != null){ - LatLon l = city.getLocation(); - cityManager.unregisterObject(l.getLatitude(), l.getLongitude(), city); - cities.get(city.getType()).remove(city); - } - } - - public City registerCity(Node c){ - City city = new City(c); - if(city.getType() != null && !Algoritms.isEmpty(city.getName())){ - cityManager.registerObject(c.getLatitude(), c.getLongitude(), city); - cities.get(city.getType()).add(city); - return city; - } - return null; - } - - public Map> getTransportRoutes() { - return routes; - } - - - public void doDataPreparation(){ - CityComparator comp = new CityComparator(false); - for(CityType t : cities.keySet()){ - Collections.sort(cities.get(t), comp); - for(City c : cities.get(t)){ - c.doDataPreparation(); - } - } - for(String s : routes.keySet()){ - List trans = routes.get(s); - Collections.sort(trans, new Comparator(){ - @Override - public int compare(TransportRoute o1, TransportRoute o2) { - int i1 = Algoritms.extractFirstIntegerNumber(o1.getRef()); - int i2 = Algoritms.extractFirstIntegerNumber(o2.getRef()); - return i1 - i2; - } - }); - } - - - } -} diff --git a/DataExtractionOSM/src/net/osmand/data/index/DataIndexWriter.java b/DataExtractionOSM/src/net/osmand/data/index/DataIndexWriter.java index b59352c304..431e658af2 100644 --- a/DataExtractionOSM/src/net/osmand/data/index/DataIndexWriter.java +++ b/DataExtractionOSM/src/net/osmand/data/index/DataIndexWriter.java @@ -1,30 +1,21 @@ package net.osmand.data.index; -import java.io.File; -import java.io.IOException; import java.sql.Connection; -import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; import net.osmand.Algoritms; -import net.osmand.LogUtil; import net.osmand.data.Amenity; import net.osmand.data.AmenityType; import net.osmand.data.Building; import net.osmand.data.City; -import net.osmand.data.Region; -import net.osmand.data.Street; import net.osmand.data.TransportRoute; import net.osmand.data.TransportStop; import net.osmand.data.City.CityType; @@ -44,75 +35,13 @@ import net.osmand.osm.Node; import net.osmand.osm.Relation; import net.osmand.osm.Way; -import org.apache.commons.logging.Log; - public class DataIndexWriter { - private final File workingDir; - private final Region region; - private static final Log log = LogUtil.getLog(DataIndexWriter.class); - - private static final int BATCH_SIZE = 1000; - public DataIndexWriter(File workingDir, Region region){ - this.workingDir = workingDir; - this.region = region; - } - - protected File checkFile(String name) throws IOException { - String fileName = name; - File f = new File(workingDir, fileName); - f.getParentFile().mkdirs(); - // remove existing file - if (f.exists()) { - log.warn("Remove existing index : " + f.getAbsolutePath()); //$NON-NLS-1$ - f.delete(); - } - return f; - } - - public DataIndexWriter writePOI() throws IOException, SQLException { - return writePOI(IndexConstants.POI_INDEX_DIR+region.getName()+IndexConstants.POI_INDEX_EXT, null); - } - - - public DataIndexWriter writePOI(String fileName, Long date) throws IOException, SQLException { - File file = checkFile(fileName); - long now = System.currentTimeMillis(); - try { - Class.forName("org.sqlite.JDBC"); //$NON-NLS-1$ - } catch (ClassNotFoundException e) { - log.error("Illegal configuration", e); //$NON-NLS-1$ - throw new IllegalStateException(e); - } - Connection conn = DriverManager.getConnection("jdbc:sqlite:"+file.getAbsolutePath()); //$NON-NLS-1$ - try { - createPoiIndexStructure(conn); - PreparedStatement prep = createStatementAmenityInsert(conn); - Map map = new LinkedHashMap(); - map.put(prep, 0); - conn.setAutoCommit(false); - for (Amenity a : region.getAmenityManager().getAllObjects()) { - insertAmenityIntoPoi(prep, map, a, BATCH_SIZE); - } - if(map.get(prep) > 0){ - prep.executeBatch(); - } - prep.close(); - conn.setAutoCommit(true); - } finally { - conn.close(); - log.info(String.format("Indexing poi done in %s ms.", System.currentTimeMillis() - now)); //$NON-NLS-1$ - } - if(date != null){ - file.setLastModified(date); - } - return this; - } public static void insertAmenityIntoPoi(PreparedStatement prep, Map map, Amenity amenity, int batchSize) throws SQLException { prep.setLong(IndexPoiTable.ID.ordinal() + 1, amenity.getId()); @@ -140,91 +69,6 @@ public class DataIndexWriter { stat.close(); } - public DataIndexWriter writeAddress() throws IOException, SQLException{ - return writeAddress(IndexConstants.ADDRESS_INDEX_DIR+region.getName()+IndexConstants.ADDRESS_INDEX_EXT, null, true); - } - - public DataIndexWriter writeAddress(String fileName, Long date, boolean writeWayNodes) throws IOException, SQLException{ - File file = checkFile(fileName); - long now = System.currentTimeMillis(); - try { - Class.forName("org.sqlite.JDBC"); //$NON-NLS-1$ - } catch (ClassNotFoundException e) { - log.error("Illegal configuration", e); //$NON-NLS-1$ - throw new IllegalStateException(e); - } - Connection conn = DriverManager.getConnection("jdbc:sqlite:"+file.getAbsolutePath()); //$NON-NLS-1$ - try { - createAddressIndexStructure(conn); - - PreparedStatement prepCity = conn.prepareStatement( - IndexConstants.generatePrepareStatementToInsert(IndexCityTable.getTable(), IndexCityTable.values().length)); - PreparedStatement prepStreet = conn.prepareStatement( - IndexConstants.generatePrepareStatementToInsert(IndexStreetTable.getTable(), IndexStreetTable.values().length)); - PreparedStatement prepBuilding = conn.prepareStatement( - IndexConstants.generatePrepareStatementToInsert(IndexBuildingTable.getTable(), IndexBuildingTable.values().length)); - PreparedStatement prepStreetNode = conn.prepareStatement( - IndexConstants.generatePrepareStatementToInsert(IndexStreetNodeTable.getTable(), IndexStreetNodeTable.values().length)); - Map count = new HashMap(); - count.put(prepStreet, 0); - count.put(prepCity, 0); - count.put(prepStreetNode, 0); - count.put(prepBuilding, 0); - conn.setAutoCommit(false); - - for(CityType t : CityType.values()){ - for(City city : region.getCitiesByType(t)) { - if(city.getId() == null || city.getLocation() == null){ - continue; - } - writeCity(prepCity, count, city, BATCH_SIZE); - - for(Street street : city.getStreets()){ - if(street.getId() == null || street.getLocation() == null){ - continue; - } - assert IndexStreetTable.values().length == 6; - prepStreet.setLong(IndexStreetTable.ID.ordinal() + 1, street.getId()); - prepStreet.setString(IndexStreetTable.NAME_EN.ordinal() + 1, street.getEnName()); - prepStreet.setDouble(IndexStreetTable.LATITUDE.ordinal() + 1, street.getLocation().getLatitude()); - prepStreet.setDouble(IndexStreetTable.LONGITUDE.ordinal() + 1, street.getLocation().getLongitude()); - prepStreet.setString(IndexStreetTable.NAME.ordinal() + 1, street.getName()); - prepStreet.setLong(IndexStreetTable.CITY.ordinal() + 1, city.getId()); - addBatch(count, prepStreet); - if (writeWayNodes) { - for (Way way : street.getWayNodes()) { - writeStreetWayNodes(prepStreetNode, count, street.getId(), way, BATCH_SIZE); - } - } - - for(Building building : street.getBuildings()){ - if(building.getId() == null || building.getLocation() == null){ - continue; - } - writeBuilding(prepBuilding, count, street.getId(), building, BATCH_SIZE); - } - } - - } - } - - for(PreparedStatement p : count.keySet()){ - if(count.get(p) > 0){ - p.executeBatch(); - } - p.close(); - } - conn.setAutoCommit(true); - } finally { - conn.close(); - log.info(String.format("Indexing address done in %s ms.", System.currentTimeMillis() - now)); //$NON-NLS-1$ - } - if(date != null){ - file.setLastModified(date); - } - return this; - } - public static void writeStreetWayNodes(PreparedStatement prepStreetNode, Map count, Long streetId, Way way, int batchSize) throws SQLException { for (Node n : way.getNodes()) { @@ -283,60 +127,6 @@ public class DataIndexWriter { } - public DataIndexWriter writeTransport() throws IOException, SQLException{ - return writeTransport(IndexConstants.TRANSPORT_INDEX_DIR+region.getName()+IndexConstants.TRANSPORT_INDEX_EXT, null); - } - - public DataIndexWriter writeTransport(String fileName, Long date) throws IOException, SQLException{ - File file = checkFile(fileName); - long now = System.currentTimeMillis(); - try { - Class.forName("org.sqlite.JDBC"); //$NON-NLS-1$ - } catch (ClassNotFoundException e) { - log.error("Illegal configuration", e); //$NON-NLS-1$ - throw new IllegalStateException(e); - } - Connection conn = DriverManager.getConnection("jdbc:sqlite:"+file.getAbsolutePath()); //$NON-NLS-1$ - try { - createTransportIndexStructure(conn); - - PreparedStatement prepRoute = conn.prepareStatement( - IndexConstants.generatePrepareStatementToInsert(IndexTransportRoute.getTable(), IndexTransportRoute.values().length)); - PreparedStatement prepRouteStops = conn.prepareStatement( - IndexConstants.generatePrepareStatementToInsert(IndexTransportRouteStop.getTable(), IndexTransportRouteStop.values().length)); - PreparedStatement prepStops = conn.prepareStatement( - IndexConstants.generatePrepareStatementToInsert(IndexTransportStop.getTable(), IndexTransportStop.values().length)); - Map count = new HashMap(); - count.put(prepRouteStops, 0); - count.put(prepRoute, 0); - count.put(prepStops, 0); - conn.setAutoCommit(false); - - - Set writtenStops = new LinkedHashSet(); - for(String t : region.getTransportRoutes().keySet()){ - for(TransportRoute r : region.getTransportRoutes().get(t)) { - insertTransportIntoIndex(prepRoute, prepRouteStops, prepStops, writtenStops, r, count, BATCH_SIZE); - } - } - - for(PreparedStatement p : count.keySet()){ - if(count.get(p) > 0){ - p.executeBatch(); - } - p.close(); - } - conn.setAutoCommit(true); - } finally { - conn.close(); - log.info(String.format("Indexing transport done in %s ms.", System.currentTimeMillis() - now)); //$NON-NLS-1$ - } - if(date != null){ - file.setLastModified(date); - } - return this; - } - private static void writeRouteStops(PreparedStatement prepRouteStops, PreparedStatement prepStops, Map count, Set writtenStops, TransportRoute r, List stops, boolean direction) throws SQLException { int i = 0; @@ -395,9 +185,9 @@ public class DataIndexWriter { Statement stat = conn.createStatement(); stat.execute(IndexConstants.generateCreateSQL(IndexMapRenderObject.values())); stat.execute(IndexConstants.generateCreateIndexSQL(IndexMapRenderObject.values())); - stat.execute("CREATE VIRTUAL TABLE "+IndexConstants.indexMapLocationsTable+" USING rtree (id, minLon, maxLon, minLat, maxLat);"); - stat.execute("CREATE VIRTUAL TABLE "+IndexConstants.indexMapLocationsTable2+" USING rtree (id, minLon, maxLon, minLat, maxLat);"); - stat.execute("CREATE VIRTUAL TABLE "+IndexConstants.indexMapLocationsTable3+" USING rtree (id, minLon, maxLon, minLat, maxLat);"); + stat.execute("CREATE VIRTUAL TABLE "+IndexConstants.indexMapLocationsTable+" USING rtree (id, minLon, maxLon, minLat, maxLat);"); //$NON-NLS-1$ //$NON-NLS-2$ + stat.execute("CREATE VIRTUAL TABLE "+IndexConstants.indexMapLocationsTable2+" USING rtree (id, minLon, maxLon, minLat, maxLat);"); //$NON-NLS-1$ //$NON-NLS-2$ + stat.execute("CREATE VIRTUAL TABLE "+IndexConstants.indexMapLocationsTable3+" USING rtree (id, minLon, maxLon, minLat, maxLat);"); //$NON-NLS-1$ //$NON-NLS-2$ stat.execute("PRAGMA user_version = " + IndexConstants.MAP_TABLE_VERSION); //$NON-NLS-1$ stat.close(); } diff --git a/DataExtractionOSM/src/net/osmand/data/index/IndexBatchCreator.java b/DataExtractionOSM/src/net/osmand/data/index/IndexBatchCreator.java index 7b26481327..06edd7c1b2 100644 --- a/DataExtractionOSM/src/net/osmand/data/index/IndexBatchCreator.java +++ b/DataExtractionOSM/src/net/osmand/data/index/IndexBatchCreator.java @@ -18,8 +18,6 @@ import java.util.zip.ZipOutputStream; import net.osmand.Algoritms; import net.osmand.LogUtil; -import net.osmand.data.Region; -import net.osmand.data.preparation.DataExtraction; import net.osmand.data.preparation.IndexCreator; import net.osmand.impl.ConsoleProgressImplementation; @@ -375,7 +373,9 @@ public class IndexBatchCreator { } System.out.println("GENERATING INDEXES FINISHED "); } - protected void generateIndexOld(File f, Set alreadyGeneratedFiles, Set alreadyUploadedFiles) { + + +/* protected void generateIndexOld(File f, Set alreadyGeneratedFiles, Set alreadyUploadedFiles) { if (!generateIndexes) { return; } @@ -410,7 +410,7 @@ public class IndexBatchCreator { } System.gc(); - } + }*/ protected void generateIndex(File f, Set alreadyGeneratedFiles, Set alreadyUploadedFiles) { if (!generateIndexes) { diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/DataExtraction.java b/DataExtractionOSM/src/net/osmand/data/preparation/DataExtraction.java deleted file mode 100644 index 15ec9daca9..0000000000 --- a/DataExtractionOSM/src/net/osmand/data/preparation/DataExtraction.java +++ /dev/null @@ -1,1065 +0,0 @@ -package net.osmand.data.preparation; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.Map.Entry; - -import javax.xml.parsers.ParserConfigurationException; - -import net.osmand.Algoritms; -import net.osmand.IProgress; -import net.osmand.data.Amenity; -import net.osmand.data.Building; -import net.osmand.data.City; -import net.osmand.data.DataTileManager; -import net.osmand.data.MapObject; -import net.osmand.data.Region; -import net.osmand.data.Street; -import net.osmand.data.TransportRoute; -import net.osmand.data.TransportStop; -import net.osmand.data.City.CityType; -import net.osmand.data.index.IndexConstants; -import net.osmand.impl.ConsoleProgressImplementation; -import net.osmand.osm.Entity; -import net.osmand.osm.LatLon; -import net.osmand.osm.MapUtils; -import net.osmand.osm.Node; -import net.osmand.osm.Relation; -import net.osmand.osm.Way; -import net.osmand.osm.Entity.EntityId; -import net.osmand.osm.Entity.EntityType; -import net.osmand.osm.OSMSettings.OSMTagKey; -import net.osmand.osm.io.IOsmStorageFilter; -import net.osmand.osm.io.OsmBaseStorage; -import net.osmand.swing.DataExtractionSettings; -import net.sf.junidecode.Junidecode; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.tools.bzip2.CBZip2InputStream; -import org.xml.sax.SAXException; - - - -/** - * http://wiki.openstreetmap.org/wiki/OSM_tags_for_routing#Is_inside.2Foutside - * http://wiki.openstreetmap.org/wiki/Relations/Proposed/Postal_Addresses - * http://wiki.openstreetmap.org/wiki/Proposed_features/House_numbers/Karlsruhe_Schema#Tags (node, way) - * - * 1. node - place : country, state, region, county, city, town, village, hamlet, suburb - * That node means label for place ! It is widely used in OSM. - * - * 2. way - highway : primary, secondary, service. - * That node means label for street if it is in city (primary, secondary, residential, tertiary, living_street), - * beware sometimes that roads could outside city. Usage : often - * - * outside city : trunk, motorway, motorway_link... - * special tags : lanes - 3, maxspeed - 90, bridge - * - * 3. relation - type = address. address:type : country, a1, a2, a3, a4, a5, a6, ... hno. - * member:node role=label : - * member:relation role=border : - * member:node role=a1,a2... : - * - * 4. node, way - addr:housenumber(+), addr:street(+), addr:country(+/-), addr:city(-) - * building=yes - * - * 5. relation - boundary=administrative, admin_level : 1, 2, .... - * - * 6. node, way - addr:postcode =? - * relation - type=postal_code (members way, node), postal_code=? - * - * 7. node, way - amenity=? - * - */ -public class DataExtraction { - private static final Log log = LogFactory.getLog(DataExtraction.class); - - public static final int BATCH_SIZE = 5000; - public static final String TEMP_NODES_DB = "nodes"+IndexConstants.MAP_INDEX_EXT; - - - private final boolean loadAllObjects; - private final boolean normalizeStreets; - private final boolean indexAddress; - private final boolean indexPOI; - private final boolean parseEntityInfo; - private final boolean indexTransport; - private File workingDir = null; - - - public DataExtraction(boolean indexAddress, boolean indexPOI, boolean indexTransport, boolean normalizeStreets, - boolean loadAllObjects, boolean parseEntityInfo, File workingDir){ - this.indexAddress = indexAddress; - this.indexPOI = indexPOI; - this.indexTransport = indexTransport; - this.normalizeStreets = normalizeStreets; - this.loadAllObjects = loadAllObjects; - this.parseEntityInfo = parseEntityInfo; - this.workingDir = workingDir; - - } - - - protected class DataExtractionOsmFilter implements IOsmStorageFilter { - ArrayList places = new ArrayList(); - ArrayList amenities = new ArrayList(); - - Map addressRelations = new LinkedHashMap(); - Map ways = new LinkedHashMap(); - Map buildings = new LinkedHashMap(); - - ArrayList transport = new ArrayList(); - Map postalCodes = new LinkedHashMap(); - - - private Connection conn; -// - private boolean preloadRelationAndWaysIntoDB = false; - private boolean createWholeOsmDB = false; - - int currentCountNode = 0; - private PreparedStatement prepNode; - int currentRelationsCount = 0; - private PreparedStatement prepRelations; - int currentWaysCount = 0; - private PreparedStatement prepWays; - int currentTagsCount = 0; - private PreparedStatement prepTags; - private final String regionName; - private File dbFile; - - - public DataExtractionOsmFilter(String regionName) { - this.regionName = regionName; - } - - public DataExtractionOsmFilter() { - createWholeOsmDB = false; - this.regionName = null; - } - - public ArrayList getPlaces() { - return places; - } - public Map getBuildings() { - return buildings; - } - public ArrayList getAmenities() { - return amenities; - } - - public Map getAddressRelations() { - return addressRelations; - } - - public Map getWays() { - return ways; - } - public ArrayList getTransport() { - return transport; - } - - public Map getPostalCodes() { - return postalCodes; - } - - public void initDatabase() throws SQLException { - try { - Class.forName("org.sqlite.JDBC"); - } catch (ClassNotFoundException e) { - log.error("Illegal configuration", e); - throw new IllegalStateException(e); - } - if(createWholeOsmDB){ - dbFile = new File(workingDir, regionName+IndexConstants.MAP_INDEX_EXT); - } else { - dbFile = new File(workingDir, TEMP_NODES_DB); - } - // to save space - if(dbFile.exists()){ - dbFile.delete(); - } - // creating nodes db to fast access for all nodes - conn = DriverManager.getConnection("jdbc:sqlite:" + dbFile.getAbsolutePath()); - - // prepare tables - Statement stat = conn.createStatement(); - stat.executeUpdate("drop table if exists node;"); - stat.executeUpdate("create table node (id long, latitude double, longitude double);"); - stat.executeUpdate("create index IdIndex ON node (id, latitude, longitude);"); - stat.executeUpdate("drop table if exists ways;"); - stat.executeUpdate("create table ways (id long, node long);"); - stat.executeUpdate("create index IdWIndex ON ways (id, node);"); - stat.executeUpdate("drop table if exists relations;"); - stat.executeUpdate("create table relations (id long, member long, type byte, role text);"); - stat.executeUpdate("create index IdRIndex ON relations (id, member, type);"); - stat.executeUpdate("drop table if exists tags;"); - stat.executeUpdate("create table tags (id long, type byte, key, value);"); - stat.executeUpdate("create index IdTIndex ON tags (id, type);"); - stat.execute("PRAGMA user_version = " + IndexConstants.MAP_TABLE_VERSION); //$NON-NLS-1$ - stat.close(); - - prepNode = conn.prepareStatement("insert into node values (?, ?, ?);"); - prepWays = conn.prepareStatement("insert into ways values (?, ?);"); - prepRelations = conn.prepareStatement("insert into relations values (?, ?, ?, ?);"); - prepTags = conn.prepareStatement("insert into tags values (?, ?, ?, ?);"); - preloadRelationAndWaysIntoDB = indexTransport || indexAddress; - conn.setAutoCommit(false); - } - - public void correlateData(OsmBaseStorage storage, IProgress progress) throws SQLException { - if (currentCountNode > 0) { - prepNode.executeBatch(); - } - prepNode.close(); - if (currentWaysCount > 0) { - prepWays.executeBatch(); - } - prepWays.close(); - if (currentRelationsCount > 0) { - prepRelations.executeBatch(); - } - prepRelations.close(); - if (currentTagsCount > 0) { - prepTags.executeBatch(); - } - prepTags.close(); - conn.setAutoCommit(true); - - final PreparedStatement pselectNode = conn.prepareStatement("select * from node where id = ?"); - final PreparedStatement pselectWay = conn.prepareStatement("select * from ways where id = ?"); - final PreparedStatement pselectRelation = conn.prepareStatement("select * from relations where id = ?"); - - Map map = new LinkedHashMap(); - progress.startTask("Correlating data...", storage.getRegisteredEntities().size()); - ArrayList values = new ArrayList(storage.getRegisteredEntities().values()); - int delayedProgress = 0; - for (int ind = 0; ind < values.size(); ind++) { - Entity e = values.get(ind); - if(delayedProgress == 0){ - progress.progress(1); - } else { - delayedProgress--; - } - if (e instanceof Node) { - continue; - } - map.clear(); - Collection ids = e instanceof Way ? ((Way) e).getEntityIds() : ((Relation) e).getMemberIds(); - for (EntityId i : ids) { - if (!storage.getRegisteredEntities().containsKey(i)) { - if (i.getType() == EntityType.NODE) { - pselectNode.setLong(1, i.getId()); - if (pselectNode.execute()) { - ResultSet rs = pselectNode.getResultSet(); - if (rs.next()) { - storage.getRegisteredEntities().put(i, new Node(rs.getDouble(2), rs.getDouble(3), rs.getLong(1))); - } - rs.close(); - } - } else if (i.getType() == EntityType.WAY) { - pselectWay.setLong(1, i.getId()); - if (pselectWay.execute()) { - ResultSet rs = pselectWay.getResultSet(); - Way way = new Way(i.getId()); - storage.getRegisteredEntities().put(i, way); - while (rs.next()) { - way.addNode(rs.getLong(2)); - } - // add way to load referred nodes - values.add(way); - delayedProgress++; - rs.close(); - } - } else if (i.getType() == EntityType.RELATION) { - pselectRelation.setLong(1, i.getId()); - if (pselectRelation.execute()) { - ResultSet rs = pselectNode.getResultSet(); - Relation rel = new Relation(i.getId()); - storage.getRegisteredEntities().put(i, rel); - while (rs.next()) { - rel.addMember(rs.getLong(1), EntityType.values()[rs.getByte(2)], rs.getString(3)); - } - // do not load relation members recursively ? It is not needed for transport, address, poi before - rs.close(); - } - } - } - if (storage.getRegisteredEntities().containsKey(i)) { - map.put(i, storage.getRegisteredEntities().get(i)); - } - } - e.initializeLinks(map); - } - - pselectNode.close(); - } - - public void close() { - if (conn != null) { - try { - conn.close(); - if(!createWholeOsmDB){ - dbFile.delete(); - } - } catch (SQLException e) { - } - } - } - - @Override - public boolean acceptEntityToLoad(OsmBaseStorage storage, EntityId entityId, Entity e) { - boolean processed = false; - if (indexPOI && Amenity.isAmenity(e)) { - amenities.add(e); - processed = true; - } - if (e instanceof Node && e.getTag(OSMTagKey.PLACE) != null) { - places.add((Node) e); - processed = true; - } - if (indexAddress) { - // index not only buildings but also addr:interpolation ways -// if ("yes".equals(e.getTag(OSMTagKey.BUILDING))) { - if (e.getTag(OSMTagKey.ADDR_HOUSE_NUMBER) != null && e.getTag(OSMTagKey.ADDR_STREET) != null) { - buildings.put(entityId, e); - processed = true; - } -// } - // suppose that streets are way for car - if (e instanceof Way /*&& OSMSettings.wayForCar(e.getTag(OSMTagKey.HIGHWAY))*/ - && e.getTag(OSMTagKey.HIGHWAY) != null - && e.getTag(OSMTagKey.NAME) != null) { - ways.put(entityId, (Way) e); - processed = true; - } - if(e instanceof Relation){ - // do not need to mark processed - if(e.getTag(OSMTagKey.POSTAL_CODE) != null){ - String tag = e.getTag(OSMTagKey.POSTAL_CODE); - for(EntityId l : ((Relation)e).getMemberIds()){ - postalCodes.put(l, tag); - } - } - - if("address".equals(e.getTag(OSMTagKey.TYPE))){ - addressRelations.put(entityId, (Relation) e); - processed = true; - } - - } - } - if(indexTransport){ - if(e instanceof Relation && e.getTag(OSMTagKey.ROUTE) != null){ - transport.add((Relation) e); - processed = true; - } - if(e instanceof Node && "bus_stop".equals(e.getTag(OSMTagKey.HIGHWAY))){ - // load all stops in order to get their names - processed = true; - } - - if (e instanceof Node && e.getTag(OSMTagKey.RAILWAY) != null) { - if ("tram_stop".equals(e.getTag(OSMTagKey.RAILWAY)) || "station".equals(e.getTag(OSMTagKey.RAILWAY))) { - // load all stops in order to get their names - processed = true; - } - } - } - // put all nodes into temporary db to get only required nodes after loading all data - try { - if (e instanceof Node) { - currentCountNode++; - prepNode.setLong(1, e.getId()); - prepNode.setDouble(2, ((Node) e).getLatitude()); - prepNode.setDouble(3, ((Node) e).getLongitude()); - prepNode.addBatch(); - if (currentCountNode >= BATCH_SIZE) { - prepNode.executeBatch(); - currentCountNode = 0; - } - } else if(preloadRelationAndWaysIntoDB || createWholeOsmDB) { - if (e instanceof Way) { - for(Long i : ((Way)e).getNodeIds()){ - currentWaysCount ++; - prepWays.setLong(1, e.getId()); - prepWays.setLong(2, i); - prepWays.addBatch(); - } - if (currentWaysCount >= BATCH_SIZE) { - prepWays.executeBatch(); - currentWaysCount = 0; - } - } else { - for(Entry i : ((Relation)e).getMembersMap().entrySet()){ - currentRelationsCount ++; - prepRelations.setLong(1, e.getId()); - prepRelations.setLong(2, i.getKey().getId()); - prepRelations.setLong(3, i.getKey().getType().ordinal()); - prepRelations.setString(4, i.getValue()); - prepRelations.addBatch(); - } - if (currentRelationsCount >= BATCH_SIZE) { - prepRelations.executeBatch(); - currentRelationsCount = 0; - } - } - } - if(createWholeOsmDB){ - for(Entry i : e.getTags().entrySet()){ - currentTagsCount ++; - prepTags.setLong(1, e.getId()); - prepTags.setLong(2, EntityType.valueOf(e).ordinal()); - prepTags.setString(3, i.getKey()); - prepTags.setString(4, i.getValue()); - prepTags.addBatch(); - } - if (currentTagsCount >= BATCH_SIZE) { - prepTags.executeBatch(); - currentTagsCount = 0; - } - } - } catch (SQLException ex) { - log.error("Could not save in db", ex); - } - return processed || loadAllObjects; - } - - } - - - public Region readCountry(String path, IProgress progress, IOsmStorageFilter addFilter) throws IOException, SAXException, SQLException{ - File f = new File(path); - InputStream stream = new FileInputStream(f); - int i = f.getName().indexOf('.'); - String regionName = Algoritms.capitalizeFirstLetterAndLowercase(f.getName().substring(0, i)); - - InputStream streamFile = stream; - long st = System.currentTimeMillis(); - if (path.endsWith(".bz2")) { - if (stream.read() != 'B' || stream.read() != 'Z') { - throw new RuntimeException("The source stream must start with the characters BZ if it is to be read as a BZip2 stream."); - } else { - stream = new CBZip2InputStream(stream); - } - } - - if(progress != null){ - progress.startTask("Loading file " + path, -1); - } - OsmBaseStorage storage = new OsmBaseStorage(); - storage.setSupressWarnings(DataExtractionSettings.getSettings().isSupressWarningsForDuplicatedId()); - if (addFilter != null) { - storage.getFilters().add(addFilter); - } - - DataExtractionOsmFilter filter = new DataExtractionOsmFilter(regionName); - // data to load & index - final ArrayList places = filter.getPlaces(); - final Map buildings = filter.getBuildings(); - final Map ways = filter.getWays(); - final ArrayList amenities = filter.getAmenities(); - final ArrayList transport = filter.getTransport(); - final Map addressRelations = filter.getAddressRelations(); - final Map postalCodes = filter.getPostalCodes(); - storage.getFilters().add(filter); - // 0. Loading osm file - - try { - // 0.1 init database to store temporary data - filter.initDatabase(); - // 0.2 parsing osm itself - progress.setGeneralProgress("[40 of 100]"); - storage.parseOSM(stream, progress, streamFile, parseEntityInfo); - if (log.isInfoEnabled()) { - log.info("File parsed : " + (System.currentTimeMillis() - st)); - } - progress.finishTask(); - - progress.setGeneralProgress("[55 of 100]"); - // 0.3 Correlating data (linking way & node) - filter.correlateData(storage, progress); - - } finally { - if (log.isInfoEnabled()) { - log.info("File indexed : " + (System.currentTimeMillis() - st)); - } - filter.close(); - } - - // 1. Initialize region - Region country = new Region(); - - country.setName(regionName); - country.setStorage(storage); - - // 2. Reading amenities - progress.setGeneralProgress("[60 of 100]"); - progress.startTask("Indexing poi...", -1); - if (indexPOI) { - readingAmenities(amenities, country); - } - - // 3. Reading cities - progress.setGeneralProgress("[65 of 100]"); - progress.startTask("Indexing cities...", -1); - LinkedHashMap registeredCities = new LinkedHashMap(); - readingCities(places, country, registeredCities); - - if (indexAddress) { - progress.setGeneralProgress("[80 of 100]"); - // 4.1 Reading address relations & remove read streets/buildings - readingAddresses(progress, addressRelations, registeredCities, ways, buildings, postalCodes, country); - - // 4. Reading streets - readingStreets(progress, ways, country); - - // 5. reading buildings - progress.setGeneralProgress("[95 of 100]"); - readingBuildings(progress, buildings, country, postalCodes); - } - - progress.setGeneralProgress("[100 of 100]"); - if(normalizeStreets){ - // 6. normalizing streets - normalizingStreets(progress, country); - } - - // 7. Indexing transport - if(indexTransport){ - readingTransport(transport, country, progress); - } - // 8. Call data preparation to sort cities, calculate center location, assign id to objects - country.doDataPreparation(); - - // 9. Transliterate names to english - - convertEnglishName(country); - for (CityType c : CityType.values()) { - for (City city : country.getCitiesByType(c)) { - convertEnglishName(city); - for (Street s : city.getStreets()) { - convertEnglishName(s); - for (Building b : s.getBuildings()) { - convertEnglishName(b); - } - } - } - } - for (Amenity a : country.getAmenityManager().getAllObjects()) { - convertEnglishName(a); - } - for(List r : country.getTransportRoutes().values()){ - for(TransportRoute route : r){ - convertEnglishName(route); - for(TransportStop s : route.getBackwardStops()){ - convertEnglishName(s); - } - for(TransportStop s : route.getForwardStops()){ - convertEnglishName(s); - } - } - } - return country; - } - // icu4j example - icu is not good in transliteration russian names -// Transliterator latin = Transliterator.getInstance("Any-Latin;NFD;[:Nonspacing Mark:] Remove;NFC"); - - private void convertEnglishName(MapObject o){ - String name = o.getName(); - if(name != null && (o.getEnName() == null || o.getEnName().isEmpty())){ - o.setEnName(Junidecode.unidecode(name)); -// o.setEnName(transliterator.transliterate(name)); - } - } - - - private void readingBuildings(IProgress progress, final Map buildings, Region country, Map postalCodes) { - // found buildings (index addresses) - progress.startTask("Indexing buildings...", buildings.size()); - for(Entity b : buildings.values()){ - LatLon center = b.getLatLon(); - progress.progress(1); - // TODO first of all tag could be checked NodeUtil.getTag(e, "addr:city") - if(center == null){ - // no nodes where loaded for this way - } else { - City city = country.getClosestCity(center); - if(city == null){ - Node n = new Node(center.getLatitude(), center.getLongitude(), -1); - n.putTag(OSMTagKey.PLACE.getValue(), CityType.TOWN.name()); - n.putTag(OSMTagKey.NAME.getValue(), "Unknown city"); - country.registerCity(n); - city = country.getClosestCity(center); - } - if (city != null) { - Building building = city.registerBuilding(b); - if (building != null) { - EntityId i = building.getEntityId(); - if (postalCodes.containsKey(i)) { - building.setPostcode(postalCodes.get(i)); - } - } - } - } - } - progress.finishTask(); - } - - private void readingAddresses(IProgress progress, Map addressRelations, Map cities, - Map ways, Map buildings, - Map postalCodes, Region country) { - progress.startTask("Indexing addresses...", addressRelations.size()); - for(Relation i : addressRelations.values()){ - progress.progress(1); - String type = i.getTag(OSMTagKey.ADDRESS_TYPE); - - boolean house = "house".equals(type); - boolean street = "a6".equals(type); - if(house || street){ - // try to find appropriate city/street - City c = null; - - Collection members = i.getMembers("is_in"); - Relation a3 = null; - Relation a6 = null; - if(!members.isEmpty()){ - if(street){ - a6 = i; - } - Entity in = members.iterator().next(); - if(in instanceof Relation){ - // go one level up for house - if(house){ - a6 = (Relation) in; - members = ((Relation)in).getMembers("is_in"); - if(!members.isEmpty()){ - in = members.iterator().next(); - if(in instanceof Relation){ - a3 = (Relation) in; - } - } - - } else { - a3 = (Relation) in; - } - } - } - - if(a3 != null){ - Collection memberIds = a3.getMemberIds("label"); - if(!memberIds.isEmpty()){ - c = cities.get(memberIds.iterator().next()); - } - } - if(c != null && a6 != null){ - String name = a6.getTag(OSMTagKey.NAME); - - if(name != null){ - Street s = c.registerStreet(name); - if(street){ - for (Map.Entry r : i.getMemberEntities().entrySet()) { - if ("street".equals(r.getValue())) { - if (r.getKey() instanceof Way) { - s.getWayNodes().add((Way) r.getKey()); - ways.remove(EntityId.valueOf(r.getKey())); - } - } else if ("house".equals(r.getValue())) { - // will be registered further in other case - if (!(r.getKey() instanceof Relation)) { - EntityId id = EntityId.valueOf(r.getKey()); - Building b = s.registerBuilding(r.getKey()); - buildings.remove(id); - if (b != null) { - if (postalCodes.containsKey(id)) { - b.setPostcode(postalCodes.get(id)); - } - } - } - } - } - } else { - String hno = i.getTag(OSMTagKey.ADDRESS_HOUSE); - if(hno == null){ - hno = i.getTag(OSMTagKey.ADDR_HOUSE_NUMBER); - } - if(hno == null){ - hno = i.getTag(OSMTagKey.NAME); - } - members = i.getMembers("border"); - if(!members.isEmpty()){ - Entity border = members.iterator().next(); - if (border != null) { - EntityId id = EntityId.valueOf(border); - // special check that address do not contain twice in a3 - border and separate a6 - if (!a6.getMemberIds().contains(id)) { - Building b = s.registerBuilding(border, hno); - if (b != null && postalCodes.containsKey(id)) { - b.setPostcode(postalCodes.get(id)); - } - buildings.remove(id); - } - } - } else { - log.info("For relation " + i.getId() + " border not found"); - } - - - } - - - } - } - } - - } - progress.finishTask(); - } - - - - private void readingStreets(IProgress progress, final Map ways, Region country) { - progress.startTask("Indexing streets...", ways.size()); - DataTileManager waysManager = new DataTileManager(); - for (Way w : ways.values()) { - progress.progress(1); - if (w.getTag(OSMTagKey.NAME) != null) { - String street = w.getTag(OSMTagKey.NAME); - LatLon center = MapUtils.getWeightCenterForNodes(w.getNodes()); - if (center != null) { - City city = country.getClosestCity(center); - if(city == null){ - Node n = new Node(center.getLatitude(), center.getLongitude(), -1); - n.putTag(OSMTagKey.PLACE.getValue(), CityType.TOWN.name()); - n.putTag(OSMTagKey.NAME.getValue(), "Uknown city"); - country.registerCity(n); - city = country.getClosestCity(center); - } - - if (city != null) { - Street str = city.registerStreet(street); - str.getWayNodes().add(w); - } - waysManager.registerObject(center.getLatitude(), center.getLongitude(), w); - } - } - } - progress.finishTask(); - /// way with name : МЗОР, ул. ..., - } - - private static Set acceptedRoutes = new HashSet(); - static { - acceptedRoutes.add("bus"); - acceptedRoutes.add("trolleybus"); - acceptedRoutes.add("share_taxi"); - - acceptedRoutes.add("subway"); - acceptedRoutes.add("train"); - - acceptedRoutes.add("tram"); - - acceptedRoutes.add("ferry"); - } - - public void readingTransport(final ArrayList transport, Region country, IProgress progress){ - progress.startTask("Reading transport...", -1); - Map> routes = country.getTransportRoutes(); - Map routeStops = new LinkedHashMap(); - for(Relation rel : transport){ - String ref = rel.getTag(OSMTagKey.REF); - String route = rel.getTag(OSMTagKey.ROUTE); - String operator = rel.getTag(OSMTagKey.OPERATOR); - if(route == null || ref == null){ - continue; - } - if(!acceptedRoutes.contains(route)){ - continue; - } - TransportRoute r = new TransportRoute(rel, ref); - r.setOperator(operator); - r.setType(route); - - - if(operator != null){ - route = operator + " : " + route; - } - if(!routes.containsKey(route)){ - routes.put(route, new ArrayList()); - } - - - final Map forwardStops = new LinkedHashMap(); - final Map backwardStops = new LinkedHashMap(); - int currentStop = 0; - int forwardStop = 0; - int backwardStop = 0; - for (Entry e : rel.getMemberEntities().entrySet()) { - if(e.getValue().contains("stop")){ - if(e.getKey() instanceof Node){ - if(!routeStops.containsKey(e.getKey().getId())){ - routeStops.put(e.getKey().getId(), new TransportStop(e.getKey())); - } - TransportStop stop = routeStops.get(e.getKey().getId()); - boolean forward = e.getValue().contains("forward"); - boolean backward = e.getValue().contains("backward"); - currentStop++; - if(forward || !backward){ - forwardStop ++; - } - if(backward){ - backwardStop ++; - } - boolean common = !forward && !backward; - int index = -1; - int i = e.getValue().length() -1; - int accum = 1; - while(i >= 0 && Character.isDigit(e.getValue().charAt(i))){ - if(index < 0){ - index = 0; - } - index = accum * Character.getNumericValue(e.getValue().charAt(i)) + index; - accum *= 10; - i --; - } - if(index < 0){ - index = forward ? forwardStop : (backward ? backwardStop : currentStop) ; - } - if(forward || common){ - forwardStops.put(stop, index); - r.getForwardStops().add(stop); - } - if(backward || common){ - if(common){ - // put with negative index - backwardStops.put(stop, -index); - } else { - backwardStops.put(stop, index); - } - - r.getBackwardStops().add(stop); - } - - } - - } else if(e.getKey() instanceof Way){ - r.addWay((Way) e.getKey()); - } - } - if(forwardStops.isEmpty() && backwardStops.isEmpty()){ - continue; - } - Collections.sort(r.getForwardStops(), new Comparator(){ - @Override - public int compare(TransportStop o1, TransportStop o2) { - return forwardStops.get(o1) - forwardStops.get(o2); - } - }); - // all common stops are with negative index (reeval them) - for(TransportStop s : new ArrayList(backwardStops.keySet())){ - if(backwardStops.get(s) < 0){ - backwardStops.put(s, backwardStops.size() + backwardStops.get(s) -1); - } - } - Collections.sort(r.getBackwardStops(), new Comparator(){ - @Override - public int compare(TransportStop o1, TransportStop o2) { - return backwardStops.get(o1) - backwardStops.get(o2); - } - }); - routes.get(route).add(r); - - // validate that all is ok -// if (validateTransportStops(r.getBackwardStops(), r) && validateTransportStops(r.getForwardStops(), r)) { -// System.out.println("Route " + r + " is valid "); -// } - - } - - progress.finishTask(); - } - - protected boolean validateTransportStops(List stops, TransportRoute r){ - boolean valid = true; - for (int i = 2; i < stops.size(); i++) { - TransportStop s1 = stops.get(i - 2); - TransportStop s2 = stops.get(i - 1); - TransportStop s3 = stops.get(i); - if (MapUtils.getDistance(s1.getLocation(), s2.getLocation()) > MapUtils.getDistance(s1.getLocation(), s3.getLocation()) * 1.3) { - System.out.println("SOMETHING WRONG with " + i + "th of " + r.getRef() +" "+ r ); - valid = false; - } - } - return valid; - } - - private void readingAmenities(final ArrayList amenities, Region country) { - for(Entity a: amenities){ - country.registerAmenity(new Amenity(a)); - } - } - - - public void readingCities(ArrayList places, Region country, Map citiesMap) { - for (Node s : places) { - String place = s.getTag(OSMTagKey.PLACE); - if (place == null) { - continue; - } - City city = country.registerCity(s); - if(city != null){ - citiesMap.put(city.getEntityId(), city); - } - } - } - - - private int checkSuffix(String name, String suffix){ - int i = -1; - boolean searchAgain = false; - do { - i = name.indexOf(suffix, i); - searchAgain = false; - if (i > 0) { - if (Character.isLetterOrDigit(name.charAt(i -1))) { - i ++; - searchAgain = true; - } - } - } while (searchAgain); - return i; - } - - private String cutSuffix(String name, int ind, int suffixLength){ - String newName = name.substring(0, ind); - if (name.length() > ind + suffixLength + 1) { - newName += name.substring(ind + suffixLength + 1); - } - return newName.trim(); - } - - private String putSuffixToEnd(String name, int ind, int suffixLength) { - if (name.length() <= ind + suffixLength) { - return name; - - } - String newName; - if(ind > 0){ - newName = name.substring(0, ind); - newName += name.substring(ind + suffixLength); - newName += name.substring(ind - 1, ind + suffixLength ); - } else { - newName = name.substring(suffixLength + 1) + name.charAt(suffixLength) + name.substring(0, suffixLength); - } - - return newName.trim(); - } - - public void normalizingStreets(IProgress progress, Region region){ - progress.startTask("Normalizing name streets...", -1); - String[] defaultSuffixes = DataExtractionSettings.getSettings().getDefaultSuffixesToNormalizeStreets(); - String[] suffixes = DataExtractionSettings.getSettings().getSuffixesToNormalizeStreets(); - for(CityType t : CityType.values()){ - for(City c : region.getCitiesByType(t)){ - ArrayList list = new ArrayList(c.getStreets()); - for (Street s : list) { - String name = s.getName(); - String newName = name.trim(); - boolean processed = newName.length() != name.length(); - for (String ch : defaultSuffixes) { - int ind = checkSuffix(newName, ch); - if (ind != -1) { - newName = cutSuffix(newName, ind, ch.length()); - processed = true; - break; - } - } - - if (!processed) { - for (String ch : suffixes) { - int ind = checkSuffix(newName, ch); - if (ind != -1) { - newName = putSuffixToEnd(newName, ind, ch.length()); - processed = true; - break; - } - } - } - if (processed) { - s.setName(newName); - } - } - } - } - } - - // Performance testing methods - public static void main(String[] args) throws IOException, SAXException, SQLException, ParserConfigurationException { - long time = System.currentTimeMillis(); - OsmBaseStorage storage = new OsmBaseStorage(); - String path = "E:\\Information\\OSM maps\\belarus_2010_06_02.osm"; -// String path = "E:\\Information\\OSM maps\\minsk_extr.bz2"; -// String path = "E:\\Information\\OSM maps\\netherlands.osm.bz2"; - String wDir = "E:\\Information\\OSM maps\\osmand\\"; - - - - File f = new File(path); - InputStream stream = new FileInputStream(f); - InputStream streamFile = stream; - if (path.endsWith(".bz2")) { - if (stream.read() != 'B' || stream.read() != 'Z') { - throw new RuntimeException("The source stream must start with the characters BZ if it is to be read as a BZip2 stream."); - } else { - stream = new CBZip2InputStream(stream); - } - } - // only sax parser -// if(true){ -// SAXParser parser = SAXParserFactory.newInstance().newSAXParser(); -// parser.parse(f, new DefaultHandler()); -// System.out.println("All time " + (System.currentTimeMillis() - time) + " ms"); // -// } - - - storage.getFilters().add(new IOsmStorageFilter(){ - @Override - public boolean acceptEntityToLoad(OsmBaseStorage storage, EntityId entityId, Entity entity) { - return false; - } - }); - DataExtraction e = new DataExtraction(true, true, true, true, false, true, new File(wDir)); - - DataExtractionOsmFilter filter = e.new DataExtractionOsmFilter(); - filter.initDatabase(); - storage.getFilters().add(filter); - - // belarus.osm - 22 843 (only sax), 33 344 (wo filter), 82 829 (filter) -// storage.parseOSM(stream, null, streamFile, false); - // belarus.osm - 46 938 (wo filter), 98 188 (filter) - // netherlands.osm - 1743 511 (wo filter) - storage.parseOSM(stream, new ConsoleProgressImplementation(), streamFile, true); - System.out.println("Total mem: " + Runtime.getRuntime().totalMemory() + " free : " + Runtime.getRuntime().freeMemory()); - System.out.println("All time " + (System.currentTimeMillis() - time) + " ms"); // -// System.out.println(amenities.size() + " " + buildings.size() + " " + places.size() + " " + ways.size()); - } - -} diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java b/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java index 429bcbd6e7..a5852c6193 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java @@ -64,12 +64,16 @@ import org.xml.sax.SAXException; /** + * http://wiki.openstreetmap.org/wiki/OSM_tags_for_routing#Is_inside.2Foutside + * http://wiki.openstreetmap.org/wiki/Relations/Proposed/Postal_Addresses + * http://wiki.openstreetmap.org/wiki/Proposed_features/House_numbers/Karlsruhe_Schema#Tags (node, way) + * That data extraction has aim, * save runtime memory and generate indexes on the fly. * It will be longer than load in memory (needed part) and save into index. */ public class IndexCreator { - private static final Log log = LogFactory.getLog(DataExtraction.class); + private static final Log log = LogFactory.getLog(IndexCreator.class); public static final int BATCH_SIZE = 5000; @@ -937,7 +941,7 @@ public class IndexCreator { } } - if(indexMap && (e instanceof Way) || (e instanceof Node)){ + if(indexMap && (e instanceof Way || e instanceof Node)){ // manipulate what kind of way to load loadEntityData(e, true); boolean inverse = "-1".equals(e.getTag(OSMTagKey.ONEWAY)); diff --git a/DataExtractionOSM/src/net/osmand/swing/OsmExtractionUI.java b/DataExtractionOSM/src/net/osmand/swing/OsmExtractionUI.java index 1cd769cdf0..f52f20d865 100644 --- a/DataExtractionOSM/src/net/osmand/swing/OsmExtractionUI.java +++ b/DataExtractionOSM/src/net/osmand/swing/OsmExtractionUI.java @@ -5,8 +5,6 @@ import java.awt.Container; import java.awt.FlowLayout; import java.awt.event.ActionEvent; import java.awt.event.ActionListener; -import java.awt.event.MouseAdapter; -import java.awt.event.MouseEvent; import java.awt.event.WindowAdapter; import java.awt.event.WindowEvent; import java.io.File; @@ -16,70 +14,33 @@ import java.io.OutputStream; import java.lang.Thread.UncaughtExceptionHandler; import java.lang.reflect.InvocationTargetException; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.EventObject; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import javax.swing.AbstractAction; -import javax.swing.Action; -import javax.swing.JButton; import javax.swing.JCheckBox; import javax.swing.JDialog; import javax.swing.JFileChooser; import javax.swing.JFrame; import javax.swing.JLabel; -import javax.swing.JList; import javax.swing.JMenu; import javax.swing.JMenuBar; import javax.swing.JMenuItem; import javax.swing.JOptionPane; import javax.swing.JPanel; -import javax.swing.JPopupMenu; import javax.swing.JScrollPane; import javax.swing.JSplitPane; -import javax.swing.JTextField; import javax.swing.JTree; import javax.swing.UIManager; -import javax.swing.event.TreeModelEvent; -import javax.swing.event.TreeModelListener; -import javax.swing.event.TreeSelectionEvent; -import javax.swing.event.TreeSelectionListener; import javax.swing.filechooser.FileFilter; import javax.swing.tree.DefaultMutableTreeNode; -import javax.swing.tree.DefaultTreeCellEditor; -import javax.swing.tree.DefaultTreeCellRenderer; import javax.swing.tree.DefaultTreeModel; -import javax.swing.tree.TreeCellEditor; -import javax.swing.tree.TreePath; import javax.xml.stream.XMLStreamException; -import net.osmand.Algoritms; import net.osmand.ExceptionHandler; -import net.osmand.data.Amenity; -import net.osmand.data.AmenityType; -import net.osmand.data.Building; -import net.osmand.data.City; -import net.osmand.data.DataTileManager; -import net.osmand.data.MapObject; -import net.osmand.data.Region; -import net.osmand.data.Street; -import net.osmand.data.TransportRoute; -import net.osmand.data.TransportStop; -import net.osmand.data.City.CityType; -import net.osmand.data.index.DataIndexWriter; -import net.osmand.data.preparation.DataExtraction; +import net.osmand.data.preparation.IndexCreator; import net.osmand.map.IMapLocationListener; import net.osmand.map.ITileSource; -import net.osmand.osm.Entity; -import net.osmand.osm.LatLon; -import net.osmand.osm.MapUtils; -import net.osmand.osm.Node; -import net.osmand.osm.Way; import net.osmand.osm.io.IOsmStorageFilter; +import net.osmand.osm.io.OsmBaseStorage; import net.osmand.osm.io.OsmBoundsFilter; import net.osmand.osm.io.OsmStorageWriter; import net.osmand.swing.MapPanel.MapSelectionArea; @@ -110,98 +71,39 @@ public class OsmExtractionUI implements IMapLocationListener { } }); - MAIN_APP = new OsmExtractionUI(null); + MAIN_APP = new OsmExtractionUI(); MAIN_APP.frame.setBounds(DataExtractionSettings.getSettings().getWindowBounds()); MAIN_APP.frame.setVisible(true); } - protected City selectedCity; - private MapPanel mapPanel; - private DataExtractionTreeNode amenitiesTree; + + private JTree treePlaces; - private JList searchList; - private JTextField searchTextField; +// private DataExtractionTreeNode amenitiesTree; +// private TreeModelListener treeModelListener; + + private MapPanel mapPanel; private JFrame frame; private JLabel statusBarLabel; - private Region region; - private JButton generateDataButton; + private JCheckBox buildPoiIndex; private JCheckBox buildAddressIndex; + private JCheckBox buildMapIndex; private JCheckBox buildTransportIndex; private JCheckBox normalizingStreets; - private TreeModelListener treeModelListener; - private JCheckBox loadingAllData; - + + private String regionName; - public OsmExtractionUI(final Region r){ - this.region = r; + public OsmExtractionUI(){ createUI(); - setRegion(r, "Region"); } - public void setRegion(Region region, String name){ - if (this.region == region) { - return; - } - this.region = region; - DefaultMutableTreeNode root = new DataExtractionTreeNode(name, region); - if (region != null) { - amenitiesTree = new DataExtractionTreeNode("Amenities", region); - amenitiesTree.add(new DataExtractionTreeNode("First 15", region)); - for (AmenityType type : AmenityType.values()) { - amenitiesTree.add(new DataExtractionTreeNode(Algoritms.capitalizeFirstLetterAndLowercase(type.toString()), type)); - } - root.add(amenitiesTree); - - DataExtractionTreeNode transport = new DataExtractionTreeNode("Transport", region); - root.add(transport); - for(String s : region.getTransportRoutes().keySet()){ - DataExtractionTreeNode trRoute = new DataExtractionTreeNode(s, s); - transport.add(trRoute); - List list = region.getTransportRoutes().get(s); - for(TransportRoute r : list){ - DataExtractionTreeNode route = new DataExtractionTreeNode(r.getRef(), r); - trRoute.add(route); - } - - } - - for (CityType t : CityType.values()) { - DefaultMutableTreeNode cityTree = new DataExtractionTreeNode(Algoritms.capitalizeFirstLetterAndLowercase(t.toString()), t); - root.add(cityTree); - for (City ct : region.getCitiesByType(t)) { - DefaultMutableTreeNode cityNodeTree = new DataExtractionTreeNode(ct.getName(), ct); - cityTree.add(cityNodeTree); - - for (Street str : ct.getStreets()) { - DefaultMutableTreeNode strTree = new DataExtractionTreeNode(str.getName(), str); - cityNodeTree.add(strTree); - for (Building b : str.getBuildings()) { - DefaultMutableTreeNode building = new DataExtractionTreeNode(b.getName(), b); - strTree.add(building); - } - } - } - } - } - - if (searchList != null) { - updateListCities(region, searchTextField.getText(), searchList); - } - mapPanel.repaint(); - DefaultTreeModel newModel = new DefaultTreeModel(root, false); - newModel.addTreeModelListener(treeModelListener); - treePlaces.setModel(newModel); - - updateButtonsBar(); - locationChanged(mapPanel.getLatitude(), mapPanel.getLongitude(), this); - } @@ -228,164 +130,31 @@ public class OsmExtractionUI implements IMapLocationListener { statusBarLabel.setText(workingDir == null ? "" : "Working directory : " + workingDir.getAbsolutePath()); - - JSplitPane panelForTreeAndMap = new JSplitPane(JSplitPane.HORIZONTAL_SPLIT, new JScrollPane(createTree(content)), mapPanel); + treePlaces = new JTree(); + treePlaces.setModel(new DefaultTreeModel(new DefaultMutableTreeNode("Region"), false)); + JSplitPane panelForTreeAndMap = new JSplitPane(JSplitPane.HORIZONTAL_SPLIT, new JScrollPane(treePlaces), mapPanel); panelForTreeAndMap.setResizeWeight(0.2); - - - + content.add(panelForTreeAndMap, BorderLayout.CENTER); createButtonsBar(content); -// createCitySearchPanel(content); - if(searchList != null){ - JSplitPane pane = new JSplitPane(JSplitPane.VERTICAL_SPLIT, new JScrollPane(searchList), panelForTreeAndMap); - pane.setResizeWeight(0.2); - content.add(pane, BorderLayout.CENTER); - } else { - content.add(panelForTreeAndMap, BorderLayout.CENTER); - } JMenuBar bar = new JMenuBar(); fillMenuWithActions(bar); - - JPopupMenu popupMenu = new JPopupMenu(); - fillPopupMenuWithActions(popupMenu); - treePlaces.add(popupMenu); - treePlaces.addMouseListener(new PopupTrigger(popupMenu)); - - + frame.setJMenuBar(bar); - - - } - - public JTree createTree(Container content) { - treePlaces = new JTree(); - treePlaces.setModel(new DefaultTreeModel(new DefaultMutableTreeNode("Region"), false)); - treePlaces.setEditable(true); - treePlaces.setCellEditor(new RegionCellEditor(treePlaces, (DefaultTreeCellRenderer) treePlaces.getCellRenderer())); - treePlaces.addTreeSelectionListener(new TreeSelectionListener() { - @Override - public void valueChanged(TreeSelectionEvent e) { - if (e.getPath() != null) { - if (e.getPath().getLastPathComponent() instanceof DataExtractionTreeNode) { - Object o = ((DataExtractionTreeNode) e.getPath().getLastPathComponent()).getModelObject(); - - if (o instanceof MapObject) { - MapObject c = (MapObject) o; - LatLon location = c.getLocation(); - if(location != null){ - if(o instanceof Street){ - DataTileManager ways = new DataTileManager(); - for(Way w : ((Street)o).getWayNodes()){ - LatLon l = w.getLatLon(); - ways.registerObject(l.getLatitude(), l.getLongitude(), w); - } - mapPanel.setPoints(ways); - mapPanel.requestFocus(); - } - mapPanel.setLatLon(location.getLatitude(), location.getLongitude()); - mapPanel.requestFocus(); - } - if(o instanceof TransportRoute){ - DataTileManager ways = new DataTileManager(); - for(Way w : ((TransportRoute)o).getWays()){ - LatLon l = w.getLatLon(); - ways.registerObject(l.getLatitude(), l.getLongitude(), w); - } - for(TransportStop w : ((TransportRoute)o).getBackwardStops()){ - LatLon l = w.getLocation(); - ways.registerObject(l.getLatitude(), l.getLongitude(), - new Node(l.getLatitude(), l.getLongitude(), w.getId())); - } - for(TransportStop w : ((TransportRoute)o).getForwardStops()){ - LatLon l = w.getLocation(); - ways.registerObject(l.getLatitude(), l.getLongitude(), - new Node(l.getLatitude(), l.getLongitude(), w.getId())); - } - mapPanel.setPoints(ways); - mapPanel.requestFocus(); - } - - } else if (o instanceof Entity) { - Entity c = (Entity) o; - LatLon latLon = c.getLatLon(); - if (latLon != null) { - mapPanel.setLatLon(latLon.getLatitude(), latLon.getLongitude()); - mapPanel.requestFocus(); - } - } - } - } - - } - }); - - treeModelListener = new TreeModelListener() { - public void treeNodesChanged(TreeModelEvent e) { - Object node = e.getTreePath().getLastPathComponent(); - if(e.getChildren() != null && e.getChildren().length > 0){ - node =e.getChildren()[0]; - } - if (node instanceof DataExtractionTreeNode) { - DataExtractionTreeNode n = ((DataExtractionTreeNode) node); - if (n.getModelObject() instanceof MapObject) { - MapObject r = (MapObject) n.getModelObject(); - String newName = n.getUserObject().toString(); - if (!r.getName().equals(newName)) { - r.setName(n.getUserObject().toString()); - } - if (r instanceof Street && !((Street) r).isRegisteredInCity()) { - DefaultMutableTreeNode parent = ((DefaultMutableTreeNode) n.getParent()); - parent.remove(n); - ((DefaultTreeModel) treePlaces.getModel()).nodeStructureChanged(parent); - } - } - } - } - public void treeNodesInserted(TreeModelEvent e) { - } - public void treeNodesRemoved(TreeModelEvent e) { - } - public void treeStructureChanged(TreeModelEvent e) { - } - }; - treePlaces.getModel().addTreeModelListener(treeModelListener); - return treePlaces; } - protected void updateButtonsBar() { - generateDataButton.setEnabled(region != null); - normalizingStreets.setVisible(region == null); - loadingAllData.setVisible(region == null); - if(region == null && !buildAddressIndex.isEnabled()){ - buildAddressIndex.setEnabled(true); - } - if(region == null && !buildPoiIndex.isEnabled()){ - buildPoiIndex.setEnabled(true); - } - if(region == null && !buildTransportIndex.isEnabled()){ - buildTransportIndex.setEnabled(true); - } - } public void createButtonsBar(Container content){ JPanel panel = new JPanel(new FlowLayout(FlowLayout.LEFT)); content.add(panel, BorderLayout.NORTH); - generateDataButton = new JButton(); - generateDataButton.setText("Generate data"); - generateDataButton.setToolTipText("Data with selected preferences will be generated in working directory." + - " The index files will be named as region in tree. All existing data will be overwritten."); - panel.add(generateDataButton); - generateDataButton.addActionListener(new ActionListener(){ - @Override - public void actionPerformed(ActionEvent e) { - generateData(); - } - }); + buildMapIndex = new JCheckBox(); + buildMapIndex.setText("Build map index"); + panel.add(buildMapIndex); + buildMapIndex.setSelected(true); buildPoiIndex = new JCheckBox(); buildPoiIndex.setText("Build POI index"); @@ -407,136 +176,17 @@ public class OsmExtractionUI implements IMapLocationListener { panel.add(buildTransportIndex); buildTransportIndex.setSelected(true); - loadingAllData = new JCheckBox(); - loadingAllData.setText("Loading all osm data"); - panel.add(loadingAllData); - loadingAllData.setSelected(false); - updateButtonsBar(); - } - - protected void generateData() { - try { - final ProgressDialog dlg = new ProgressDialog(frame, "Generating data"); - dlg.setRunnable(new Runnable(){ - - @Override - public void run() { - dlg.startTask("Generating indices...", -1); - DataIndexWriter builder = new DataIndexWriter(DataExtractionSettings.getSettings().getDefaultWorkingDir(), region); - StringBuilder msg = new StringBuilder(); - try { - msg.append("Indices for ").append(region.getName()); - if(buildPoiIndex.isSelected()){ - dlg.startTask("Generating POI index...", -1); - builder.writePOI(); - msg.append(", POI index ").append("successfully created"); - } - if(buildAddressIndex.isSelected()){ - dlg.startTask("Generating address index...", -1); - builder.writeAddress(); - msg.append(", address index ").append("successfully created"); - } - if(buildTransportIndex.isSelected()){ - dlg.startTask("Generating transport index...", -1); - builder.writeTransport(); - msg.append(", transport index ").append("successfully created"); - } - -// new DataIndexReader().testIndex(new File( -// DataExtractionSettings.getSettings().getDefaultWorkingDir(), -// IndexConstants.ADDRESS_INDEX_DIR+region.getName()+IndexConstants.ADDRESS_INDEX_EXT)); - msg.append("."); - JOptionPane pane = new JOptionPane(msg); - JDialog dialog = pane.createDialog(frame, "Generation data"); - dialog.setVisible(true); - } catch (SQLException e1) { - throw new IllegalArgumentException(e1); - } catch (IOException e1) { - throw new IllegalArgumentException(e1); - } - } - }); - dlg.run(); - } catch (InterruptedException e1) { - log.error("Interrupted", e1); - } catch (InvocationTargetException e1) { - ExceptionHandler.handle((Exception) e1.getCause()); - } } - - public void fillPopupMenuWithActions(JPopupMenu menu) { - Action delete = new AbstractAction("Delete") { - private static final long serialVersionUID = 7476603434847164396L; - - public void actionPerformed(ActionEvent e) { - TreePath[] p = treePlaces.getSelectionPaths(); - if(p != null && - JOptionPane.OK_OPTION == - JOptionPane.showConfirmDialog(frame, "Are you sure about deleting " +p.length + " resources ? ")){ - for(TreePath path : treePlaces.getSelectionPaths()){ - Object node = path.getLastPathComponent(); - if (node instanceof DataExtractionTreeNode) { - DataExtractionTreeNode n = ((DataExtractionTreeNode) node); - if(n.getParent() instanceof DataExtractionTreeNode){ - DataExtractionTreeNode parent = ((DataExtractionTreeNode) n.getParent()); - boolean remove = false; - if (n.getModelObject() instanceof Street) { - ((City)parent.getModelObject()).unregisterStreet(((Street)n.getModelObject()).getName()); - remove = true; - } else if (n.getModelObject() instanceof Building) { - ((Street)parent.getModelObject()).getBuildings().remove(n.getModelObject()); - remove = true; - } else if (n.getModelObject() instanceof City) { - Region r = (Region) ((DataExtractionTreeNode)parent.getParent()).getModelObject(); - r.unregisterCity((City) n.getModelObject()); - remove = true; - } else if (n.getModelObject() instanceof Amenity) { - Region r = (Region) ((DataExtractionTreeNode)parent.getParent().getParent()).getModelObject(); - Amenity am = (Amenity) n.getModelObject(); - r.getAmenityManager().unregisterObject(am.getLocation().getLatitude(), am.getLocation().getLongitude(), am); - remove = true; - } - if(remove){ - parent.remove(n); - ((DefaultTreeModel) treePlaces.getModel()).nodeStructureChanged(parent); - } - } - } - } - - } - } - }; - menu.add(delete); - Action rename= new AbstractAction("Rename") { - private static final long serialVersionUID = -8257594433235073767L; - - public void actionPerformed(ActionEvent e) { - TreePath path = treePlaces.getSelectionPath(); - if(path != null){ - treePlaces.startEditingAtPath(path); - } - } - }; - menu.add(rename); - - } - public void fillMenuWithActions(final JMenuBar bar){ JMenu menu = new JMenu("File"); bar.add(menu); - JMenuItem loadFile = new JMenuItem("Load osm file..."); + JMenuItem loadFile = new JMenuItem("Select osm file..."); menu.add(loadFile); - JMenuItem loadSpecifiedAreaFile = new JMenuItem("Load osm file for specifed area..."); + JMenuItem loadSpecifiedAreaFile = new JMenuItem("Select osm file for specifed area..."); menu.add(loadSpecifiedAreaFile); - JMenuItem closeCurrentFile = new JMenuItem("Close current file"); - menu.add(closeCurrentFile); - menu.addSeparator(); - JMenuItem saveOsmFile = new JMenuItem("Save data to osm file..."); - menu.add(saveOsmFile); JMenuItem specifyWorkingDir = new JMenuItem("Specify working directory..."); menu.add(specifyWorkingDir); menu.addSeparator(); @@ -582,7 +232,7 @@ public class OsmExtractionUI implements IMapLocationListener { sqliteDB.addActionListener(new ActionListener(){ @Override public void actionPerformed(ActionEvent e) { - final String regionName = region == null ? "Region" : region.getName(); + final String regionName = OsmExtractionUI.this.regionName == null ? "Region" : OsmExtractionUI.this.regionName; final ITileSource map = mapPanel.getMap(); if(map != null){ try { @@ -619,14 +269,6 @@ public class OsmExtractionUI implements IMapLocationListener { frame.setVisible(false); } }); - closeCurrentFile.addActionListener(new ActionListener(){ - - @Override - public void actionPerformed(ActionEvent e) { - setRegion(null, "Region"); - frame.setTitle("OsmAnd Map Creator"); - } - }); settings.addActionListener(new ActionListener(){ @Override public void actionPerformed(ActionEvent e) { @@ -655,7 +297,6 @@ public class OsmExtractionUI implements IMapLocationListener { tileSource.add(sqliteDB); bar.remove(1); bar.add(tileSource, 1); - updateButtonsBar(); } } @@ -714,21 +355,7 @@ public class OsmExtractionUI implements IMapLocationListener { } }); - saveOsmFile.addActionListener(new ActionListener(){ - @Override - public void actionPerformed(ActionEvent e) { - if(region == null){ - return; - } - JFileChooser fc = getOsmFileChooser(); - int answer = fc.showSaveDialog(frame); - if (answer == JFileChooser.APPROVE_OPTION && fc.getSelectedFile() != null){ - saveCountry(fc.getSelectedFile()); - } - } - - }); } public JFileChooser getOsmFileChooser(){ @@ -763,22 +390,14 @@ public class OsmExtractionUI implements IMapLocationListener { @Override public void run() { - Region res; + IndexCreator creator = new IndexCreator(DataExtractionSettings.getSettings().getDefaultWorkingDir()); try { - DataExtraction dataExtraction = new DataExtraction(buildAddressIndex.isSelected(), buildPoiIndex.isSelected(), - buildTransportIndex.isSelected(), normalizingStreets.isSelected(), loadingAllData.isSelected(), - DataExtractionSettings.getSettings().getLoadEntityInfo(), - DataExtractionSettings.getSettings().getDefaultWorkingDir()); - if(!buildAddressIndex.isSelected()){ - buildAddressIndex.setEnabled(false); - } - if(!buildTransportIndex.isSelected()){ - buildTransportIndex.setEnabled(false); - } - if(!buildPoiIndex.isSelected()){ - buildPoiIndex.setEnabled(false); - } - res = dataExtraction.readCountry(f.getAbsolutePath(), dlg, filter); + creator.setIndexAddress(buildAddressIndex.isSelected()); + creator.setIndexPOI(buildPoiIndex.isSelected()); + creator.setNormalizeStreets(normalizingStreets.isSelected()); + creator.setIndexTransport(buildTransportIndex.isSelected()); + creator.setIndexMap(buildMapIndex.isSelected()); + creator.generateIndexes(f, dlg, filter); } catch (IOException e) { throw new IllegalArgumentException(e); } catch (SAXException e) { @@ -786,25 +405,47 @@ public class OsmExtractionUI implements IMapLocationListener { } catch (SQLException e) { throw new IllegalStateException(e); } - dlg.setResult(res); + regionName = creator.getRegionName(); + StringBuilder msg = new StringBuilder(); + msg.append("Indexes for ").append(regionName).append(" : "); + boolean comma = false; + if (buildMapIndex.isSelected()) { + if(comma) msg.append(", "); + comma = true; + msg.append("map"); + } + if (buildPoiIndex.isSelected()) { + if(comma) msg.append(", "); + comma = true; + msg.append("POI"); + } + if (buildAddressIndex.isSelected()) { + if(comma) msg.append(", "); + comma = true; + msg.append("address"); + } + if (buildTransportIndex.isSelected()) { + if(comma) msg.append(", "); + comma = true; + msg.append("transport"); + } + msg.append(" - successfully created in working directory."); + JOptionPane pane = new JOptionPane(msg); + JDialog dialog = pane.createDialog(frame, "Generation data"); + dialog.setVisible(true); } }); - Region region = (Region) dlg.run(); - if(region != null){ - setRegion(region, region.getName()); - frame.setTitle("OsmAnd Map Creator - " + f.getName()); - } else { - //frame.setTitle("OsmAnd Map Creator"); - } + + dlg.run(); + frame.setTitle("OsmAnd Map Creator - " + f.getName()); } catch (InterruptedException e1) { log.error("Interrupted", e1); - updateButtonsBar(); } catch (InvocationTargetException e1) { ExceptionHandler.handle("Exception during operation", e1.getCause()); } } - public void saveCountry(final File f){ + public void saveCountry(final File f, final OsmBaseStorage storage){ final OsmStorageWriter writer = new OsmStorageWriter(); try { final ProgressDialog dlg = new ProgressDialog(frame, "Saving osm file"); @@ -819,7 +460,7 @@ public class OsmExtractionUI implements IMapLocationListener { output.write('Z'); output = new CBZip2OutputStream(output); } - writer.saveStorage(output, region.getStorage(), null, false); + writer.saveStorage(output, storage, null, false); } finally { output.close(); } @@ -840,6 +481,258 @@ public class OsmExtractionUI implements IMapLocationListener { @Override public void locationChanged(final double newLatitude, final double newLongitude, Object source){ +// recalculateAmenities(newLatitude, newLongitude); + } + + + + + public class ExitListener extends WindowAdapter { + public void windowClosing(WindowEvent event) { + // save preferences + DataExtractionSettings settings = DataExtractionSettings.getSettings(); + settings.saveDefaultLocation(mapPanel.getLatitude(), mapPanel.getLongitude()); + settings.saveDefaultZoom(mapPanel.getZoom()); + settings.saveWindowBounds(frame.getBounds()); + System.exit(0); + } + } + + + + + + // OLD CODE +/* + + public JTree createTree(Container content) { + treePlaces.setEditable(true); + treePlaces.setCellEditor(new RegionCellEditor(treePlaces, (DefaultTreeCellRenderer) treePlaces.getCellRenderer())); + treePlaces.addTreeSelectionListener(new TreeSelectionListener() { + @Override + public void valueChanged(TreeSelectionEvent e) { + if (e.getPath() != null) { + if (e.getPath().getLastPathComponent() instanceof DataExtractionTreeNode) { + Object o = ((DataExtractionTreeNode) e.getPath().getLastPathComponent()).getModelObject(); + + if (o instanceof MapObject) { + MapObject c = (MapObject) o; + LatLon location = c.getLocation(); + if (location != null) { + if (o instanceof Street) { + DataTileManager ways = new DataTileManager(); + for (Way w : ((Street) o).getWayNodes()) { + LatLon l = w.getLatLon(); + ways.registerObject(l.getLatitude(), l.getLongitude(), w); + } + mapPanel.setPoints(ways); + mapPanel.requestFocus(); + } + mapPanel.setLatLon(location.getLatitude(), location.getLongitude()); + mapPanel.requestFocus(); + } + if (o instanceof TransportRoute) { + DataTileManager ways = new DataTileManager(); + for (Way w : ((TransportRoute) o).getWays()) { + LatLon l = w.getLatLon(); + ways.registerObject(l.getLatitude(), l.getLongitude(), w); + } + for (TransportStop w : ((TransportRoute) o).getBackwardStops()) { + LatLon l = w.getLocation(); + ways.registerObject(l.getLatitude(), l.getLongitude(), new Node(l.getLatitude(), l.getLongitude(), w + .getId())); + } + for (TransportStop w : ((TransportRoute) o).getForwardStops()) { + LatLon l = w.getLocation(); + ways.registerObject(l.getLatitude(), l.getLongitude(), new Node(l.getLatitude(), l.getLongitude(), w + .getId())); + } + mapPanel.setPoints(ways); + mapPanel.requestFocus(); + } + + } else if (o instanceof Entity) { + Entity c = (Entity) o; + LatLon latLon = c.getLatLon(); + if (latLon != null) { + mapPanel.setLatLon(latLon.getLatitude(), latLon.getLongitude()); + mapPanel.requestFocus(); + } + } + } + } + + } + }); + + treeModelListener = new TreeModelListener() { + public void treeNodesChanged(TreeModelEvent e) { + Object node = e.getTreePath().getLastPathComponent(); + if (e.getChildren() != null && e.getChildren().length > 0) { + node = e.getChildren()[0]; + } + if (node instanceof DataExtractionTreeNode) { + DataExtractionTreeNode n = ((DataExtractionTreeNode) node); + if (n.getModelObject() instanceof MapObject) { + MapObject r = (MapObject) n.getModelObject(); + String newName = n.getUserObject().toString(); + if (!r.getName().equals(newName)) { + r.setName(n.getUserObject().toString()); + } + if (r instanceof Street && !((Street) r).isRegisteredInCity()) { + DefaultMutableTreeNode parent = ((DefaultMutableTreeNode) n.getParent()); + parent.remove(n); + ((DefaultTreeModel) treePlaces.getModel()).nodeStructureChanged(parent); + } + } + } + } + + public void treeNodesInserted(TreeModelEvent e) { + } + + public void treeNodesRemoved(TreeModelEvent e) { + } + + public void treeStructureChanged(TreeModelEvent e) { + } + }; + treePlaces.getModel().addTreeModelListener(treeModelListener); + return treePlaces; + } + + public void fillPopupMenuWithActions(JPopupMenu menu) { + Action delete = new AbstractAction("Delete") { + private static final long serialVersionUID = 7476603434847164396L; + + public void actionPerformed(ActionEvent e) { + TreePath[] p = treePlaces.getSelectionPaths(); + if(p != null && + JOptionPane.OK_OPTION == + JOptionPane.showConfirmDialog(frame, "Are you sure about deleting " +p.length + " resources ? ")){ + for(TreePath path : treePlaces.getSelectionPaths()){ + Object node = path.getLastPathComponent(); + if (node instanceof DataExtractionTreeNode) { + DataExtractionTreeNode n = ((DataExtractionTreeNode) node); + if(n.getParent() instanceof DataExtractionTreeNode){ + DataExtractionTreeNode parent = ((DataExtractionTreeNode) n.getParent()); + boolean remove = false; + if (n.getModelObject() instanceof Street) { + ((City)parent.getModelObject()).unregisterStreet(((Street)n.getModelObject()).getName()); + remove = true; + } else if (n.getModelObject() instanceof Building) { + ((Street)parent.getModelObject()).getBuildings().remove(n.getModelObject()); + remove = true; + } else if (n.getModelObject() instanceof City) { + Region r = (Region) ((DataExtractionTreeNode)parent.getParent()).getModelObject(); + r.unregisterCity((City) n.getModelObject()); + remove = true; + } else if (n.getModelObject() instanceof Amenity) { + Region r = (Region) ((DataExtractionTreeNode)parent.getParent().getParent()).getModelObject(); + Amenity am = (Amenity) n.getModelObject(); + r.getAmenityManager().unregisterObject(am.getLocation().getLatitude(), am.getLocation().getLongitude(), am); + remove = true; + } + if(remove){ + parent.remove(n); + ((DefaultTreeModel) treePlaces.getModel()).nodeStructureChanged(parent); + } + } + } + } + + } + } + }; + menu.add(delete); + Action rename= new AbstractAction("Rename") { + private static final long serialVersionUID = -8257594433235073767L; + + public void actionPerformed(ActionEvent e) { + TreePath path = treePlaces.getSelectionPath(); + if(path != null){ + treePlaces.startEditingAtPath(path); + } + } + }; + menu.add(rename); + } + + + public void setRegion(Region region, String name){ + if (this.region == region) { + return; + } + this.region = region; + DefaultMutableTreeNode root = new DataExtractionTreeNode(name, region); + if (region != null) { + amenitiesTree = new DataExtractionTreeNode("Amenities", region); + amenitiesTree.add(new DataExtractionTreeNode("First 15", region)); + for (AmenityType type : AmenityType.values()) { + amenitiesTree.add(new DataExtractionTreeNode(Algoritms.capitalizeFirstLetterAndLowercase(type.toString()), type)); + } + root.add(amenitiesTree); + + DataExtractionTreeNode transport = new DataExtractionTreeNode("Transport", region); + root.add(transport); + for(String s : region.getTransportRoutes().keySet()){ + DataExtractionTreeNode trRoute = new DataExtractionTreeNode(s, s); + transport.add(trRoute); + List list = region.getTransportRoutes().get(s); + for(TransportRoute r : list){ + DataExtractionTreeNode route = new DataExtractionTreeNode(r.getRef(), r); + trRoute.add(route); + } + + } + + for (CityType t : CityType.values()) { + DefaultMutableTreeNode cityTree = new DataExtractionTreeNode(Algoritms.capitalizeFirstLetterAndLowercase(t.toString()), t); + root.add(cityTree); + for (City ct : region.getCitiesByType(t)) { + DefaultMutableTreeNode cityNodeTree = new DataExtractionTreeNode(ct.getName(), ct); + cityTree.add(cityNodeTree); + + for (Street str : ct.getStreets()) { + DefaultMutableTreeNode strTree = new DataExtractionTreeNode(str.getName(), str); + cityNodeTree.add(strTree); + for (Building b : str.getBuildings()) { + DefaultMutableTreeNode building = new DataExtractionTreeNode(b.getName(), b); + strTree.add(building); + } + } + } + } + } + + if (searchList != null) { + updateListCities(region, searchTextField.getText(), searchList); + } + mapPanel.repaint(); + DefaultTreeModel newModel = new DefaultTreeModel(root, false); + newModel.addTreeModelListener(treeModelListener); + treePlaces.setModel(newModel); + + updateButtonsBar(); + locationChanged(mapPanel.getLatitude(), mapPanel.getLongitude(), this); + } + + public static class DataExtractionTreeNode extends DefaultMutableTreeNode { + private static final long serialVersionUID = 1L; + private final Object modelObject; + + public DataExtractionTreeNode(String name, Object modelObject){ + super(name); + this.modelObject = modelObject; + } + + public Object getModelObject(){ + return modelObject; + } + + } + + private void recalculateAmenities(final double newLatitude, final double newLongitude) { if (amenitiesTree != null) { Region reg = (Region) amenitiesTree.getModelObject(); List closestAmenities = reg.getAmenityManager().getClosestObjects(newLatitude, newLongitude, 0, 5); @@ -878,8 +771,6 @@ public class OsmExtractionUI implements IMapLocationListener { ((DefaultMutableTreeNode) amenitiesTree.getChildAt(0)).add(new DataExtractionTreeNode(str, n)); ((DefaultTreeModel)treePlaces.getModel()).nodeStructureChanged(amenitiesTree.getChildAt(0)); } - - } } @@ -897,23 +788,7 @@ public class OsmExtractionUI implements IMapLocationListener { } jList.setListData(names); } - - - public static class DataExtractionTreeNode extends DefaultMutableTreeNode { - private static final long serialVersionUID = 1L; - private final Object modelObject; - public DataExtractionTreeNode(String name, Object modelObject){ - super(name); - this.modelObject = modelObject; - } - - public Object getModelObject(){ - return modelObject; - } - - } - public static class RegionCellEditor extends DefaultTreeCellEditor { public RegionCellEditor(JTree tree, DefaultTreeCellRenderer renderer) { @@ -939,17 +814,6 @@ public class OsmExtractionUI implements IMapLocationListener { } } - public class ExitListener extends WindowAdapter { - public void windowClosing(WindowEvent event) { - // save preferences - DataExtractionSettings settings = DataExtractionSettings.getSettings(); - settings.saveDefaultLocation(mapPanel.getLatitude(), mapPanel.getLongitude()); - settings.saveDefaultZoom(mapPanel.getZoom()); - settings.saveWindowBounds(frame.getBounds()); - System.exit(0); - } - } - private class PopupTrigger extends MouseAdapter { private final JPopupMenu popupMenu; @@ -973,6 +837,7 @@ public class OsmExtractionUI implements IMapLocationListener { } } } +*/ + - } diff --git a/OsmAnd/src/net/osmand/RegionAddressRepository.java b/OsmAnd/src/net/osmand/RegionAddressRepository.java index ac0c09af2b..f67877058c 100644 --- a/OsmAnd/src/net/osmand/RegionAddressRepository.java +++ b/OsmAnd/src/net/osmand/RegionAddressRepository.java @@ -17,9 +17,9 @@ import java.util.TreeSet; import net.osmand.data.Building; import net.osmand.data.City; +import net.osmand.data.CityComparator; import net.osmand.data.MapObject; import net.osmand.data.PostCode; -import net.osmand.data.Region; import net.osmand.data.Street; import net.osmand.data.City.CityType; import net.osmand.data.index.IndexConstants; @@ -183,7 +183,7 @@ public class RegionAddressRepository { } // sort cities ArrayList list = new ArrayList(cities.values()); - Collections.sort(list, new Region.CityComparator(useEnglishNames)); + Collections.sort(list, new CityComparator(useEnglishNames)); cities.clear(); cityTypes.clear(); for(City c : list){