diff --git a/DataExtractionOSM/src/net/osmand/data/index/DataIndexReader.java b/DataExtractionOSM/src/net/osmand/data/index/DataIndexReader.java deleted file mode 100644 index 3fd0a343bc..0000000000 --- a/DataExtractionOSM/src/net/osmand/data/index/DataIndexReader.java +++ /dev/null @@ -1,219 +0,0 @@ -package net.osmand.data.index; - -import java.io.File; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -import net.osmand.data.Building; -import net.osmand.data.City; -import net.osmand.data.Street; -import net.osmand.data.City.CityType; -import net.osmand.osm.Node; -import net.osmand.LogUtil; - -import org.apache.commons.logging.Log; - - -public class DataIndexReader { - private static final Log log = LogUtil.getLog(DataIndexReader.class); - - public Connection getConnection(File file) throws SQLException{ - try { - Class.forName("org.sqlite.JDBC"); //$NON-NLS-1$ - } catch (ClassNotFoundException e) { - log.error("Illegal configuration", e); //$NON-NLS-1$ - throw new IllegalStateException(e); - } - return DriverManager.getConnection("jdbc:sqlite:"+file.getAbsolutePath()); //$NON-NLS-1$ - } - - - public List readCities(Connection c) throws SQLException{ - List cities = new ArrayList(); - Statement stat = c.createStatement(); - ResultSet set = stat.executeQuery("select id, latitude, longitude , name , name_en , city_type from city"); //$NON-NLS-1$ - while(set.next()){ - City city = new City(CityType.valueFromString(set.getString(6))); - city.setName(set.getString(4)); - city.setEnName(set.getString(5)); - city.setLocation(set.getDouble(2), - set.getDouble(3)); - city.setId(set.getLong(1)); - cities.add(city); - - } - set.close(); - stat.close(); - return cities; - } - - - - - - public PreparedStatement getStreetsBuildingPreparedStatement(Connection c) throws SQLException{ - return c.prepareStatement("SELECT A.id, A.name, A.name_en, A.latitude, A.longitude, "+ //$NON-NLS-1$ - "B.id, B.name, B.name_en, B.latitude, B.longitude, B.postcode "+ //$NON-NLS-1$ - "FROM street A left JOIN building B ON B.street = A.id WHERE A.city = ?"); //$NON-NLS-1$ - } - - public List readStreetsBuildings(PreparedStatement streetBuildingsStat, City city, List streets) throws SQLException { - return readStreetsBuildings(streetBuildingsStat, city, streets, null, null, null); - } - - public PreparedStatement getStreetsWayNodesPreparedStatement(Connection c) throws SQLException{ - return c.prepareStatement("SELECT A.id, A.latitude, A.longitude FROM street_node A WHERE A.street = ? "); //$NON-NLS-1$ - } - - public List readStreetsBuildings(PreparedStatement streetBuildingsStat, City city, List streets, - PreparedStatement waynodesStat, Map> streetNodes, List citySuburbs) throws SQLException { - Map visitedStreets = new LinkedHashMap(); - //read streets for city - readStreatsByBuildingsForCity(streetBuildingsStat, city, streets, - waynodesStat, streetNodes, visitedStreets); - //read streets for suburbs of the city - if (citySuburbs != null) { - for (City suburb : citySuburbs) { - readStreatsByBuildingsForCity(streetBuildingsStat, suburb, streets, waynodesStat, streetNodes, visitedStreets); - } - } - return streets; - } - - - private void readStreatsByBuildingsForCity( - PreparedStatement streetBuildingsStat, City city, - List streets, PreparedStatement waynodesStat, - Map> streetNodes, - Map visitedStreets) throws SQLException { - streetBuildingsStat.setLong(1, city.getId()); - ResultSet set = streetBuildingsStat.executeQuery(); - while (set.next()) { - long streetId = set.getLong(1); - if (!visitedStreets.containsKey(streetId)) { - Street street = new Street(null); - street.setName(set.getString(2)); - street.setEnName(set.getString(3)); - street.setLocation(set.getDouble(4), set.getDouble(5)); - street.setId(streetId); - streets.add(street); - visitedStreets.put(streetId, street); - if (waynodesStat != null && streetNodes != null) { - ArrayList list = new ArrayList(); - streetNodes.put(street, list); - waynodesStat.setLong(1, street.getId()); - ResultSet rs = waynodesStat.executeQuery(); - while (rs.next()) { - list.add(new Node(rs.getDouble(2), rs.getDouble(3), rs.getLong(1))); - } - rs.close(); - } - } - if (set.getObject(6) != null) { - Street s = visitedStreets.get(streetId); - Building b = new Building(); - b.setId(set.getLong(6)); - b.setName(set.getString(7)); - b.setEnName(set.getString(8)); - b.setLocation(set.getDouble(9), set.getDouble(10)); - b.setPostcode(set.getString(11)); - s.registerBuilding(b); - } - } - - set.close(); - } - - public PreparedStatement getStreetsPreparedStatement(Connection c) throws SQLException{ - return c.prepareStatement("select id, latitude, longitude , name, name_en, city from street where city = ?"); //$NON-NLS-1$ - } - - public List readStreets(PreparedStatement streetsStat, City city, List streets) throws SQLException{ - streetsStat.setLong(1, city.getId()); - ResultSet set = streetsStat.executeQuery(); - while(set.next()){ - Street street = new Street(city); - street.setName(set.getString(4)); - street.setEnName(set.getString(5)); - street.setLocation(set.getDouble(2), - set.getDouble(3)); - street.setId(set.getLong(1)); - streets.add(street); - } - set.close(); - return streets; - } - - public PreparedStatement getBuildingsPreparedStatement(Connection c) throws SQLException{ - return c.prepareStatement("select id, latitude, longitude, name, name_en, street, postcode from building where street = ?"); //$NON-NLS-1$ - } - - - public List readBuildings(PreparedStatement buildingStat, Street street, List buildings) throws SQLException{ - buildingStat.setLong(1, street.getId()); - ResultSet set = buildingStat.executeQuery(); - while(set.next()){ - Building building = new Building(); - building.setName(set.getString(4)); - building.setEnName(set.getString(5)); - building.setLocation(set.getDouble(2), - set.getDouble(3)); - building.setId(set.getLong(1)); - building.setPostcode(set.getString(7)); - buildings.add(building); - } - set.close(); - return buildings; - } - - public void testIndex(File f) throws SQLException { - Connection c = getConnection(f); - try { - ArrayList streets = new ArrayList(); -// ArrayList buildings = new ArrayList(); - PreparedStatement streetstat = getStreetsBuildingPreparedStatement(c); - int countCity = 0; - int countStreets = 0; - int countBuildings = 0; - List cities = readCities(c); - for (City city : cities) { - countCity ++; -// System.out.println("CITY " + city.getName()); //$NON-NLS-1$ - if(city.getType() != CityType.CITY){ - continue; - } - streets.clear(); -// long time = System.currentTimeMillis(); - readStreetsBuildings(streetstat, city, streets); - if(!streets.isEmpty()){ - System.out.println(city.getName()); - } else { - System.out.print("."); - } - for (Street s : streets) { - countStreets ++; -// System.out.println("\tSTREET " + s.getName()); //$NON-NLS-1$ -// buildings.clear(); - countBuildings += s.getBuildings().size(); -// for (Building b : s.getBuildings()) { -// countBuildings ++; -// System.out.println("\t\tBULDING " + b.getName()); //$NON-NLS-1$ -// } - } - - } - System.out.println(countCity + " " + countStreets + " " + countBuildings); - } finally { - c.close(); - } - } - -} diff --git a/DataExtractionOSM/src/net/osmand/data/index/DataIndexWriter.java b/DataExtractionOSM/src/net/osmand/data/index/DataIndexWriter.java deleted file mode 100644 index b3e68d2044..0000000000 --- a/DataExtractionOSM/src/net/osmand/data/index/DataIndexWriter.java +++ /dev/null @@ -1,344 +0,0 @@ -package net.osmand.data.index; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import net.osmand.Algoritms; -import net.osmand.data.Amenity; -import net.osmand.data.AmenityType; -import net.osmand.data.Building; -import net.osmand.data.City; -import net.osmand.data.City.CityType; -import net.osmand.data.preparation.DBDialect; -import net.osmand.osm.Entity; -import net.osmand.osm.LatLon; -import net.osmand.osm.MapUtils; -import net.osmand.osm.Node; -import net.osmand.osm.Relation; -import net.osmand.osm.Way; -import rtree.IllegalValueException; -import rtree.LeafElement; -import rtree.RTree; -import rtree.RTreeInsertException; -import rtree.Rect; - - - -public class DataIndexWriter { - - private static final int BATCH_SIZE = 1000; - - - - public static PreparedStatement getStreetNodeInsertPreparedStatement(Connection conn) throws SQLException { - assert IndexConstants.STREET_NODE_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - return conn.prepareStatement("insert into street_node (id, latitude, longitude, street, way) values (?, ?, ?, ?, ?)"); - } - - public static void writeStreetWayNodes(PreparedStatement prepStreetNode, Map count, Long streetId, Way way, int batchSize) - throws SQLException { - assert IndexConstants.STREET_NODE_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - for (Node n : way.getNodes()) { - if (n == null) { - continue; - } - prepStreetNode.setLong(1, n.getId()); - prepStreetNode.setDouble(2, n.getLatitude()); - prepStreetNode.setDouble(3, n.getLongitude()); - prepStreetNode.setLong(5, way.getId()); - prepStreetNode.setLong(4, streetId); - addBatch(count, prepStreetNode, BATCH_SIZE); - } - } - - public static PreparedStatement getBuildingInsertPreparedStatement(Connection conn) throws SQLException { - assert IndexConstants.BUILDING_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - return conn.prepareStatement("insert into building (id, latitude, longitude, name, name_en, street, postcode) values (?, ?, ?, ?, ?, ?, ?)"); - } - - public static void writeBuilding(PreparedStatement prepBuilding, Map count, Long streetId, - Building building, int batchSize) - throws SQLException { - assert IndexConstants.BUILDING_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - prepBuilding.setLong(1, building.getId()); - prepBuilding.setDouble(2, building.getLocation().getLatitude()); - prepBuilding.setDouble(3, building.getLocation().getLongitude()); - prepBuilding.setString(4, building.getName()); - prepBuilding.setString(5, building.getEnName()); - prepBuilding.setLong(6, streetId); - prepBuilding.setString(7, building.getPostcode() == null ? null : building.getPostcode().toUpperCase()); - - addBatch(count, prepBuilding); - } - - - public static PreparedStatement getSearchStreetPreparedStatement(Connection mapConnection) throws SQLException { - assert IndexConstants.STREET_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - return mapConnection.prepareStatement("SELECT ID FROM street WHERE ? = city AND ? = name"); - } - - public static PreparedStatement getSearchBuildingPreparedStatement(Connection mapConnection) throws SQLException { - assert IndexConstants.BUILDING_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - return mapConnection.prepareStatement("SELECT id FROM building where ? = id"); - } - - public static PreparedStatement getStreeNodeSearchPreparedStatement(Connection mapConnection) throws SQLException { - assert IndexConstants.STREET_NODE_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - return mapConnection.prepareStatement("SELECT way FROM street_node WHERE ? = way"); - } - - public static PreparedStatement getUpdateBuildingPostcodePreparedStatement(Connection mapConnection) throws SQLException { - assert IndexConstants.BUILDING_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - return mapConnection.prepareStatement("UPDATE building SET postcode = ? WHERE id = ?"); - } - - - public static PreparedStatement getCityInsertPreparedStatement(Connection conn) throws SQLException{ - assert IndexConstants.CITY_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - return conn.prepareStatement("insert into city (id, latitude, longitude, name, name_en, city_type) values (?, ?, ?, ?, ?, ?)"); - } - - - public static void writeCity(PreparedStatement prepCity, Map count, City city, int batchSize) throws SQLException { - assert IndexConstants.CITY_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - prepCity.setLong(1, city.getId()); - prepCity.setDouble(2, city.getLocation().getLatitude()); - prepCity.setDouble(3, city.getLocation().getLongitude()); - prepCity.setString(4, city.getName()); - prepCity.setString(5, city.getEnName()); - prepCity.setString(6, CityType.valueToString(city.getType())); - addBatch(count, prepCity, batchSize); - } - - public static PreparedStatement getStreetInsertPreparedStatement(Connection conn) throws SQLException{ - assert IndexConstants.STREET_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - return conn.prepareStatement("insert into street (id, latitude, longitude, name, name_en, city) values (?, ?, ?, ?, ?, ?)"); - } - - public static void insertStreetData(PreparedStatement addressStreetStat, long id, String name, String nameEn, double latitude, - double longitude, Long cityId) throws SQLException { - assert IndexConstants.STREET_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - addressStreetStat.setLong(1, id); - addressStreetStat.setString(4, name); - addressStreetStat.setString(5, nameEn); - addressStreetStat.setDouble(2, latitude); - addressStreetStat.setDouble(3, longitude); - addressStreetStat.setLong(6, cityId); - } - - - public static void createAddressIndexStructure(Connection conn, DBDialect dialect) throws SQLException{ - Statement stat = conn.createStatement(); - assert IndexConstants.CITY_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - assert IndexConstants.STREET_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - assert IndexConstants.STREET_NODE_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - assert IndexConstants.STREET_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - - stat.executeUpdate("create table city (id bigint primary key, latitude double, longitude double, " + - "name varchar(255), name_en varchar(255), city_type varchar(32))"); - stat.executeUpdate("create index city_ind on city (id, city_type)"); - - stat.executeUpdate("create table street (id bigint primary key, latitude double, longitude double, " + - "name varchar(255), name_en varchar(255), city bigint)"); - stat.executeUpdate("create index street_city on street (city)"); - stat.executeUpdate("create index street_id on street (id)"); - // create index on name ? - - stat.executeUpdate("create table building (id bigint, latitude double, longitude double, " + - "name varchar(255), name_en varchar(255), street bigint, postcode varchar(255), primary key(street, id))"); - stat.executeUpdate("create index building_postcode on building (postcode)"); - stat.executeUpdate("create index building_street on building (street)"); - stat.executeUpdate("create index building_id on building (id)"); - - - stat.executeUpdate("create table street_node (id bigint, latitude double, longitude double, " + - "street bigint, way bigint)"); - stat.executeUpdate("create index street_node_street on street_node (street)"); - stat.executeUpdate("create index street_node_way on street_node (way)"); - - if(dialect == DBDialect.SQLITE){ - stat.execute("PRAGMA user_version = " + IndexConstants.ADDRESS_TABLE_VERSION); //$NON-NLS-1$ - } - stat.close(); - } - - - - - - public static void createMapIndexStructure(Connection conn) throws SQLException{ - Statement stat = conn.createStatement(); - assert IndexConstants.BINARY_MAP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - assert IndexConstants.LOW_LEVEL_MAP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - stat.executeUpdate("create table binary_map_objects (id bigint primary key, name varchar(255), " + - "types binary, restrictions binary, nodes binary, highway int)"); - stat.executeUpdate("create index binary_map_objects_ind on binary_map_objects (id)"); - - stat.executeUpdate("create table low_level_map_objects (id bigint primary key, start_node bigint, " + - "end_node bigint, name varchar(255), nodes binary, type bigint, level smallint)"); - stat.executeUpdate("create index low_level_map_objects_ind on low_level_map_objects (id)"); - stat.executeUpdate("create index low_level_map_objects_ind_st on low_level_map_objects (start_node, type)"); - stat.executeUpdate("create index low_level_map_objects_ind_end on low_level_map_objects (end_node, type)"); - stat.close(); - } - - public static PreparedStatement createStatementMapBinaryInsert(Connection conn) throws SQLException{ - assert IndexConstants.BINARY_MAP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - return conn.prepareStatement("insert into binary_map_objects(id, name, types, restrictions, nodes, highway) values(?, ?, ?, ?, ?, ?)"); - } - - public static PreparedStatement createStatementLowLevelMapBinaryInsert(Connection conn) throws SQLException{ - assert IndexConstants.LOW_LEVEL_MAP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - return conn.prepareStatement("insert into low_level_map_objects(id, start_node, end_node, name, nodes, type, level) values(?, ?, ?, ?, ?, ?, ?)"); - } - - public static void insertLowLevelMapBinaryObject(Map statements, - PreparedStatement mapLowLevelBinaryStat, int level,long types, long id, List nodes, String name) throws SQLException{ - assert IndexConstants.LOW_LEVEL_MAP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - boolean first = true; - long firstId = -1; - long lastId = -1; - ByteArrayOutputStream bnodes = new ByteArrayOutputStream(); - try { - for (Node n : nodes) { - if (n != null) { - if (first) { - firstId = n.getId(); - first = false; - } - lastId = n.getId(); - Algoritms.writeInt(bnodes, Float.floatToRawIntBits((float) n.getLatitude())); - Algoritms.writeInt(bnodes, Float.floatToRawIntBits((float) n.getLongitude())); - } - } - } catch (IOException e) { - throw new IllegalStateException(e); - } - if(firstId == -1){ - return; - } - // conn.prepareStatement("insert into binary_map_objects(id, name, types, restrictions, nodes, highway) values(?, ?, ?, ?, ?, ?)"); - mapLowLevelBinaryStat.setLong(1, id); - mapLowLevelBinaryStat.setLong(2, firstId); - mapLowLevelBinaryStat.setLong(3, lastId); - mapLowLevelBinaryStat.setString(4, name); - mapLowLevelBinaryStat.setBytes(5, bnodes.toByteArray()); - mapLowLevelBinaryStat.setLong(6, types); - mapLowLevelBinaryStat.setShort(7, (short) level); - - addBatch(statements, mapLowLevelBinaryStat); - } - public static void insertBinaryMapRenderObjectIndex(Map statements, - PreparedStatement mapBinaryStat, RTree mapTree, Entity e, String name, - long id, int type, List typeUse, int highwayAttributes, List restrictions, - boolean inversePath, boolean writeAsPoint, boolean commit) throws SQLException { - if(e instanceof Relation){ - throw new IllegalArgumentException(); - } - boolean init = false; - int minX = Integer.MAX_VALUE; - int maxX = 0; - int minY = Integer.MAX_VALUE; - int maxY = 0; - Collection nodes; - if (e instanceof Way) { - if (writeAsPoint) { - LatLon center = MapUtils.getCenter(((Way) e)); - nodes = Collections.singleton(new Node(center.getLatitude(), center.getLongitude(), -1)); - } else { - nodes = ((Way) e).getNodes(); - } - } else { - nodes = Collections.singleton((Node) e); - } - if(inversePath){ - nodes = new ArrayList(nodes); - Collections.reverse((List) nodes); - } - - ByteArrayOutputStream bnodes = new ByteArrayOutputStream(); - ByteArrayOutputStream btypes = new ByteArrayOutputStream(); - ByteArrayOutputStream brestrictions = new ByteArrayOutputStream(); - - try { - Algoritms.writeSmallInt(btypes, type); - for (Integer i : typeUse) { - Algoritms.writeSmallInt(btypes, i); - } - for (Long i : restrictions) { - Algoritms.writeLongInt(brestrictions, i); - } - - for (Node n : nodes) { - if (n != null) { - int y = MapUtils.get31TileNumberY(n.getLatitude()); - int x = MapUtils.get31TileNumberX(n.getLongitude()); - minX = Math.min(minX, x); - maxX = Math.max(maxX, x); - minY = Math.min(minY, y); - maxY = Math.max(maxY, y); - init = true; - Algoritms.writeInt(bnodes, x); - Algoritms.writeInt(bnodes, y); - } - } - } catch (IOException es) { - throw new IllegalStateException(es); - } - if (init) { - assert IndexConstants.BINARY_MAP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - // conn.prepareStatement("insert into binary_map_objects(id, name, types, restrictions, nodes, highway) values(?, ?, ?, ?, ?, ?)"); - mapBinaryStat.setLong(1, id); - mapBinaryStat.setString(2, name); - mapBinaryStat.setBytes(3, btypes.toByteArray()); - mapBinaryStat.setBytes(4, brestrictions.toByteArray()); - mapBinaryStat.setBytes(5, bnodes.toByteArray()); - mapBinaryStat.setInt(6, highwayAttributes); - - addBatch(statements, mapBinaryStat, commit); - try { - mapTree.insert(new LeafElement(new Rect(minX, minY, maxX, maxY), id)); - } catch (RTreeInsertException e1) { - throw new IllegalArgumentException(e1); - } catch (IllegalValueException e1) { - throw new IllegalArgumentException(e1); - } - } - } - public static void addBatch(Map count, PreparedStatement p) throws SQLException { - addBatch(count, p, BATCH_SIZE, true); - } - - public static void addBatch(Map count, PreparedStatement p, boolean commit) throws SQLException{ - addBatch(count, p, BATCH_SIZE, commit); - } - - public static void addBatch(Map count, PreparedStatement p, int batchSize) throws SQLException{ - addBatch(count, p, batchSize, true); - } - - public static void addBatch(Map count, PreparedStatement p, int batchSize, boolean commit) throws SQLException{ - p.addBatch(); - if(count.get(p) >= batchSize){ - p.executeBatch(); - if(commit){ - p.getConnection().commit(); - } - count.put(p, 0); - } else { - count.put(p, count.get(p) + 1); - } - } - - - -} diff --git a/DataExtractionOSM/src/net/osmand/data/index/IndexBatchCreator.java b/DataExtractionOSM/src/net/osmand/data/index/IndexBatchCreator.java index 71e549e17a..7a0581db72 100644 --- a/DataExtractionOSM/src/net/osmand/data/index/IndexBatchCreator.java +++ b/DataExtractionOSM/src/net/osmand/data/index/IndexBatchCreator.java @@ -484,10 +484,6 @@ public class IndexBatchCreator { String regionName = f.getName().substring(0, f.getName().lastIndexOf('_', f.getName().indexOf('.'))); if(f.getName().endsWith(IndexConstants.POI_INDEX_EXT) || f.getName().endsWith(IndexConstants.POI_INDEX_EXT_ZIP)){ summary = "POI index for " ; - } else if(f.getName().endsWith(IndexConstants.ADDRESS_INDEX_EXT) || f.getName().endsWith(IndexConstants.ADDRESS_INDEX_EXT_ZIP)){ - summary = "Address index for " ; - } else if(f.getName().endsWith(IndexConstants.TRANSPORT_INDEX_EXT) || f.getName().endsWith(IndexConstants.TRANSPORT_INDEX_EXT_ZIP)){ - summary = "Transport index for "; } else if(f.getName().endsWith(IndexConstants.BINARY_MAP_INDEX_EXT) || f.getName().endsWith(IndexConstants.BINARY_MAP_INDEX_EXT_ZIP)){ boolean addr = indexAddress; boolean trans = indexTransport; diff --git a/DataExtractionOSM/src/net/osmand/data/index/IndexConstants.java b/DataExtractionOSM/src/net/osmand/data/index/IndexConstants.java index fa91382f60..171a0d6d26 100644 --- a/DataExtractionOSM/src/net/osmand/data/index/IndexConstants.java +++ b/DataExtractionOSM/src/net/osmand/data/index/IndexConstants.java @@ -5,52 +5,23 @@ public class IndexConstants { // Important : Every time you change schema of db upgrade version!!! // If you want that new application support old index : put upgrade code in android app ResourceManager - - public final static int POI_TABLE_VERSION = 1; public final static int BINARY_MAP_VERSION = 1; // starts with 1 public final static int VOICE_VERSION = 0; - // these indexes are deprecated - public final static int TRANSPORT_TABLE_VERSION = 0; - public final static int ADDRESS_TABLE_VERSION = 1; - - public static final String POI_INDEX_DIR = "POI/"; //$NON-NLS-1$ - public static final String ADDRESS_INDEX_DIR = "Address/"; //$NON-NLS-1$ public static final String VOICE_INDEX_DIR = "voice/"; //$NON-NLS-1$ - public static final String TRANSPORT_INDEX_DIR = "Transport/"; //$NON-NLS-1$ public static final String RENDERERS_DIR = "rendering/"; //$NON-NLS-1$ public static final String POI_INDEX_EXT = ".poi.odb"; //$NON-NLS-1$ - public static final String ADDRESS_INDEX_EXT = ".addr.odb"; //$NON-NLS-1$ - public static final String TRANSPORT_INDEX_EXT = ".trans.odb"; //$NON-NLS-1$ public static final String BINARY_MAP_INDEX_EXT = ".obf"; //$NON-NLS-1$ public static final String POI_INDEX_EXT_ZIP = ".poi.zip"; //$NON-NLS-1$ - public static final String ADDRESS_INDEX_EXT_ZIP = ".addr.zip"; //$NON-NLS-1$ - public static final String TRANSPORT_INDEX_EXT_ZIP = ".trans.zip"; //$NON-NLS-1$ public static final String VOICE_INDEX_EXT_ZIP = ".voice.zip"; //$NON-NLS-1$ public static final String BINARY_MAP_INDEX_EXT_ZIP = ".obf.zip"; //$NON-NLS-1$ public static final String RENDERER_INDEX_EXT = ".render.xml"; //$NON-NLS-1$ - - public final static String STREET_NODE_TABLE = "street_node"; //$NON-NLS-1$ - public final static String STREET_TABLE = "street"; //$NON-NLS-1$ - public final static String CITY_TABLE = "city"; //$NON-NLS-1$ - public final static String BUILDING_TABLE = "building"; //$NON-NLS-1$ - - public final static String POI_TABLE = "poi"; //$NON-NLS-1$ - public final static String BINARY_MAP_TABLE = "binary_map_objects"; //$NON-NLS-1$ - public final static String LOW_LEVEL_MAP_TABLE = "low_level_map_objects"; //$NON-NLS-1$ - - public final static String TRANSPORT_STOP_TABLE = "transport_stop"; //$NON-NLS-1$ - public final static String TRANSPORT_ROUTE_STOP_TABLE = "transport_route_stop"; //$NON-NLS-1$ - public final static String TRANSPORT_ROUTE_TABLE = "transport_route"; //$NON-NLS-1$ - - - } diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/AbstractIndexPartCreator.java b/DataExtractionOSM/src/net/osmand/data/preparation/AbstractIndexPartCreator.java new file mode 100644 index 0000000000..954d26e942 --- /dev/null +++ b/DataExtractionOSM/src/net/osmand/data/preparation/AbstractIndexPartCreator.java @@ -0,0 +1,105 @@ +package net.osmand.data.preparation; + +import java.io.File; +import java.io.IOException; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import rtree.Element; +import rtree.Pack; +import rtree.RTree; +import rtree.RTreeException; + +public class AbstractIndexPartCreator { + + private final static Log log = LogFactory.getLog(AbstractIndexPartCreator.class); + protected int BATCH_SIZE = 1000; + + protected Map pStatements = new LinkedHashMap(); + + protected void closePreparedStatements(PreparedStatement... preparedStatements) throws SQLException { + for (PreparedStatement p : preparedStatements) { + if (p != null) { + p.executeBatch(); + p.close(); + pStatements.remove(p); + } + } + } + + protected void closeAllPreparedStatements() throws SQLException { + for (PreparedStatement p : pStatements.keySet()) { + if (pStatements.get(p) > 0) { + p.executeBatch(); + } + p.close(); + } + } + + protected void addBatch(Map count, PreparedStatement p) throws SQLException { + addBatch(count, p, BATCH_SIZE, true); + } + + protected void addBatch(Map count, PreparedStatement p, boolean commit) throws SQLException{ + addBatch(count, p, BATCH_SIZE, commit); + } + + protected void addBatch(Map count, PreparedStatement p, int batchSize) throws SQLException{ + addBatch(count, p, batchSize, true); + } + + protected void addBatch(Map count, PreparedStatement p, int batchSize, boolean commit) throws SQLException{ + p.addBatch(); + if(count.get(p) >= batchSize){ + p.executeBatch(); + if(commit){ + p.getConnection().commit(); + } + count.put(p, 0); + } else { + count.put(p, count.get(p) + 1); + } + } + + protected boolean nodeIsLastSubTree(RTree tree, long ptr) throws RTreeException { + rtree.Node parent = tree.getReadNode(ptr); + Element[] e = parent.getAllElements(); + for (int i = 0; i < parent.getTotalElements(); i++) { + if (e[i].getElementType() != rtree.Node.LEAF_NODE) { + return false; + } + } + return true; + + } + + protected RTree packRtreeFile(RTree tree, String nonPackFileName, String packFileName) throws IOException { + try { + assert rtree.Node.MAX < 50 : "It is better for search performance"; //$NON-NLS-1$ + tree.flush(); + File file = new File(packFileName); + if (file.exists()) { + file.delete(); + } + long rootIndex = tree.getFileHdr().getRootIndex(); + if (!nodeIsLastSubTree(tree, rootIndex)) { + // there is a bug for small files in packing method + new Pack().packTree(tree, packFileName); + tree.getFileHdr().getFile().close(); + file = new File(nonPackFileName); + file.delete(); + + return new RTree(packFileName); + } + } catch (RTreeException e) { + log.error("Error flushing", e); //$NON-NLS-1$ + throw new IOException(e); + } + return tree; + } +} diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/DBDialect.java b/DataExtractionOSM/src/net/osmand/data/preparation/DBDialect.java index eb16911330..761de54aa8 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/DBDialect.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/DBDialect.java @@ -1,9 +1,13 @@ package net.osmand.data.preparation; import java.io.File; +import java.sql.Connection; +import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; +import org.apache.commons.logging.Log; + import net.osmand.Algoritms; public enum DBDialect { @@ -45,4 +49,42 @@ public enum DBDialect { } } + + protected Connection getDatabaseConnection(String fileName, Log log) throws SQLException { + if (DBDialect.SQLITE == this) { + try { + Class.forName("org.sqlite.JDBC"); + } catch (ClassNotFoundException e) { + log.error("Illegal configuration", e); + throw new IllegalStateException(e); + } + Connection connection = DriverManager.getConnection("jdbc:sqlite:" + fileName); + Statement statement = connection.createStatement(); + statement.executeUpdate("PRAGMA synchronous = 0"); + statement.close(); + return connection; + } else if (DBDialect.DERBY == this) { + try { + Class.forName("org.apache.derby.jdbc.EmbeddedDriver"); + } catch (ClassNotFoundException e) { + log.error("Illegal configuration", e); + throw new IllegalStateException(e); + } + Connection conn = DriverManager.getConnection("jdbc:derby:" + fileName + ";create=true"); + conn.setAutoCommit(false); + return conn; + } else if (DBDialect.H2 == this) { + try { + Class.forName("org.h2.Driver"); + } catch (ClassNotFoundException e) { + log.error("Illegal configuration", e); + throw new IllegalStateException(e); + } + + return DriverManager.getConnection("jdbc:h2:file:" + fileName); + } else { + throw new UnsupportedOperationException(); + } + + } } diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/IndexAddressCreator.java b/DataExtractionOSM/src/net/osmand/data/preparation/IndexAddressCreator.java new file mode 100644 index 0000000000..d397ea4547 --- /dev/null +++ b/DataExtractionOSM/src/net/osmand/data/preparation/IndexAddressCreator.java @@ -0,0 +1,905 @@ +package net.osmand.data.preparation; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.text.Collator; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import net.osmand.Algoritms; +import net.osmand.IProgress; +import net.osmand.binary.BinaryMapIndexWriter; +import net.osmand.data.Boundary; +import net.osmand.data.Building; +import net.osmand.data.City; +import net.osmand.data.DataTileManager; +import net.osmand.data.Street; +import net.osmand.data.City.CityType; +import net.osmand.osm.Entity; +import net.osmand.osm.LatLon; +import net.osmand.osm.MapUtils; +import net.osmand.osm.Node; +import net.osmand.osm.Relation; +import net.osmand.osm.Way; +import net.osmand.osm.Entity.EntityId; +import net.osmand.osm.OSMSettings.OSMTagKey; +import net.osmand.swing.Messages; +import net.sf.junidecode.Junidecode; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class IndexAddressCreator extends AbstractIndexPartCreator{ + + private static final Log log = LogFactory.getLog(IndexAddressCreator.class); + + + private PreparedStatement addressCityStat; + private PreparedStatement addressStreetStat; + private PreparedStatement addressBuildingStat; + private PreparedStatement addressStreetNodeStat; + + // MEMORY address : choose what to use ? + private boolean loadInMemory = true; + private PreparedStatement addressSearchStreetStat; + private PreparedStatement addressSearchBuildingStat; + private PreparedStatement addressSearchStreetNodeStat; + + private Map addressStreetLocalMap = new LinkedHashMap(); + private Set addressBuildingLocalSet = new LinkedHashSet(); + private Set addressStreetNodeLocalSet = new LinkedHashSet(); + + // MEMORY address : address structure + // load it in memory + private Map cities = new LinkedHashMap(); + private DataTileManager cityVillageManager = new DataTileManager(13); + private DataTileManager cityManager = new DataTileManager(10); + private List postalCodeRelations = new ArrayList(); + private Map citiBoundaries = new LinkedHashMap(); + private Set visitedBoundaryWays = new HashSet(); + + private boolean normalizeStreets; + private String[] normalizeDefaultSuffixes; + private String[] normalizeSuffixes; + + private String cityAdminLevel; + private boolean saveAddressWays; + + // TODO + Connection mapConnection; + + + public IndexAddressCreator(){ + } + + + public void initSettings(boolean normalizeStreets, String[] normalizeDefaultSuffixes, String[] normalizeSuffixes, + boolean saveAddressWays, String cityAdminLevel) { + cities.clear(); + cityManager.clear(); + postalCodeRelations.clear(); + citiBoundaries.clear(); + this.normalizeStreets = normalizeStreets; + this.normalizeDefaultSuffixes = normalizeDefaultSuffixes; + this.normalizeSuffixes = normalizeSuffixes; + this.cityAdminLevel = cityAdminLevel; + this.saveAddressWays = saveAddressWays; + } + + public void registerCityIfNeeded(Entity e) { + if (e instanceof Node && e.getTag(OSMTagKey.PLACE) != null) { + City city = new City((Node) e); + if (city.getType() != null && !Algoritms.isEmpty(city.getName())) { + if (city.getType() == CityType.CITY || city.getType() == CityType.TOWN) { + cityManager.registerObject(((Node) e).getLatitude(), ((Node) e).getLongitude(), city); + } else { + cityVillageManager.registerObject(((Node) e).getLatitude(), ((Node) e).getLongitude(), city); + } + cities.put(city.getEntityId(), city); + } + } + } + + public void indexBoundariesRelation(Entity e, OsmDbAccessorContext ctx) throws SQLException { + if ("administrative".equals(e.getTag(OSMTagKey.BOUNDARY)) && (e instanceof Relation || e instanceof Way)) { + String adminLevel = e.getTag("admin_level"); + Boundary boundary = null; + if (cityAdminLevel.equals(adminLevel)) { + if (e instanceof Relation) { + Relation i = (Relation) e; + ctx.loadEntityData(i, true); + boundary = new Boundary(); + if (i.getTag(OSMTagKey.NAME) != null) { + boundary.setName(i.getTag(OSMTagKey.NAME)); + } + boundary.setBoundaryId(i.getId()); + Map entities = i.getMemberEntities(); + for (Entity es : entities.keySet()) { + if (es instanceof Way) { + boolean inner = "inner".equals(entities.get(es)); //$NON-NLS-1$ + if (inner) { + boundary.getInnerWays().add((Way) es); + } else { + String wName = es.getTag(OSMTagKey.NAME); + // if name are not equal keep the way for further check (it could be different suburb) + if (Algoritms.objectEquals(wName, boundary.getName()) || wName == null) { + visitedBoundaryWays.add(es.getId()); + } + boundary.getOuterWays().add((Way) es); + } + } + } + } else if (e instanceof Way) { + if (!visitedBoundaryWays.contains(e.getId())) { + boundary = new Boundary(); + if (e.getTag(OSMTagKey.NAME) != null) { + boundary.setName(e.getTag(OSMTagKey.NAME)); + } + boundary.setBoundaryId(e.getId()); + boundary.getOuterWays().add((Way) e); + + } + + } + } + + if (boundary != null && boundary.getCenterPoint() != null) { + LatLon point = boundary.getCenterPoint(); + boolean cityFound = false; + boolean containsCityInside = false; + for (City c : cityManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) { + if (boundary.containsPoint(c.getLocation())) { + if (boundary.getName() == null || boundary.getName().equalsIgnoreCase(c.getName())) { + citiBoundaries.put(c, boundary); + cityFound = true; + containsCityInside = true; + } + } + } + // TODO mark all suburbs inside city as is_in tag (!) or use another solution + if (!cityFound) { + for (City c : cityVillageManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) { + if (boundary.containsPoint(c.getLocation())) { + if (boundary.getName() == null || boundary.getName().equalsIgnoreCase(c.getName())) { + citiBoundaries.put(c, boundary); + cityFound = true; + } + } + } + } + if (!cityFound && boundary.getName() != null) { + // / create new city for named boundary very rare case that's why do not proper generate id + // however it could be a problem + City nCity = new City(containsCityInside ? CityType.CITY : CityType.SUBURB); + nCity.setLocation(point.getLatitude(), point.getLongitude()); + nCity.setId(-boundary.getBoundaryId()); + nCity.setName(boundary.getName()); + citiBoundaries.put(nCity, boundary); + cityVillageManager.registerObject(point.getLatitude(), point.getLongitude(), nCity); + + writeCity(addressCityStat, pStatements, nCity); + // commit to put all cities + if (pStatements.get(addressCityStat) > 0) { + addressCityStat.executeBatch(); + pStatements.put(addressCityStat, 0); + } + } + } + } + } + + + public void indexAddressRelation(Relation i, OsmDbAccessorContext ctx) throws SQLException { + if (i instanceof Relation && "address".equals(i.getTag(OSMTagKey.TYPE))) { //$NON-NLS-1$ + String type = i.getTag(OSMTagKey.ADDRESS_TYPE); + boolean house = "house".equals(type); //$NON-NLS-1$ + boolean street = "a6".equals(type); //$NON-NLS-1$ + if (house || street) { + // try to find appropriate city/street + City c = null; + // load with member ways with their nodes and tags ! + ctx.loadEntityData(i, true); + + Collection members = i.getMembers("is_in"); //$NON-NLS-1$ + Relation a3 = null; + Relation a6 = null; + if (!members.isEmpty()) { + if (street) { + a6 = i; + } + Entity in = members.iterator().next(); + ctx.loadEntityData(in, true); + if (in instanceof Relation) { + // go one level up for house + if (house) { + a6 = (Relation) in; + members = ((Relation) in).getMembers("is_in"); //$NON-NLS-1$ + if (!members.isEmpty()) { + in = members.iterator().next(); + ctx.loadEntityData(in, true); + if (in instanceof Relation) { + a3 = (Relation) in; + } + } + + } else { + a3 = (Relation) in; + } + } + } + + if (a3 != null) { + Collection memberIds = a3.getMemberIds("label"); //$NON-NLS-1$ + if (!memberIds.isEmpty()) { + c = cities.get(memberIds.iterator().next()); + } + } + if (c != null && a6 != null) { + String name = a6.getTag(OSMTagKey.NAME); + + if (name != null) { + LatLon location = c.getLocation(); + for (Entity e : i.getMembers(null)) { + if (e instanceof Way) { + LatLon l = ((Way) e).getLatLon(); + if (l != null) { + location = l; + break; + } + } + } + + Long streetId = getStreetInCity(c, name, location, (a6.getId() << 2) | 2); + if (streetId == null) { + return; + } + if (street) { + for (Map.Entry r : i.getMemberEntities().entrySet()) { + if ("street".equals(r.getValue())) { //$NON-NLS-1$ + if (r.getKey() instanceof Way && saveAddressWays) { + writeStreetWayNodes(addressStreetNodeStat, pStatements, streetId, (Way) r.getKey()); + if (loadInMemory) { + addressStreetNodeLocalSet.add(r.getKey().getId()); + } + } + } else if ("house".equals(r.getValue())) { //$NON-NLS-1$ + // will be registered further in other case + if (!(r.getKey() instanceof Relation)) { + String hno = r.getKey().getTag(OSMTagKey.ADDR_HOUSE_NUMBER); + if (hno != null) { + Building building = new Building(r.getKey()); + building.setName(hno); + writeBuilding(addressBuildingStat, pStatements, streetId, building); + if (loadInMemory) { + addressBuildingLocalSet.add(r.getKey().getId()); + } + } + } + } + } + } else { + String hno = i.getTag(OSMTagKey.ADDRESS_HOUSE); + if (hno == null) { + hno = i.getTag(OSMTagKey.ADDR_HOUSE_NUMBER); + } + if (hno == null) { + hno = i.getTag(OSMTagKey.NAME); + } + members = i.getMembers("border"); //$NON-NLS-1$ + if (!members.isEmpty()) { + Entity border = members.iterator().next(); + if (border != null) { + EntityId id = EntityId.valueOf(border); + // special check that address do not contain twice in a3 - border and separate a6 + if (!a6.getMemberIds().contains(id)) { + Building building = new Building(border); + if (building.getLocation() != null) { + building.setName(hno); + writeBuilding(addressBuildingStat, pStatements, streetId, building); + if (loadInMemory) { + addressBuildingLocalSet.add(id.getId()); + } + } else { + log.error("Strange border " + id + " location couldn't be found"); + } + } + } + } else { + log.info("For relation " + i.getId() + " border not found"); //$NON-NLS-1$ //$NON-NLS-2$ + } + + } + } + } + } + } + } + + + public String normalizeStreetName(String name) { + name = name.trim(); + if (normalizeStreets) { + String newName = name; + boolean processed = newName.length() != name.length(); + for (String ch : normalizeDefaultSuffixes) { + int ind = checkSuffix(newName, ch); + if (ind != -1) { + newName = cutSuffix(newName, ind, ch.length()); + processed = true; + break; + } + } + + if (!processed) { + for (String ch : normalizeSuffixes) { + int ind = checkSuffix(newName, ch); + if (ind != -1) { + newName = putSuffixToEnd(newName, ind, ch.length()); + processed = true; + break; + } + } + } + if (processed) { + return newName; + } + } + return name; + } + + private int checkSuffix(String name, String suffix) { + int i = -1; + boolean searchAgain = false; + do { + i = name.indexOf(suffix, i); + searchAgain = false; + if (i > 0) { + if (Character.isLetterOrDigit(name.charAt(i - 1))) { + i++; + searchAgain = true; + } + } + } while (searchAgain); + return i; + } + + private String cutSuffix(String name, int ind, int suffixLength) { + String newName = name.substring(0, ind); + if (name.length() > ind + suffixLength + 1) { + newName += name.substring(ind + suffixLength + 1); + } + return newName.trim(); + } + + private String putSuffixToEnd(String name, int ind, int suffixLength) { + if (name.length() <= ind + suffixLength) { + return name; + + } + String newName; + if (ind > 0) { + newName = name.substring(0, ind); + newName += name.substring(ind + suffixLength); + newName += name.substring(ind - 1, ind + suffixLength); + } else { + newName = name.substring(suffixLength + 1) + name.charAt(suffixLength) + name.substring(0, suffixLength); + } + + return newName.trim(); + } + + public Long getStreetInCity(City city, String name, LatLon location, long initId) throws SQLException { + if (name == null || city == null) { + return null; + } + Long foundId = null; + + name = normalizeStreetName(name); + if (loadInMemory) { + foundId = addressStreetLocalMap.get(name + "_" + city.getId()); //$NON-NLS-1$ + } else { + addressSearchStreetStat.setLong(1, city.getId()); + addressSearchStreetStat.setString(2, name); + ResultSet rs = addressSearchStreetStat.executeQuery(); + if (rs.next()) { + foundId = rs.getLong(1); + } + rs.close(); + } + + if (foundId == null) { + insertStreetData(addressStreetStat, initId, name, Junidecode.unidecode(name), + location.getLatitude(), location.getLongitude(), city.getId()); + if (loadInMemory) { + addBatch(pStatements, addressStreetStat, BATCH_SIZE); + addressStreetLocalMap.put(name + "_" + city.getId(), initId); //$NON-NLS-1$ + } else { + addressStreetStat.execute(); + // commit immediately to search after + mapConnection.commit(); + } + foundId = initId; + } + return foundId; + } + + public City getClosestCity(LatLon point) { + if (point == null) { + return null; + } + + for (City c : cityManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) { + Boundary boundary = citiBoundaries.get(c); + if(boundary != null){ + if(boundary.containsPoint(point)){ + return c; + } + } + } + for (City c : cityVillageManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) { + Boundary boundary = citiBoundaries.get(c); + if(boundary != null){ + if(boundary.containsPoint(point)){ + return c; + } + } + } + City closest = null; + double relDist = Double.POSITIVE_INFINITY; + for (City c : cityManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) { + double rel = MapUtils.getDistance(c.getLocation(), point) / c.getType().getRadius(); + if (rel < relDist) { + closest = c; + relDist = rel; + if (relDist < 0.2d) { + break; + } + } + } + if (relDist < 0.2d) { + return closest; + } + for (City c : cityVillageManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) { + double rel = MapUtils.getDistance(c.getLocation(), point) / c.getType().getRadius(); + if (rel < relDist) { + closest = c; + relDist = rel; + if (relDist < 0.2d) { + break; + } + } + } + return closest; + } + + + public void iterateMainEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException { + // index not only buildings but also nodes that belongs to addr:interpolation ways + if (e.getTag(OSMTagKey.ADDR_HOUSE_NUMBER) != null && e.getTag(OSMTagKey.ADDR_STREET) != null) { + // TODO e.getTag(OSMTagKey.ADDR_CITY) could be used to find city however many cities could have same name! + // check that building is not registered already + boolean exist = false; + if (loadInMemory) { + exist = addressBuildingLocalSet.contains(e.getId()); + } else { + addressSearchBuildingStat.setLong(1, e.getId()); + ResultSet rs = addressSearchBuildingStat.executeQuery(); + exist = rs.next(); + rs.close(); + + } + if (!exist) { + ctx.loadEntityData(e, false); + LatLon l = e.getLatLon(); + City city = getClosestCity(l); + Long idStreet = getStreetInCity(city, e.getTag(OSMTagKey.ADDR_STREET), l, (e.getId() << 2)); + if (idStreet != null) { + Building building = new Building(e); + building.setName(e.getTag(OSMTagKey.ADDR_HOUSE_NUMBER)); + writeBuilding(addressBuildingStat, pStatements, idStreet, building); + } + } + } else if (e instanceof Way /* && OSMSettings.wayForCar(e.getTag(OSMTagKey.HIGHWAY)) */ + && e.getTag(OSMTagKey.HIGHWAY) != null && e.getTag(OSMTagKey.NAME) != null) { + // suppose that streets with names are ways for car + // Ignore all ways that have house numbers and highway type + boolean exist = false; + + // if we saved address ways we could checked that we registered before + if (saveAddressWays) { + if (loadInMemory) { + exist = addressStreetNodeLocalSet.contains(e.getId()); + } else { + addressSearchStreetNodeStat.setLong(1, e.getId()); + ResultSet rs = addressSearchStreetNodeStat.executeQuery(); + exist = rs.next(); + rs.close(); + } + } + + // check that street way is not registered already + if (!exist) { + ctx.loadEntityData(e, false); + LatLon l = e.getLatLon(); + City city = getClosestCity(l); + Long idStreet = getStreetInCity(city, e.getTag(OSMTagKey.NAME), l, (e.getId() << 2) | 1); + if (idStreet != null && saveAddressWays) { + writeStreetWayNodes(addressStreetNodeStat, pStatements, idStreet, (Way) e); + } + } + } + if (e instanceof Relation) { + if (e.getTag(OSMTagKey.POSTAL_CODE) != null) { + ctx.loadEntityData(e, false); + postalCodeRelations.add((Relation) e); + } + } + } + + + private void writeStreetWayNodes(PreparedStatement prepStreetNode, Map count, Long streetId, Way way) + throws SQLException { + for (Node n : way.getNodes()) { + if (n == null) { + continue; + } + prepStreetNode.setLong(1, n.getId()); + prepStreetNode.setDouble(2, n.getLatitude()); + prepStreetNode.setDouble(3, n.getLongitude()); + prepStreetNode.setLong(5, way.getId()); + prepStreetNode.setLong(4, streetId); + addBatch(count, prepStreetNode, BATCH_SIZE); + } + } + + + private void writeBuilding(PreparedStatement prepBuilding, Map count, Long streetId, + Building building) + throws SQLException { + prepBuilding.setLong(1, building.getId()); + prepBuilding.setDouble(2, building.getLocation().getLatitude()); + prepBuilding.setDouble(3, building.getLocation().getLongitude()); + prepBuilding.setString(4, building.getName()); + prepBuilding.setString(5, building.getEnName()); + prepBuilding.setLong(6, streetId); + prepBuilding.setString(7, building.getPostcode() == null ? null : building.getPostcode().toUpperCase()); + + addBatch(count, prepBuilding); + } + + + private void writeCity(PreparedStatement prepCity, Map count, City city) throws SQLException { + prepCity.setLong(1, city.getId()); + prepCity.setDouble(2, city.getLocation().getLatitude()); + prepCity.setDouble(3, city.getLocation().getLongitude()); + prepCity.setString(4, city.getName()); + prepCity.setString(5, city.getEnName()); + prepCity.setString(6, CityType.valueToString(city.getType())); + addBatch(count, prepCity, BATCH_SIZE); + } + + + private void insertStreetData(PreparedStatement addressStreetStat, long id, String name, String nameEn, double latitude, + double longitude, Long cityId) throws SQLException { + addressStreetStat.setLong(1, id); + addressStreetStat.setString(4, name); + addressStreetStat.setString(5, nameEn); + addressStreetStat.setDouble(2, latitude); + addressStreetStat.setDouble(3, longitude); + addressStreetStat.setLong(6, cityId); + } + + + public void writeCitiesIntoDb() throws SQLException { + for (City c : cities.values()) { + writeCity(addressCityStat, pStatements, c); + } + // commit to put all cities + if (pStatements.get(addressCityStat) > 0) { + addressCityStat.executeBatch(); + pStatements.put(addressCityStat, 0); + mapConnection.commit(); + } + } + + public void processingPostcodes() throws SQLException { + if (pStatements.get(addressBuildingStat) > 0) { + addressBuildingStat.executeBatch(); + pStatements.put(addressBuildingStat, 0); + mapConnection.commit(); + } + PreparedStatement pstat = mapConnection.prepareStatement("UPDATE building SET postcode = ? WHERE id = ?"); + pStatements.put(pstat, 0); + for (Relation r : postalCodeRelations) { + String tag = r.getTag(OSMTagKey.POSTAL_CODE); + for (EntityId l : r.getMemberIds()) { + pstat.setString(1, tag); + pstat.setLong(2, l.getId()); + addBatch(pStatements, pstat, BATCH_SIZE); + } + } + if (pStatements.get(pstat) > 0) { + pstat.executeBatch(); + } + pStatements.remove(pstat); + } + + + public void writeBinaryAddressIndex(BinaryMapIndexWriter writer, String regionName, IProgress progress) throws IOException, SQLException { + closePreparedStatements(addressCityStat, addressStreetStat, addressStreetNodeStat, addressBuildingStat); + mapConnection.commit(); + boolean readWayNodes = saveAddressWays; + + writer.startWriteAddressIndex(regionName); + List cities = readCities(mapConnection); + List streets = new ArrayList(); + Collections.sort(cities, new Comparator() { + + @Override + public int compare(City o1, City o2) { + if (o1.getType() != o2.getType()) { + return (o1.getType().ordinal() - o2.getType().ordinal()); + } + return Collator.getInstance().compare(o1.getName(), o2.getName()); + } + }); + PreparedStatement streetstat = mapConnection.prepareStatement("SELECT A.id, A.name, A.name_en, A.latitude, A.longitude, "+ //$NON-NLS-1$ + "B.id, B.name, B.name_en, B.latitude, B.longitude, B.postcode "+ //$NON-NLS-1$ + "FROM street A left JOIN building B ON B.street = A.id WHERE A.city = ?"); //$NON-NLS-1$ + PreparedStatement waynodesStat = null; + if (readWayNodes) { + waynodesStat = mapConnection.prepareStatement("SELECT A.id, A.latitude, A.longitude FROM street_node A WHERE A.street = ? "); //$NON-NLS-1$ + } + + int j = 0; + for (; j < cities.size(); j++) { + City c = cities.get(j); + if (c.getType() != CityType.CITY && c.getType() != CityType.TOWN) { + break; + } + } + progress.startTask(Messages.getString("IndexCreator.SERIALIZING_ADRESS"), j + ((cities.size() - j) / 100 + 1)); //$NON-NLS-1$ + + Map> postcodes = new TreeMap>(); + boolean writeCities = true; + + // collect suburbs with is in value + List suburbs = new ArrayList(); + for(City s : cities){ + if(s.getType() == CityType.SUBURB && s.getIsInValue() != null){ + suburbs.add(s); + } + } + + // write cities and after villages + writer.startCityIndexes(false); + for (int i = 0; i < cities.size(); i++) { + City c = cities.get(i); + List listSuburbs = null; + for (City suburb : suburbs) { + if (suburb.getIsInValue().contains(c.getName().toLowerCase())) { + if(listSuburbs == null){ + listSuburbs = new ArrayList(); + } + listSuburbs.add(suburb); + } + } + if (writeCities) { + progress.progress(1); + } else if ((cities.size() - i) % 100 == 0) { + progress.progress(1); + } + if (writeCities && c.getType() != CityType.CITY && c.getType() != CityType.TOWN) { + writer.endCityIndexes(false); + writer.startCityIndexes(true); + writeCities = false; + } + + streets.clear(); + Map> streetNodes = null; + if (readWayNodes) { + streetNodes = new LinkedHashMap>(); + } + long time = System.currentTimeMillis(); + readStreetsBuildings(streetstat, c, streets, waynodesStat, streetNodes, listSuburbs); + long f = System.currentTimeMillis() - time; + writer.writeCityIndex(c, streets, streetNodes); + int bCount = 0; + for (Street s : streets) { + bCount++; + for (Building b : s.getBuildings()) { + bCount++; + if (b.getPostcode() != null) { + if (!postcodes.containsKey(b.getPostcode())) { + postcodes.put(b.getPostcode(), new LinkedHashSet(3)); + } + postcodes.get(b.getPostcode()).add(s); + } + } + } + if (f > 500) { + System.out.println("! " + c.getName() + " ! " + f + " " + bCount + " streets " + streets.size()); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ + } + } + writer.endCityIndexes(!writeCities); + + // write postcodes + writer.startPostcodes(); + for (String s : postcodes.keySet()) { + writer.writePostcode(s, postcodes.get(s)); + } + writer.endPostcodes(); + + progress.finishTask(); + + writer.endWriteAddressIndex(); + writer.flush(); + streetstat.close(); + if (readWayNodes) { + waynodesStat.close(); + } + + } + + public void commitToPutAllCities() throws SQLException { + // commit to put all cities + if (pStatements.get(addressBuildingStat) > 0) { + addressBuildingStat.executeBatch(); + pStatements.put(addressBuildingStat, 0); + } + if (pStatements.get(addressStreetNodeStat) > 0) { + addressStreetNodeStat.executeBatch(); + pStatements.put(addressStreetNodeStat, 0); + } + mapConnection.commit(); + + } + + public void createDatabaseStructure(Connection mapConnection, DBDialect dialect) throws SQLException { + this.mapConnection = mapConnection; + createAddressIndexStructure(mapConnection, dialect); + addressCityStat = mapConnection.prepareStatement("insert into city (id, latitude, longitude, name, name_en, city_type) values (?, ?, ?, ?, ?, ?)"); + addressStreetStat = mapConnection.prepareStatement("insert into street (id, latitude, longitude, name, name_en, city) values (?, ?, ?, ?, ?, ?)"); + addressBuildingStat = mapConnection.prepareStatement("insert into building (id, latitude, longitude, name, name_en, street, postcode) values (?, ?, ?, ?, ?, ?, ?)"); + addressStreetNodeStat = mapConnection.prepareStatement("insert into street_node (id, latitude, longitude, street, way) values (?, ?, ?, ?, ?)"); + + addressSearchStreetStat = mapConnection.prepareStatement("SELECT ID FROM street WHERE ? = city AND ? = name"); + addressSearchBuildingStat = mapConnection.prepareStatement("SELECT id FROM building where ? = id"); + addressSearchStreetNodeStat = mapConnection.prepareStatement("SELECT way FROM street_node WHERE ? = way"); + + pStatements.put(addressCityStat, 0); + pStatements.put(addressStreetStat, 0); + pStatements.put(addressStreetNodeStat, 0); + pStatements.put(addressBuildingStat, 0); + // put search statements to close them after all + pStatements.put(addressSearchBuildingStat, 0); + pStatements.put(addressSearchStreetNodeStat, 0); + pStatements.put(addressSearchStreetStat, 0); + + } + + private void createAddressIndexStructure(Connection conn, DBDialect dialect) throws SQLException{ + Statement stat = conn.createStatement(); + + stat.executeUpdate("create table city (id bigint primary key, latitude double, longitude double, " + + "name varchar(255), name_en varchar(255), city_type varchar(32))"); + stat.executeUpdate("create index city_ind on city (id, city_type)"); + + stat.executeUpdate("create table street (id bigint primary key, latitude double, longitude double, " + + "name varchar(255), name_en varchar(255), city bigint)"); + stat.executeUpdate("create index street_city on street (city)"); + stat.executeUpdate("create index street_id on street (id)"); + // create index on name ? + + stat.executeUpdate("create table building (id bigint, latitude double, longitude double, " + + "name varchar(255), name_en varchar(255), street bigint, postcode varchar(255), primary key(street, id))"); + stat.executeUpdate("create index building_postcode on building (postcode)"); + stat.executeUpdate("create index building_street on building (street)"); + stat.executeUpdate("create index building_id on building (id)"); + + + stat.executeUpdate("create table street_node (id bigint, latitude double, longitude double, " + + "street bigint, way bigint)"); + stat.executeUpdate("create index street_node_street on street_node (street)"); + stat.executeUpdate("create index street_node_way on street_node (way)"); + +// if(dialect == DBDialect.SQLITE){ +// stat.execute("PRAGMA user_version = " + IndexConstants.ADDRESS_TABLE_VERSION); //$NON-NLS-1$ +// } + stat.close(); + } + + private List readStreetsBuildings(PreparedStatement streetBuildingsStat, City city, List streets, + PreparedStatement waynodesStat, Map> streetNodes, List citySuburbs) throws SQLException { + Map visitedStreets = new LinkedHashMap(); + //read streets for city + readStreatsByBuildingsForCity(streetBuildingsStat, city, streets, + waynodesStat, streetNodes, visitedStreets); + //read streets for suburbs of the city + if (citySuburbs != null) { + for (City suburb : citySuburbs) { + readStreatsByBuildingsForCity(streetBuildingsStat, suburb, streets, waynodesStat, streetNodes, visitedStreets); + } + } + return streets; + } + + + private void readStreatsByBuildingsForCity( + PreparedStatement streetBuildingsStat, City city, + List streets, PreparedStatement waynodesStat, + Map> streetNodes, + Map visitedStreets) throws SQLException { + streetBuildingsStat.setLong(1, city.getId()); + ResultSet set = streetBuildingsStat.executeQuery(); + while (set.next()) { + long streetId = set.getLong(1); + if (!visitedStreets.containsKey(streetId)) { + Street street = new Street(null); + street.setName(set.getString(2)); + street.setEnName(set.getString(3)); + street.setLocation(set.getDouble(4), set.getDouble(5)); + street.setId(streetId); + streets.add(street); + visitedStreets.put(streetId, street); + if (waynodesStat != null && streetNodes != null) { + ArrayList list = new ArrayList(); + streetNodes.put(street, list); + waynodesStat.setLong(1, street.getId()); + ResultSet rs = waynodesStat.executeQuery(); + while (rs.next()) { + list.add(new Node(rs.getDouble(2), rs.getDouble(3), rs.getLong(1))); + } + rs.close(); + } + } + if (set.getObject(6) != null) { + Street s = visitedStreets.get(streetId); + Building b = new Building(); + b.setId(set.getLong(6)); + b.setName(set.getString(7)); + b.setEnName(set.getString(8)); + b.setLocation(set.getDouble(9), set.getDouble(10)); + b.setPostcode(set.getString(11)); + s.registerBuilding(b); + } + } + + set.close(); + } + + + public List readCities(Connection c) throws SQLException{ + List cities = new ArrayList(); + Statement stat = c.createStatement(); + ResultSet set = stat.executeQuery("select id, latitude, longitude , name , name_en , city_type from city"); //$NON-NLS-1$ + while(set.next()){ + City city = new City(CityType.valueFromString(set.getString(6))); + city.setName(set.getString(4)); + city.setEnName(set.getString(5)); + city.setLocation(set.getDouble(2), + set.getDouble(3)); + city.setId(set.getLong(1)); + cities.add(city); + + } + set.close(); + stat.close(); + return cities; + } + + +} diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java b/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java index a77464a343..038ce2cea5 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java @@ -1,7 +1,5 @@ package net.osmand.data.preparation; -import java.awt.geom.Line2D; -import java.awt.geom.Point2D; import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; @@ -12,66 +10,31 @@ import java.io.RandomAccessFile; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; -import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; -import java.text.Collator; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashSet; import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.TreeMap; import net.osmand.Algoritms; import net.osmand.IProgress; import net.osmand.binary.BinaryMapIndexWriter; -import net.osmand.data.Boundary; -import net.osmand.data.Building; -import net.osmand.data.City; -import net.osmand.data.DataTileManager; -import net.osmand.data.MapAlgorithms; -import net.osmand.data.Street; -import net.osmand.data.City.CityType; -import net.osmand.data.index.DataIndexReader; -import net.osmand.data.index.DataIndexWriter; import net.osmand.data.index.IndexConstants; import net.osmand.data.preparation.OsmDbAccessor.OsmDbVisitor; import net.osmand.impl.ConsoleProgressImplementation; import net.osmand.osm.Entity; -import net.osmand.osm.LatLon; import net.osmand.osm.MapRenderingTypes; -import net.osmand.osm.MapUtils; -import net.osmand.osm.Node; import net.osmand.osm.Relation; -import net.osmand.osm.Way; -import net.osmand.osm.Entity.EntityId; import net.osmand.osm.Entity.EntityType; -import net.osmand.osm.OSMSettings.OSMTagKey; import net.osmand.osm.io.IOsmStorageFilter; import net.osmand.osm.io.OsmBaseStorage; import net.osmand.swing.DataExtractionSettings; import net.osmand.swing.Messages; -import net.sf.junidecode.Junidecode; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.tools.bzip2.CBZip2InputStream; import org.xml.sax.SAXException; -import rtree.Element; -import rtree.IllegalValueException; -import rtree.LeafElement; -import rtree.NonLeafElement; -import rtree.Pack; -import rtree.RTree; import rtree.RTreeException; -import rtree.Rect; /** * http://wiki.openstreetmap.org/wiki/OSM_tags_for_routing#Is_inside.2Foutside @@ -81,7 +44,6 @@ import rtree.Rect; * That data extraction has aim, save runtime memory and generate indexes on the fly. It will be longer than load in memory (needed part) * and save into index. */ -@SuppressWarnings("unchecked") public class IndexCreator { private static final Log log = LogFactory.getLog(IndexCreator.class); @@ -112,6 +74,8 @@ public class IndexCreator { private IndexTransportCreator indexTransportCreator; private IndexPoiCreator indexPoiCreator; + private IndexAddressCreator indexAddressCreator; + private IndexVectorMapCreator indexMapCreator; private OsmDbAccessor accessor; // constants to start process from the middle and save temporary results private boolean recreateOnlyBinaryFile = false; // false; @@ -126,59 +90,15 @@ public class IndexCreator { private File mapFile; private RandomAccessFile mapRAFile; private Connection mapConnection; - - private PreparedStatement mapBinaryStat; - private PreparedStatement mapLowLevelBinaryStat; - private int lowLevelWays = -1; - - private PreparedStatement addressCityStat; - private PreparedStatement addressStreetStat; - private PreparedStatement addressBuildingStat; - private PreparedStatement addressStreetNodeStat; - - private RTree[] mapTree = null; - private MapZooms mapZooms = MapZooms.getDefault(); - private MapRenderingTypes renderingTypes = MapRenderingTypes.getDefault(); private String cityAdminLevel = "8"; - // MEMORY map : save it in memory while that is allowed - private Map>[] multiPolygonsWays; - private Map multiPolygonsNames = new LinkedHashMap(); - private Map> highwayRestrictions = new LinkedHashMap>(); - - // MEMORY address : choose what to use ? - private boolean loadInMemory = true; - private PreparedStatement addressSearchStreetStat; - private PreparedStatement addressSearchBuildingStat; - private PreparedStatement addressSearchStreetNodeStat; - - private Map addressStreetLocalMap = new LinkedHashMap(); - private Set addressBuildingLocalSet = new LinkedHashSet(); - private Set addressStreetNodeLocalSet = new LinkedHashSet(); - - // MEMORY address : address structure - // load it in memory - private Map cities = new LinkedHashMap(); - private DataTileManager cityVillageManager = new DataTileManager(13); - private DataTileManager cityManager = new DataTileManager(10); - private List postalCodeRelations = new ArrayList(); - private Map citiBoundaries = new LinkedHashMap(); - private Set visitedBoundaryWays = new HashSet(); - - private String[] normalizeDefaultSuffixes; - private String[] normalizeSuffixes; - - // local purpose - List typeUse = new ArrayList(8); - List restrictionsUse = new ArrayList(8); - - - public IndexCreator(File workingDir) { this.workingDir = workingDir; - this.indexTransportCreator = new IndexTransportCreator(this); - this.indexPoiCreator = new IndexPoiCreator(this); + this.indexTransportCreator = new IndexTransportCreator(); + this.indexPoiCreator = new IndexPoiCreator(); + this.indexAddressCreator = new IndexAddressCreator(); + this.indexMapCreator = new IndexVectorMapCreator(); this.accessor = new OsmDbAccessor(this); } @@ -206,20 +126,6 @@ public class IndexCreator { this.normalizeStreets = normalizeStreets; } -/* protected static int defineLevel(int minZoom) { - int level = 0; - if (minZoom < 15) { - for (int i = 1; i < MAP_ZOOMS.length; i++) { - if (minZoom <= MAP_ZOOMS[i]) { - level = MAP_ZOOMS.length - 1 - i; - break; - } - } - } - return level; - } -*/ - public String getRegionName() { if (regionName == null) { return "Region"; //$NON-NLS-1$ @@ -232,49 +138,9 @@ public class IndexCreator { } private Connection getDatabaseConnection(String fileName) throws SQLException { - return getDatabaseConnection(fileName, dialect); + return dialect.getDatabaseConnection(fileName, log); } - - protected Connection getDatabaseConnection(String fileName, DBDialect dialect) throws SQLException { - if (DBDialect.SQLITE == dialect) { - try { - Class.forName("org.sqlite.JDBC"); - } catch (ClassNotFoundException e) { - log.error("Illegal configuration", e); - throw new IllegalStateException(e); - } - Connection connection = DriverManager.getConnection("jdbc:sqlite:" + fileName); - Statement statement = connection.createStatement(); - statement.executeUpdate("PRAGMA synchronous = 0"); - statement.close(); - return connection; - } else if (DBDialect.DERBY == dialect) { - try { - Class.forName("org.apache.derby.jdbc.EmbeddedDriver"); - } catch (ClassNotFoundException e) { - log.error("Illegal configuration", e); - throw new IllegalStateException(e); - } - Connection conn = DriverManager.getConnection("jdbc:derby:" + fileName + ";create=true"); - conn.setAutoCommit(false); - return conn; - } else if (DBDialect.H2 == dialect) { - try { - Class.forName("org.h2.Driver"); - } catch (ClassNotFoundException e) { - log.error("Illegal configuration", e); - throw new IllegalStateException(e); - } - - return DriverManager.getConnection("jdbc:h2:file:" + fileName); - } else { - throw new UnsupportedOperationException(); - } - - } - - public void setPoiFileName(String poiFileName) { this.poiFileName = poiFileName; } @@ -312,8 +178,6 @@ public class IndexCreator { } return poiFileName; } - - public String getCityAdminLevel() { return cityAdminLevel; @@ -323,1278 +187,27 @@ public class IndexCreator { this.cityAdminLevel = cityAdminLevel; } - public void indexBoundariesRelation(Entity e, OsmDbAccessorContext ctx) throws SQLException { - String adminLevel = e.getTag("admin_level"); - Boundary boundary = null; - if (cityAdminLevel.equals(adminLevel)) { - if (e instanceof Relation) { - Relation i = (Relation) e; - ctx.loadEntityData(i, true); - boundary = new Boundary(); - if (i.getTag(OSMTagKey.NAME) != null) { - boundary.setName(i.getTag(OSMTagKey.NAME)); - } - boundary.setBoundaryId(i.getId()); - Map entities = i.getMemberEntities(); - for (Entity es : entities.keySet()) { - if (es instanceof Way) { - boolean inner = "inner".equals(entities.get(es)); //$NON-NLS-1$ - if (inner) { - boundary.getInnerWays().add((Way) es); - } else { - String wName = es.getTag(OSMTagKey.NAME); - // if name are not equal keep the way for further check (it could be different suburb) - if (Algoritms.objectEquals(wName, boundary.getName()) || wName == null) { - visitedBoundaryWays.add(es.getId()); - } - boundary.getOuterWays().add((Way) es); - } - } - } - } else if (e instanceof Way) { - if (!visitedBoundaryWays.contains(e.getId())) { - boundary = new Boundary(); - if (e.getTag(OSMTagKey.NAME) != null) { - boundary.setName(e.getTag(OSMTagKey.NAME)); - } - boundary.setBoundaryId(e.getId()); - boundary.getOuterWays().add((Way) e); - - } - - } - } - - if (boundary != null && boundary.getCenterPoint() != null) { - LatLon point = boundary.getCenterPoint(); - boolean cityFound = false; - boolean containsCityInside = false; - for (City c : cityManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) { - if (boundary.containsPoint(c.getLocation())) { - if (boundary.getName() == null || boundary.getName().equalsIgnoreCase(c.getName())) { - citiBoundaries.put(c, boundary); - cityFound = true; - containsCityInside = true; - } - } - } - // TODO mark all suburbs inside city as is_in tag (!) or use another solution - if (!cityFound) { - for (City c : cityVillageManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) { - if (boundary.containsPoint(c.getLocation())) { - if (boundary.getName() == null || boundary.getName().equalsIgnoreCase(c.getName())) { - citiBoundaries.put(c, boundary); - cityFound = true; - } - } - } - } - if (!cityFound && boundary.getName() != null) { - // / create new city for named boundary very rare case that's why do not proper generate id - // however it could be a problem - City nCity = new City(containsCityInside ? CityType.CITY : CityType.SUBURB); - nCity.setLocation(point.getLatitude(), point.getLongitude()); - nCity.setId(-boundary.getBoundaryId()); - nCity.setName(boundary.getName()); - citiBoundaries.put(nCity, boundary); - cityVillageManager.registerObject(point.getLatitude(), point.getLongitude(), nCity); - - DataIndexWriter.writeCity(addressCityStat, pStatements, nCity, BATCH_SIZE); - // commit to put all cities - if (pStatements.get(addressCityStat) > 0) { - addressCityStat.executeBatch(); - pStatements.put(addressCityStat, 0); - } - } - } - } - - public void indexAddressRelation(Relation i, OsmDbAccessorContext ctx) throws SQLException { - String type = i.getTag(OSMTagKey.ADDRESS_TYPE); - boolean house = "house".equals(type); //$NON-NLS-1$ - boolean street = "a6".equals(type); //$NON-NLS-1$ - if (house || street) { - // try to find appropriate city/street - City c = null; - // load with member ways with their nodes and tags ! - ctx.loadEntityData(i, true); - - Collection members = i.getMembers("is_in"); //$NON-NLS-1$ - Relation a3 = null; - Relation a6 = null; - if (!members.isEmpty()) { - if (street) { - a6 = i; - } - Entity in = members.iterator().next(); - ctx.loadEntityData(in, true); - if (in instanceof Relation) { - // go one level up for house - if (house) { - a6 = (Relation) in; - members = ((Relation) in).getMembers("is_in"); //$NON-NLS-1$ - if (!members.isEmpty()) { - in = members.iterator().next(); - ctx.loadEntityData(in, true); - if (in instanceof Relation) { - a3 = (Relation) in; - } - } - - } else { - a3 = (Relation) in; - } - } - } - - if (a3 != null) { - Collection memberIds = a3.getMemberIds("label"); //$NON-NLS-1$ - if (!memberIds.isEmpty()) { - c = cities.get(memberIds.iterator().next()); - } - } - if (c != null && a6 != null) { - String name = a6.getTag(OSMTagKey.NAME); - - if (name != null) { - LatLon location = c.getLocation(); - for (Entity e : i.getMembers(null)) { - if (e instanceof Way) { - LatLon l = ((Way) e).getLatLon(); - if (l != null) { - location = l; - break; - } - } - } - - Long streetId = getStreetInCity(c, name, location, (a6.getId() << 2) | 2); - if (streetId == null) { - return; - } - if (street) { - for (Map.Entry r : i.getMemberEntities().entrySet()) { - if ("street".equals(r.getValue())) { //$NON-NLS-1$ - if (r.getKey() instanceof Way && saveAddressWays) { - DataIndexWriter.writeStreetWayNodes(addressStreetNodeStat, - pStatements, streetId, (Way) r.getKey(), BATCH_SIZE); - if (loadInMemory) { - addressStreetNodeLocalSet.add(r.getKey().getId()); - } - } - } else if ("house".equals(r.getValue())) { //$NON-NLS-1$ - // will be registered further in other case - if (!(r.getKey() instanceof Relation)) { - String hno = r.getKey().getTag(OSMTagKey.ADDR_HOUSE_NUMBER); - if (hno != null) { - Building building = new Building(r.getKey()); - building.setName(hno); - DataIndexWriter.writeBuilding(addressBuildingStat, pStatements, - streetId, building, BATCH_SIZE); - if (loadInMemory) { - addressBuildingLocalSet.add(r.getKey().getId()); - } - } - } - } - } - } else { - String hno = i.getTag(OSMTagKey.ADDRESS_HOUSE); - if (hno == null) { - hno = i.getTag(OSMTagKey.ADDR_HOUSE_NUMBER); - } - if (hno == null) { - hno = i.getTag(OSMTagKey.NAME); - } - members = i.getMembers("border"); //$NON-NLS-1$ - if (!members.isEmpty()) { - Entity border = members.iterator().next(); - if (border != null) { - EntityId id = EntityId.valueOf(border); - // special check that address do not contain twice in a3 - border and separate a6 - if (!a6.getMemberIds().contains(id)) { - Building building = new Building(border); - if (building.getLocation() != null) { - building.setName(hno); - DataIndexWriter.writeBuilding(addressBuildingStat, pStatements, streetId, building, BATCH_SIZE); - if (loadInMemory) { - addressBuildingLocalSet.add(id.getId()); - } - } else { - log.error("Strange border " + id + " location couldn't be found"); - } - } - } - } else { - log.info("For relation " + i.getId() + " border not found"); //$NON-NLS-1$ //$NON-NLS-2$ - } - - } - } - } - } - } - - public City getClosestCity(LatLon point) { - if (point == null) { - return null; - } - - for (City c : cityManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) { - Boundary boundary = citiBoundaries.get(c); - if(boundary != null){ - if(boundary.containsPoint(point)){ - return c; - } - } - } - for (City c : cityVillageManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) { - Boundary boundary = citiBoundaries.get(c); - if(boundary != null){ - if(boundary.containsPoint(point)){ - return c; - } - } - } - City closest = null; - double relDist = Double.POSITIVE_INFINITY; - for (City c : cityManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) { - double rel = MapUtils.getDistance(c.getLocation(), point) / c.getType().getRadius(); - if (rel < relDist) { - closest = c; - relDist = rel; - if (relDist < 0.2d) { - break; - } - } - } - if (relDist < 0.2d) { - return closest; - } - for (City c : cityVillageManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) { - double rel = MapUtils.getDistance(c.getLocation(), point) / c.getType().getRadius(); - if (rel < relDist) { - closest = c; - relDist = rel; - if (relDist < 0.2d) { - break; - } - } - } - return closest; - } - - public String normalizeStreetName(String name) { - name = name.trim(); - if (normalizeStreets) { - String newName = name; - boolean processed = newName.length() != name.length(); - for (String ch : normalizeDefaultSuffixes) { - int ind = checkSuffix(newName, ch); - if (ind != -1) { - newName = cutSuffix(newName, ind, ch.length()); - processed = true; - break; - } - } - - if (!processed) { - for (String ch : normalizeSuffixes) { - int ind = checkSuffix(newName, ch); - if (ind != -1) { - newName = putSuffixToEnd(newName, ind, ch.length()); - processed = true; - break; - } - } - } - if (processed) { - return newName; - } - } - return name; - } - - private int checkSuffix(String name, String suffix) { - int i = -1; - boolean searchAgain = false; - do { - i = name.indexOf(suffix, i); - searchAgain = false; - if (i > 0) { - if (Character.isLetterOrDigit(name.charAt(i - 1))) { - i++; - searchAgain = true; - } - } - } while (searchAgain); - return i; - } - - private String cutSuffix(String name, int ind, int suffixLength) { - String newName = name.substring(0, ind); - if (name.length() > ind + suffixLength + 1) { - newName += name.substring(ind + suffixLength + 1); - } - return newName.trim(); - } - - private String putSuffixToEnd(String name, int ind, int suffixLength) { - if (name.length() <= ind + suffixLength) { - return name; - - } - String newName; - if (ind > 0) { - newName = name.substring(0, ind); - newName += name.substring(ind + suffixLength); - newName += name.substring(ind - 1, ind + suffixLength); - } else { - newName = name.substring(suffixLength + 1) + name.charAt(suffixLength) + name.substring(0, suffixLength); - } - - return newName.trim(); - } - - public Long getStreetInCity(City city, String name, LatLon location, long initId) throws SQLException { - if (name == null || city == null) { - return null; - } - Long foundId = null; - - name = normalizeStreetName(name); - if (loadInMemory) { - foundId = addressStreetLocalMap.get(name + "_" + city.getId()); //$NON-NLS-1$ - } else { - addressSearchStreetStat.setLong(1, city.getId()); - addressSearchStreetStat.setString(2, name); - ResultSet rs = addressSearchStreetStat.executeQuery(); - if (rs.next()) { - foundId = rs.getLong(1); - } - rs.close(); - } - - if (foundId == null) { - DataIndexWriter.insertStreetData(addressStreetStat, initId, name, Junidecode.unidecode(name), - location.getLatitude(), location.getLongitude(), city.getId()); - if (loadInMemory) { - DataIndexWriter.addBatch(pStatements, addressStreetStat, BATCH_SIZE); - addressStreetLocalMap.put(name + "_" + city.getId(), initId); //$NON-NLS-1$ - } else { - addressStreetStat.execute(); - // commit immediately to search after - mapConnection.commit(); - } - foundId = initId; - } - return foundId; - } - - private void iterateMainEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException { if (indexPOI) { - indexPoiCreator.iterateEntity(e, pStatements); + indexPoiCreator.iterateEntity(e, ctx); } if (indexTransport) { - indexTransportCreator.visitEntityMainStep(e, ctx, pStatements); + indexTransportCreator.visitEntityMainStep(e, ctx); } - - if (indexMap && (e instanceof Way || e instanceof Node)) { - // manipulate what kind of way to load - ctx.loadEntityData(e, false); - boolean inverse = "-1".equals(e.getTag(OSMTagKey.ONEWAY)); //$NON-NLS-1$ - for (int i = 0; i < mapZooms.size(); i++) { - writeBinaryEntityToMapDatabase(e, e.getId(), i == 0 ? inverse : false, i); - } - + if (indexMap) { + indexMapCreator.iterateMainEntity(e, ctx); } - if (indexAddress) { - // index not only buildings but also nodes that belongs to addr:interpolation ways - if (e.getTag(OSMTagKey.ADDR_HOUSE_NUMBER) != null && e.getTag(OSMTagKey.ADDR_STREET) != null) { - // TODO e.getTag(OSMTagKey.ADDR_CITY) could be used to find city however many cities could have same name! - // check that building is not registered already - boolean exist = false; - if (loadInMemory) { - exist = addressBuildingLocalSet.contains(e.getId()); - } else { - addressSearchBuildingStat.setLong(1, e.getId()); - ResultSet rs = addressSearchBuildingStat.executeQuery(); - exist = rs.next(); - rs.close(); - - } - if (!exist) { - ctx.loadEntityData(e, false); - LatLon l = e.getLatLon(); - City city = getClosestCity(l); - Long idStreet = getStreetInCity(city, e.getTag(OSMTagKey.ADDR_STREET), l, (e.getId() << 2)); - if (idStreet != null) { - Building building = new Building(e); - building.setName(e.getTag(OSMTagKey.ADDR_HOUSE_NUMBER)); - DataIndexWriter.writeBuilding(addressBuildingStat, pStatements, idStreet, building, BATCH_SIZE); - } - } - } else if (e instanceof Way /* && OSMSettings.wayForCar(e.getTag(OSMTagKey.HIGHWAY)) */ - && e.getTag(OSMTagKey.HIGHWAY) != null && e.getTag(OSMTagKey.NAME) != null) { - // suppose that streets with names are ways for car - // Ignore all ways that have house numbers and highway type - boolean exist = false; - - // if we saved address ways we could checked that we registered before - if (saveAddressWays) { - if (loadInMemory) { - exist = addressStreetNodeLocalSet.contains(e.getId()); - } else { - addressSearchStreetNodeStat.setLong(1, e.getId()); - ResultSet rs = addressSearchStreetNodeStat.executeQuery(); - exist = rs.next(); - rs.close(); - } - } - - // check that street way is not registered already - if (!exist) { - ctx.loadEntityData(e, false); - LatLon l = e.getLatLon(); - City city = getClosestCity(l); - Long idStreet = getStreetInCity(city, e.getTag(OSMTagKey.NAME), l, (e.getId() << 2) | 1); - if (idStreet != null && saveAddressWays) { - DataIndexWriter.writeStreetWayNodes(addressStreetNodeStat, pStatements, idStreet, (Way) e, BATCH_SIZE); - } - } - } - if (e instanceof Relation) { - if (e.getTag(OSMTagKey.POSTAL_CODE) != null) { - ctx.loadEntityData(e, false); - postalCodeRelations.add((Relation) e); - } - } + indexAddressCreator.iterateMainEntity(e, ctx); } } - public void indexAddressRelationsAndMultiPolygons(Entity e, OsmDbAccessorContext ctx) throws SQLException { - if (indexAddress) { - if (e instanceof Relation && "address".equals(e.getTag(OSMTagKey.TYPE))) { //$NON-NLS-1$ - indexAddressRelation((Relation) e, ctx); - } - - if (e instanceof Relation && "administrative".equals(e.getTag(OSMTagKey.BOUNDARY))) { //$NON-NLS-1$ - indexBoundariesRelation((Relation) e, ctx); - } - } - if (indexMap && e instanceof Relation && "restriction".equals(e.getTag(OSMTagKey.TYPE))) { //$NON-NLS-1$ - String val = e.getTag("restriction"); //$NON-NLS-1$ - if (val != null) { - byte type = -1; - if ("no_right_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$ - type = MapRenderingTypes.RESTRICTION_NO_RIGHT_TURN; - } else if ("no_left_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$ - type = MapRenderingTypes.RESTRICTION_NO_LEFT_TURN; - } else if ("no_u_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$ - type = MapRenderingTypes.RESTRICTION_NO_U_TURN; - } else if ("no_straight_on".equalsIgnoreCase(val)) { //$NON-NLS-1$ - type = MapRenderingTypes.RESTRICTION_NO_STRAIGHT_ON; - } else if ("only_right_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$ - type = MapRenderingTypes.RESTRICTION_ONLY_RIGHT_TURN; - } else if ("only_left_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$ - type = MapRenderingTypes.RESTRICTION_ONLY_LEFT_TURN; - } else if ("only_straight_on".equalsIgnoreCase(val)) { //$NON-NLS-1$ - type = MapRenderingTypes.RESTRICTION_ONLY_STRAIGHT_ON; - } - if (type != -1) { - ctx.loadEntityData(e, true); - Collection fromL = ((Relation) e).getMemberIds("from"); //$NON-NLS-1$ - Collection toL = ((Relation) e).getMemberIds("to"); //$NON-NLS-1$ - if (!fromL.isEmpty() && !toL.isEmpty()) { - EntityId from = fromL.iterator().next(); - EntityId to = toL.iterator().next(); - if (from.getType() == EntityType.WAY) { - if (!highwayRestrictions.containsKey(from.getId())) { - highwayRestrictions.put(from.getId(), new ArrayList(4)); - } - highwayRestrictions.get(from.getId()).add((to.getId() << 3) | (long) type); - } - } - } - } - } - if (indexMap && e instanceof Relation && "multipolygon".equals(e.getTag(OSMTagKey.TYPE))) { //$NON-NLS-1$ - ctx.loadEntityData(e, true); - Map entities = ((Relation) e).getMemberEntities(); - - boolean outerFound = false; - for (Entity es : entities.keySet()) { - if (es instanceof Way) { - boolean inner = "inner".equals(entities.get(es)); //$NON-NLS-1$ - if (!inner) { - outerFound = true; - for (String t : es.getTagKeySet()) { - e.putTag(t, es.getTag(t)); - } - break; - } - } - } - if(!outerFound){ - log.warn("Probably map bug: Multipoligon id=" + e.getId() + " contains only inner ways : "); //$NON-NLS-1$ //$NON-NLS-2$ - return; - } - - int mtType = findMultiPolygonType(e, 0); - if (mtType != 0) { - - String name = renderingTypes.getEntityName(e); - List> completedRings = new ArrayList>(); - List> incompletedRings = new ArrayList>(); - for (Entity es : entities.keySet()) { - if (es instanceof Way) { - if (!((Way) es).getNodeIds().isEmpty()) { - combineMultiPolygons((Way) es, completedRings, incompletedRings); - } - } - } - // skip incompleted rings and do not add whole relation ? - if (!incompletedRings.isEmpty()) { - // log.warn("In multipolygon " + e.getId() + " there are incompleted ways : " + incompletedRings); - return; - // completedRings.addAll(incompletedRings); - } - - // skip completed rings that are not one type - for (List l : completedRings) { - boolean innerType = "inner".equals(entities.get(l.get(0))); //$NON-NLS-1$ - for (Way way : l) { - boolean inner = "inner".equals(entities.get(way)); //$NON-NLS-1$ - if (innerType != inner) { - log.warn("Probably map bug: Multipoligon contains outer and inner ways.\n" + //$NON-NLS-1$ - "Way:" + way.getId() + " is strange part of completed ring. InnerType:" + innerType + " way inner: " + inner + " way inner string:" + entities.get(way)); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ - return; - } - } - } - - for (List l : completedRings) { - boolean innerType = "inner".equals(entities.get(l.get(0))); //$NON-NLS-1$ - boolean clockwise = isClockwiseWay(l); - // clockwise - outer (like coastline), anticlockwise - inner - boolean inverse = clockwise != !innerType; - for (Way way : l) { - boolean inner = "inner".equals(entities.get(way)); //$NON-NLS-1$ - if (!inner && name != null) { - multiPolygonsNames.put(way.getId(), name); - } - putMultipolygonType(multiPolygonsWays[0], way.getId(), mtType, inverse); - for (int i = 1; i < multiPolygonsWays.length; i++) { - int type = findMultiPolygonType(e, i); - if (type != 0) { - putMultipolygonType(multiPolygonsWays[i], way.getId(), type, inverse); - } - } - } - } - - } - } - - } - - public void combineMultiPolygons(Way w, List> completedRings, List> incompletedRings) { - long lId = w.getEntityIds().get(w.getEntityIds().size() - 1).getId().longValue(); - long fId = w.getEntityIds().get(0).getId().longValue(); - if (fId == lId) { - completedRings.add(Collections.singletonList(w)); - } else { - List l = new ArrayList(); - l.add(w); - boolean add = true; - for (int k = 0; k < incompletedRings.size();) { - boolean remove = false; - List i = incompletedRings.get(k); - Way last = i.get(i.size() - 1); - Way first = i.get(0); - long lastId = last.getEntityIds().get(last.getEntityIds().size() - 1).getId().longValue(); - long firstId = first.getEntityIds().get(0).getId().longValue(); - if (fId == lastId) { - i.addAll(l); - remove = true; - l = i; - fId = firstId; - } else if (lId == firstId) { - l.addAll(i); - remove = true; - lId = lastId; - } - if (remove) { - incompletedRings.remove(k); - } else { - k++; - } - if (fId == lId) { - completedRings.add(l); - add = false; - break; - } - } - if (add) { - incompletedRings.add(l); - } - } - } - - private void putMultipolygonType(Map> multiPolygonsWays, long baseId, int mtType, boolean inverse) { - if (mtType == 0) { - return; - } - if (!multiPolygonsWays.containsKey(baseId)) { - multiPolygonsWays.put(baseId, new LinkedHashSet()); - } - if (inverse) { - multiPolygonsWays.get(baseId).add(mtType | (1 << 15)); - } else { - multiPolygonsWays.get(baseId).add(mtType); - } - } - - private int findMultiPolygonType(Entity e, int level) { - int t = renderingTypes.encodeEntityWithType(e, mapZooms.getLevel(level).getMaxZoom(), true, typeUse); - int mtType = 0; - if (t != 0) { - if ((t & 3) == MapRenderingTypes.MULTY_POLYGON_TYPE) { - mtType = t; - } else { - for (Integer i : typeUse) { - if ((i & 3) == MapRenderingTypes.MULTY_POLYGON_TYPE) { - mtType = i; - break; - } - } - } - } - return mtType; - } - - public Point2D.Float getIntersectionPoint(Line2D.Float line1, Line2D.Float line2) { - if (!line1.intersectsLine(line2)) - return null; - double px = line1.getX1(), py = line1.getY1(), rx = line1.getX2() - px, ry = line1.getY2() - py; - double qx = line2.getX1(), qy = line2.getY1(), sx = line2.getX2() - qx, sy = line2.getY2() - qy; - - double det = sx * ry - sy * rx; - if (det == 0) { - return null; - } else { - double z = (sx * (qy - py) + sy * (px - qx)) / det; - if (z <= 0 || z >= 1) - return null; // intersection at end point! - return new Point2D.Float((float) (px + z * rx), (float) (py + z * ry)); - } - } // end intersection line-line - - private boolean isClockwiseWay(List ways) { - if (ways.isEmpty()) { - return false; - } - List nodes; - if (ways.size() == 1) { - nodes = ways.get(0).getNodes(); - } else { - nodes = new ArrayList(); - boolean first = true; - for (Way e : ways) { - if (first) { - first = false; - nodes.addAll(e.getNodes()); - } else { - nodes.addAll(e.getNodes().subList(1, e.getNodes().size())); - } - } - } - if (nodes.isEmpty()) { - return false; - } - double angle = 0; - double prevAng = 0; - double firstAng = 0; - double selfIntersection = 0; - boolean open = nodes.get(nodes.size() - 1).getId() != nodes.get(0).getId(); - - for (int i = 1; open ? i < nodes.size() : i <= nodes.size(); i++) {// nodes.get(i).getId() - double ang; - if (i < nodes.size()) { - ang = Math.atan2(nodes.get(i).getLatitude() - nodes.get(i - 1).getLatitude(), - nodes.get(i).getLongitude() - nodes.get(i - 1).getLongitude()); - // find self intersection - Line2D.Float l = new Line2D.Float((float) nodes.get(i).getLongitude(), (float) nodes.get(i).getLatitude(), - (float) nodes.get(i - 1).getLongitude(), (float) nodes.get(i -1).getLatitude()); - for (int j = i - 2; j > i - 7; j--) { - if (j < 1) { - break; - } - Line2D.Float l2 = new Line2D.Float((float) nodes.get(j).getLongitude(), (float) nodes.get(j).getLatitude(), - (float) nodes.get(j - 1).getLongitude(), (float) nodes.get(j - 1).getLatitude()); - java.awt.geom.Point2D.Float point = getIntersectionPoint(l, l2); - if (point != null) { - double dang = Math.atan2(nodes.get(j).getLatitude() - nodes.get(j - 1).getLatitude(), - nodes.get(j).getLongitude() - nodes.get(j - 1).getLongitude()); - if (adjustDirection(ang - dang) < 0) { - selfIntersection += 2 * Math.PI; - } else { - selfIntersection -= 2 * Math.PI; - } - } - - } - } else { - ang = firstAng; - } - if (i > 1) { - angle += adjustDirection(ang - prevAng); - prevAng = ang; - } else { - prevAng = ang; - firstAng = ang; - } - - } - return (angle - selfIntersection) < 0; - } - - private double adjustDirection(double ang) { - if (ang < -Math.PI) { - ang += 2 * Math.PI; - } else if (ang > Math.PI) { - ang -= 2 * Math.PI; - } - return ang; - } - public void registerCityIfNeeded(Entity e) { - if (e instanceof Node && e.getTag(OSMTagKey.PLACE) != null) { - City city = new City((Node) e); - if (city.getType() != null && !Algoritms.isEmpty(city.getName())) { - if (city.getType() == CityType.CITY || city.getType() == CityType.TOWN) { - cityManager.registerObject(((Node) e).getLatitude(), ((Node) e).getLongitude(), city); - } else { - cityVillageManager.registerObject(((Node) e).getLatitude(), ((Node) e).getLongitude(), city); - } - cities.put(city.getEntityId(), city); - } - } - } - - private void writeBinaryEntityToMapDatabase(Entity e, long baseId, boolean inverse, int level) throws SQLException { - int type = renderingTypes.encodeEntityWithType(e, mapZooms.getLevel(level).getMaxZoom(), false, typeUse); - Map> multiPolygonsWays = this.multiPolygonsWays[level]; - boolean hasMulti = e instanceof Way && multiPolygonsWays.containsKey(e.getId()); - if (hasMulti) { - Set set = multiPolygonsWays.get(e.getId()); - boolean first = true; - for (Integer i : set) { - if (first && type == 0) { - type = i; - first = false; - } else { - // do not compare direction - int k = i & 0x7fff; - int ks = k | MapRenderingTypes.POLYGON_TYPE; - // turn of polygon type 3 ^ (suppose polygon = multipolygon) - if (ks == type) { - type = i; - } else { - int ind = typeUse.indexOf(ks); - if (ind == -1) { - typeUse.add(i); - } else { - typeUse.set(ind, i); - } - } - } - } - } - - if (type == 0) { - return; - } - - restrictionsUse.clear(); - // try to find restrictions only for max zoom level - if (level == 0 && highwayRestrictions.containsKey(baseId)) { - restrictionsUse.addAll(highwayRestrictions.get(baseId)); - } - - boolean point = (type & 3) == MapRenderingTypes.POINT_TYPE; - RTree rtree = null; - int zoom; - long id = (baseId << 3) | ((level & 3) << 1); - rtree = mapTree[level]; - zoom = mapZooms.getLevel(level).getMaxZoom() - 1; - boolean skip = false; - - String eName = renderingTypes.getEntityName(e); - if (eName == null) { - eName = multiPolygonsNames.get(baseId); - } - int highwayAttributes = 0; - if (e.getTag(OSMTagKey.HIGHWAY) != null) { - highwayAttributes = MapRenderingTypes.getHighwayAttributes(e); - } - if (e instanceof Way) { - id |= 1; - // simplify route - if (level > 0) { - e = simplifyWay((Way) e, id, hasMulti, zoom, eName, type, level); - skip = e == null; - } - - } - if (!skip) { - DataIndexWriter.insertBinaryMapRenderObjectIndex(pStatements, mapBinaryStat, rtree, e, eName, id, type, typeUse, - highwayAttributes, restrictionsUse, inverse, point, true); - } - } - - protected long encodeTypesToOneLong(int mainType) { - long i = 0; - int ind = 0; - int sh = 0; - if(typeUse.size() > 3){ - log.error("Types for low index way more than 4"); //$NON-NLS-1$ - } - i |= (mainType << sh); - if (typeUse.size() > ind) { - sh += 16; - i |= ((long)typeUse.get(ind++) << sh ); - if (typeUse.size() > ind) { - sh += 16; - i |= ((long)typeUse.get(ind++) << sh ); - if (typeUse.size() > ind) { - sh += 16; - i |= ((long)typeUse.get(ind++) << sh); - } - } - } - return i; - } - - protected int decodeTypesFromOneLong(long i) { - typeUse.clear(); - int mask = (1 << 16) - 1; - int k = (int) (i & mask); - int r = 0; - if (k > 0) { - r = k; - i >>= 16; - k = (int) (i & mask); - if (k > 0) { - typeUse.add(k); - i >>= 16; - k = (int) (i & mask); - if (k > 0) { - typeUse.add(k); - i >>= 16; - k = (int) (i & mask); - if (k > 0) { - typeUse.add(k); - i >>= 16; - } - } - } - } - return r; + indexAddressCreator.registerCityIfNeeded(e); } - - protected Way simplifyWay(Way originalE, long id, boolean hasMulti, int zoom, String name, int type, int level) throws SQLException { - List nodes = originalE.getNodes(); - Way way = new Way(id); - for (String t : originalE.getTagKeySet()) { - way.putTag(t, originalE.getTag(t)); - } - boolean cycle = originalE.getNodeIds().get(0).longValue() == originalE.getNodeIds().get(nodes.size() - 1).longValue(); - long longType = encodeTypesToOneLong(type); - - boolean skip = checkForSmallAreas(nodes, zoom, 3, 3); - if (skip && cycle/* || !hasMulti)*/) { - return null; - } - - MapAlgorithms.simplifyDouglasPeucker(nodes, zoom + 8, 3, way); - if (way.getNodes().size() < 2) { - return null; - } - if (cycle) { - // nothing to do - return way; - } else { - lowLevelWays ++; - DataIndexWriter.insertLowLevelMapBinaryObject(pStatements, mapLowLevelBinaryStat, level, longType, id, way.getNodes(), name); - return null; - } - - } - - private void loadNodes(byte[] nodes, List toPut){ - toPut.clear(); - for (int i = 0; i < nodes.length;) { - int lat = Algoritms.parseIntFromBytes(nodes, i); - i += 4; - int lon = Algoritms.parseIntFromBytes(nodes, i); - i += 4; - toPut.add(Float.intBitsToFloat(lat)); - toPut.add(Float.intBitsToFloat(lon)); - } - } - - private void processingLowLevelWays(IProgress progress) throws SQLException { - restrictionsUse.clear(); - mapLowLevelBinaryStat.executeBatch(); - mapLowLevelBinaryStat.close(); - pStatements.remove(mapLowLevelBinaryStat); - mapLowLevelBinaryStat = null; - mapConnection.commit(); - - PreparedStatement startStat = mapConnection.prepareStatement("SELECT id, start_node, end_node, nodes FROM " - + IndexConstants.LOW_LEVEL_MAP_TABLE + " WHERE start_node = ? AND type=? AND level = ? AND name=?"); - PreparedStatement endStat = mapConnection.prepareStatement("SELECT id, start_node, end_node, nodes FROM " - + IndexConstants.LOW_LEVEL_MAP_TABLE + " WHERE end_node = ? AND type=? AND level = ? AND name=?"); - Statement selectStatement = mapConnection.createStatement(); - ResultSet rs = selectStatement.executeQuery("SELECT id, start_node, end_node, name, nodes, type, level FROM " + IndexConstants.LOW_LEVEL_MAP_TABLE); - Set visitedWays = new LinkedHashSet(); - ArrayList list = new ArrayList(100); - while(rs.next()){ - if(lowLevelWays != -1){ - progress.progress(1); - } - long id = rs.getLong(1); - if(visitedWays.contains(id)){ - continue; - } - - visitedWays.add(id); - int level = rs.getInt(7); - int zoom = mapZooms.getLevel(level).getMaxZoom(); - - long startNode = rs.getLong(2); - long endNode = rs.getLong(3); - - String name = rs.getString(4); - long ltype = rs.getLong(6); - loadNodes(rs.getBytes(5), list); - ArrayList wayNodes = new ArrayList(list); - - - // combine startPoint with EndPoint - boolean combined = true; - while (combined) { - combined = false; - endStat.setLong(1, startNode); - endStat.setLong(2, ltype); - endStat.setShort(3, (short) level); - endStat.setString(4, name); - ResultSet fs = endStat.executeQuery(); - while (fs.next()) { - if (!visitedWays.contains(fs.getLong(1))) { - combined = true; - long lid = fs.getLong(1); - startNode = fs.getLong(2); - visitedWays.add(lid); - loadNodes(fs.getBytes(4), list); - ArrayList li = new ArrayList(list); - // remove first lat/lon point - wayNodes.remove(0); - wayNodes.remove(0); - li.addAll(wayNodes); - wayNodes = li; - } - } - fs.close(); - } - - // combined end point - combined = true; - while (combined) { - combined = false; - startStat.setLong(1, endNode); - startStat.setLong(2, ltype); - startStat.setShort(3, (short) level); - startStat.setString(4, name); - ResultSet fs = startStat.executeQuery(); - while (fs.next()) { - if (!visitedWays.contains(fs.getLong(1))) { - combined = true; - long lid = fs.getLong(1); - endNode = fs.getLong(3); - visitedWays.add(lid); - loadNodes(fs.getBytes(4), list); - for (int i = 2; i < list.size(); i++) { - wayNodes.add(list.get(i)); - } - } - } - fs.close(); - } - List wNodes = new ArrayList(); - for (int i = 0; i < wayNodes.size(); i += 2) { - wNodes.add(new Node(wayNodes.get(i), wayNodes.get(i + 1), i == 0 ? startNode : endNode)); - } - boolean skip = false; - boolean cycle = startNode == endNode; - boolean hasMulti = multiPolygonsWays[level].containsKey(id >> 3); - if(cycle || !hasMulti){ - skip = checkForSmallAreas(wNodes, zoom - 1, 1, 4); - } - - if (!skip) { - Way newWs = new Way(id); - MapAlgorithms.simplifyDouglasPeucker(wNodes, zoom - 1 + 8, 3, newWs); - - int type = decodeTypesFromOneLong(ltype); - DataIndexWriter.insertBinaryMapRenderObjectIndex(pStatements, mapBinaryStat, mapTree[level], newWs, name, - id, type, typeUse, 0, restrictionsUse, false, false, false); - } - - } - - } - - private boolean checkForSmallAreas(List nodes, int zoom, int minz, int maxz) { - int minX = Integer.MAX_VALUE; - int maxX = Integer.MIN_VALUE; - int minY = Integer.MAX_VALUE; - int maxY = Integer.MIN_VALUE; - int c = 0; - for (int i = 0; i < nodes.size(); i++) { - if (nodes.get(i) != null) { - c++; - int x = (int) (MapUtils.getTileNumberX(zoom, nodes.get(i).getLongitude()) * 256d); - int y = (int) (MapUtils.getTileNumberY(zoom, nodes.get(i).getLatitude()) * 256d); - minX = Math.min(minX, x); - maxX = Math.max(maxX, x); - minY = Math.min(minY, y); - maxY = Math.max(maxY, y); - } - } - if (c < 2) { - return true; - } - return ((maxX - minX) <= minz && (maxY - minY) <= maxz) || - ((maxX - minX) <= maxz && (maxY - minY) <= minz); - - } - - - - public boolean nodeIsLastSubTree(RTree tree, long ptr) throws RTreeException { - rtree.Node parent = tree.getReadNode(ptr); - Element[] e = parent.getAllElements(); - for (int i = 0; i < parent.getTotalElements(); i++) { - if (e[i].getElementType() != rtree.Node.LEAF_NODE) { - return false; - } - } - return true; - - } - - public void writeBinaryAddressIndex(BinaryMapIndexWriter writer, IProgress progress) throws IOException, SQLException { - boolean readWayNodes = saveAddressWays; - - writer.startWriteAddressIndex(getRegionName()); - DataIndexReader reader = new DataIndexReader(); - List cities = reader.readCities(mapConnection); - List streets = new ArrayList(); - Collections.sort(cities, new Comparator() { - - @Override - public int compare(City o1, City o2) { - if (o1.getType() != o2.getType()) { - return (o1.getType().ordinal() - o2.getType().ordinal()); - } - return Collator.getInstance().compare(o1.getName(), o2.getName()); - } - }); - PreparedStatement streetstat = reader.getStreetsBuildingPreparedStatement(mapConnection); - PreparedStatement waynodesStat = null; - if (readWayNodes) { - waynodesStat = reader.getStreetsWayNodesPreparedStatement(mapConnection); - } - - int j = 0; - for (; j < cities.size(); j++) { - City c = cities.get(j); - if (c.getType() != CityType.CITY && c.getType() != CityType.TOWN) { - break; - } - } - progress.startTask(Messages.getString("IndexCreator.SERIALIZING_ADRESS"), j + ((cities.size() - j) / 100 + 1)); //$NON-NLS-1$ - - Map> postcodes = new TreeMap>(); - boolean writeCities = true; - - // collect suburbs with is in value - List suburbs = new ArrayList(); - for(City s : cities){ - if(s.getType() == CityType.SUBURB && s.getIsInValue() != null){ - suburbs.add(s); - } - } - - // write cities and after villages - writer.startCityIndexes(false); - for (int i = 0; i < cities.size(); i++) { - City c = cities.get(i); - List listSuburbs = null; - for (City suburb : suburbs) { - if (suburb.getIsInValue().contains(c.getName().toLowerCase())) { - if(listSuburbs == null){ - listSuburbs = new ArrayList(); - } - listSuburbs.add(suburb); - } - } - if (writeCities) { - progress.progress(1); - } else if ((cities.size() - i) % 100 == 0) { - progress.progress(1); - } - if (writeCities && c.getType() != CityType.CITY && c.getType() != CityType.TOWN) { - writer.endCityIndexes(false); - writer.startCityIndexes(true); - writeCities = false; - } - - streets.clear(); - Map> streetNodes = null; - if (readWayNodes) { - streetNodes = new LinkedHashMap>(); - } - long time = System.currentTimeMillis(); - reader.readStreetsBuildings(streetstat, c, streets, waynodesStat, streetNodes, listSuburbs); - long f = System.currentTimeMillis() - time; - writer.writeCityIndex(c, streets, streetNodes); - int bCount = 0; - for (Street s : streets) { - bCount++; - for (Building b : s.getBuildings()) { - bCount++; - if (b.getPostcode() != null) { - if (!postcodes.containsKey(b.getPostcode())) { - postcodes.put(b.getPostcode(), new LinkedHashSet(3)); - } - postcodes.get(b.getPostcode()).add(s); - } - } - } - if (f > 500) { - System.out.println("! " + c.getName() + " ! " + f + " " + bCount + " streets " + streets.size()); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ - } - } - writer.endCityIndexes(!writeCities); - - // write postcodes - writer.startPostcodes(); - for (String s : postcodes.keySet()) { - writer.writePostcode(s, postcodes.get(s)); - } - writer.endPostcodes(); - - progress.finishTask(); - - writer.endWriteAddressIndex(); - writer.flush(); - streetstat.close(); - if (readWayNodes) { - waynodesStat.close(); - } - - } - - public void writeBinaryMapIndex(BinaryMapIndexWriter writer) throws IOException, SQLException { - try { - PreparedStatement selectData = mapConnection.prepareStatement("SELECT nodes, types, name, highway, restrictions FROM binary_map_objects WHERE id = ?"); //$NON-NLS-1$ - - writer.startWriteMapIndex(regionName); - - for (int i = 0; i < mapZooms.size(); i++) { - RTree rtree = mapTree[i]; - long rootIndex = rtree.getFileHdr().getRootIndex(); - rtree.Node root = rtree.getReadNode(rootIndex); - Rect rootBounds = calcBounds(root); - if (rootBounds != null) { - boolean last = nodeIsLastSubTree(rtree, rootIndex); - writer.startWriteMapLevelIndex(mapZooms.getLevel(i).getMinZoom(), mapZooms.getLevel(i).getMaxZoom(), rootBounds - .getMinX(), rootBounds.getMaxX(), rootBounds.getMinY(), rootBounds.getMaxY()); - if (last) { - writer.startMapTreeElement(rootBounds.getMinX(), rootBounds.getMaxX(), rootBounds.getMinY(), rootBounds.getMaxY()); - - } - writeBinaryMapTree(root, rtree, writer, selectData); - if (last) { - writer.endWriteMapTreeElement(); - } - - writer.endWriteMapLevelIndex(); - } - } - selectData.close(); - writer.writeMapEncodingRules(renderingTypes.getEncodingRuleTypes()); - writer.endWriteMapIndex(); - writer.flush(); - } catch (RTreeException e) { - throw new IllegalStateException(e); - } - } - - public void writeBinaryMapTree(rtree.Node parent, RTree r, BinaryMapIndexWriter writer, PreparedStatement selectData) throws IOException, RTreeException, SQLException { - Element[] e = parent.getAllElements(); - - for (int i = 0; i < parent.getTotalElements(); i++) { - Rect re = e[i].getRect(); - if (e[i].getElementType() == rtree.Node.LEAF_NODE) { - long id = ((LeafElement) e[i]).getPtr(); - selectData.setLong(1, id); - ResultSet rs = selectData.executeQuery(); - if (rs.next()) { - // mapConnection.prepareStatement("SELECT nodes, types, name, highway, restrictions FROM binary_map_objects WHERE id = ?"); - writer.writeMapData(id, rs.getBytes(1), - rs.getBytes(2), rs.getString(3), - rs.getInt(4), rs.getBytes(5)); - } else { - log.error("Something goes wrong with id = " + id); //$NON-NLS-1$ - } - } else { - long ptr = ((NonLeafElement) e[i]).getPtr(); - rtree.Node ns = r.getReadNode(ptr); - - writer.startMapTreeElement(re.getMinX(), re.getMaxX(), re.getMinY(), re.getMaxY()); - - writeBinaryMapTree(ns, r, writer, selectData); - writer.endWriteMapTreeElement(); - } - } - } - - public Rect calcBounds(rtree.Node n) { - Rect r = null; - Element[] e = n.getAllElements(); - for (int i = 0; i < n.getTotalElements(); i++) { - Rect re = e[i].getRect(); - if (r == null) { - try { - r = new Rect(re.getMinX(), re.getMinY(), re.getMaxX(), re.getMaxY()); - } catch (IllegalValueException ex) { - } - } else { - r.expandToInclude(re); - } - } - return r; - } - public String getRTreeMapIndexNonPackFileName() { return mapFile.getAbsolutePath() + ".rtree"; //$NON-NLS-1$ } @@ -1618,18 +231,11 @@ public class IndexCreator { public void generateIndexes(File readFile, IProgress progress, IOsmStorageFilter addFilter, MapZooms mapZooms, MapRenderingTypes renderingTypes) throws IOException, SAXException, SQLException { - if(renderingTypes != null){ - this.renderingTypes = renderingTypes; + if(renderingTypes == null){ + renderingTypes = MapRenderingTypes.getDefault(); } - - if(mapZooms != null){ - this.mapZooms = mapZooms; - } else { - mapZooms = this.mapZooms; - } - multiPolygonsWays = new Map[mapZooms.size()]; - for (int i = 0; i < multiPolygonsWays.length; i++) { - multiPolygonsWays[i] = new LinkedHashMap>(); + if(mapZooms == null){ + mapZooms = MapZooms.getDefault(); } // clear previous results and setting variables @@ -1639,15 +245,18 @@ public class IndexCreator { regionName = Algoritms.capitalizeFirstLetterAndLowercase(readFile.getName().substring(0, i)); } } - - cities.clear(); - cityManager.clear(); - lowLevelWays = -1; - postalCodeRelations.clear(); + + indexMapCreator.initSettings(mapZooms, renderingTypes); + + // init address + String[] normalizeDefaultSuffixes = null; + String[] normalizeSuffixes = null; if (normalizeStreets) { normalizeDefaultSuffixes = DataExtractionSettings.getSettings().getDefaultSuffixesToNormalizeStreets(); normalizeSuffixes = DataExtractionSettings.getSettings().getSuffixesToNormalizeStreets(); } + indexAddressCreator.initSettings(normalizeStreets, normalizeDefaultSuffixes, normalizeSuffixes, + saveAddressWays, cityAdminLevel); // Main generation method try { @@ -1687,11 +296,7 @@ public class IndexCreator { mapConnection.setAutoCommit(false); try { if (indexMap) { - mapTree = new RTree[mapZooms.size()]; - for (int i = 0; i < mapZooms.size(); i++) { - mapTree[i] = new RTree(getRTreeMapIndexPackFileName() + i); - } - + indexMapCreator.createRTreeFiles(getRTreeMapIndexPackFileName()); } if (indexTransport) { indexTransportCreator.createRTreeFile(getRTreeTransportStopsPackFileName()); @@ -1716,21 +321,11 @@ public class IndexCreator { allNodes = accessor.iterateOverEntities(progress, EntityType.NODE, allNodes, new OsmDbVisitor() { @Override public void iterateEntity(Entity e, OsmDbAccessorContext ctx) { - registerCityIfNeeded(e); + indexAddressCreator.registerCityIfNeeded(e); } }); } - - for (City c : cities.values()) { - DataIndexWriter.writeCity(addressCityStat, pStatements, c, BATCH_SIZE); - } - // commit to put all cities - if (pStatements.get(addressCityStat) > 0) { - addressCityStat.executeBatch(); - pStatements.put(addressCityStat, 0); - mapConnection.commit(); - } - + indexAddressCreator.writeCitiesIntoDb(); } // 3.2 index address relations @@ -1740,7 +335,13 @@ public class IndexCreator { allRelations = accessor.iterateOverEntities(progress, EntityType.RELATION, allRelations, new OsmDbVisitor() { @Override public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException { - indexAddressRelationsAndMultiPolygons(e, ctx); + if (indexAddress) { + indexAddressCreator.indexAddressRelation((Relation) e, ctx); + indexAddressCreator.indexBoundariesRelation((Relation) e, ctx); + } + if (indexMap) { + indexMapCreator.indexMapRelationsAndMultiPolygons(e, ctx); + } } }); if (indexAddress) { @@ -1749,26 +350,13 @@ public class IndexCreator { allWays = accessor.iterateOverEntities(progress, EntityType.WAY, allWays, new OsmDbVisitor() { @Override public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException { - if (e instanceof Way && "administrative".equals(e.getTag(OSMTagKey.BOUNDARY))) { //$NON-NLS-1$ - indexBoundariesRelation(e ,ctx); - } + indexAddressCreator.indexBoundariesRelation(e ,ctx); } }); + + indexAddressCreator.commitToPutAllCities(); } - // commit to put all cities - if (indexAddress) { - if (pStatements.get(addressBuildingStat) > 0) { - addressBuildingStat.executeBatch(); - pStatements.put(addressBuildingStat, 0); - } - if (pStatements.get(addressStreetNodeStat) > 0) { - addressStreetNodeStat.executeBatch(); - pStatements.put(addressStreetNodeStat, 0); - } - mapConnection.commit(); - } - } // 3.3 MAIN iterate over all entities @@ -1802,35 +390,28 @@ public class IndexCreator { // 3.4 combine all low level ways and simplify them if(indexMap){ progress.setGeneralProgress("[90 / 100]"); - progress.startTask(Messages.getString("IndexCreator.INDEX_LO_LEVEL_WAYS"), lowLevelWays); - processingLowLevelWays(progress); + progress.startTask(Messages.getString("IndexCreator.INDEX_LO_LEVEL_WAYS"), indexMapCreator.getLowLevelWays()); + indexMapCreator.processingLowLevelWays(progress); } // 3.5 update all postal codes from relations - if (indexAddress && !postalCodeRelations.isEmpty()) { + if (indexAddress) { progress.setGeneralProgress("[90 / 100]"); progress.startTask(Messages.getString("IndexCreator.REGISTER_PCODES"), -1); - if (pStatements.get(addressBuildingStat) > 0) { - addressBuildingStat.executeBatch(); - pStatements.put(addressBuildingStat, 0); - mapConnection.commit(); - } - processingPostcodes(); + indexAddressCreator.processingPostcodes(); } // 4. packing map rtree indexes if (indexMap) { progress.setGeneralProgress("[90 / 100]"); //$NON-NLS-1$ progress.startTask(Messages.getString("IndexCreator.PACK_RTREE_MAP"), -1); //$NON-NLS-1$ - for (int i = 0; i < mapZooms.size(); i++) { - mapTree[i] = packRtreeFile(mapTree[i], getRTreeMapIndexNonPackFileName() + i, getRTreeMapIndexPackFileName() + i); - } + indexMapCreator.packRtreeFiles(getRTreeMapIndexNonPackFileName(), getRTreeMapIndexPackFileName()); } if (indexTransport) { progress.setGeneralProgress("[90 / 100]"); //$NON-NLS-1$ progress.startTask(Messages.getString("IndexCreator.PACK_RTREE_TRANSP"), -1); //$NON-NLS-1$ - indexTransportCreator.transportStopsTree = packRtreeFile(indexTransportCreator.transportStopsTree, getRTreeTransportStopsFileName(), getRTreeTransportStopsPackFileName()); + indexTransportCreator.packRTree(getRTreeTransportStopsFileName(), getRTreeTransportStopsPackFileName()); } } @@ -1844,20 +425,15 @@ public class IndexCreator { if (indexMap) { progress.setGeneralProgress("[95 of 100]"); progress.startTask("Writing map index to binary file...", -1); - closePreparedStatements(mapBinaryStat, mapLowLevelBinaryStat); - mapConnection.commit(); - writeBinaryMapIndex(writer); + indexMapCreator.writeBinaryMapIndex(writer, regionName); } if (indexAddress) { progress.setGeneralProgress("[95 of 100]"); progress.startTask("Writing address index to binary file...", -1); - closePreparedStatements(addressCityStat, addressStreetStat, addressStreetNodeStat, addressBuildingStat); - mapConnection.commit(); - writeBinaryAddressIndex(writer, progress); + indexAddressCreator.writeBinaryAddressIndex(writer, regionName, progress); } - if (indexTransport) { progress.setGeneralProgress("[95 of 100]"); progress.startTask("Writing transport index to binary file...", -1); @@ -1884,14 +460,12 @@ public class IndexCreator { try { accessor.closeReadingConnection(); - for (PreparedStatement p : pStatements.keySet()) { - if (pStatements.get(p) > 0) { - p.executeBatch(); - } - p.close(); - } - indexPoiCreator.commitAndClosePoiFile(lastModifiedDate); + indexAddressCreator.closeAllPreparedStatements(); + indexTransportCreator.commitAndCloseFiles(getRTreeTransportStopsFileName(), + getRTreeTransportStopsPackFileName(), deleteDatabaseIndexes); + indexMapCreator.commitAndCloseFiles(getRTreeMapIndexNonPackFileName(), + getRTreeMapIndexPackFileName(), deleteDatabaseIndexes); if (mapConnection != null) { mapConnection.commit(); @@ -1903,27 +477,7 @@ public class IndexCreator { dialect.removeDatabase(tempDBFile); } } - - // delete map rtree files - if (mapTree != null) { - for (int i = 0; i < mapTree.length; i++) { - if (mapTree[i] != null) { - RandomAccessFile file = mapTree[i].getFileHdr().getFile(); - file.close(); - } - - } - for (int i = 0; i < mapTree.length; i++) { - File f = new File(getRTreeMapIndexNonPackFileName() + i); - if (f.exists() && deleteDatabaseIndexes) { - f.delete(); - } - f = new File(getRTreeMapIndexPackFileName() + i); - if (f.exists() && deleteDatabaseIndexes) { - f.delete(); - } - } - } + // do not delete first db connection if (dbConn != null) { @@ -1949,47 +503,7 @@ public class IndexCreator { } - private void processingPostcodes() throws SQLException { - PreparedStatement pstat = DataIndexWriter.getUpdateBuildingPostcodePreparedStatement(mapConnection);; - pStatements.put(pstat, 0); - for (Relation r : postalCodeRelations) { - String tag = r.getTag(OSMTagKey.POSTAL_CODE); - for (EntityId l : r.getMemberIds()) { - pstat.setString(1, tag); - pstat.setLong(2, l.getId()); - DataIndexWriter.addBatch(pStatements, pstat, BATCH_SIZE); - } - } - if (pStatements.get(pstat) > 0) { - pstat.executeBatch(); - } - pStatements.remove(pstat); - } - - public RTree packRtreeFile(RTree tree, String nonPackFileName, String packFileName) throws IOException { - try { - assert rtree.Node.MAX < 50 : "It is better for search performance"; //$NON-NLS-1$ - tree.flush(); - File file = new File(packFileName); - if (file.exists()) { - file.delete(); - } - long rootIndex = tree.getFileHdr().getRootIndex(); - if (!nodeIsLastSubTree(tree, rootIndex)) { - // there is a bug for small files in packing method - new Pack().packTree(tree, packFileName); - tree.getFileHdr().getFile().close(); - file = new File(nonPackFileName); - file.delete(); - - return new RTree(packFileName); - } - } catch (RTreeException e) { - log.error("Error flushing", e); //$NON-NLS-1$ - throw new IOException(e); - } - return tree; - } + private OsmDbCreator extractOsmToNodesDB(File readFile, IProgress progress, IOsmStorageFilter addFilter) throws FileNotFoundException, @@ -2054,70 +568,25 @@ public class IndexCreator { // 2.2 create rtree map if (indexMap) { - DataIndexWriter.createMapIndexStructure(mapConnection); - mapBinaryStat = DataIndexWriter.createStatementMapBinaryInsert(mapConnection); - mapLowLevelBinaryStat = DataIndexWriter.createStatementLowLevelMapBinaryInsert(mapConnection); - try { - mapTree = new RTree[mapZooms.size()]; - for (int i = 0; i < mapZooms.size(); i++) { - File file = new File(getRTreeMapIndexNonPackFileName() + i); - if (file.exists()) { - file.delete(); - } - mapTree[i] = new RTree(getRTreeMapIndexNonPackFileName() + i); - // very slow - // mapTree[i].getFileHdr().setBufferPolicy(true); - } - } catch (RTreeException e) { - throw new IOException(e); - } - pStatements.put(mapBinaryStat, 0); - pStatements.put(mapLowLevelBinaryStat, 0); + indexMapCreator.createDatabaseStructure(mapConnection, dialect, getRTreeMapIndexNonPackFileName()); } if (indexAddress) { - DataIndexWriter.createAddressIndexStructure(mapConnection, dialect); - addressCityStat = DataIndexWriter.getCityInsertPreparedStatement(mapConnection); - addressStreetStat = DataIndexWriter.getStreetInsertPreparedStatement(mapConnection); - addressBuildingStat = DataIndexWriter.getBuildingInsertPreparedStatement(mapConnection); - addressStreetNodeStat = DataIndexWriter.getStreetNodeInsertPreparedStatement(mapConnection); - addressSearchStreetStat = DataIndexWriter.getSearchStreetPreparedStatement(mapConnection); - addressSearchBuildingStat = DataIndexWriter.getSearchBuildingPreparedStatement(mapConnection); - addressSearchStreetNodeStat = DataIndexWriter.getStreeNodeSearchPreparedStatement(mapConnection); - - pStatements.put(addressCityStat, 0); - pStatements.put(addressStreetStat, 0); - pStatements.put(addressStreetNodeStat, 0); - pStatements.put(addressBuildingStat, 0); - // put search statements to close them after all - pStatements.put(addressSearchBuildingStat, 0); - pStatements.put(addressSearchStreetNodeStat, 0); - pStatements.put(addressSearchStreetStat, 0); - + indexAddressCreator.createDatabaseStructure(mapConnection, dialect); } if (indexPOI) { - indexPoiCreator.createDatabaseStructure(new File(workingDir, getPoiFileName()), pStatements); + indexPoiCreator.createDatabaseStructure(new File(workingDir, getPoiFileName())); } if (indexTransport) { - indexTransportCreator.createTransportIndexStructure(mapConnection, dialect, getRTreeTransportStopsFileName(), pStatements); + indexTransportCreator.createTransportIndexStructure(mapConnection, dialect, getRTreeTransportStopsFileName()); } } - protected void closePreparedStatements(PreparedStatement... preparedStatements) throws SQLException { - for (PreparedStatement p : preparedStatements) { - if (p != null) { - p.executeBatch(); - p.close(); - pStatements.remove(p); - } - } - } public static void main(String[] args) throws IOException, SAXException, SQLException { - long time = System.currentTimeMillis(); IndexCreator creator = new IndexCreator(new File("/home/victor/projects/OsmAnd/data/osm-gen/")); //$NON-NLS-1$ creator.setIndexMap(true); diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/IndexPoiCreator.java b/DataExtractionOSM/src/net/osmand/data/preparation/IndexPoiCreator.java index acf12bb2fe..63dd58984e 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/IndexPoiCreator.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/IndexPoiCreator.java @@ -7,7 +7,6 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; import java.util.Collection; -import java.util.Date; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -15,7 +14,6 @@ import java.util.Map; import net.osmand.Algoritms; import net.osmand.data.Amenity; import net.osmand.data.AmenityType; -import net.osmand.data.index.DataIndexWriter; import net.osmand.data.index.IndexConstants; import net.osmand.osm.Entity; import net.osmand.osm.MapUtils; @@ -24,11 +22,9 @@ import net.osmand.osm.OSMSettings.OSMTagKey; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -public class IndexPoiCreator { +public class IndexPoiCreator extends AbstractIndexPartCreator { - private static final int BATCH_SIZE = 1000; private static final Log log = LogFactory.getLog(IndexPoiCreator.class); - private final IndexCreator creator; private Connection poiConnection; private File poiIndexFile; @@ -36,16 +32,15 @@ public class IndexPoiCreator { private List tempAmenityList = new ArrayList(); - public IndexPoiCreator(IndexCreator creator){ - this.creator = creator; + public IndexPoiCreator(){ } - public void iterateEntity(Entity e, Map pStatements) throws SQLException{ + public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException{ tempAmenityList.clear(); tempAmenityList = Amenity.parseAmenities(e, tempAmenityList); if (!tempAmenityList.isEmpty() && poiPreparedStatement != null) { // load data for way (location etc...) - creator.loadEntityData(e, false); + ctx.loadEntityData(e, false); for (Amenity a : tempAmenityList) { checkEntity(e); a.setEntity(e); @@ -68,6 +63,7 @@ public class IndexPoiCreator { } } + closeAllPreparedStatements(); } public void insertAmenityIntoPoi( Map map, Amenity amenity) throws SQLException { @@ -83,7 +79,7 @@ public class IndexPoiCreator { poiPreparedStatement.setString(8, amenity.getOpeningHours()); poiPreparedStatement.setString(9, amenity.getSite()); poiPreparedStatement.setString(10, amenity.getPhone()); - DataIndexWriter.addBatch(map, poiPreparedStatement, BATCH_SIZE); + addBatch(map, poiPreparedStatement, BATCH_SIZE); } public void checkEntity(Entity e){ @@ -109,7 +105,7 @@ public class IndexPoiCreator { } } - public void createDatabaseStructure(File poiIndexFile, Map pStatements) throws SQLException { + public void createDatabaseStructure(File poiIndexFile) throws SQLException { this.poiIndexFile = poiIndexFile; // to save space if (poiIndexFile.exists()) { @@ -117,7 +113,7 @@ public class IndexPoiCreator { } poiIndexFile.getParentFile().mkdirs(); // creating nodes db to fast access for all nodes - poiConnection = creator.getDatabaseConnection(poiIndexFile.getAbsolutePath(), DBDialect.SQLITE); + poiConnection = DBDialect.SQLITE.getDatabaseConnection(poiIndexFile.getAbsolutePath(), log); createPoiIndexStructure(poiConnection); poiPreparedStatement = createStatementAmenityInsert(poiConnection); pStatements.put(poiPreparedStatement, 0); diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/IndexTransportCreator.java b/DataExtractionOSM/src/net/osmand/data/preparation/IndexTransportCreator.java index f4d7267cdd..d68fa50305 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/IndexTransportCreator.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/IndexTransportCreator.java @@ -20,8 +20,6 @@ import java.util.Map.Entry; import net.osmand.binary.BinaryMapIndexWriter; import net.osmand.data.TransportRoute; import net.osmand.data.TransportStop; -import net.osmand.data.index.DataIndexWriter; -import net.osmand.data.index.IndexConstants; import net.osmand.osm.Entity; import net.osmand.osm.MapUtils; import net.osmand.osm.Node; @@ -42,17 +40,15 @@ import rtree.RTreeException; import rtree.RTreeInsertException; import rtree.Rect; -public class IndexTransportCreator { +public class IndexTransportCreator extends AbstractIndexPartCreator { - private final IndexCreator creator; - private static final int BATCH_SIZE = 1000; private static final Log log = LogFactory.getLog(IndexTransportCreator.class); private Set visitedStops = new HashSet(); private PreparedStatement transRouteStat; private PreparedStatement transRouteStopsStat; private PreparedStatement transStopsStat; - protected RTree transportStopsTree; + private RTree transportStopsTree; private static Set acceptedRoutes = new HashSet(); @@ -69,8 +65,7 @@ public class IndexTransportCreator { acceptedRoutes.add("ferry"); //$NON-NLS-1$ } - public IndexTransportCreator(IndexCreator creator){ - this.creator = creator; + public IndexTransportCreator(){ } @@ -78,11 +73,8 @@ public class IndexTransportCreator { transportStopsTree = new RTree(rtreeTransportStopFile); } - public void createTransportIndexStructure(Connection conn, DBDialect dialect, String rtreeStopsFileName, Map pStatements) throws SQLException, IOException{ + public void createTransportIndexStructure(Connection conn, DBDialect dialect, String rtreeStopsFileName) throws SQLException, IOException{ Statement stat = conn.createStatement(); - assert IndexConstants.TRANSPORT_ROUTE_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - assert IndexConstants.TRANSPORT_STOP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - assert IndexConstants.TRANSPORT_ROUTE_STOP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ stat.executeUpdate("create table transport_route (id bigint primary key, type varchar(255), operator varchar(255)," + "ref varchar(255), name varchar(255), name_en varchar(255), dist int)"); @@ -96,9 +88,9 @@ public class IndexTransportCreator { stat.executeUpdate("create index transport_stop_id on transport_stop (id)"); stat.executeUpdate("create index transport_stop_location on transport_stop (latitude, longitude)"); - if(dialect == DBDialect.SQLITE){ - stat.execute("PRAGMA user_version = " + IndexConstants.TRANSPORT_TABLE_VERSION); //$NON-NLS-1$ - } +// if(dialect == DBDialect.SQLITE){ +// stat.execute("PRAGMA user_version = " + IndexConstants.TRANSPORT_TABLE_VERSION); //$NON-NLS-1$ +// } stat.close(); try { @@ -119,7 +111,11 @@ public class IndexTransportCreator { } - public void visitEntityMainStep(Entity e, OsmDbAccessorContext ctx, Map pStatements) throws SQLException { + public void packRTree(String rtreeTransportStopsFileName, String rtreeTransportStopsPackFileName) throws IOException { + transportStopsTree = packRtreeFile(transportStopsTree, rtreeTransportStopsFileName, rtreeTransportStopsPackFileName); + } + + public void visitEntityMainStep(Entity e, OsmDbAccessorContext ctx) throws SQLException { if (e instanceof Relation && e.getTag(OSMTagKey.ROUTE) != null) { ctx.loadEntityData(e, true); TransportRoute route = indexTransportRoute((Relation) e); @@ -130,11 +126,11 @@ public class IndexTransportCreator { } } - public void insertTransportIntoIndex(PreparedStatement prepRoute, PreparedStatement prepRouteStops, + + private void insertTransportIntoIndex(PreparedStatement prepRoute, PreparedStatement prepRouteStops, PreparedStatement prepStops, RTree transportStopsTree, Set writtenStops, TransportRoute route, Map statements, int batchSize) throws SQLException { - assert IndexConstants.TRANSPORT_ROUTE_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ prepRoute.setLong(1, route.getId()); prepRoute.setString(2, route.getType()); prepRoute.setString(3, route.getOperator()); @@ -142,26 +138,22 @@ public class IndexTransportCreator { prepRoute.setString(5, route.getName()); prepRoute.setString(6, route.getEnName()); prepRoute.setInt(7, route.getAvgBothDistance()); - DataIndexWriter.addBatch(statements, prepRoute); + addBatch(statements, prepRoute); writeRouteStops(transportStopsTree, prepRouteStops, prepStops, statements, writtenStops, route, route.getForwardStops(), true); writeRouteStops(transportStopsTree, prepRouteStops, prepStops, statements, writtenStops, route, route.getBackwardStops(), false); } - public static PreparedStatement createStatementTransportStopInsert(Connection conn) throws SQLException{ - assert IndexConstants.TRANSPORT_STOP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ + private PreparedStatement createStatementTransportStopInsert(Connection conn) throws SQLException{ return conn.prepareStatement("insert into transport_stop(id, latitude, longitude, name, name_en) values(?, ?, ?, ?, ?)"); } - public static PreparedStatement createStatementTransportRouteStopInsert(Connection conn) throws SQLException{ - assert IndexConstants.TRANSPORT_ROUTE_STOP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ + private PreparedStatement createStatementTransportRouteStopInsert(Connection conn) throws SQLException{ return conn.prepareStatement("insert into transport_route_stop(route, stop, direction, ord) values(?, ?, ?, ?)"); } - private static void writeRouteStops(RTree transportStopsTree, PreparedStatement prepRouteStops, PreparedStatement prepStops, Map count, + private void writeRouteStops(RTree transportStopsTree, PreparedStatement prepRouteStops, PreparedStatement prepStops, Map count, Set writtenStops, TransportRoute r, List stops, boolean direction) throws SQLException { - assert IndexConstants.TRANSPORT_STOP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ - assert IndexConstants.TRANSPORT_ROUTE_STOP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ int i = 0; for(TransportStop s : stops){ if (!writtenStops.contains(s.getId())) { @@ -172,7 +164,7 @@ public class IndexTransportCreator { prepStops.setString(5, s.getEnName()); int x = (int) MapUtils.getTileNumberX(24, s.getLocation().getLongitude()); int y = (int) MapUtils.getTileNumberY(24, s.getLocation().getLatitude()); - DataIndexWriter.addBatch(count, prepStops); + addBatch(count, prepStops); try { transportStopsTree.insert(new LeafElement(new Rect(x, y, x, y), s.getId())); } catch (RTreeInsertException e) { @@ -186,12 +178,11 @@ public class IndexTransportCreator { prepRouteStops.setLong(2, s.getId()); prepRouteStops.setInt(3, direction ? 1 : 0); prepRouteStops.setInt(4, i++); - DataIndexWriter.addBatch(count, prepRouteStops); + addBatch(count, prepRouteStops); } } - public static PreparedStatement createStatementTransportRouteInsert(Connection conn) throws SQLException{ - assert IndexConstants.TRANSPORT_ROUTE_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$ + private PreparedStatement createStatementTransportRouteInsert(Connection conn) throws SQLException{ return conn.prepareStatement("insert into transport_route(id, type, operator, ref, name, name_en, dist) values(?, ?, ?, ?, ?, ?, ?)"); } @@ -199,7 +190,7 @@ public class IndexTransportCreator { public void writeBinaryTransportIndex(BinaryMapIndexWriter writer, String regionName, Connection mapConnection) throws IOException, SQLException { try { - creator.closePreparedStatements(transRouteStat, transRouteStopsStat, transStopsStat); + closePreparedStatements(transRouteStat, transRouteStopsStat, transStopsStat); mapConnection.commit(); transportStopsTree.flush(); @@ -332,7 +323,7 @@ public class IndexTransportCreator { } - public void commitAndCloseFiles(String rtreeStopsFileName, String rtreeStopsPackFileName, boolean deleteDatabaseIndexes) throws IOException { + public void commitAndCloseFiles(String rtreeStopsFileName, String rtreeStopsPackFileName, boolean deleteDatabaseIndexes) throws IOException, SQLException { // delete transport rtree files if (transportStopsTree != null) { transportStopsTree.getFileHdr().getFile().close(); @@ -345,6 +336,7 @@ public class IndexTransportCreator { f.delete(); } } + closeAllPreparedStatements(); } diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/IndexVectorMapCreator.java b/DataExtractionOSM/src/net/osmand/data/preparation/IndexVectorMapCreator.java new file mode 100644 index 0000000000..ef701bc1e8 --- /dev/null +++ b/DataExtractionOSM/src/net/osmand/data/preparation/IndexVectorMapCreator.java @@ -0,0 +1,877 @@ +package net.osmand.data.preparation; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import net.osmand.Algoritms; +import net.osmand.IProgress; +import net.osmand.binary.BinaryMapIndexWriter; +import net.osmand.data.MapAlgorithms; +import net.osmand.osm.Entity; +import net.osmand.osm.LatLon; +import net.osmand.osm.MapRenderingTypes; +import net.osmand.osm.MapUtils; +import net.osmand.osm.Node; +import net.osmand.osm.Relation; +import net.osmand.osm.Way; +import net.osmand.osm.Entity.EntityId; +import net.osmand.osm.Entity.EntityType; +import net.osmand.osm.OSMSettings.OSMTagKey; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import rtree.Element; +import rtree.IllegalValueException; +import rtree.LeafElement; +import rtree.NonLeafElement; +import rtree.RTree; +import rtree.RTreeException; +import rtree.RTreeInsertException; +import rtree.Rect; + +public class IndexVectorMapCreator extends AbstractIndexPartCreator { + + private static final Log log = LogFactory.getLog(IndexVectorMapCreator.class); + private MapRenderingTypes renderingTypes; + + // MEMORY map : save it in memory while that is allowed + private Map>[] multiPolygonsWays; + private Map multiPolygonsNames = new LinkedHashMap(); + private Map> highwayRestrictions = new LinkedHashMap>(); + + // local purpose + List typeUse = new ArrayList(8); + List restrictionsUse = new ArrayList(8); + private MapZooms mapZooms; + + private PreparedStatement mapBinaryStat; + private PreparedStatement mapLowLevelBinaryStat; + private int lowLevelWays = -1; + private RTree[] mapTree = null; + private Connection mapConnection; + + + public IndexVectorMapCreator() { + } + + public void indexMapRelationsAndMultiPolygons(Entity e, OsmDbAccessorContext ctx) throws SQLException { + if (e instanceof Relation && "restriction".equals(e.getTag(OSMTagKey.TYPE))) { //$NON-NLS-1$ + String val = e.getTag("restriction"); //$NON-NLS-1$ + if (val != null) { + byte type = -1; + if ("no_right_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$ + type = MapRenderingTypes.RESTRICTION_NO_RIGHT_TURN; + } else if ("no_left_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$ + type = MapRenderingTypes.RESTRICTION_NO_LEFT_TURN; + } else if ("no_u_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$ + type = MapRenderingTypes.RESTRICTION_NO_U_TURN; + } else if ("no_straight_on".equalsIgnoreCase(val)) { //$NON-NLS-1$ + type = MapRenderingTypes.RESTRICTION_NO_STRAIGHT_ON; + } else if ("only_right_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$ + type = MapRenderingTypes.RESTRICTION_ONLY_RIGHT_TURN; + } else if ("only_left_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$ + type = MapRenderingTypes.RESTRICTION_ONLY_LEFT_TURN; + } else if ("only_straight_on".equalsIgnoreCase(val)) { //$NON-NLS-1$ + type = MapRenderingTypes.RESTRICTION_ONLY_STRAIGHT_ON; + } + if (type != -1) { + ctx.loadEntityData(e, true); + Collection fromL = ((Relation) e).getMemberIds("from"); //$NON-NLS-1$ + Collection toL = ((Relation) e).getMemberIds("to"); //$NON-NLS-1$ + if (!fromL.isEmpty() && !toL.isEmpty()) { + EntityId from = fromL.iterator().next(); + EntityId to = toL.iterator().next(); + if (from.getType() == EntityType.WAY) { + if (!highwayRestrictions.containsKey(from.getId())) { + highwayRestrictions.put(from.getId(), new ArrayList(4)); + } + highwayRestrictions.get(from.getId()).add((to.getId() << 3) | (long) type); + } + } + } + } + } + if (e instanceof Relation && "multipolygon".equals(e.getTag(OSMTagKey.TYPE))) { //$NON-NLS-1$ + ctx.loadEntityData(e, true); + Map entities = ((Relation) e).getMemberEntities(); + + boolean outerFound = false; + for (Entity es : entities.keySet()) { + if (es instanceof Way) { + boolean inner = "inner".equals(entities.get(es)); //$NON-NLS-1$ + if (!inner) { + outerFound = true; + for (String t : es.getTagKeySet()) { + e.putTag(t, es.getTag(t)); + } + break; + } + } + } + if(!outerFound){ + log.warn("Probably map bug: Multipoligon id=" + e.getId() + " contains only inner ways : "); //$NON-NLS-1$ //$NON-NLS-2$ + return; + } + + int mtType = findMultiPolygonType(e, 0); + if (mtType != 0) { + + String name = renderingTypes.getEntityName(e); + List> completedRings = new ArrayList>(); + List> incompletedRings = new ArrayList>(); + for (Entity es : entities.keySet()) { + if (es instanceof Way) { + if (!((Way) es).getNodeIds().isEmpty()) { + combineMultiPolygons((Way) es, completedRings, incompletedRings); + } + } + } + // skip incompleted rings and do not add whole relation ? + if (!incompletedRings.isEmpty()) { + // log.warn("In multipolygon " + e.getId() + " there are incompleted ways : " + incompletedRings); + return; + // completedRings.addAll(incompletedRings); + } + + // skip completed rings that are not one type + for (List l : completedRings) { + boolean innerType = "inner".equals(entities.get(l.get(0))); //$NON-NLS-1$ + for (Way way : l) { + boolean inner = "inner".equals(entities.get(way)); //$NON-NLS-1$ + if (innerType != inner) { + log.warn("Probably map bug: Multipoligon contains outer and inner ways.\n" + //$NON-NLS-1$ + "Way:" + way.getId() + " is strange part of completed ring. InnerType:" + innerType + " way inner: " + inner + " way inner string:" + entities.get(way)); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ + return; + } + } + } + + for (List l : completedRings) { + boolean innerType = "inner".equals(entities.get(l.get(0))); //$NON-NLS-1$ + boolean clockwise = MapSwingAlgorithms.isClockwiseWay(l); + // clockwise - outer (like coastline), anticlockwise - inner + boolean inverse = clockwise != !innerType; + for (Way way : l) { + boolean inner = "inner".equals(entities.get(way)); //$NON-NLS-1$ + if (!inner && name != null) { + multiPolygonsNames.put(way.getId(), name); + } + putMultipolygonType(multiPolygonsWays[0], way.getId(), mtType, inverse); + for (int i = 1; i < multiPolygonsWays.length; i++) { + int type = findMultiPolygonType(e, i); + if (type != 0) { + putMultipolygonType(multiPolygonsWays[i], way.getId(), type, inverse); + } + } + } + } + + } + } + + } + + private void putMultipolygonType(Map> multiPolygonsWays, long baseId, int mtType, boolean inverse) { + if (mtType == 0) { + return; + } + if (!multiPolygonsWays.containsKey(baseId)) { + multiPolygonsWays.put(baseId, new LinkedHashSet()); + } + if (inverse) { + multiPolygonsWays.get(baseId).add(mtType | (1 << 15)); + } else { + multiPolygonsWays.get(baseId).add(mtType); + } + } + + private int findMultiPolygonType(Entity e, int level) { + int t = renderingTypes.encodeEntityWithType(e, mapZooms.getLevel(level).getMaxZoom(), true, typeUse); + int mtType = 0; + if (t != 0) { + if ((t & 3) == MapRenderingTypes.MULTY_POLYGON_TYPE) { + mtType = t; + } else { + for (Integer i : typeUse) { + if ((i & 3) == MapRenderingTypes.MULTY_POLYGON_TYPE) { + mtType = i; + break; + } + } + } + } + return mtType; + } + + + public void combineMultiPolygons(Way w, List> completedRings, List> incompletedRings) { + long lId = w.getEntityIds().get(w.getEntityIds().size() - 1).getId().longValue(); + long fId = w.getEntityIds().get(0).getId().longValue(); + if (fId == lId) { + completedRings.add(Collections.singletonList(w)); + } else { + List l = new ArrayList(); + l.add(w); + boolean add = true; + for (int k = 0; k < incompletedRings.size();) { + boolean remove = false; + List i = incompletedRings.get(k); + Way last = i.get(i.size() - 1); + Way first = i.get(0); + long lastId = last.getEntityIds().get(last.getEntityIds().size() - 1).getId().longValue(); + long firstId = first.getEntityIds().get(0).getId().longValue(); + if (fId == lastId) { + i.addAll(l); + remove = true; + l = i; + fId = firstId; + } else if (lId == firstId) { + l.addAll(i); + remove = true; + lId = lastId; + } + if (remove) { + incompletedRings.remove(k); + } else { + k++; + } + if (fId == lId) { + completedRings.add(l); + add = false; + break; + } + } + if (add) { + incompletedRings.add(l); + } + } + } + + private void writeBinaryEntityToMapDatabase(Entity e, long baseId, boolean inverse, int level) throws SQLException { + int type = renderingTypes.encodeEntityWithType(e, mapZooms.getLevel(level).getMaxZoom(), false, typeUse); + Map> multiPolygonsWays = this.multiPolygonsWays[level]; + boolean hasMulti = e instanceof Way && multiPolygonsWays.containsKey(e.getId()); + if (hasMulti) { + Set set = multiPolygonsWays.get(e.getId()); + boolean first = true; + for (Integer i : set) { + if (first && type == 0) { + type = i; + first = false; + } else { + // do not compare direction + int k = i & 0x7fff; + int ks = k | MapRenderingTypes.POLYGON_TYPE; + // turn of polygon type 3 ^ (suppose polygon = multipolygon) + if (ks == type) { + type = i; + } else { + int ind = typeUse.indexOf(ks); + if (ind == -1) { + typeUse.add(i); + } else { + typeUse.set(ind, i); + } + } + } + } + } + + if (type == 0) { + return; + } + + restrictionsUse.clear(); + // try to find restrictions only for max zoom level + if (level == 0 && highwayRestrictions.containsKey(baseId)) { + restrictionsUse.addAll(highwayRestrictions.get(baseId)); + } + + boolean point = (type & 3) == MapRenderingTypes.POINT_TYPE; + RTree rtree = null; + int zoom; + long id = (baseId << 3) | ((level & 3) << 1); + rtree = mapTree[level]; + zoom = mapZooms.getLevel(level).getMaxZoom() - 1; + boolean skip = false; + + String eName = renderingTypes.getEntityName(e); + if (eName == null) { + eName = multiPolygonsNames.get(baseId); + } + int highwayAttributes = 0; + if (e.getTag(OSMTagKey.HIGHWAY) != null) { + highwayAttributes = MapRenderingTypes.getHighwayAttributes(e); + } + if (e instanceof Way) { + id |= 1; + // simplify route + if (level > 0) { + e = simplifyWay((Way) e, id, hasMulti, zoom, eName, type, level); + skip = e == null; + } + + } + if (!skip) { + insertBinaryMapRenderObjectIndex(pStatements, mapBinaryStat, rtree, e, eName, id, type, typeUse, + highwayAttributes, restrictionsUse, inverse, point, true); + } + } + + + protected long encodeTypesToOneLong(int mainType) { + long i = 0; + int ind = 0; + int sh = 0; + if(typeUse.size() > 3){ + log.error("Types for low index way more than 4"); //$NON-NLS-1$ + } + i |= (mainType << sh); + if (typeUse.size() > ind) { + sh += 16; + i |= ((long)typeUse.get(ind++) << sh ); + if (typeUse.size() > ind) { + sh += 16; + i |= ((long)typeUse.get(ind++) << sh ); + if (typeUse.size() > ind) { + sh += 16; + i |= ((long)typeUse.get(ind++) << sh); + } + } + } + return i; + } + + protected int decodeTypesFromOneLong(long i) { + typeUse.clear(); + int mask = (1 << 16) - 1; + int k = (int) (i & mask); + int r = 0; + if (k > 0) { + r = k; + i >>= 16; + k = (int) (i & mask); + if (k > 0) { + typeUse.add(k); + i >>= 16; + k = (int) (i & mask); + if (k > 0) { + typeUse.add(k); + i >>= 16; + k = (int) (i & mask); + if (k > 0) { + typeUse.add(k); + i >>= 16; + } + } + } + } + return r; + } + + + protected Way simplifyWay(Way originalE, long id, boolean hasMulti, int zoom, String name, int type, int level) throws SQLException { + List nodes = originalE.getNodes(); + Way way = new Way(id); + for (String t : originalE.getTagKeySet()) { + way.putTag(t, originalE.getTag(t)); + } + boolean cycle = originalE.getNodeIds().get(0).longValue() == originalE.getNodeIds().get(nodes.size() - 1).longValue(); + long longType = encodeTypesToOneLong(type); + + boolean skip = checkForSmallAreas(nodes, zoom, 3, 3); + if (skip && cycle/* || !hasMulti)*/) { + return null; + } + + MapAlgorithms.simplifyDouglasPeucker(nodes, zoom + 8, 3, way); + if (way.getNodes().size() < 2) { + return null; + } + if (cycle) { + // nothing to do + return way; + } else { + lowLevelWays ++; + insertLowLevelMapBinaryObject(pStatements, mapLowLevelBinaryStat, level, longType, id, way.getNodes(), name); + return null; + } + + } + + public int getLowLevelWays() { + return lowLevelWays; + } + + private void loadNodes(byte[] nodes, List toPut){ + toPut.clear(); + for (int i = 0; i < nodes.length;) { + int lat = Algoritms.parseIntFromBytes(nodes, i); + i += 4; + int lon = Algoritms.parseIntFromBytes(nodes, i); + i += 4; + toPut.add(Float.intBitsToFloat(lat)); + toPut.add(Float.intBitsToFloat(lon)); + } + } + + public void processingLowLevelWays(IProgress progress) throws SQLException { + restrictionsUse.clear(); + mapLowLevelBinaryStat.executeBatch(); + mapLowLevelBinaryStat.close(); + pStatements.remove(mapLowLevelBinaryStat); + mapLowLevelBinaryStat = null; + mapConnection.commit(); + + PreparedStatement startStat = mapConnection.prepareStatement("SELECT id, start_node, end_node, nodes FROM low_level_map_objects" + + " WHERE start_node = ? AND type=? AND level = ? AND name=?"); + PreparedStatement endStat = mapConnection.prepareStatement("SELECT id, start_node, end_node, nodes FROM low_level_map_objects" + + " WHERE end_node = ? AND type=? AND level = ? AND name=?"); + Statement selectStatement = mapConnection.createStatement(); + ResultSet rs = selectStatement.executeQuery("SELECT id, start_node, end_node, name, nodes, type, level FROM low_level_map_objects"); + Set visitedWays = new LinkedHashSet(); + ArrayList list = new ArrayList(100); + while(rs.next()){ + if(lowLevelWays != -1){ + progress.progress(1); + } + long id = rs.getLong(1); + if(visitedWays.contains(id)){ + continue; + } + + visitedWays.add(id); + int level = rs.getInt(7); + int zoom = mapZooms.getLevel(level).getMaxZoom(); + + long startNode = rs.getLong(2); + long endNode = rs.getLong(3); + + String name = rs.getString(4); + long ltype = rs.getLong(6); + loadNodes(rs.getBytes(5), list); + ArrayList wayNodes = new ArrayList(list); + + + // combine startPoint with EndPoint + boolean combined = true; + while (combined) { + combined = false; + endStat.setLong(1, startNode); + endStat.setLong(2, ltype); + endStat.setShort(3, (short) level); + endStat.setString(4, name); + ResultSet fs = endStat.executeQuery(); + while (fs.next()) { + if (!visitedWays.contains(fs.getLong(1))) { + combined = true; + long lid = fs.getLong(1); + startNode = fs.getLong(2); + visitedWays.add(lid); + loadNodes(fs.getBytes(4), list); + ArrayList li = new ArrayList(list); + // remove first lat/lon point + wayNodes.remove(0); + wayNodes.remove(0); + li.addAll(wayNodes); + wayNodes = li; + } + } + fs.close(); + } + + // combined end point + combined = true; + while (combined) { + combined = false; + startStat.setLong(1, endNode); + startStat.setLong(2, ltype); + startStat.setShort(3, (short) level); + startStat.setString(4, name); + ResultSet fs = startStat.executeQuery(); + while (fs.next()) { + if (!visitedWays.contains(fs.getLong(1))) { + combined = true; + long lid = fs.getLong(1); + endNode = fs.getLong(3); + visitedWays.add(lid); + loadNodes(fs.getBytes(4), list); + for (int i = 2; i < list.size(); i++) { + wayNodes.add(list.get(i)); + } + } + } + fs.close(); + } + List wNodes = new ArrayList(); + for (int i = 0; i < wayNodes.size(); i += 2) { + wNodes.add(new Node(wayNodes.get(i), wayNodes.get(i + 1), i == 0 ? startNode : endNode)); + } + boolean skip = false; + boolean cycle = startNode == endNode; + boolean hasMulti = multiPolygonsWays[level].containsKey(id >> 3); + if(cycle || !hasMulti){ + skip = checkForSmallAreas(wNodes, zoom - 1, 1, 4); + } + + if (!skip) { + Way newWs = new Way(id); + MapAlgorithms.simplifyDouglasPeucker(wNodes, zoom - 1 + 8, 3, newWs); + + int type = decodeTypesFromOneLong(ltype); + insertBinaryMapRenderObjectIndex(pStatements, mapBinaryStat, mapTree[level], newWs, name, + id, type, typeUse, 0, restrictionsUse, false, false, false); + } + + } + + } + + private boolean checkForSmallAreas(List nodes, int zoom, int minz, int maxz) { + int minX = Integer.MAX_VALUE; + int maxX = Integer.MIN_VALUE; + int minY = Integer.MAX_VALUE; + int maxY = Integer.MIN_VALUE; + int c = 0; + for (int i = 0; i < nodes.size(); i++) { + if (nodes.get(i) != null) { + c++; + int x = (int) (MapUtils.getTileNumberX(zoom, nodes.get(i).getLongitude()) * 256d); + int y = (int) (MapUtils.getTileNumberY(zoom, nodes.get(i).getLatitude()) * 256d); + minX = Math.min(minX, x); + maxX = Math.max(maxX, x); + minY = Math.min(minY, y); + maxY = Math.max(maxY, y); + } + } + if (c < 2) { + return true; + } + return ((maxX - minX) <= minz && (maxY - minY) <= maxz) || + ((maxX - minX) <= maxz && (maxY - minY) <= minz); + + } + + public void initSettings(MapZooms mapZooms, MapRenderingTypes renderingTypes) { + this.mapZooms = mapZooms; + this.renderingTypes = renderingTypes; + // init map + multiPolygonsWays = new Map[mapZooms.size()]; + for (int i = 0; i < multiPolygonsWays.length; i++) { + multiPolygonsWays[i] = new LinkedHashMap>(); + } + lowLevelWays = -1; + + } + + public void iterateMainEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException { + if (e instanceof Way || e instanceof Node) { + // manipulate what kind of way to load + ctx.loadEntityData(e, false); + boolean inverse = "-1".equals(e.getTag(OSMTagKey.ONEWAY)); //$NON-NLS-1$ + for (int i = 0; i < mapZooms.size(); i++) { + writeBinaryEntityToMapDatabase(e, e.getId(), i == 0 ? inverse : false, i); + } + } + } + + public void writeBinaryMapIndex(BinaryMapIndexWriter writer, String regionName) throws IOException, SQLException { + closePreparedStatements(mapBinaryStat, mapLowLevelBinaryStat); + mapConnection.commit(); + try { + PreparedStatement selectData = mapConnection.prepareStatement("SELECT nodes, types, name, highway, restrictions FROM binary_map_objects WHERE id = ?"); //$NON-NLS-1$ + + writer.startWriteMapIndex(regionName); + + for (int i = 0; i < mapZooms.size(); i++) { + RTree rtree = mapTree[i]; + long rootIndex = rtree.getFileHdr().getRootIndex(); + rtree.Node root = rtree.getReadNode(rootIndex); + Rect rootBounds = calcBounds(root); + if (rootBounds != null) { + boolean last = nodeIsLastSubTree(rtree, rootIndex); + writer.startWriteMapLevelIndex(mapZooms.getLevel(i).getMinZoom(), mapZooms.getLevel(i).getMaxZoom(), rootBounds + .getMinX(), rootBounds.getMaxX(), rootBounds.getMinY(), rootBounds.getMaxY()); + if (last) { + writer.startMapTreeElement(rootBounds.getMinX(), rootBounds.getMaxX(), rootBounds.getMinY(), rootBounds.getMaxY()); + + } + writeBinaryMapTree(root, rtree, writer, selectData); + if (last) { + writer.endWriteMapTreeElement(); + } + + writer.endWriteMapLevelIndex(); + } + } + selectData.close(); + writer.writeMapEncodingRules(renderingTypes.getEncodingRuleTypes()); + writer.endWriteMapIndex(); + writer.flush(); + } catch (RTreeException e) { + throw new IllegalStateException(e); + } + } + + public void writeBinaryMapTree(rtree.Node parent, RTree r, BinaryMapIndexWriter writer, PreparedStatement selectData) throws IOException, RTreeException, SQLException { + Element[] e = parent.getAllElements(); + + for (int i = 0; i < parent.getTotalElements(); i++) { + Rect re = e[i].getRect(); + if (e[i].getElementType() == rtree.Node.LEAF_NODE) { + long id = ((LeafElement) e[i]).getPtr(); + selectData.setLong(1, id); + ResultSet rs = selectData.executeQuery(); + if (rs.next()) { + // mapConnection.prepareStatement("SELECT nodes, types, name, highway, restrictions FROM binary_map_objects WHERE id = ?"); + writer.writeMapData(id, rs.getBytes(1), + rs.getBytes(2), rs.getString(3), + rs.getInt(4), rs.getBytes(5)); + } else { + log.error("Something goes wrong with id = " + id); //$NON-NLS-1$ + } + } else { + long ptr = ((NonLeafElement) e[i]).getPtr(); + rtree.Node ns = r.getReadNode(ptr); + + writer.startMapTreeElement(re.getMinX(), re.getMaxX(), re.getMinY(), re.getMaxY()); + + writeBinaryMapTree(ns, r, writer, selectData); + writer.endWriteMapTreeElement(); + } + } + } + + public Rect calcBounds(rtree.Node n) { + Rect r = null; + Element[] e = n.getAllElements(); + for (int i = 0; i < n.getTotalElements(); i++) { + Rect re = e[i].getRect(); + if (r == null) { + try { + r = new Rect(re.getMinX(), re.getMinY(), re.getMaxX(), re.getMaxY()); + } catch (IllegalValueException ex) { + } + } else { + r.expandToInclude(re); + } + } + return r; + } + + public void createDatabaseStructure(Connection mapConnection, DBDialect dialect, + String rtreeMapIndexNonPackFileName) throws SQLException, IOException { + createMapIndexStructure(mapConnection); + this.mapConnection = mapConnection; + mapBinaryStat = createStatementMapBinaryInsert(mapConnection); + mapLowLevelBinaryStat = createStatementLowLevelMapBinaryInsert(mapConnection); + try { + mapTree = new RTree[mapZooms.size()]; + for (int i = 0; i < mapZooms.size(); i++) { + File file = new File(rtreeMapIndexNonPackFileName + i); + if (file.exists()) { + file.delete(); + } + mapTree[i] = new RTree(rtreeMapIndexNonPackFileName + i); + // very slow + // mapTree[i].getFileHdr().setBufferPolicy(true); + } + } catch (RTreeException e) { + throw new IOException(e); + } + pStatements.put(mapBinaryStat, 0); + pStatements.put(mapLowLevelBinaryStat, 0); + } + + + + private void createMapIndexStructure(Connection conn) throws SQLException{ + Statement stat = conn.createStatement(); + stat.executeUpdate("create table binary_map_objects (id bigint primary key, name varchar(255), " + + "types binary, restrictions binary, nodes binary, highway int)"); + stat.executeUpdate("create index binary_map_objects_ind on binary_map_objects (id)"); + + stat.executeUpdate("create table low_level_map_objects (id bigint primary key, start_node bigint, " + + "end_node bigint, name varchar(255), nodes binary, type bigint, level smallint)"); + stat.executeUpdate("create index low_level_map_objects_ind on low_level_map_objects (id)"); + stat.executeUpdate("create index low_level_map_objects_ind_st on low_level_map_objects (start_node, type)"); + stat.executeUpdate("create index low_level_map_objects_ind_end on low_level_map_objects (end_node, type)"); + stat.close(); + } + + private PreparedStatement createStatementMapBinaryInsert(Connection conn) throws SQLException{ + return conn.prepareStatement("insert into binary_map_objects(id, name, types, restrictions, nodes, highway) values(?, ?, ?, ?, ?, ?)"); + } + + private PreparedStatement createStatementLowLevelMapBinaryInsert(Connection conn) throws SQLException{ + return conn.prepareStatement("insert into low_level_map_objects(id, start_node, end_node, name, nodes, type, level) values(?, ?, ?, ?, ?, ?, ?)"); + } + + private void insertLowLevelMapBinaryObject(Map statements, + PreparedStatement mapLowLevelBinaryStat, int level,long types, long id, List nodes, String name) throws SQLException{ + boolean first = true; + long firstId = -1; + long lastId = -1; + ByteArrayOutputStream bnodes = new ByteArrayOutputStream(); + try { + for (Node n : nodes) { + if (n != null) { + if (first) { + firstId = n.getId(); + first = false; + } + lastId = n.getId(); + Algoritms.writeInt(bnodes, Float.floatToRawIntBits((float) n.getLatitude())); + Algoritms.writeInt(bnodes, Float.floatToRawIntBits((float) n.getLongitude())); + } + } + } catch (IOException e) { + throw new IllegalStateException(e); + } + if(firstId == -1){ + return; + } + // conn.prepareStatement("insert into binary_map_objects(id, name, types, restrictions, nodes, highway) values(?, ?, ?, ?, ?, ?)"); + mapLowLevelBinaryStat.setLong(1, id); + mapLowLevelBinaryStat.setLong(2, firstId); + mapLowLevelBinaryStat.setLong(3, lastId); + mapLowLevelBinaryStat.setString(4, name); + mapLowLevelBinaryStat.setBytes(5, bnodes.toByteArray()); + mapLowLevelBinaryStat.setLong(6, types); + mapLowLevelBinaryStat.setShort(7, (short) level); + + addBatch(statements, mapLowLevelBinaryStat); + } + private void insertBinaryMapRenderObjectIndex(Map statements, + PreparedStatement mapBinaryStat, RTree mapTree, Entity e, String name, + long id, int type, List typeUse, int highwayAttributes, List restrictions, + boolean inversePath, boolean writeAsPoint, boolean commit) throws SQLException { + if(e instanceof Relation){ + throw new IllegalArgumentException(); + } + boolean init = false; + int minX = Integer.MAX_VALUE; + int maxX = 0; + int minY = Integer.MAX_VALUE; + int maxY = 0; + Collection nodes; + if (e instanceof Way) { + if (writeAsPoint) { + LatLon center = MapUtils.getCenter(((Way) e)); + nodes = Collections.singleton(new Node(center.getLatitude(), center.getLongitude(), -1)); + } else { + nodes = ((Way) e).getNodes(); + } + } else { + nodes = Collections.singleton((Node) e); + } + if(inversePath){ + nodes = new ArrayList(nodes); + Collections.reverse((List) nodes); + } + + ByteArrayOutputStream bnodes = new ByteArrayOutputStream(); + ByteArrayOutputStream btypes = new ByteArrayOutputStream(); + ByteArrayOutputStream brestrictions = new ByteArrayOutputStream(); + + try { + Algoritms.writeSmallInt(btypes, type); + for (Integer i : typeUse) { + Algoritms.writeSmallInt(btypes, i); + } + for (Long i : restrictions) { + Algoritms.writeLongInt(brestrictions, i); + } + + for (Node n : nodes) { + if (n != null) { + int y = MapUtils.get31TileNumberY(n.getLatitude()); + int x = MapUtils.get31TileNumberX(n.getLongitude()); + minX = Math.min(minX, x); + maxX = Math.max(maxX, x); + minY = Math.min(minY, y); + maxY = Math.max(maxY, y); + init = true; + Algoritms.writeInt(bnodes, x); + Algoritms.writeInt(bnodes, y); + } + } + } catch (IOException es) { + throw new IllegalStateException(es); + } + if (init) { + // conn.prepareStatement("insert into binary_map_objects(id, name, types, restrictions, nodes, highway) values(?, ?, ?, ?, ?, ?)"); + mapBinaryStat.setLong(1, id); + mapBinaryStat.setString(2, name); + mapBinaryStat.setBytes(3, btypes.toByteArray()); + mapBinaryStat.setBytes(4, brestrictions.toByteArray()); + mapBinaryStat.setBytes(5, bnodes.toByteArray()); + mapBinaryStat.setInt(6, highwayAttributes); + + addBatch(statements, mapBinaryStat, commit); + try { + mapTree.insert(new LeafElement(new Rect(minX, minY, maxX, maxY), id)); + } catch (RTreeInsertException e1) { + throw new IllegalArgumentException(e1); + } catch (IllegalValueException e1) { + throw new IllegalArgumentException(e1); + } + } + } + + public void createRTreeFiles(String rTreeMapIndexPackFileName) throws RTreeException { + mapTree = new RTree[mapZooms.size()]; + for (int i = 0; i < mapZooms.size(); i++) { + mapTree[i] = new RTree(rTreeMapIndexPackFileName + i); + } + + } + + public void packRtreeFiles(String rTreeMapIndexNonPackFileName, String rTreeMapIndexPackFileName) throws IOException { + for (int i = 0; i < mapZooms.size(); i++) { + mapTree[i] = packRtreeFile(mapTree[i], rTreeMapIndexNonPackFileName + i, rTreeMapIndexPackFileName + i); + } + } + + public void commitAndCloseFiles(String rTreeMapIndexNonPackFileName, String rTreeMapIndexPackFileName, boolean deleteDatabaseIndexes) throws IOException, SQLException { + + // delete map rtree files + if (mapTree != null) { + for (int i = 0; i < mapTree.length; i++) { + if (mapTree[i] != null) { + RandomAccessFile file = mapTree[i].getFileHdr().getFile(); + file.close(); + } + + } + for (int i = 0; i < mapTree.length; i++) { + File f = new File(rTreeMapIndexNonPackFileName + i); + if (f.exists() && deleteDatabaseIndexes) { + f.delete(); + } + f = new File(rTreeMapIndexPackFileName + i); + if (f.exists() && deleteDatabaseIndexes) { + f.delete(); + } + } + } + closeAllPreparedStatements(); + + } +} diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/MapSwingAlgorithms.java b/DataExtractionOSM/src/net/osmand/data/preparation/MapSwingAlgorithms.java new file mode 100644 index 0000000000..1ffe2f4d86 --- /dev/null +++ b/DataExtractionOSM/src/net/osmand/data/preparation/MapSwingAlgorithms.java @@ -0,0 +1,108 @@ +package net.osmand.data.preparation; + +import java.awt.geom.Line2D; +import java.awt.geom.Point2D; +import java.util.ArrayList; +import java.util.List; + +import net.osmand.osm.Node; +import net.osmand.osm.Way; + +public class MapSwingAlgorithms { + + public static Point2D.Float getIntersectionPoint(Line2D.Float line1, Line2D.Float line2) { + if (!line1.intersectsLine(line2)) + return null; + double px = line1.getX1(), py = line1.getY1(), rx = line1.getX2() - px, ry = line1.getY2() - py; + double qx = line2.getX1(), qy = line2.getY1(), sx = line2.getX2() - qx, sy = line2.getY2() - qy; + + double det = sx * ry - sy * rx; + if (det == 0) { + return null; + } else { + double z = (sx * (qy - py) + sy * (px - qx)) / det; + if (z <= 0 || z >= 1) + return null; // intersection at end point! + return new Point2D.Float((float) (px + z * rx), (float) (py + z * ry)); + } + } // end intersection line-line + + public static boolean isClockwiseWay(List ways) { + if (ways.isEmpty()) { + return false; + } + List nodes; + if (ways.size() == 1) { + nodes = ways.get(0).getNodes(); + } else { + nodes = new ArrayList(); + boolean first = true; + for (Way e : ways) { + if (first) { + first = false; + nodes.addAll(e.getNodes()); + } else { + nodes.addAll(e.getNodes().subList(1, e.getNodes().size())); + } + } + } + if (nodes.isEmpty()) { + return false; + } + double angle = 0; + double prevAng = 0; + double firstAng = 0; + double selfIntersection = 0; + boolean open = nodes.get(nodes.size() - 1).getId() != nodes.get(0).getId(); + + for (int i = 1; open ? i < nodes.size() : i <= nodes.size(); i++) {// nodes.get(i).getId() + double ang; + if (i < nodes.size()) { + ang = Math.atan2(nodes.get(i).getLatitude() - nodes.get(i - 1).getLatitude(), nodes.get(i).getLongitude() + - nodes.get(i - 1).getLongitude()); + // find self intersection + Line2D.Float l = new Line2D.Float((float) nodes.get(i).getLongitude(), (float) nodes.get(i).getLatitude(), (float) nodes + .get(i - 1).getLongitude(), (float) nodes.get(i - 1).getLatitude()); + for (int j = i - 2; j > i - 7; j--) { + if (j < 1) { + break; + } + Line2D.Float l2 = new Line2D.Float((float) nodes.get(j).getLongitude(), (float) nodes.get(j).getLatitude(), + (float) nodes.get(j - 1).getLongitude(), (float) nodes.get(j - 1).getLatitude()); + java.awt.geom.Point2D.Float point = getIntersectionPoint(l, l2); + if (point != null) { + double dang = Math.atan2(nodes.get(j).getLatitude() - nodes.get(j - 1).getLatitude(), nodes.get(j).getLongitude() + - nodes.get(j - 1).getLongitude()); + if (adjustDirection(ang - dang) < 0) { + selfIntersection += 2 * Math.PI; + } else { + selfIntersection -= 2 * Math.PI; + } + } + + } + } else { + ang = firstAng; + } + if (i > 1) { + angle += adjustDirection(ang - prevAng); + prevAng = ang; + } else { + prevAng = ang; + firstAng = ang; + } + + } + return (angle - selfIntersection) < 0; + } + + private static double adjustDirection(double ang) { + if (ang < -Math.PI) { + ang += 2 * Math.PI; + } else if (ang > Math.PI) { + ang -= 2 * Math.PI; + } + return ang; + } + +}