Android-builders-club/OsmAnd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Refactoring IndexCreator (3)

Browse source
This commit is contained in:
Victor Shcherb 2011-05-02 21:18:53 +02:00
parent bcbee2496c
commit c34eb50799
12 changed files with 2126 additions and 2228 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

219
DataExtractionOSM/src/net/osmand/data/index/DataIndexReader.java
View file

@ -1,219 +0,0 @@
package net.osmand.data.index;
import java.io.File;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import net.osmand.data.Building;
import net.osmand.data.City;
import net.osmand.data.Street;
import net.osmand.data.City.CityType;
import net.osmand.osm.Node;
import net.osmand.LogUtil;
import org.apache.commons.logging.Log;
public class DataIndexReader {
private static final Log log = LogUtil.getLog(DataIndexReader.class);
public Connection getConnection(File file) throws SQLException{
try {
Class.forName("org.sqlite.JDBC"); //$NON-NLS-1$
} catch (ClassNotFoundException e) {
log.error("Illegal configuration", e); //$NON-NLS-1$
throw new IllegalStateException(e);
}
return DriverManager.getConnection("jdbc:sqlite:"+file.getAbsolutePath()); //$NON-NLS-1$
}
public List<City> readCities(Connection c) throws SQLException{
List<City> cities = new ArrayList<City>();
Statement stat = c.createStatement();
ResultSet set = stat.executeQuery("select id, latitude, longitude , name , name_en , city_type from city"); //$NON-NLS-1$
while(set.next()){
City city = new City(CityType.valueFromString(set.getString(6)));
city.setName(set.getString(4));
city.setEnName(set.getString(5));
city.setLocation(set.getDouble(2),
set.getDouble(3));
city.setId(set.getLong(1));
cities.add(city);
}
set.close();
stat.close();
return cities;
}
public PreparedStatement getStreetsBuildingPreparedStatement(Connection c) throws SQLException{
return c.prepareStatement("SELECT A.id, A.name, A.name_en, A.latitude, A.longitude, "+ //$NON-NLS-1$
"B.id, B.name, B.name_en, B.latitude, B.longitude, B.postcode "+ //$NON-NLS-1$
"FROM street A left JOIN building B ON B.street = A.id WHERE A.city = ?"); //$NON-NLS-1$
}
public List<Street> readStreetsBuildings(PreparedStatement streetBuildingsStat, City city, List<Street> streets) throws SQLException {
return readStreetsBuildings(streetBuildingsStat, city, streets, null, null, null);
}
public PreparedStatement getStreetsWayNodesPreparedStatement(Connection c) throws SQLException{
return c.prepareStatement("SELECT A.id, A.latitude, A.longitude FROM street_node A WHERE A.street = ? "); //$NON-NLS-1$
}
public List<Street> readStreetsBuildings(PreparedStatement streetBuildingsStat, City city, List<Street> streets,
PreparedStatement waynodesStat, Map<Street, List<Node>> streetNodes, List<City> citySuburbs) throws SQLException {
Map<Long, Street> visitedStreets = new LinkedHashMap<Long, Street>();
//read streets for city
readStreatsByBuildingsForCity(streetBuildingsStat, city, streets,
waynodesStat, streetNodes, visitedStreets);
//read streets for suburbs of the city
if (citySuburbs != null) {
for (City suburb : citySuburbs) {
readStreatsByBuildingsForCity(streetBuildingsStat, suburb, streets, waynodesStat, streetNodes, visitedStreets);
}
}
return streets;
}
private void readStreatsByBuildingsForCity(
PreparedStatement streetBuildingsStat, City city,
List<Street> streets, PreparedStatement waynodesStat,
Map<Street, List<Node>> streetNodes,
Map<Long, Street> visitedStreets) throws SQLException {
streetBuildingsStat.setLong(1, city.getId());
ResultSet set = streetBuildingsStat.executeQuery();
while (set.next()) {
long streetId = set.getLong(1);
if (!visitedStreets.containsKey(streetId)) {
Street street = new Street(null);
street.setName(set.getString(2));
street.setEnName(set.getString(3));
street.setLocation(set.getDouble(4), set.getDouble(5));
street.setId(streetId);
streets.add(street);
visitedStreets.put(streetId, street);
if (waynodesStat != null && streetNodes != null) {
ArrayList<Node> list = new ArrayList<Node>();
streetNodes.put(street, list);
waynodesStat.setLong(1, street.getId());
ResultSet rs = waynodesStat.executeQuery();
while (rs.next()) {
list.add(new Node(rs.getDouble(2), rs.getDouble(3), rs.getLong(1)));
}
rs.close();
}
}
if (set.getObject(6) != null) {
Street s = visitedStreets.get(streetId);
Building b = new Building();
b.setId(set.getLong(6));
b.setName(set.getString(7));
b.setEnName(set.getString(8));
b.setLocation(set.getDouble(9), set.getDouble(10));
b.setPostcode(set.getString(11));
s.registerBuilding(b);
}
}
set.close();
}
public PreparedStatement getStreetsPreparedStatement(Connection c) throws SQLException{
return c.prepareStatement("select id, latitude, longitude , name, name_en, city from street where city = ?"); //$NON-NLS-1$
}
public List<Street> readStreets(PreparedStatement streetsStat, City city, List<Street> streets) throws SQLException{
streetsStat.setLong(1, city.getId());
ResultSet set = streetsStat.executeQuery();
while(set.next()){
Street street = new Street(city);
street.setName(set.getString(4));
street.setEnName(set.getString(5));
street.setLocation(set.getDouble(2),
set.getDouble(3));
street.setId(set.getLong(1));
streets.add(street);
}
set.close();
return streets;
}
public PreparedStatement getBuildingsPreparedStatement(Connection c) throws SQLException{
return c.prepareStatement("select id, latitude, longitude, name, name_en, street, postcode from building where street = ?"); //$NON-NLS-1$
}
public List<Building> readBuildings(PreparedStatement buildingStat, Street street, List<Building> buildings) throws SQLException{
buildingStat.setLong(1, street.getId());
ResultSet set = buildingStat.executeQuery();
while(set.next()){
Building building = new Building();
building.setName(set.getString(4));
building.setEnName(set.getString(5));
building.setLocation(set.getDouble(2),
set.getDouble(3));
building.setId(set.getLong(1));
building.setPostcode(set.getString(7));
buildings.add(building);
}
set.close();
return buildings;
}
public void testIndex(File f) throws SQLException {
Connection c = getConnection(f);
try {
ArrayList<Street> streets = new ArrayList<Street>();
// ArrayList<Building> buildings = new ArrayList<Building>();
PreparedStatement streetstat = getStreetsBuildingPreparedStatement(c);
int countCity = 0;
int countStreets = 0;
int countBuildings = 0;
List<City> cities = readCities(c);
for (City city : cities) {
countCity ++;
// System.out.println("CITY " + city.getName()); //$NON-NLS-1$
if(city.getType() != CityType.CITY){
continue;
}
streets.clear();
// long time = System.currentTimeMillis();
readStreetsBuildings(streetstat, city, streets);
if(!streets.isEmpty()){
System.out.println(city.getName());
} else {
System.out.print(".");
}
for (Street s : streets) {
countStreets ++;
// System.out.println("\tSTREET " + s.getName()); //$NON-NLS-1$
// buildings.clear();
countBuildings += s.getBuildings().size();
// for (Building b : s.getBuildings()) {
// countBuildings ++;
// System.out.println("\t\tBULDING " + b.getName()); //$NON-NLS-1$
// }
}
}
System.out.println(countCity + " " + countStreets + " " + countBuildings);
} finally {
c.close();
}
}
}

344
DataExtractionOSM/src/net/osmand/data/index/DataIndexWriter.java
View file

@ -1,344 +0,0 @@
package net.osmand.data.index;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import net.osmand.Algoritms;
import net.osmand.data.Amenity;
import net.osmand.data.AmenityType;
import net.osmand.data.Building;
import net.osmand.data.City;
import net.osmand.data.City.CityType;
import net.osmand.data.preparation.DBDialect;
import net.osmand.osm.Entity;
import net.osmand.osm.LatLon;
import net.osmand.osm.MapUtils;
import net.osmand.osm.Node;
import net.osmand.osm.Relation;
import net.osmand.osm.Way;
import rtree.IllegalValueException;
import rtree.LeafElement;
import rtree.RTree;
import rtree.RTreeInsertException;
import rtree.Rect;
public class DataIndexWriter {
private static final int BATCH_SIZE = 1000;
public static PreparedStatement getStreetNodeInsertPreparedStatement(Connection conn) throws SQLException {
assert IndexConstants.STREET_NODE_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
return conn.prepareStatement("insert into street_node (id, latitude, longitude, street, way) values (?, ?, ?, ?, ?)");
}
public static void writeStreetWayNodes(PreparedStatement prepStreetNode, Map<PreparedStatement, Integer> count, Long streetId, Way way, int batchSize)
throws SQLException {
assert IndexConstants.STREET_NODE_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
for (Node n : way.getNodes()) {
if (n == null) {
continue;
}
prepStreetNode.setLong(1, n.getId());
prepStreetNode.setDouble(2, n.getLatitude());
prepStreetNode.setDouble(3, n.getLongitude());
prepStreetNode.setLong(5, way.getId());
prepStreetNode.setLong(4, streetId);
addBatch(count, prepStreetNode, BATCH_SIZE);
}
}
public static PreparedStatement getBuildingInsertPreparedStatement(Connection conn) throws SQLException {
assert IndexConstants.BUILDING_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
return conn.prepareStatement("insert into building (id, latitude, longitude, name, name_en, street, postcode) values (?, ?, ?, ?, ?, ?, ?)");
}
public static void writeBuilding(PreparedStatement prepBuilding, Map<PreparedStatement, Integer> count, Long streetId,
Building building, int batchSize)
throws SQLException {
assert IndexConstants.BUILDING_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
prepBuilding.setLong(1, building.getId());
prepBuilding.setDouble(2, building.getLocation().getLatitude());
prepBuilding.setDouble(3, building.getLocation().getLongitude());
prepBuilding.setString(4, building.getName());
prepBuilding.setString(5, building.getEnName());
prepBuilding.setLong(6, streetId);
prepBuilding.setString(7, building.getPostcode() == null ? null : building.getPostcode().toUpperCase());
addBatch(count, prepBuilding);
}
public static PreparedStatement getSearchStreetPreparedStatement(Connection mapConnection) throws SQLException {
assert IndexConstants.STREET_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
return mapConnection.prepareStatement("SELECT ID FROM street WHERE ? = city AND ? = name");
}
public static PreparedStatement getSearchBuildingPreparedStatement(Connection mapConnection) throws SQLException {
assert IndexConstants.BUILDING_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
return mapConnection.prepareStatement("SELECT id FROM building where ? = id");
}
public static PreparedStatement getStreeNodeSearchPreparedStatement(Connection mapConnection) throws SQLException {
assert IndexConstants.STREET_NODE_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
return mapConnection.prepareStatement("SELECT way FROM street_node WHERE ? = way");
}
public static PreparedStatement getUpdateBuildingPostcodePreparedStatement(Connection mapConnection) throws SQLException {
assert IndexConstants.BUILDING_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
return mapConnection.prepareStatement("UPDATE building SET postcode = ? WHERE id = ?");
}
public static PreparedStatement getCityInsertPreparedStatement(Connection conn) throws SQLException{
assert IndexConstants.CITY_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
return conn.prepareStatement("insert into city (id, latitude, longitude, name, name_en, city_type) values (?, ?, ?, ?, ?, ?)");
}
public static void writeCity(PreparedStatement prepCity, Map<PreparedStatement, Integer> count, City city, int batchSize) throws SQLException {
assert IndexConstants.CITY_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
prepCity.setLong(1, city.getId());
prepCity.setDouble(2, city.getLocation().getLatitude());
prepCity.setDouble(3, city.getLocation().getLongitude());
prepCity.setString(4, city.getName());
prepCity.setString(5, city.getEnName());
prepCity.setString(6, CityType.valueToString(city.getType()));
addBatch(count, prepCity, batchSize);
}
public static PreparedStatement getStreetInsertPreparedStatement(Connection conn) throws SQLException{
assert IndexConstants.STREET_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
return conn.prepareStatement("insert into street (id, latitude, longitude, name, name_en, city) values (?, ?, ?, ?, ?, ?)");
}
public static void insertStreetData(PreparedStatement addressStreetStat, long id, String name, String nameEn, double latitude,
double longitude, Long cityId) throws SQLException {
assert IndexConstants.STREET_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
addressStreetStat.setLong(1, id);
addressStreetStat.setString(4, name);
addressStreetStat.setString(5, nameEn);
addressStreetStat.setDouble(2, latitude);
addressStreetStat.setDouble(3, longitude);
addressStreetStat.setLong(6, cityId);
}
public static void createAddressIndexStructure(Connection conn, DBDialect dialect) throws SQLException{
Statement stat = conn.createStatement();
assert IndexConstants.CITY_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
assert IndexConstants.STREET_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
assert IndexConstants.STREET_NODE_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
assert IndexConstants.STREET_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
stat.executeUpdate("create table city (id bigint primary key, latitude double, longitude double, " +
"name varchar(255), name_en varchar(255), city_type varchar(32))");
stat.executeUpdate("create index city_ind on city (id, city_type)");
stat.executeUpdate("create table street (id bigint primary key, latitude double, longitude double, " +
"name varchar(255), name_en varchar(255), city bigint)");
stat.executeUpdate("create index street_city on street (city)");
stat.executeUpdate("create index street_id on street (id)");
// create index on name ?
stat.executeUpdate("create table building (id bigint, latitude double, longitude double, " +
"name varchar(255), name_en varchar(255), street bigint, postcode varchar(255), primary key(street, id))");
stat.executeUpdate("create index building_postcode on building (postcode)");
stat.executeUpdate("create index building_street on building (street)");
stat.executeUpdate("create index building_id on building (id)");
stat.executeUpdate("create table street_node (id bigint, latitude double, longitude double, " +
"street bigint, way bigint)");
stat.executeUpdate("create index street_node_street on street_node (street)");
stat.executeUpdate("create index street_node_way on street_node (way)");
if(dialect == DBDialect.SQLITE){
stat.execute("PRAGMA user_version = " + IndexConstants.ADDRESS_TABLE_VERSION); //$NON-NLS-1$
}
stat.close();
}
public static void createMapIndexStructure(Connection conn) throws SQLException{
Statement stat = conn.createStatement();
assert IndexConstants.BINARY_MAP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
assert IndexConstants.LOW_LEVEL_MAP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
stat.executeUpdate("create table binary_map_objects (id bigint primary key, name varchar(255), " +
"types binary, restrictions binary, nodes binary, highway int)");
stat.executeUpdate("create index binary_map_objects_ind on binary_map_objects (id)");
stat.executeUpdate("create table low_level_map_objects (id bigint primary key, start_node bigint, " +
"end_node bigint, name varchar(255), nodes binary, type bigint, level smallint)");
stat.executeUpdate("create index low_level_map_objects_ind on low_level_map_objects (id)");
stat.executeUpdate("create index low_level_map_objects_ind_st on low_level_map_objects (start_node, type)");
stat.executeUpdate("create index low_level_map_objects_ind_end on low_level_map_objects (end_node, type)");
stat.close();
}
public static PreparedStatement createStatementMapBinaryInsert(Connection conn) throws SQLException{
assert IndexConstants.BINARY_MAP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
return conn.prepareStatement("insert into binary_map_objects(id, name, types, restrictions, nodes, highway) values(?, ?, ?, ?, ?, ?)");
}
public static PreparedStatement createStatementLowLevelMapBinaryInsert(Connection conn) throws SQLException{
assert IndexConstants.LOW_LEVEL_MAP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
return conn.prepareStatement("insert into low_level_map_objects(id, start_node, end_node, name, nodes, type, level) values(?, ?, ?, ?, ?, ?, ?)");
}
public static void insertLowLevelMapBinaryObject(Map<PreparedStatement, Integer> statements,
PreparedStatement mapLowLevelBinaryStat, int level,long types, long id, List<Node> nodes, String name) throws SQLException{
assert IndexConstants.LOW_LEVEL_MAP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
boolean first = true;
long firstId = -1;
long lastId = -1;
ByteArrayOutputStream bnodes = new ByteArrayOutputStream();
try {
for (Node n : nodes) {
if (n != null) {
if (first) {
firstId = n.getId();
first = false;
}
lastId = n.getId();
Algoritms.writeInt(bnodes, Float.floatToRawIntBits((float) n.getLatitude()));
Algoritms.writeInt(bnodes, Float.floatToRawIntBits((float) n.getLongitude()));
}
}
} catch (IOException e) {
throw new IllegalStateException(e);
}
if(firstId == -1){
return;
}
// conn.prepareStatement("insert into binary_map_objects(id, name, types, restrictions, nodes, highway) values(?, ?, ?, ?, ?, ?)");
mapLowLevelBinaryStat.setLong(1, id);
mapLowLevelBinaryStat.setLong(2, firstId);
mapLowLevelBinaryStat.setLong(3, lastId);
mapLowLevelBinaryStat.setString(4, name);
mapLowLevelBinaryStat.setBytes(5, bnodes.toByteArray());
mapLowLevelBinaryStat.setLong(6, types);
mapLowLevelBinaryStat.setShort(7, (short) level);
addBatch(statements, mapLowLevelBinaryStat);
}
public static void insertBinaryMapRenderObjectIndex(Map<PreparedStatement, Integer> statements,
PreparedStatement mapBinaryStat, RTree mapTree, Entity e, String name,
long id, int type, List<Integer> typeUse, int highwayAttributes, List<Long> restrictions,
boolean inversePath, boolean writeAsPoint, boolean commit) throws SQLException {
if(e instanceof Relation){
throw new IllegalArgumentException();
}
boolean init = false;
int minX = Integer.MAX_VALUE;
int maxX = 0;
int minY = Integer.MAX_VALUE;
int maxY = 0;
Collection<Node> nodes;
if (e instanceof Way) {
if (writeAsPoint) {
LatLon center = MapUtils.getCenter(((Way) e));
nodes = Collections.singleton(new Node(center.getLatitude(), center.getLongitude(), -1));
} else {
nodes = ((Way) e).getNodes();
}
} else {
nodes = Collections.singleton((Node) e);
}
if(inversePath){
nodes = new ArrayList<Node>(nodes);
Collections.reverse((List<?>) nodes);
}
ByteArrayOutputStream bnodes = new ByteArrayOutputStream();
ByteArrayOutputStream btypes = new ByteArrayOutputStream();
ByteArrayOutputStream brestrictions = new ByteArrayOutputStream();
try {
Algoritms.writeSmallInt(btypes, type);
for (Integer i : typeUse) {
Algoritms.writeSmallInt(btypes, i);
}
for (Long i : restrictions) {
Algoritms.writeLongInt(brestrictions, i);
}
for (Node n : nodes) {
if (n != null) {
int y = MapUtils.get31TileNumberY(n.getLatitude());
int x = MapUtils.get31TileNumberX(n.getLongitude());
minX = Math.min(minX, x);
maxX = Math.max(maxX, x);
minY = Math.min(minY, y);
maxY = Math.max(maxY, y);
init = true;
Algoritms.writeInt(bnodes, x);
Algoritms.writeInt(bnodes, y);
}
}
} catch (IOException es) {
throw new IllegalStateException(es);
}
if (init) {
assert IndexConstants.BINARY_MAP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
// conn.prepareStatement("insert into binary_map_objects(id, name, types, restrictions, nodes, highway) values(?, ?, ?, ?, ?, ?)");
mapBinaryStat.setLong(1, id);
mapBinaryStat.setString(2, name);
mapBinaryStat.setBytes(3, btypes.toByteArray());
mapBinaryStat.setBytes(4, brestrictions.toByteArray());
mapBinaryStat.setBytes(5, bnodes.toByteArray());
mapBinaryStat.setInt(6, highwayAttributes);
addBatch(statements, mapBinaryStat, commit);
try {
mapTree.insert(new LeafElement(new Rect(minX, minY, maxX, maxY), id));
} catch (RTreeInsertException e1) {
throw new IllegalArgumentException(e1);
} catch (IllegalValueException e1) {
throw new IllegalArgumentException(e1);
}
}
}
public static void addBatch(Map<PreparedStatement, Integer> count, PreparedStatement p) throws SQLException {
addBatch(count, p, BATCH_SIZE, true);
}
public static void addBatch(Map<PreparedStatement, Integer> count, PreparedStatement p, boolean commit) throws SQLException{
addBatch(count, p, BATCH_SIZE, commit);
}
public static void addBatch(Map<PreparedStatement, Integer> count, PreparedStatement p, int batchSize) throws SQLException{
addBatch(count, p, batchSize, true);
}
public static void addBatch(Map<PreparedStatement, Integer> count, PreparedStatement p, int batchSize, boolean commit) throws SQLException{
p.addBatch();
if(count.get(p) >= batchSize){
p.executeBatch();
if(commit){
p.getConnection().commit();
}
count.put(p, 0);
} else {
count.put(p, count.get(p) + 1);
}
}
}

4
DataExtractionOSM/src/net/osmand/data/index/IndexBatchCreator.java
View file

@ -484,10 +484,6 @@ public class IndexBatchCreator {
String regionName = f.getName().substring(0, f.getName().lastIndexOf('_', f.getName().indexOf('.')));
if(f.getName().endsWith(IndexConstants.POI_INDEX_EXT) || f.getName().endsWith(IndexConstants.POI_INDEX_EXT_ZIP)){
summary = "POI index for " ;
} else if(f.getName().endsWith(IndexConstants.ADDRESS_INDEX_EXT) || f.getName().endsWith(IndexConstants.ADDRESS_INDEX_EXT_ZIP)){
summary = "Address index for " ;
} else if(f.getName().endsWith(IndexConstants.TRANSPORT_INDEX_EXT) || f.getName().endsWith(IndexConstants.TRANSPORT_INDEX_EXT_ZIP)){
summary = "Transport index for ";
} else if(f.getName().endsWith(IndexConstants.BINARY_MAP_INDEX_EXT) || f.getName().endsWith(IndexConstants.BINARY_MAP_INDEX_EXT_ZIP)){
boolean addr = indexAddress;
boolean trans = indexTransport;

29
DataExtractionOSM/src/net/osmand/data/index/IndexConstants.java
View file

@ -5,52 +5,23 @@ public class IndexConstants {
// Important : Every time you change schema of db upgrade version!!!
// If you want that new application support old index : put upgrade code in android app ResourceManager
public final static int POI_TABLE_VERSION = 1;
public final static int BINARY_MAP_VERSION = 1; // starts with 1
public final static int VOICE_VERSION = 0;
// these indexes are deprecated
public final static int TRANSPORT_TABLE_VERSION = 0;
public final static int ADDRESS_TABLE_VERSION = 1;
public static final String POI_INDEX_DIR = "POI/"; //$NON-NLS-1$
public static final String ADDRESS_INDEX_DIR = "Address/"; //$NON-NLS-1$
public static final String VOICE_INDEX_DIR = "voice/"; //$NON-NLS-1$
public static final String TRANSPORT_INDEX_DIR = "Transport/"; //$NON-NLS-1$
public static final String RENDERERS_DIR = "rendering/"; //$NON-NLS-1$
public static final String POI_INDEX_EXT = ".poi.odb"; //$NON-NLS-1$
public static final String ADDRESS_INDEX_EXT = ".addr.odb"; //$NON-NLS-1$
public static final String TRANSPORT_INDEX_EXT = ".trans.odb"; //$NON-NLS-1$
public static final String BINARY_MAP_INDEX_EXT = ".obf"; //$NON-NLS-1$
public static final String POI_INDEX_EXT_ZIP = ".poi.zip"; //$NON-NLS-1$
public static final String ADDRESS_INDEX_EXT_ZIP = ".addr.zip"; //$NON-NLS-1$
public static final String TRANSPORT_INDEX_EXT_ZIP = ".trans.zip"; //$NON-NLS-1$
public static final String VOICE_INDEX_EXT_ZIP = ".voice.zip"; //$NON-NLS-1$
public static final String BINARY_MAP_INDEX_EXT_ZIP = ".obf.zip"; //$NON-NLS-1$
public static final String RENDERER_INDEX_EXT = ".render.xml"; //$NON-NLS-1$
public final static String STREET_NODE_TABLE = "street_node"; //$NON-NLS-1$
public final static String STREET_TABLE = "street"; //$NON-NLS-1$
public final static String CITY_TABLE = "city"; //$NON-NLS-1$
public final static String BUILDING_TABLE = "building"; //$NON-NLS-1$
public final static String POI_TABLE = "poi"; //$NON-NLS-1$
public final static String BINARY_MAP_TABLE = "binary_map_objects"; //$NON-NLS-1$
public final static String LOW_LEVEL_MAP_TABLE = "low_level_map_objects"; //$NON-NLS-1$
public final static String TRANSPORT_STOP_TABLE = "transport_stop"; //$NON-NLS-1$
public final static String TRANSPORT_ROUTE_STOP_TABLE = "transport_route_stop"; //$NON-NLS-1$
public final static String TRANSPORT_ROUTE_TABLE = "transport_route"; //$NON-NLS-1$
}

105
DataExtractionOSM/src/net/osmand/data/preparation/AbstractIndexPartCreator.java Normal file
View file

@ -0,0 +1,105 @@
package net.osmand.data.preparation;
import java.io.File;
import java.io.IOException;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.LinkedHashMap;
import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import rtree.Element;
import rtree.Pack;
import rtree.RTree;
import rtree.RTreeException;
public class AbstractIndexPartCreator {
private final static Log log = LogFactory.getLog(AbstractIndexPartCreator.class);
protected int BATCH_SIZE = 1000;
protected Map<PreparedStatement, Integer> pStatements = new LinkedHashMap<PreparedStatement, Integer>();
protected void closePreparedStatements(PreparedStatement... preparedStatements) throws SQLException {
for (PreparedStatement p : preparedStatements) {
if (p != null) {
p.executeBatch();
p.close();
pStatements.remove(p);
}
}
}
protected void closeAllPreparedStatements() throws SQLException {
for (PreparedStatement p : pStatements.keySet()) {
if (pStatements.get(p) > 0) {
p.executeBatch();
}
p.close();
}
}
protected void addBatch(Map<PreparedStatement, Integer> count, PreparedStatement p) throws SQLException {
addBatch(count, p, BATCH_SIZE, true);
}
protected void addBatch(Map<PreparedStatement, Integer> count, PreparedStatement p, boolean commit) throws SQLException{
addBatch(count, p, BATCH_SIZE, commit);
}
protected void addBatch(Map<PreparedStatement, Integer> count, PreparedStatement p, int batchSize) throws SQLException{
addBatch(count, p, batchSize, true);
}
protected void addBatch(Map<PreparedStatement, Integer> count, PreparedStatement p, int batchSize, boolean commit) throws SQLException{
p.addBatch();
if(count.get(p) >= batchSize){
p.executeBatch();
if(commit){
p.getConnection().commit();
}
count.put(p, 0);
} else {
count.put(p, count.get(p) + 1);
}
}
protected boolean nodeIsLastSubTree(RTree tree, long ptr) throws RTreeException {
rtree.Node parent = tree.getReadNode(ptr);
Element[] e = parent.getAllElements();
for (int i = 0; i < parent.getTotalElements(); i++) {
if (e[i].getElementType() != rtree.Node.LEAF_NODE) {
return false;
}
}
return true;
}
protected RTree packRtreeFile(RTree tree, String nonPackFileName, String packFileName) throws IOException {
try {
assert rtree.Node.MAX < 50 : "It is better for search performance"; //$NON-NLS-1$
tree.flush();
File file = new File(packFileName);
if (file.exists()) {
file.delete();
}
long rootIndex = tree.getFileHdr().getRootIndex();
if (!nodeIsLastSubTree(tree, rootIndex)) {
// there is a bug for small files in packing method
new Pack().packTree(tree, packFileName);
tree.getFileHdr().getFile().close();
file = new File(nonPackFileName);
file.delete();
return new RTree(packFileName);
}
} catch (RTreeException e) {
log.error("Error flushing", e); //$NON-NLS-1$
throw new IOException(e);
}
return tree;
}
}

42
DataExtractionOSM/src/net/osmand/data/preparation/DBDialect.java
View file

@ -1,9 +1,13 @@
package net.osmand.data.preparation;
import java.io.File;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
import org.apache.commons.logging.Log;
import net.osmand.Algoritms;
public enum DBDialect {
@ -45,4 +49,42 @@ public enum DBDialect {
}
}
protected Connection getDatabaseConnection(String fileName, Log log) throws SQLException {
if (DBDialect.SQLITE == this) {
try {
Class.forName("org.sqlite.JDBC");
} catch (ClassNotFoundException e) {
log.error("Illegal configuration", e);
throw new IllegalStateException(e);
}
Connection connection = DriverManager.getConnection("jdbc:sqlite:" + fileName);
Statement statement = connection.createStatement();
statement.executeUpdate("PRAGMA synchronous = 0");
statement.close();
return connection;
} else if (DBDialect.DERBY == this) {
try {
Class.forName("org.apache.derby.jdbc.EmbeddedDriver");
} catch (ClassNotFoundException e) {
log.error("Illegal configuration", e);
throw new IllegalStateException(e);
}
Connection conn = DriverManager.getConnection("jdbc:derby:" + fileName + ";create=true");
conn.setAutoCommit(false);
return conn;
} else if (DBDialect.H2 == this) {
try {
Class.forName("org.h2.Driver");
} catch (ClassNotFoundException e) {
log.error("Illegal configuration", e);
throw new IllegalStateException(e);
}
return DriverManager.getConnection("jdbc:h2:file:" + fileName);
} else {
throw new UnsupportedOperationException();
}
}
}

905
DataExtractionOSM/src/net/osmand/data/preparation/IndexAddressCreator.java Normal file
View file

@ -0,0 +1,905 @@
package net.osmand.data.preparation;
import java.io.IOException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.text.Collator;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import net.osmand.Algoritms;
import net.osmand.IProgress;
import net.osmand.binary.BinaryMapIndexWriter;
import net.osmand.data.Boundary;
import net.osmand.data.Building;
import net.osmand.data.City;
import net.osmand.data.DataTileManager;
import net.osmand.data.Street;
import net.osmand.data.City.CityType;
import net.osmand.osm.Entity;
import net.osmand.osm.LatLon;
import net.osmand.osm.MapUtils;
import net.osmand.osm.Node;
import net.osmand.osm.Relation;
import net.osmand.osm.Way;
import net.osmand.osm.Entity.EntityId;
import net.osmand.osm.OSMSettings.OSMTagKey;
import net.osmand.swing.Messages;
import net.sf.junidecode.Junidecode;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
public class IndexAddressCreator extends AbstractIndexPartCreator{
private static final Log log = LogFactory.getLog(IndexAddressCreator.class);
private PreparedStatement addressCityStat;
private PreparedStatement addressStreetStat;
private PreparedStatement addressBuildingStat;
private PreparedStatement addressStreetNodeStat;
// MEMORY address : choose what to use ?
private boolean loadInMemory = true;
private PreparedStatement addressSearchStreetStat;
private PreparedStatement addressSearchBuildingStat;
private PreparedStatement addressSearchStreetNodeStat;
private Map<String, Long> addressStreetLocalMap = new LinkedHashMap<String, Long>();
private Set<Long> addressBuildingLocalSet = new LinkedHashSet<Long>();
private Set<Long> addressStreetNodeLocalSet = new LinkedHashSet<Long>();
// MEMORY address : address structure
// load it in memory
private Map<EntityId, City> cities = new LinkedHashMap<EntityId, City>();
private DataTileManager<City> cityVillageManager = new DataTileManager<City>(13);
private DataTileManager<City> cityManager = new DataTileManager<City>(10);
private List<Relation> postalCodeRelations = new ArrayList<Relation>();
private Map<City, Boundary> citiBoundaries = new LinkedHashMap<City, Boundary>();
private Set<Long> visitedBoundaryWays = new HashSet<Long>();
private boolean normalizeStreets;
private String[] normalizeDefaultSuffixes;
private String[] normalizeSuffixes;
private String cityAdminLevel;
private boolean saveAddressWays;
// TODO
Connection mapConnection;
public IndexAddressCreator(){
}
public void initSettings(boolean normalizeStreets, String[] normalizeDefaultSuffixes, String[] normalizeSuffixes,
boolean saveAddressWays, String cityAdminLevel) {
cities.clear();
cityManager.clear();
postalCodeRelations.clear();
citiBoundaries.clear();
this.normalizeStreets = normalizeStreets;
this.normalizeDefaultSuffixes = normalizeDefaultSuffixes;
this.normalizeSuffixes = normalizeSuffixes;
this.cityAdminLevel = cityAdminLevel;
this.saveAddressWays = saveAddressWays;
}
public void registerCityIfNeeded(Entity e) {
if (e instanceof Node && e.getTag(OSMTagKey.PLACE) != null) {
City city = new City((Node) e);
if (city.getType() != null && !Algoritms.isEmpty(city.getName())) {
if (city.getType() == CityType.CITY || city.getType() == CityType.TOWN) {
cityManager.registerObject(((Node) e).getLatitude(), ((Node) e).getLongitude(), city);
} else {
cityVillageManager.registerObject(((Node) e).getLatitude(), ((Node) e).getLongitude(), city);
}
cities.put(city.getEntityId(), city);
}
}
}
public void indexBoundariesRelation(Entity e, OsmDbAccessorContext ctx) throws SQLException {
if ("administrative".equals(e.getTag(OSMTagKey.BOUNDARY)) && (e instanceof Relation || e instanceof Way)) {
String adminLevel = e.getTag("admin_level");
Boundary boundary = null;
if (cityAdminLevel.equals(adminLevel)) {
if (e instanceof Relation) {
Relation i = (Relation) e;
ctx.loadEntityData(i, true);
boundary = new Boundary();
if (i.getTag(OSMTagKey.NAME) != null) {
boundary.setName(i.getTag(OSMTagKey.NAME));
}
boundary.setBoundaryId(i.getId());
Map<Entity, String> entities = i.getMemberEntities();
for (Entity es : entities.keySet()) {
if (es instanceof Way) {
boolean inner = "inner".equals(entities.get(es)); //$NON-NLS-1$
if (inner) {
boundary.getInnerWays().add((Way) es);
} else {
String wName = es.getTag(OSMTagKey.NAME);
// if name are not equal keep the way for further check (it could be different suburb)
if (Algoritms.objectEquals(wName, boundary.getName()) || wName == null) {
visitedBoundaryWays.add(es.getId());
}
boundary.getOuterWays().add((Way) es);
}
}
}
} else if (e instanceof Way) {
if (!visitedBoundaryWays.contains(e.getId())) {
boundary = new Boundary();
if (e.getTag(OSMTagKey.NAME) != null) {
boundary.setName(e.getTag(OSMTagKey.NAME));
}
boundary.setBoundaryId(e.getId());
boundary.getOuterWays().add((Way) e);
}
}
}
if (boundary != null && boundary.getCenterPoint() != null) {
LatLon point = boundary.getCenterPoint();
boolean cityFound = false;
boolean containsCityInside = false;
for (City c : cityManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) {
if (boundary.containsPoint(c.getLocation())) {
if (boundary.getName() == null || boundary.getName().equalsIgnoreCase(c.getName())) {
citiBoundaries.put(c, boundary);
cityFound = true;
containsCityInside = true;
}
}
}
// TODO mark all suburbs inside city as is_in tag (!) or use another solution
if (!cityFound) {
for (City c : cityVillageManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) {
if (boundary.containsPoint(c.getLocation())) {
if (boundary.getName() == null || boundary.getName().equalsIgnoreCase(c.getName())) {
citiBoundaries.put(c, boundary);
cityFound = true;
}
}
}
}
if (!cityFound && boundary.getName() != null) {
// / create new city for named boundary very rare case that's why do not proper generate id
// however it could be a problem
City nCity = new City(containsCityInside ? CityType.CITY : CityType.SUBURB);
nCity.setLocation(point.getLatitude(), point.getLongitude());
nCity.setId(-boundary.getBoundaryId());
nCity.setName(boundary.getName());
citiBoundaries.put(nCity, boundary);
cityVillageManager.registerObject(point.getLatitude(), point.getLongitude(), nCity);
writeCity(addressCityStat, pStatements, nCity);
// commit to put all cities
if (pStatements.get(addressCityStat) > 0) {
addressCityStat.executeBatch();
pStatements.put(addressCityStat, 0);
}
}
}
}
}
public void indexAddressRelation(Relation i, OsmDbAccessorContext ctx) throws SQLException {
if (i instanceof Relation && "address".equals(i.getTag(OSMTagKey.TYPE))) { //$NON-NLS-1$
String type = i.getTag(OSMTagKey.ADDRESS_TYPE);
boolean house = "house".equals(type); //$NON-NLS-1$
boolean street = "a6".equals(type); //$NON-NLS-1$
if (house || street) {
// try to find appropriate city/street
City c = null;
// load with member ways with their nodes and tags !
ctx.loadEntityData(i, true);
Collection<Entity> members = i.getMembers("is_in"); //$NON-NLS-1$
Relation a3 = null;
Relation a6 = null;
if (!members.isEmpty()) {
if (street) {
a6 = i;
}
Entity in = members.iterator().next();
ctx.loadEntityData(in, true);
if (in instanceof Relation) {
// go one level up for house
if (house) {
a6 = (Relation) in;
members = ((Relation) in).getMembers("is_in"); //$NON-NLS-1$
if (!members.isEmpty()) {
in = members.iterator().next();
ctx.loadEntityData(in, true);
if (in instanceof Relation) {
a3 = (Relation) in;
}
}
} else {
a3 = (Relation) in;
}
}
}
if (a3 != null) {
Collection<EntityId> memberIds = a3.getMemberIds("label"); //$NON-NLS-1$
if (!memberIds.isEmpty()) {
c = cities.get(memberIds.iterator().next());
}
}
if (c != null && a6 != null) {
String name = a6.getTag(OSMTagKey.NAME);
if (name != null) {
LatLon location = c.getLocation();
for (Entity e : i.getMembers(null)) {
if (e instanceof Way) {
LatLon l = ((Way) e).getLatLon();
if (l != null) {
location = l;
break;
}
}
}
Long streetId = getStreetInCity(c, name, location, (a6.getId() << 2) | 2);
if (streetId == null) {
return;
}
if (street) {
for (Map.Entry<Entity, String> r : i.getMemberEntities().entrySet()) {
if ("street".equals(r.getValue())) { //$NON-NLS-1$
if (r.getKey() instanceof Way && saveAddressWays) {
writeStreetWayNodes(addressStreetNodeStat, pStatements, streetId, (Way) r.getKey());
if (loadInMemory) {
addressStreetNodeLocalSet.add(r.getKey().getId());
}
}
} else if ("house".equals(r.getValue())) { //$NON-NLS-1$
// will be registered further in other case
if (!(r.getKey() instanceof Relation)) {
String hno = r.getKey().getTag(OSMTagKey.ADDR_HOUSE_NUMBER);
if (hno != null) {
Building building = new Building(r.getKey());
building.setName(hno);
writeBuilding(addressBuildingStat, pStatements, streetId, building);
if (loadInMemory) {
addressBuildingLocalSet.add(r.getKey().getId());
}
}
}
}
}
} else {
String hno = i.getTag(OSMTagKey.ADDRESS_HOUSE);
if (hno == null) {
hno = i.getTag(OSMTagKey.ADDR_HOUSE_NUMBER);
}
if (hno == null) {
hno = i.getTag(OSMTagKey.NAME);
}
members = i.getMembers("border"); //$NON-NLS-1$
if (!members.isEmpty()) {
Entity border = members.iterator().next();
if (border != null) {
EntityId id = EntityId.valueOf(border);
// special check that address do not contain twice in a3 - border and separate a6
if (!a6.getMemberIds().contains(id)) {
Building building = new Building(border);
if (building.getLocation() != null) {
building.setName(hno);
writeBuilding(addressBuildingStat, pStatements, streetId, building);
if (loadInMemory) {
addressBuildingLocalSet.add(id.getId());
}
} else {
log.error("Strange border " + id + " location couldn't be found");
}
}
}
} else {
log.info("For relation " + i.getId() + " border not found"); //$NON-NLS-1$ //$NON-NLS-2$
}
}
}
}
}
}
}
public String normalizeStreetName(String name) {
name = name.trim();
if (normalizeStreets) {
String newName = name;
boolean processed = newName.length() != name.length();
for (String ch : normalizeDefaultSuffixes) {
int ind = checkSuffix(newName, ch);
if (ind != -1) {
newName = cutSuffix(newName, ind, ch.length());
processed = true;
break;
}
}
if (!processed) {
for (String ch : normalizeSuffixes) {
int ind = checkSuffix(newName, ch);
if (ind != -1) {
newName = putSuffixToEnd(newName, ind, ch.length());
processed = true;
break;
}
}
}
if (processed) {
return newName;
}
}
return name;
}
private int checkSuffix(String name, String suffix) {
int i = -1;
boolean searchAgain = false;
do {
i = name.indexOf(suffix, i);
searchAgain = false;
if (i > 0) {
if (Character.isLetterOrDigit(name.charAt(i - 1))) {
i++;
searchAgain = true;
}
}
} while (searchAgain);
return i;
}
private String cutSuffix(String name, int ind, int suffixLength) {
String newName = name.substring(0, ind);
if (name.length() > ind + suffixLength + 1) {
newName += name.substring(ind + suffixLength + 1);
}
return newName.trim();
}
private String putSuffixToEnd(String name, int ind, int suffixLength) {
if (name.length() <= ind + suffixLength) {
return name;
}
String newName;
if (ind > 0) {
newName = name.substring(0, ind);
newName += name.substring(ind + suffixLength);
newName += name.substring(ind - 1, ind + suffixLength);
} else {
newName = name.substring(suffixLength + 1) + name.charAt(suffixLength) + name.substring(0, suffixLength);
}
return newName.trim();
}
public Long getStreetInCity(City city, String name, LatLon location, long initId) throws SQLException {
if (name == null || city == null) {
return null;
}
Long foundId = null;
name = normalizeStreetName(name);
if (loadInMemory) {
foundId = addressStreetLocalMap.get(name + "_" + city.getId()); //$NON-NLS-1$
} else {
addressSearchStreetStat.setLong(1, city.getId());
addressSearchStreetStat.setString(2, name);
ResultSet rs = addressSearchStreetStat.executeQuery();
if (rs.next()) {
foundId = rs.getLong(1);
}
rs.close();
}
if (foundId == null) {
insertStreetData(addressStreetStat, initId, name, Junidecode.unidecode(name),
location.getLatitude(), location.getLongitude(), city.getId());
if (loadInMemory) {
addBatch(pStatements, addressStreetStat, BATCH_SIZE);
addressStreetLocalMap.put(name + "_" + city.getId(), initId); //$NON-NLS-1$
} else {
addressStreetStat.execute();
// commit immediately to search after
mapConnection.commit();
}
foundId = initId;
}
return foundId;
}
public City getClosestCity(LatLon point) {
if (point == null) {
return null;
}
for (City c : cityManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) {
Boundary boundary = citiBoundaries.get(c);
if(boundary != null){
if(boundary.containsPoint(point)){
return c;
}
}
}
for (City c : cityVillageManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) {
Boundary boundary = citiBoundaries.get(c);
if(boundary != null){
if(boundary.containsPoint(point)){
return c;
}
}
}
City closest = null;
double relDist = Double.POSITIVE_INFINITY;
for (City c : cityManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) {
double rel = MapUtils.getDistance(c.getLocation(), point) / c.getType().getRadius();
if (rel < relDist) {
closest = c;
relDist = rel;
if (relDist < 0.2d) {
break;
}
}
}
if (relDist < 0.2d) {
return closest;
}
for (City c : cityVillageManager.getClosestObjects(point.getLatitude(), point.getLongitude(), 3)) {
double rel = MapUtils.getDistance(c.getLocation(), point) / c.getType().getRadius();
if (rel < relDist) {
closest = c;
relDist = rel;
if (relDist < 0.2d) {
break;
}
}
}
return closest;
}
public void iterateMainEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException {
// index not only buildings but also nodes that belongs to addr:interpolation ways
if (e.getTag(OSMTagKey.ADDR_HOUSE_NUMBER) != null && e.getTag(OSMTagKey.ADDR_STREET) != null) {
// TODO e.getTag(OSMTagKey.ADDR_CITY) could be used to find city however many cities could have same name!
// check that building is not registered already
boolean exist = false;
if (loadInMemory) {
exist = addressBuildingLocalSet.contains(e.getId());
} else {
addressSearchBuildingStat.setLong(1, e.getId());
ResultSet rs = addressSearchBuildingStat.executeQuery();
exist = rs.next();
rs.close();
}
if (!exist) {
ctx.loadEntityData(e, false);
LatLon l = e.getLatLon();
City city = getClosestCity(l);
Long idStreet = getStreetInCity(city, e.getTag(OSMTagKey.ADDR_STREET), l, (e.getId() << 2));
if (idStreet != null) {
Building building = new Building(e);
building.setName(e.getTag(OSMTagKey.ADDR_HOUSE_NUMBER));
writeBuilding(addressBuildingStat, pStatements, idStreet, building);
}
}
} else if (e instanceof Way /* && OSMSettings.wayForCar(e.getTag(OSMTagKey.HIGHWAY)) */
&& e.getTag(OSMTagKey.HIGHWAY) != null && e.getTag(OSMTagKey.NAME) != null) {
// suppose that streets with names are ways for car
// Ignore all ways that have house numbers and highway type
boolean exist = false;
// if we saved address ways we could checked that we registered before
if (saveAddressWays) {
if (loadInMemory) {
exist = addressStreetNodeLocalSet.contains(e.getId());
} else {
addressSearchStreetNodeStat.setLong(1, e.getId());
ResultSet rs = addressSearchStreetNodeStat.executeQuery();
exist = rs.next();
rs.close();
}
}
// check that street way is not registered already
if (!exist) {
ctx.loadEntityData(e, false);
LatLon l = e.getLatLon();
City city = getClosestCity(l);
Long idStreet = getStreetInCity(city, e.getTag(OSMTagKey.NAME), l, (e.getId() << 2) | 1);
if (idStreet != null && saveAddressWays) {
writeStreetWayNodes(addressStreetNodeStat, pStatements, idStreet, (Way) e);
}
}
}
if (e instanceof Relation) {
if (e.getTag(OSMTagKey.POSTAL_CODE) != null) {
ctx.loadEntityData(e, false);
postalCodeRelations.add((Relation) e);
}
}
}
private void writeStreetWayNodes(PreparedStatement prepStreetNode, Map<PreparedStatement, Integer> count, Long streetId, Way way)
throws SQLException {
for (Node n : way.getNodes()) {
if (n == null) {
continue;
}
prepStreetNode.setLong(1, n.getId());
prepStreetNode.setDouble(2, n.getLatitude());
prepStreetNode.setDouble(3, n.getLongitude());
prepStreetNode.setLong(5, way.getId());
prepStreetNode.setLong(4, streetId);
addBatch(count, prepStreetNode, BATCH_SIZE);
}
}
private void writeBuilding(PreparedStatement prepBuilding, Map<PreparedStatement, Integer> count, Long streetId,
Building building)
throws SQLException {
prepBuilding.setLong(1, building.getId());
prepBuilding.setDouble(2, building.getLocation().getLatitude());
prepBuilding.setDouble(3, building.getLocation().getLongitude());
prepBuilding.setString(4, building.getName());
prepBuilding.setString(5, building.getEnName());
prepBuilding.setLong(6, streetId);
prepBuilding.setString(7, building.getPostcode() == null ? null : building.getPostcode().toUpperCase());
addBatch(count, prepBuilding);
}
private void writeCity(PreparedStatement prepCity, Map<PreparedStatement, Integer> count, City city) throws SQLException {
prepCity.setLong(1, city.getId());
prepCity.setDouble(2, city.getLocation().getLatitude());
prepCity.setDouble(3, city.getLocation().getLongitude());
prepCity.setString(4, city.getName());
prepCity.setString(5, city.getEnName());
prepCity.setString(6, CityType.valueToString(city.getType()));
addBatch(count, prepCity, BATCH_SIZE);
}
private void insertStreetData(PreparedStatement addressStreetStat, long id, String name, String nameEn, double latitude,
double longitude, Long cityId) throws SQLException {
addressStreetStat.setLong(1, id);
addressStreetStat.setString(4, name);
addressStreetStat.setString(5, nameEn);
addressStreetStat.setDouble(2, latitude);
addressStreetStat.setDouble(3, longitude);
addressStreetStat.setLong(6, cityId);
}
public void writeCitiesIntoDb() throws SQLException {
for (City c : cities.values()) {
writeCity(addressCityStat, pStatements, c);
}
// commit to put all cities
if (pStatements.get(addressCityStat) > 0) {
addressCityStat.executeBatch();
pStatements.put(addressCityStat, 0);
mapConnection.commit();
}
}
public void processingPostcodes() throws SQLException {
if (pStatements.get(addressBuildingStat) > 0) {
addressBuildingStat.executeBatch();
pStatements.put(addressBuildingStat, 0);
mapConnection.commit();
}
PreparedStatement pstat = mapConnection.prepareStatement("UPDATE building SET postcode = ? WHERE id = ?");
pStatements.put(pstat, 0);
for (Relation r : postalCodeRelations) {
String tag = r.getTag(OSMTagKey.POSTAL_CODE);
for (EntityId l : r.getMemberIds()) {
pstat.setString(1, tag);
pstat.setLong(2, l.getId());
addBatch(pStatements, pstat, BATCH_SIZE);
}
}
if (pStatements.get(pstat) > 0) {
pstat.executeBatch();
}
pStatements.remove(pstat);
}
public void writeBinaryAddressIndex(BinaryMapIndexWriter writer, String regionName, IProgress progress) throws IOException, SQLException {
closePreparedStatements(addressCityStat, addressStreetStat, addressStreetNodeStat, addressBuildingStat);
mapConnection.commit();
boolean readWayNodes = saveAddressWays;
writer.startWriteAddressIndex(regionName);
List<City> cities = readCities(mapConnection);
List<Street> streets = new ArrayList<Street>();
Collections.sort(cities, new Comparator<City>() {
@Override
public int compare(City o1, City o2) {
if (o1.getType() != o2.getType()) {
return (o1.getType().ordinal() - o2.getType().ordinal());
}
return Collator.getInstance().compare(o1.getName(), o2.getName());
}
});
PreparedStatement streetstat = mapConnection.prepareStatement("SELECT A.id, A.name, A.name_en, A.latitude, A.longitude, "+ //$NON-NLS-1$
"B.id, B.name, B.name_en, B.latitude, B.longitude, B.postcode "+ //$NON-NLS-1$
"FROM street A left JOIN building B ON B.street = A.id WHERE A.city = ?"); //$NON-NLS-1$
PreparedStatement waynodesStat = null;
if (readWayNodes) {
waynodesStat = mapConnection.prepareStatement("SELECT A.id, A.latitude, A.longitude FROM street_node A WHERE A.street = ? "); //$NON-NLS-1$
}
int j = 0;
for (; j < cities.size(); j++) {
City c = cities.get(j);
if (c.getType() != CityType.CITY && c.getType() != CityType.TOWN) {
break;
}
}
progress.startTask(Messages.getString("IndexCreator.SERIALIZING_ADRESS"), j + ((cities.size() - j) / 100 + 1)); //$NON-NLS-1$
Map<String, Set<Street>> postcodes = new TreeMap<String, Set<Street>>();
boolean writeCities = true;
// collect suburbs with is in value
List<City> suburbs = new ArrayList<City>();
for(City s : cities){
if(s.getType() == CityType.SUBURB && s.getIsInValue() != null){
suburbs.add(s);
}
}
// write cities and after villages
writer.startCityIndexes(false);
for (int i = 0; i < cities.size(); i++) {
City c = cities.get(i);
List<City> listSuburbs = null;
for (City suburb : suburbs) {
if (suburb.getIsInValue().contains(c.getName().toLowerCase())) {
if(listSuburbs == null){
listSuburbs = new ArrayList<City>();
}
listSuburbs.add(suburb);
}
}
if (writeCities) {
progress.progress(1);
} else if ((cities.size() - i) % 100 == 0) {
progress.progress(1);
}
if (writeCities && c.getType() != CityType.CITY && c.getType() != CityType.TOWN) {
writer.endCityIndexes(false);
writer.startCityIndexes(true);
writeCities = false;
}
streets.clear();
Map<Street, List<Node>> streetNodes = null;
if (readWayNodes) {
streetNodes = new LinkedHashMap<Street, List<Node>>();
}
long time = System.currentTimeMillis();
readStreetsBuildings(streetstat, c, streets, waynodesStat, streetNodes, listSuburbs);
long f = System.currentTimeMillis() - time;
writer.writeCityIndex(c, streets, streetNodes);
int bCount = 0;
for (Street s : streets) {
bCount++;
for (Building b : s.getBuildings()) {
bCount++;
if (b.getPostcode() != null) {
if (!postcodes.containsKey(b.getPostcode())) {
postcodes.put(b.getPostcode(), new LinkedHashSet<Street>(3));
}
postcodes.get(b.getPostcode()).add(s);
}
}
}
if (f > 500) {
System.out.println("! " + c.getName() + " ! " + f + " " + bCount + " streets " + streets.size()); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
}
}
writer.endCityIndexes(!writeCities);
// write postcodes
writer.startPostcodes();
for (String s : postcodes.keySet()) {
writer.writePostcode(s, postcodes.get(s));
}
writer.endPostcodes();
progress.finishTask();
writer.endWriteAddressIndex();
writer.flush();
streetstat.close();
if (readWayNodes) {
waynodesStat.close();
}
}
public void commitToPutAllCities() throws SQLException {
// commit to put all cities
if (pStatements.get(addressBuildingStat) > 0) {
addressBuildingStat.executeBatch();
pStatements.put(addressBuildingStat, 0);
}
if (pStatements.get(addressStreetNodeStat) > 0) {
addressStreetNodeStat.executeBatch();
pStatements.put(addressStreetNodeStat, 0);
}
mapConnection.commit();
}
public void createDatabaseStructure(Connection mapConnection, DBDialect dialect) throws SQLException {
this.mapConnection = mapConnection;
createAddressIndexStructure(mapConnection, dialect);
addressCityStat = mapConnection.prepareStatement("insert into city (id, latitude, longitude, name, name_en, city_type) values (?, ?, ?, ?, ?, ?)");
addressStreetStat = mapConnection.prepareStatement("insert into street (id, latitude, longitude, name, name_en, city) values (?, ?, ?, ?, ?, ?)");
addressBuildingStat = mapConnection.prepareStatement("insert into building (id, latitude, longitude, name, name_en, street, postcode) values (?, ?, ?, ?, ?, ?, ?)");
addressStreetNodeStat = mapConnection.prepareStatement("insert into street_node (id, latitude, longitude, street, way) values (?, ?, ?, ?, ?)");
addressSearchStreetStat = mapConnection.prepareStatement("SELECT ID FROM street WHERE ? = city AND ? = name");
addressSearchBuildingStat = mapConnection.prepareStatement("SELECT id FROM building where ? = id");
addressSearchStreetNodeStat = mapConnection.prepareStatement("SELECT way FROM street_node WHERE ? = way");
pStatements.put(addressCityStat, 0);
pStatements.put(addressStreetStat, 0);
pStatements.put(addressStreetNodeStat, 0);
pStatements.put(addressBuildingStat, 0);
// put search statements to close them after all
pStatements.put(addressSearchBuildingStat, 0);
pStatements.put(addressSearchStreetNodeStat, 0);
pStatements.put(addressSearchStreetStat, 0);
}
private void createAddressIndexStructure(Connection conn, DBDialect dialect) throws SQLException{
Statement stat = conn.createStatement();
stat.executeUpdate("create table city (id bigint primary key, latitude double, longitude double, " +
"name varchar(255), name_en varchar(255), city_type varchar(32))");
stat.executeUpdate("create index city_ind on city (id, city_type)");
stat.executeUpdate("create table street (id bigint primary key, latitude double, longitude double, " +
"name varchar(255), name_en varchar(255), city bigint)");
stat.executeUpdate("create index street_city on street (city)");
stat.executeUpdate("create index street_id on street (id)");
// create index on name ?
stat.executeUpdate("create table building (id bigint, latitude double, longitude double, " +
"name varchar(255), name_en varchar(255), street bigint, postcode varchar(255), primary key(street, id))");
stat.executeUpdate("create index building_postcode on building (postcode)");
stat.executeUpdate("create index building_street on building (street)");
stat.executeUpdate("create index building_id on building (id)");
stat.executeUpdate("create table street_node (id bigint, latitude double, longitude double, " +
"street bigint, way bigint)");
stat.executeUpdate("create index street_node_street on street_node (street)");
stat.executeUpdate("create index street_node_way on street_node (way)");
// if(dialect == DBDialect.SQLITE){
// stat.execute("PRAGMA user_version = " + IndexConstants.ADDRESS_TABLE_VERSION); //$NON-NLS-1$
// }
stat.close();
}
private List<Street> readStreetsBuildings(PreparedStatement streetBuildingsStat, City city, List<Street> streets,
PreparedStatement waynodesStat, Map<Street, List<Node>> streetNodes, List<City> citySuburbs) throws SQLException {
Map<Long, Street> visitedStreets = new LinkedHashMap<Long, Street>();
//read streets for city
readStreatsByBuildingsForCity(streetBuildingsStat, city, streets,
waynodesStat, streetNodes, visitedStreets);
//read streets for suburbs of the city
if (citySuburbs != null) {
for (City suburb : citySuburbs) {
readStreatsByBuildingsForCity(streetBuildingsStat, suburb, streets, waynodesStat, streetNodes, visitedStreets);
}
}
return streets;
}
private void readStreatsByBuildingsForCity(
PreparedStatement streetBuildingsStat, City city,
List<Street> streets, PreparedStatement waynodesStat,
Map<Street, List<Node>> streetNodes,
Map<Long, Street> visitedStreets) throws SQLException {
streetBuildingsStat.setLong(1, city.getId());
ResultSet set = streetBuildingsStat.executeQuery();
while (set.next()) {
long streetId = set.getLong(1);
if (!visitedStreets.containsKey(streetId)) {
Street street = new Street(null);
street.setName(set.getString(2));
street.setEnName(set.getString(3));
street.setLocation(set.getDouble(4), set.getDouble(5));
street.setId(streetId);
streets.add(street);
visitedStreets.put(streetId, street);
if (waynodesStat != null && streetNodes != null) {
ArrayList<Node> list = new ArrayList<Node>();
streetNodes.put(street, list);
waynodesStat.setLong(1, street.getId());
ResultSet rs = waynodesStat.executeQuery();
while (rs.next()) {
list.add(new Node(rs.getDouble(2), rs.getDouble(3), rs.getLong(1)));
}
rs.close();
}
}
if (set.getObject(6) != null) {
Street s = visitedStreets.get(streetId);
Building b = new Building();
b.setId(set.getLong(6));
b.setName(set.getString(7));
b.setEnName(set.getString(8));
b.setLocation(set.getDouble(9), set.getDouble(10));
b.setPostcode(set.getString(11));
s.registerBuilding(b);
}
}
set.close();
}
public List<City> readCities(Connection c) throws SQLException{
List<City> cities = new ArrayList<City>();
Statement stat = c.createStatement();
ResultSet set = stat.executeQuery("select id, latitude, longitude , name , name_en , city_type from city"); //$NON-NLS-1$
while(set.next()){
City city = new City(CityType.valueFromString(set.getString(6)));
city.setName(set.getString(4));
city.setEnName(set.getString(5));
city.setLocation(set.getDouble(2),
set.getDouble(3));
city.setId(set.getLong(1));
cities.add(city);
}
set.close();
stat.close();
return cities;
}
}

1645
DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java
View file

File diff suppressed because it is too large Load diff

20
DataExtractionOSM/src/net/osmand/data/preparation/IndexPoiCreator.java
View file

@ -7,7 +7,6 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@ -15,7 +14,6 @@ import java.util.Map;
import net.osmand.Algoritms;
import net.osmand.data.Amenity;
import net.osmand.data.AmenityType;
import net.osmand.data.index.DataIndexWriter;
import net.osmand.data.index.IndexConstants;
import net.osmand.osm.Entity;
import net.osmand.osm.MapUtils;
@ -24,11 +22,9 @@ import net.osmand.osm.OSMSettings.OSMTagKey;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
public class IndexPoiCreator {
public class IndexPoiCreator extends AbstractIndexPartCreator {
private static final int BATCH_SIZE = 1000;
private static final Log log = LogFactory.getLog(IndexPoiCreator.class);
private final IndexCreator creator;
private Connection poiConnection;
private File poiIndexFile;
@ -36,16 +32,15 @@ public class IndexPoiCreator {
private List<Amenity> tempAmenityList = new ArrayList<Amenity>();
public IndexPoiCreator(IndexCreator creator){
this.creator = creator;
public IndexPoiCreator(){
}
public void iterateEntity(Entity e, Map<PreparedStatement, Integer> pStatements) throws SQLException{
public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException{
tempAmenityList.clear();
tempAmenityList = Amenity.parseAmenities(e, tempAmenityList);
if (!tempAmenityList.isEmpty() && poiPreparedStatement != null) {
// load data for way (location etc...)
creator.loadEntityData(e, false);
ctx.loadEntityData(e, false);
for (Amenity a : tempAmenityList) {
checkEntity(e);
a.setEntity(e);
@ -68,6 +63,7 @@ public class IndexPoiCreator {
}
}
closeAllPreparedStatements();
}
public void insertAmenityIntoPoi( Map<PreparedStatement, Integer> map, Amenity amenity) throws SQLException {
@ -83,7 +79,7 @@ public class IndexPoiCreator {
poiPreparedStatement.setString(8, amenity.getOpeningHours());
poiPreparedStatement.setString(9, amenity.getSite());
poiPreparedStatement.setString(10, amenity.getPhone());
DataIndexWriter.addBatch(map, poiPreparedStatement, BATCH_SIZE);
addBatch(map, poiPreparedStatement, BATCH_SIZE);
}
public void checkEntity(Entity e){
@ -109,7 +105,7 @@ public class IndexPoiCreator {
}
}
public void createDatabaseStructure(File poiIndexFile, Map<PreparedStatement, Integer> pStatements) throws SQLException {
public void createDatabaseStructure(File poiIndexFile) throws SQLException {
this.poiIndexFile = poiIndexFile;
// to save space
if (poiIndexFile.exists()) {
@ -117,7 +113,7 @@ public class IndexPoiCreator {
}
poiIndexFile.getParentFile().mkdirs();
// creating nodes db to fast access for all nodes
poiConnection = creator.getDatabaseConnection(poiIndexFile.getAbsolutePath(), DBDialect.SQLITE);
poiConnection = DBDialect.SQLITE.getDatabaseConnection(poiIndexFile.getAbsolutePath(), log);
createPoiIndexStructure(poiConnection);
poiPreparedStatement = createStatementAmenityInsert(poiConnection);
pStatements.put(poiPreparedStatement, 0);

56
DataExtractionOSM/src/net/osmand/data/preparation/IndexTransportCreator.java
View file

@ -20,8 +20,6 @@ import java.util.Map.Entry;
import net.osmand.binary.BinaryMapIndexWriter;
import net.osmand.data.TransportRoute;
import net.osmand.data.TransportStop;
import net.osmand.data.index.DataIndexWriter;
import net.osmand.data.index.IndexConstants;
import net.osmand.osm.Entity;
import net.osmand.osm.MapUtils;
import net.osmand.osm.Node;
@ -42,17 +40,15 @@ import rtree.RTreeException;
import rtree.RTreeInsertException;
import rtree.Rect;
public class IndexTransportCreator {
public class IndexTransportCreator extends AbstractIndexPartCreator {
private final IndexCreator creator;
private static final int BATCH_SIZE = 1000;
private static final Log log = LogFactory.getLog(IndexTransportCreator.class);
private Set<Long> visitedStops = new HashSet<Long>();
private PreparedStatement transRouteStat;
private PreparedStatement transRouteStopsStat;
private PreparedStatement transStopsStat;
protected RTree transportStopsTree;
private RTree transportStopsTree;
private static Set<String> acceptedRoutes = new HashSet<String>();
@ -69,8 +65,7 @@ public class IndexTransportCreator {
acceptedRoutes.add("ferry"); //$NON-NLS-1$
}
public IndexTransportCreator(IndexCreator creator){
this.creator = creator;
public IndexTransportCreator(){
}
@ -78,11 +73,8 @@ public class IndexTransportCreator {
transportStopsTree = new RTree(rtreeTransportStopFile);
}
public void createTransportIndexStructure(Connection conn, DBDialect dialect, String rtreeStopsFileName, Map<PreparedStatement, Integer> pStatements) throws SQLException, IOException{
public void createTransportIndexStructure(Connection conn, DBDialect dialect, String rtreeStopsFileName) throws SQLException, IOException{
Statement stat = conn.createStatement();
assert IndexConstants.TRANSPORT_ROUTE_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
assert IndexConstants.TRANSPORT_STOP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
assert IndexConstants.TRANSPORT_ROUTE_STOP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
stat.executeUpdate("create table transport_route (id bigint primary key, type varchar(255), operator varchar(255)," +
"ref varchar(255), name varchar(255), name_en varchar(255), dist int)");
@ -96,9 +88,9 @@ public class IndexTransportCreator {
stat.executeUpdate("create index transport_stop_id on transport_stop (id)");
stat.executeUpdate("create index transport_stop_location on transport_stop (latitude, longitude)");
if(dialect == DBDialect.SQLITE){
stat.execute("PRAGMA user_version = " + IndexConstants.TRANSPORT_TABLE_VERSION); //$NON-NLS-1$
}
// if(dialect == DBDialect.SQLITE){
// stat.execute("PRAGMA user_version = " + IndexConstants.TRANSPORT_TABLE_VERSION); //$NON-NLS-1$
// }
stat.close();
try {
@ -119,7 +111,11 @@ public class IndexTransportCreator {
}
public void visitEntityMainStep(Entity e, OsmDbAccessorContext ctx, Map<PreparedStatement, Integer> pStatements) throws SQLException {
public void packRTree(String rtreeTransportStopsFileName, String rtreeTransportStopsPackFileName) throws IOException {
transportStopsTree = packRtreeFile(transportStopsTree, rtreeTransportStopsFileName, rtreeTransportStopsPackFileName);
}
public void visitEntityMainStep(Entity e, OsmDbAccessorContext ctx) throws SQLException {
if (e instanceof Relation && e.getTag(OSMTagKey.ROUTE) != null) {
ctx.loadEntityData(e, true);
TransportRoute route = indexTransportRoute((Relation) e);
@ -130,11 +126,11 @@ public class IndexTransportCreator {
}
}
public void insertTransportIntoIndex(PreparedStatement prepRoute, PreparedStatement prepRouteStops,
private void insertTransportIntoIndex(PreparedStatement prepRoute, PreparedStatement prepRouteStops,
PreparedStatement prepStops, RTree transportStopsTree,
Set<Long> writtenStops, TransportRoute route, Map<PreparedStatement, Integer> statements,
int batchSize) throws SQLException {
assert IndexConstants.TRANSPORT_ROUTE_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
prepRoute.setLong(1, route.getId());
prepRoute.setString(2, route.getType());
prepRoute.setString(3, route.getOperator());
@ -142,26 +138,22 @@ public class IndexTransportCreator {
prepRoute.setString(5, route.getName());
prepRoute.setString(6, route.getEnName());
prepRoute.setInt(7, route.getAvgBothDistance());
DataIndexWriter.addBatch(statements, prepRoute);
addBatch(statements, prepRoute);
writeRouteStops(transportStopsTree, prepRouteStops, prepStops, statements, writtenStops, route, route.getForwardStops(), true);
writeRouteStops(transportStopsTree, prepRouteStops, prepStops, statements, writtenStops, route, route.getBackwardStops(), false);
}
public static PreparedStatement createStatementTransportStopInsert(Connection conn) throws SQLException{
assert IndexConstants.TRANSPORT_STOP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
private PreparedStatement createStatementTransportStopInsert(Connection conn) throws SQLException{
return conn.prepareStatement("insert into transport_stop(id, latitude, longitude, name, name_en) values(?, ?, ?, ?, ?)");
}
public static PreparedStatement createStatementTransportRouteStopInsert(Connection conn) throws SQLException{
assert IndexConstants.TRANSPORT_ROUTE_STOP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
private PreparedStatement createStatementTransportRouteStopInsert(Connection conn) throws SQLException{
return conn.prepareStatement("insert into transport_route_stop(route, stop, direction, ord) values(?, ?, ?, ?)");
}
private static void writeRouteStops(RTree transportStopsTree, PreparedStatement prepRouteStops, PreparedStatement prepStops, Map<PreparedStatement, Integer> count,
private void writeRouteStops(RTree transportStopsTree, PreparedStatement prepRouteStops, PreparedStatement prepStops, Map<PreparedStatement, Integer> count,
Set<Long> writtenStops, TransportRoute r, List<TransportStop> stops, boolean direction) throws SQLException {
assert IndexConstants.TRANSPORT_STOP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
assert IndexConstants.TRANSPORT_ROUTE_STOP_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
int i = 0;
for(TransportStop s : stops){
if (!writtenStops.contains(s.getId())) {
@ -172,7 +164,7 @@ public class IndexTransportCreator {
prepStops.setString(5, s.getEnName());
int x = (int) MapUtils.getTileNumberX(24, s.getLocation().getLongitude());
int y = (int) MapUtils.getTileNumberY(24, s.getLocation().getLatitude());
DataIndexWriter.addBatch(count, prepStops);
addBatch(count, prepStops);
try {
transportStopsTree.insert(new LeafElement(new Rect(x, y, x, y), s.getId()));
} catch (RTreeInsertException e) {
@ -186,12 +178,11 @@ public class IndexTransportCreator {
prepRouteStops.setLong(2, s.getId());
prepRouteStops.setInt(3, direction ? 1 : 0);
prepRouteStops.setInt(4, i++);
DataIndexWriter.addBatch(count, prepRouteStops);
addBatch(count, prepRouteStops);
}
}
public static PreparedStatement createStatementTransportRouteInsert(Connection conn) throws SQLException{
assert IndexConstants.TRANSPORT_ROUTE_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
private PreparedStatement createStatementTransportRouteInsert(Connection conn) throws SQLException{
return conn.prepareStatement("insert into transport_route(id, type, operator, ref, name, name_en, dist) values(?, ?, ?, ?, ?, ?, ?)");
}
@ -199,7 +190,7 @@ public class IndexTransportCreator {
public void writeBinaryTransportIndex(BinaryMapIndexWriter writer, String regionName,
Connection mapConnection) throws IOException, SQLException {
try {
creator.closePreparedStatements(transRouteStat, transRouteStopsStat, transStopsStat);
closePreparedStatements(transRouteStat, transRouteStopsStat, transStopsStat);
mapConnection.commit();
transportStopsTree.flush();
@ -332,7 +323,7 @@ public class IndexTransportCreator {
}
public void commitAndCloseFiles(String rtreeStopsFileName, String rtreeStopsPackFileName, boolean deleteDatabaseIndexes) throws IOException {
public void commitAndCloseFiles(String rtreeStopsFileName, String rtreeStopsPackFileName, boolean deleteDatabaseIndexes) throws IOException, SQLException {
// delete transport rtree files
if (transportStopsTree != null) {
transportStopsTree.getFileHdr().getFile().close();
@ -345,6 +336,7 @@ public class IndexTransportCreator {
f.delete();
}
}
closeAllPreparedStatements();
}

877
DataExtractionOSM/src/net/osmand/data/preparation/IndexVectorMapCreator.java Normal file
View file

@ -0,0 +1,877 @@
package net.osmand.data.preparation;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import net.osmand.Algoritms;
import net.osmand.IProgress;
import net.osmand.binary.BinaryMapIndexWriter;
import net.osmand.data.MapAlgorithms;
import net.osmand.osm.Entity;
import net.osmand.osm.LatLon;
import net.osmand.osm.MapRenderingTypes;
import net.osmand.osm.MapUtils;
import net.osmand.osm.Node;
import net.osmand.osm.Relation;
import net.osmand.osm.Way;
import net.osmand.osm.Entity.EntityId;
import net.osmand.osm.Entity.EntityType;
import net.osmand.osm.OSMSettings.OSMTagKey;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import rtree.Element;
import rtree.IllegalValueException;
import rtree.LeafElement;
import rtree.NonLeafElement;
import rtree.RTree;
import rtree.RTreeException;
import rtree.RTreeInsertException;
import rtree.Rect;
public class IndexVectorMapCreator extends AbstractIndexPartCreator {
private static final Log log = LogFactory.getLog(IndexVectorMapCreator.class);
private MapRenderingTypes renderingTypes;
// MEMORY map : save it in memory while that is allowed
private Map<Long, Set<Integer>>[] multiPolygonsWays;
private Map<Long, String> multiPolygonsNames = new LinkedHashMap<Long, String>();
private Map<Long, List<Long>> highwayRestrictions = new LinkedHashMap<Long, List<Long>>();
// local purpose
List<Integer> typeUse = new ArrayList<Integer>(8);
List<Long> restrictionsUse = new ArrayList<Long>(8);
private MapZooms mapZooms;
private PreparedStatement mapBinaryStat;
private PreparedStatement mapLowLevelBinaryStat;
private int lowLevelWays = -1;
private RTree[] mapTree = null;
private Connection mapConnection;
public IndexVectorMapCreator() {
}
public void indexMapRelationsAndMultiPolygons(Entity e, OsmDbAccessorContext ctx) throws SQLException {
if (e instanceof Relation && "restriction".equals(e.getTag(OSMTagKey.TYPE))) { //$NON-NLS-1$
String val = e.getTag("restriction"); //$NON-NLS-1$
if (val != null) {
byte type = -1;
if ("no_right_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_NO_RIGHT_TURN;
} else if ("no_left_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_NO_LEFT_TURN;
} else if ("no_u_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_NO_U_TURN;
} else if ("no_straight_on".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_NO_STRAIGHT_ON;
} else if ("only_right_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_ONLY_RIGHT_TURN;
} else if ("only_left_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_ONLY_LEFT_TURN;
} else if ("only_straight_on".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_ONLY_STRAIGHT_ON;
}
if (type != -1) {
ctx.loadEntityData(e, true);
Collection<EntityId> fromL = ((Relation) e).getMemberIds("from"); //$NON-NLS-1$
Collection<EntityId> toL = ((Relation) e).getMemberIds("to"); //$NON-NLS-1$
if (!fromL.isEmpty() && !toL.isEmpty()) {
EntityId from = fromL.iterator().next();
EntityId to = toL.iterator().next();
if (from.getType() == EntityType.WAY) {
if (!highwayRestrictions.containsKey(from.getId())) {
highwayRestrictions.put(from.getId(), new ArrayList<Long>(4));
}
highwayRestrictions.get(from.getId()).add((to.getId() << 3) | (long) type);
}
}
}
}
}
if (e instanceof Relation && "multipolygon".equals(e.getTag(OSMTagKey.TYPE))) { //$NON-NLS-1$
ctx.loadEntityData(e, true);
Map<Entity, String> entities = ((Relation) e).getMemberEntities();
boolean outerFound = false;
for (Entity es : entities.keySet()) {
if (es instanceof Way) {
boolean inner = "inner".equals(entities.get(es)); //$NON-NLS-1$
if (!inner) {
outerFound = true;
for (String t : es.getTagKeySet()) {
e.putTag(t, es.getTag(t));
}
break;
}
}
}
if(!outerFound){
log.warn("Probably map bug: Multipoligon id=" + e.getId() + " contains only inner ways : "); //$NON-NLS-1$ //$NON-NLS-2$
return;
}
int mtType = findMultiPolygonType(e, 0);
if (mtType != 0) {
String name = renderingTypes.getEntityName(e);
List<List<Way>> completedRings = new ArrayList<List<Way>>();
List<List<Way>> incompletedRings = new ArrayList<List<Way>>();
for (Entity es : entities.keySet()) {
if (es instanceof Way) {
if (!((Way) es).getNodeIds().isEmpty()) {
combineMultiPolygons((Way) es, completedRings, incompletedRings);
}
}
}
// skip incompleted rings and do not add whole relation ?
if (!incompletedRings.isEmpty()) {
// log.warn("In multipolygon " + e.getId() + " there are incompleted ways : " + incompletedRings);
return;
// completedRings.addAll(incompletedRings);
}
// skip completed rings that are not one type
for (List<Way> l : completedRings) {
boolean innerType = "inner".equals(entities.get(l.get(0))); //$NON-NLS-1$
for (Way way : l) {
boolean inner = "inner".equals(entities.get(way)); //$NON-NLS-1$
if (innerType != inner) {
log.warn("Probably map bug: Multipoligon contains outer and inner ways.\n" + //$NON-NLS-1$
"Way:" + way.getId() + " is strange part of completed ring. InnerType:" + innerType + " way inner: " + inner + " way inner string:" + entities.get(way)); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
return;
}
}
}
for (List<Way> l : completedRings) {
boolean innerType = "inner".equals(entities.get(l.get(0))); //$NON-NLS-1$
boolean clockwise = MapSwingAlgorithms.isClockwiseWay(l);
// clockwise - outer (like coastline), anticlockwise - inner
boolean inverse = clockwise != !innerType;
for (Way way : l) {
boolean inner = "inner".equals(entities.get(way)); //$NON-NLS-1$
if (!inner && name != null) {
multiPolygonsNames.put(way.getId(), name);
}
putMultipolygonType(multiPolygonsWays[0], way.getId(), mtType, inverse);
for (int i = 1; i < multiPolygonsWays.length; i++) {
int type = findMultiPolygonType(e, i);
if (type != 0) {
putMultipolygonType(multiPolygonsWays[i], way.getId(), type, inverse);
}
}
}
}
}
}
}
private void putMultipolygonType(Map<Long, Set<Integer>> multiPolygonsWays, long baseId, int mtType, boolean inverse) {
if (mtType == 0) {
return;
}
if (!multiPolygonsWays.containsKey(baseId)) {
multiPolygonsWays.put(baseId, new LinkedHashSet<Integer>());
}
if (inverse) {
multiPolygonsWays.get(baseId).add(mtType | (1 << 15));
} else {
multiPolygonsWays.get(baseId).add(mtType);
}
}
private int findMultiPolygonType(Entity e, int level) {
int t = renderingTypes.encodeEntityWithType(e, mapZooms.getLevel(level).getMaxZoom(), true, typeUse);
int mtType = 0;
if (t != 0) {
if ((t & 3) == MapRenderingTypes.MULTY_POLYGON_TYPE) {
mtType = t;
} else {
for (Integer i : typeUse) {
if ((i & 3) == MapRenderingTypes.MULTY_POLYGON_TYPE) {
mtType = i;
break;
}
}
}
}
return mtType;
}
public void combineMultiPolygons(Way w, List<List<Way>> completedRings, List<List<Way>> incompletedRings) {
long lId = w.getEntityIds().get(w.getEntityIds().size() - 1).getId().longValue();
long fId = w.getEntityIds().get(0).getId().longValue();
if (fId == lId) {
completedRings.add(Collections.singletonList(w));
} else {
List<Way> l = new ArrayList<Way>();
l.add(w);
boolean add = true;
for (int k = 0; k < incompletedRings.size();) {
boolean remove = false;
List<Way> i = incompletedRings.get(k);
Way last = i.get(i.size() - 1);
Way first = i.get(0);
long lastId = last.getEntityIds().get(last.getEntityIds().size() - 1).getId().longValue();
long firstId = first.getEntityIds().get(0).getId().longValue();
if (fId == lastId) {
i.addAll(l);
remove = true;
l = i;
fId = firstId;
} else if (lId == firstId) {
l.addAll(i);
remove = true;
lId = lastId;
}
if (remove) {
incompletedRings.remove(k);
} else {
k++;
}
if (fId == lId) {
completedRings.add(l);
add = false;
break;
}
}
if (add) {
incompletedRings.add(l);
}
}
}
private void writeBinaryEntityToMapDatabase(Entity e, long baseId, boolean inverse, int level) throws SQLException {
int type = renderingTypes.encodeEntityWithType(e, mapZooms.getLevel(level).getMaxZoom(), false, typeUse);
Map<Long, Set<Integer>> multiPolygonsWays = this.multiPolygonsWays[level];
boolean hasMulti = e instanceof Way && multiPolygonsWays.containsKey(e.getId());
if (hasMulti) {
Set<Integer> set = multiPolygonsWays.get(e.getId());
boolean first = true;
for (Integer i : set) {
if (first && type == 0) {
type = i;
first = false;
} else {
// do not compare direction
int k = i & 0x7fff;
int ks = k | MapRenderingTypes.POLYGON_TYPE;
// turn of polygon type 3 ^ (suppose polygon = multipolygon)
if (ks == type) {
type = i;
} else {
int ind = typeUse.indexOf(ks);
if (ind == -1) {
typeUse.add(i);
} else {
typeUse.set(ind, i);
}
}
}
}
}
if (type == 0) {
return;
}
restrictionsUse.clear();
// try to find restrictions only for max zoom level
if (level == 0 && highwayRestrictions.containsKey(baseId)) {
restrictionsUse.addAll(highwayRestrictions.get(baseId));
}
boolean point = (type & 3) == MapRenderingTypes.POINT_TYPE;
RTree rtree = null;
int zoom;
long id = (baseId << 3) | ((level & 3) << 1);
rtree = mapTree[level];
zoom = mapZooms.getLevel(level).getMaxZoom() - 1;
boolean skip = false;
String eName = renderingTypes.getEntityName(e);
if (eName == null) {
eName = multiPolygonsNames.get(baseId);
}
int highwayAttributes = 0;
if (e.getTag(OSMTagKey.HIGHWAY) != null) {
highwayAttributes = MapRenderingTypes.getHighwayAttributes(e);
}
if (e instanceof Way) {
id |= 1;
// simplify route
if (level > 0) {
e = simplifyWay((Way) e, id, hasMulti, zoom, eName, type, level);
skip = e == null;
}
}
if (!skip) {
insertBinaryMapRenderObjectIndex(pStatements, mapBinaryStat, rtree, e, eName, id, type, typeUse,
highwayAttributes, restrictionsUse, inverse, point, true);
}
}
protected long encodeTypesToOneLong(int mainType) {
long i = 0;
int ind = 0;
int sh = 0;
if(typeUse.size() > 3){
log.error("Types for low index way more than 4"); //$NON-NLS-1$
}
i |= (mainType << sh);
if (typeUse.size() > ind) {
sh += 16;
i |= ((long)typeUse.get(ind++) << sh );
if (typeUse.size() > ind) {
sh += 16;
i |= ((long)typeUse.get(ind++) << sh );
if (typeUse.size() > ind) {
sh += 16;
i |= ((long)typeUse.get(ind++) << sh);
}
}
}
return i;
}
protected int decodeTypesFromOneLong(long i) {
typeUse.clear();
int mask = (1 << 16) - 1;
int k = (int) (i & mask);
int r = 0;
if (k > 0) {
r = k;
i >>= 16;
k = (int) (i & mask);
if (k > 0) {
typeUse.add(k);
i >>= 16;
k = (int) (i & mask);
if (k > 0) {
typeUse.add(k);
i >>= 16;
k = (int) (i & mask);
if (k > 0) {
typeUse.add(k);
i >>= 16;
}
}
}
}
return r;
}
protected Way simplifyWay(Way originalE, long id, boolean hasMulti, int zoom, String name, int type, int level) throws SQLException {
List<Node> nodes = originalE.getNodes();
Way way = new Way(id);
for (String t : originalE.getTagKeySet()) {
way.putTag(t, originalE.getTag(t));
}
boolean cycle = originalE.getNodeIds().get(0).longValue() == originalE.getNodeIds().get(nodes.size() - 1).longValue();
long longType = encodeTypesToOneLong(type);
boolean skip = checkForSmallAreas(nodes, zoom, 3, 3);
if (skip && cycle/* || !hasMulti)*/) {
return null;
}
MapAlgorithms.simplifyDouglasPeucker(nodes, zoom + 8, 3, way);
if (way.getNodes().size() < 2) {
return null;
}
if (cycle) {
// nothing to do
return way;
} else {
lowLevelWays ++;
insertLowLevelMapBinaryObject(pStatements, mapLowLevelBinaryStat, level, longType, id, way.getNodes(), name);
return null;
}
}
public int getLowLevelWays() {
return lowLevelWays;
}
private void loadNodes(byte[] nodes, List<Float> toPut){
toPut.clear();
for (int i = 0; i < nodes.length;) {
int lat = Algoritms.parseIntFromBytes(nodes, i);
i += 4;
int lon = Algoritms.parseIntFromBytes(nodes, i);
i += 4;
toPut.add(Float.intBitsToFloat(lat));
toPut.add(Float.intBitsToFloat(lon));
}
}
public void processingLowLevelWays(IProgress progress) throws SQLException {
restrictionsUse.clear();
mapLowLevelBinaryStat.executeBatch();
mapLowLevelBinaryStat.close();
pStatements.remove(mapLowLevelBinaryStat);
mapLowLevelBinaryStat = null;
mapConnection.commit();
PreparedStatement startStat = mapConnection.prepareStatement("SELECT id, start_node, end_node, nodes FROM low_level_map_objects"
+ " WHERE start_node = ? AND type=? AND level = ? AND name=?");
PreparedStatement endStat = mapConnection.prepareStatement("SELECT id, start_node, end_node, nodes FROM low_level_map_objects"
+ " WHERE end_node = ? AND type=? AND level = ? AND name=?");
Statement selectStatement = mapConnection.createStatement();
ResultSet rs = selectStatement.executeQuery("SELECT id, start_node, end_node, name, nodes, type, level FROM low_level_map_objects");
Set<Long> visitedWays = new LinkedHashSet<Long>();
ArrayList<Float> list = new ArrayList<Float>(100);
while(rs.next()){
if(lowLevelWays != -1){
progress.progress(1);
}
long id = rs.getLong(1);
if(visitedWays.contains(id)){
continue;
}
visitedWays.add(id);
int level = rs.getInt(7);
int zoom = mapZooms.getLevel(level).getMaxZoom();
long startNode = rs.getLong(2);
long endNode = rs.getLong(3);
String name = rs.getString(4);
long ltype = rs.getLong(6);
loadNodes(rs.getBytes(5), list);
ArrayList<Float> wayNodes = new ArrayList<Float>(list);
// combine startPoint with EndPoint
boolean combined = true;
while (combined) {
combined = false;
endStat.setLong(1, startNode);
endStat.setLong(2, ltype);
endStat.setShort(3, (short) level);
endStat.setString(4, name);
ResultSet fs = endStat.executeQuery();
while (fs.next()) {
if (!visitedWays.contains(fs.getLong(1))) {
combined = true;
long lid = fs.getLong(1);
startNode = fs.getLong(2);
visitedWays.add(lid);
loadNodes(fs.getBytes(4), list);
ArrayList li = new ArrayList<Float>(list);
// remove first lat/lon point
wayNodes.remove(0);
wayNodes.remove(0);
li.addAll(wayNodes);
wayNodes = li;
}
}
fs.close();
}
// combined end point
combined = true;
while (combined) {
combined = false;
startStat.setLong(1, endNode);
startStat.setLong(2, ltype);
startStat.setShort(3, (short) level);
startStat.setString(4, name);
ResultSet fs = startStat.executeQuery();
while (fs.next()) {
if (!visitedWays.contains(fs.getLong(1))) {
combined = true;
long lid = fs.getLong(1);
endNode = fs.getLong(3);
visitedWays.add(lid);
loadNodes(fs.getBytes(4), list);
for (int i = 2; i < list.size(); i++) {
wayNodes.add(list.get(i));
}
}
}
fs.close();
}
List<Node> wNodes = new ArrayList<Node>();
for (int i = 0; i < wayNodes.size(); i += 2) {
wNodes.add(new Node(wayNodes.get(i), wayNodes.get(i + 1), i == 0 ? startNode : endNode));
}
boolean skip = false;
boolean cycle = startNode == endNode;
boolean hasMulti = multiPolygonsWays[level].containsKey(id >> 3);
if(cycle || !hasMulti){
skip = checkForSmallAreas(wNodes, zoom - 1, 1, 4);
}
if (!skip) {
Way newWs = new Way(id);
MapAlgorithms.simplifyDouglasPeucker(wNodes, zoom - 1 + 8, 3, newWs);
int type = decodeTypesFromOneLong(ltype);
insertBinaryMapRenderObjectIndex(pStatements, mapBinaryStat, mapTree[level], newWs, name,
id, type, typeUse, 0, restrictionsUse, false, false, false);
}
}
}
private boolean checkForSmallAreas(List<Node> nodes, int zoom, int minz, int maxz) {
int minX = Integer.MAX_VALUE;
int maxX = Integer.MIN_VALUE;
int minY = Integer.MAX_VALUE;
int maxY = Integer.MIN_VALUE;
int c = 0;
for (int i = 0; i < nodes.size(); i++) {
if (nodes.get(i) != null) {
c++;
int x = (int) (MapUtils.getTileNumberX(zoom, nodes.get(i).getLongitude()) * 256d);
int y = (int) (MapUtils.getTileNumberY(zoom, nodes.get(i).getLatitude()) * 256d);
minX = Math.min(minX, x);
maxX = Math.max(maxX, x);
minY = Math.min(minY, y);
maxY = Math.max(maxY, y);
}
}
if (c < 2) {
return true;
}
return ((maxX - minX) <= minz && (maxY - minY) <= maxz) ||
((maxX - minX) <= maxz && (maxY - minY) <= minz);
}
public void initSettings(MapZooms mapZooms, MapRenderingTypes renderingTypes) {
this.mapZooms = mapZooms;
this.renderingTypes = renderingTypes;
// init map
multiPolygonsWays = new Map[mapZooms.size()];
for (int i = 0; i < multiPolygonsWays.length; i++) {
multiPolygonsWays[i] = new LinkedHashMap<Long, Set<Integer>>();
}
lowLevelWays = -1;
}
public void iterateMainEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException {
if (e instanceof Way || e instanceof Node) {
// manipulate what kind of way to load
ctx.loadEntityData(e, false);
boolean inverse = "-1".equals(e.getTag(OSMTagKey.ONEWAY)); //$NON-NLS-1$
for (int i = 0; i < mapZooms.size(); i++) {
writeBinaryEntityToMapDatabase(e, e.getId(), i == 0 ? inverse : false, i);
}
}
}
public void writeBinaryMapIndex(BinaryMapIndexWriter writer, String regionName) throws IOException, SQLException {
closePreparedStatements(mapBinaryStat, mapLowLevelBinaryStat);
mapConnection.commit();
try {
PreparedStatement selectData = mapConnection.prepareStatement("SELECT nodes, types, name, highway, restrictions FROM binary_map_objects WHERE id = ?"); //$NON-NLS-1$
writer.startWriteMapIndex(regionName);
for (int i = 0; i < mapZooms.size(); i++) {
RTree rtree = mapTree[i];
long rootIndex = rtree.getFileHdr().getRootIndex();
rtree.Node root = rtree.getReadNode(rootIndex);
Rect rootBounds = calcBounds(root);
if (rootBounds != null) {
boolean last = nodeIsLastSubTree(rtree, rootIndex);
writer.startWriteMapLevelIndex(mapZooms.getLevel(i).getMinZoom(), mapZooms.getLevel(i).getMaxZoom(), rootBounds
.getMinX(), rootBounds.getMaxX(), rootBounds.getMinY(), rootBounds.getMaxY());
if (last) {
writer.startMapTreeElement(rootBounds.getMinX(), rootBounds.getMaxX(), rootBounds.getMinY(), rootBounds.getMaxY());
}
writeBinaryMapTree(root, rtree, writer, selectData);
if (last) {
writer.endWriteMapTreeElement();
}
writer.endWriteMapLevelIndex();
}
}
selectData.close();
writer.writeMapEncodingRules(renderingTypes.getEncodingRuleTypes());
writer.endWriteMapIndex();
writer.flush();
} catch (RTreeException e) {
throw new IllegalStateException(e);
}
}
public void writeBinaryMapTree(rtree.Node parent, RTree r, BinaryMapIndexWriter writer, PreparedStatement selectData) throws IOException, RTreeException, SQLException {
Element[] e = parent.getAllElements();
for (int i = 0; i < parent.getTotalElements(); i++) {
Rect re = e[i].getRect();
if (e[i].getElementType() == rtree.Node.LEAF_NODE) {
long id = ((LeafElement) e[i]).getPtr();
selectData.setLong(1, id);
ResultSet rs = selectData.executeQuery();
if (rs.next()) {
// mapConnection.prepareStatement("SELECT nodes, types, name, highway, restrictions FROM binary_map_objects WHERE id = ?");
writer.writeMapData(id, rs.getBytes(1),
rs.getBytes(2), rs.getString(3),
rs.getInt(4), rs.getBytes(5));
} else {
log.error("Something goes wrong with id = " + id); //$NON-NLS-1$
}
} else {
long ptr = ((NonLeafElement) e[i]).getPtr();
rtree.Node ns = r.getReadNode(ptr);
writer.startMapTreeElement(re.getMinX(), re.getMaxX(), re.getMinY(), re.getMaxY());
writeBinaryMapTree(ns, r, writer, selectData);
writer.endWriteMapTreeElement();
}
}
}
public Rect calcBounds(rtree.Node n) {
Rect r = null;
Element[] e = n.getAllElements();
for (int i = 0; i < n.getTotalElements(); i++) {
Rect re = e[i].getRect();
if (r == null) {
try {
r = new Rect(re.getMinX(), re.getMinY(), re.getMaxX(), re.getMaxY());
} catch (IllegalValueException ex) {
}
} else {
r.expandToInclude(re);
}
}
return r;
}
public void createDatabaseStructure(Connection mapConnection, DBDialect dialect,
String rtreeMapIndexNonPackFileName) throws SQLException, IOException {
createMapIndexStructure(mapConnection);
this.mapConnection = mapConnection;
mapBinaryStat = createStatementMapBinaryInsert(mapConnection);
mapLowLevelBinaryStat = createStatementLowLevelMapBinaryInsert(mapConnection);
try {
mapTree = new RTree[mapZooms.size()];
for (int i = 0; i < mapZooms.size(); i++) {
File file = new File(rtreeMapIndexNonPackFileName + i);
if (file.exists()) {
file.delete();
}
mapTree[i] = new RTree(rtreeMapIndexNonPackFileName + i);
// very slow
// mapTree[i].getFileHdr().setBufferPolicy(true);
}
} catch (RTreeException e) {
throw new IOException(e);
}
pStatements.put(mapBinaryStat, 0);
pStatements.put(mapLowLevelBinaryStat, 0);
}
private void createMapIndexStructure(Connection conn) throws SQLException{
Statement stat = conn.createStatement();
stat.executeUpdate("create table binary_map_objects (id bigint primary key, name varchar(255), " +
"types binary, restrictions binary, nodes binary, highway int)");
stat.executeUpdate("create index binary_map_objects_ind on binary_map_objects (id)");
stat.executeUpdate("create table low_level_map_objects (id bigint primary key, start_node bigint, " +
"end_node bigint, name varchar(255), nodes binary, type bigint, level smallint)");
stat.executeUpdate("create index low_level_map_objects_ind on low_level_map_objects (id)");
stat.executeUpdate("create index low_level_map_objects_ind_st on low_level_map_objects (start_node, type)");
stat.executeUpdate("create index low_level_map_objects_ind_end on low_level_map_objects (end_node, type)");
stat.close();
}
private PreparedStatement createStatementMapBinaryInsert(Connection conn) throws SQLException{
return conn.prepareStatement("insert into binary_map_objects(id, name, types, restrictions, nodes, highway) values(?, ?, ?, ?, ?, ?)");
}
private PreparedStatement createStatementLowLevelMapBinaryInsert(Connection conn) throws SQLException{
return conn.prepareStatement("insert into low_level_map_objects(id, start_node, end_node, name, nodes, type, level) values(?, ?, ?, ?, ?, ?, ?)");
}
private void insertLowLevelMapBinaryObject(Map<PreparedStatement, Integer> statements,
PreparedStatement mapLowLevelBinaryStat, int level,long types, long id, List<Node> nodes, String name) throws SQLException{
boolean first = true;
long firstId = -1;
long lastId = -1;
ByteArrayOutputStream bnodes = new ByteArrayOutputStream();
try {
for (Node n : nodes) {
if (n != null) {
if (first) {
firstId = n.getId();
first = false;
}
lastId = n.getId();
Algoritms.writeInt(bnodes, Float.floatToRawIntBits((float) n.getLatitude()));
Algoritms.writeInt(bnodes, Float.floatToRawIntBits((float) n.getLongitude()));
}
}
} catch (IOException e) {
throw new IllegalStateException(e);
}
if(firstId == -1){
return;
}
// conn.prepareStatement("insert into binary_map_objects(id, name, types, restrictions, nodes, highway) values(?, ?, ?, ?, ?, ?)");
mapLowLevelBinaryStat.setLong(1, id);
mapLowLevelBinaryStat.setLong(2, firstId);
mapLowLevelBinaryStat.setLong(3, lastId);
mapLowLevelBinaryStat.setString(4, name);
mapLowLevelBinaryStat.setBytes(5, bnodes.toByteArray());
mapLowLevelBinaryStat.setLong(6, types);
mapLowLevelBinaryStat.setShort(7, (short) level);
addBatch(statements, mapLowLevelBinaryStat);
}
private void insertBinaryMapRenderObjectIndex(Map<PreparedStatement, Integer> statements,
PreparedStatement mapBinaryStat, RTree mapTree, Entity e, String name,
long id, int type, List<Integer> typeUse, int highwayAttributes, List<Long> restrictions,
boolean inversePath, boolean writeAsPoint, boolean commit) throws SQLException {
if(e instanceof Relation){
throw new IllegalArgumentException();
}
boolean init = false;
int minX = Integer.MAX_VALUE;
int maxX = 0;
int minY = Integer.MAX_VALUE;
int maxY = 0;
Collection<Node> nodes;
if (e instanceof Way) {
if (writeAsPoint) {
LatLon center = MapUtils.getCenter(((Way) e));
nodes = Collections.singleton(new Node(center.getLatitude(), center.getLongitude(), -1));
} else {
nodes = ((Way) e).getNodes();
}
} else {
nodes = Collections.singleton((Node) e);
}
if(inversePath){
nodes = new ArrayList<Node>(nodes);
Collections.reverse((List<?>) nodes);
}
ByteArrayOutputStream bnodes = new ByteArrayOutputStream();
ByteArrayOutputStream btypes = new ByteArrayOutputStream();
ByteArrayOutputStream brestrictions = new ByteArrayOutputStream();
try {
Algoritms.writeSmallInt(btypes, type);
for (Integer i : typeUse) {
Algoritms.writeSmallInt(btypes, i);
}
for (Long i : restrictions) {
Algoritms.writeLongInt(brestrictions, i);
}
for (Node n : nodes) {
if (n != null) {
int y = MapUtils.get31TileNumberY(n.getLatitude());
int x = MapUtils.get31TileNumberX(n.getLongitude());
minX = Math.min(minX, x);
maxX = Math.max(maxX, x);
minY = Math.min(minY, y);
maxY = Math.max(maxY, y);
init = true;
Algoritms.writeInt(bnodes, x);
Algoritms.writeInt(bnodes, y);
}
}
} catch (IOException es) {
throw new IllegalStateException(es);
}
if (init) {
// conn.prepareStatement("insert into binary_map_objects(id, name, types, restrictions, nodes, highway) values(?, ?, ?, ?, ?, ?)");
mapBinaryStat.setLong(1, id);
mapBinaryStat.setString(2, name);
mapBinaryStat.setBytes(3, btypes.toByteArray());
mapBinaryStat.setBytes(4, brestrictions.toByteArray());
mapBinaryStat.setBytes(5, bnodes.toByteArray());
mapBinaryStat.setInt(6, highwayAttributes);
addBatch(statements, mapBinaryStat, commit);
try {
mapTree.insert(new LeafElement(new Rect(minX, minY, maxX, maxY), id));
} catch (RTreeInsertException e1) {
throw new IllegalArgumentException(e1);
} catch (IllegalValueException e1) {
throw new IllegalArgumentException(e1);
}
}
}
public void createRTreeFiles(String rTreeMapIndexPackFileName) throws RTreeException {
mapTree = new RTree[mapZooms.size()];
for (int i = 0; i < mapZooms.size(); i++) {
mapTree[i] = new RTree(rTreeMapIndexPackFileName + i);
}
}
public void packRtreeFiles(String rTreeMapIndexNonPackFileName, String rTreeMapIndexPackFileName) throws IOException {
for (int i = 0; i < mapZooms.size(); i++) {
mapTree[i] = packRtreeFile(mapTree[i], rTreeMapIndexNonPackFileName + i, rTreeMapIndexPackFileName + i);
}
}
public void commitAndCloseFiles(String rTreeMapIndexNonPackFileName, String rTreeMapIndexPackFileName, boolean deleteDatabaseIndexes) throws IOException, SQLException {
// delete map rtree files
if (mapTree != null) {
for (int i = 0; i < mapTree.length; i++) {
if (mapTree[i] != null) {
RandomAccessFile file = mapTree[i].getFileHdr().getFile();
file.close();
}
}
for (int i = 0; i < mapTree.length; i++) {
File f = new File(rTreeMapIndexNonPackFileName + i);
if (f.exists() && deleteDatabaseIndexes) {
f.delete();
}
f = new File(rTreeMapIndexPackFileName + i);
if (f.exists() && deleteDatabaseIndexes) {
f.delete();
}
}
}
closeAllPreparedStatements();
}
}

108
DataExtractionOSM/src/net/osmand/data/preparation/MapSwingAlgorithms.java Normal file
View file

@ -0,0 +1,108 @@
package net.osmand.data.preparation;
import java.awt.geom.Line2D;
import java.awt.geom.Point2D;
import java.util.ArrayList;
import java.util.List;
import net.osmand.osm.Node;
import net.osmand.osm.Way;
public class MapSwingAlgorithms {
public static Point2D.Float getIntersectionPoint(Line2D.Float line1, Line2D.Float line2) {
if (!line1.intersectsLine(line2))
return null;
double px = line1.getX1(), py = line1.getY1(), rx = line1.getX2() - px, ry = line1.getY2() - py;
double qx = line2.getX1(), qy = line2.getY1(), sx = line2.getX2() - qx, sy = line2.getY2() - qy;
double det = sx * ry - sy * rx;
if (det == 0) {
return null;
} else {
double z = (sx * (qy - py) + sy * (px - qx)) / det;
if (z <= 0 || z >= 1)
return null; // intersection at end point!
return new Point2D.Float((float) (px + z * rx), (float) (py + z * ry));
}
} // end intersection line-line
public static boolean isClockwiseWay(List<Way> ways) {
if (ways.isEmpty()) {
return false;
}
List<Node> nodes;
if (ways.size() == 1) {
nodes = ways.get(0).getNodes();
} else {
nodes = new ArrayList<Node>();
boolean first = true;
for (Way e : ways) {
if (first) {
first = false;
nodes.addAll(e.getNodes());
} else {
nodes.addAll(e.getNodes().subList(1, e.getNodes().size()));
}
}
}
if (nodes.isEmpty()) {
return false;
}
double angle = 0;
double prevAng = 0;
double firstAng = 0;
double selfIntersection = 0;
boolean open = nodes.get(nodes.size() - 1).getId() != nodes.get(0).getId();
for (int i = 1; open ? i < nodes.size() : i <= nodes.size(); i++) {// nodes.get(i).getId()
double ang;
if (i < nodes.size()) {
ang = Math.atan2(nodes.get(i).getLatitude() - nodes.get(i - 1).getLatitude(), nodes.get(i).getLongitude()
- nodes.get(i - 1).getLongitude());
// find self intersection
Line2D.Float l = new Line2D.Float((float) nodes.get(i).getLongitude(), (float) nodes.get(i).getLatitude(), (float) nodes
.get(i - 1).getLongitude(), (float) nodes.get(i - 1).getLatitude());
for (int j = i - 2; j > i - 7; j--) {
if (j < 1) {
break;
}
Line2D.Float l2 = new Line2D.Float((float) nodes.get(j).getLongitude(), (float) nodes.get(j).getLatitude(),
(float) nodes.get(j - 1).getLongitude(), (float) nodes.get(j - 1).getLatitude());
java.awt.geom.Point2D.Float point = getIntersectionPoint(l, l2);
if (point != null) {
double dang = Math.atan2(nodes.get(j).getLatitude() - nodes.get(j - 1).getLatitude(), nodes.get(j).getLongitude()
- nodes.get(j - 1).getLongitude());
if (adjustDirection(ang - dang) < 0) {
selfIntersection += 2 * Math.PI;
} else {
selfIntersection -= 2 * Math.PI;
}
}
}
} else {
ang = firstAng;
}
if (i > 1) {
angle += adjustDirection(ang - prevAng);
prevAng = ang;
} else {
prevAng = ang;
firstAng = ang;
}
}
return (angle - selfIntersection) < 0;
}
private static double adjustDirection(double ang) {
if (ang < -Math.PI) {
ang += 2 * Math.PI;
} else if (ang > Math.PI) {
ang -= 2 * Math.PI;
}
return ang;
}
}