Android-builders-club/OsmAnd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

fix performance for address index

Browse source 
git-svn-id: https://osmand.googlecode.com/svn/trunk@653 e29c36b1-1cfa-d876-8d93-3434fc2bb7b8
This commit is contained in:
Victor Shcherb 2010-11-11 14:54:18 +00:00
parent c81d9dbed4
commit 48f7a63670
5 changed files with 184 additions and 293 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

66
DataExtractionOSM/src/net/osmand/data/index/DataIndexReader.java
View file

@ -17,9 +17,6 @@ import net.osmand.data.Building;
import net.osmand.data.City; import net.osmand.data.City;
import net.osmand.data.Street; import net.osmand.data.Street;
import net.osmand.data.City.CityType; import net.osmand.data.City.CityType;
import net.osmand.data.index.IndexConstants.IndexBuildingTable;
import net.osmand.data.index.IndexConstants.IndexCityTable;
import net.osmand.data.index.IndexConstants.IndexStreetTable;
import net.osmand.osm.Node; import net.osmand.osm.Node;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
@ -42,14 +39,14 @@ public class DataIndexReader {
public List<City> readCities(Connection c) throws SQLException{ public List<City> readCities(Connection c) throws SQLException{
List<City> cities = new ArrayList<City>(); List<City> cities = new ArrayList<City>();
Statement stat = c.createStatement(); Statement stat = c.createStatement();
ResultSet set = stat.executeQuery(IndexConstants.generateSelectSQL(IndexCityTable.values())); ResultSet set = stat.executeQuery("select id, latitude, longitude , name , name_en , city_type from city"); //$NON-NLS-1$
while(set.next()){ while(set.next()){
City city = new City(CityType.valueFromString(set.getString(IndexCityTable.CITY_TYPE.ordinal() + 1))); City city = new City(CityType.valueFromString(set.getString(6)));
city.setName(set.getString(IndexCityTable.NAME.ordinal() + 1)); city.setName(set.getString(4));
city.setEnName(set.getString(IndexCityTable.NAME_EN.ordinal() + 1)); city.setEnName(set.getString(5));
city.setLocation(set.getDouble(IndexCityTable.LATITUDE.ordinal() + 1), city.setLocation(set.getDouble(2),
set.getDouble(IndexCityTable.LONGITUDE.ordinal() + 1)); set.getDouble(3));
city.setId(set.getLong(IndexCityTable.ID.ordinal() + 1)); city.setId(set.getLong(1));
cities.add(city); cities.add(city);
} }
@ -58,25 +55,24 @@ public class DataIndexReader {
return cities; return cities;
} }
public PreparedStatement getStreetsPreparedStatement(Connection c) throws SQLException{
return c.prepareStatement(IndexConstants.generateSelectSQL(IndexStreetTable.values(),
IndexStreetTable.CITY.toString() +" = ? ")); //$NON-NLS-1$
}
public PreparedStatement getStreetsBuildingPreparedStatement(Connection c) throws SQLException{ public PreparedStatement getStreetsBuildingPreparedStatement(Connection c) throws SQLException{
return c.prepareStatement("SELECT A.id, A.name, A.name_en, A.latitude, A.longitude, "+ return c.prepareStatement("SELECT A.id, A.name, A.name_en, A.latitude, A.longitude, "+ //$NON-NLS-1$
"B.id, B.name, B.name_en, B.latitude, B.longitude, B.postcode "+ "B.id, B.name, B.name_en, B.latitude, B.longitude, B.postcode "+ //$NON-NLS-1$
"FROM street A LEFT JOIN building B ON B.street = A.id WHERE A.city = ? "); //$NON-NLS-1$ "FROM street A left JOIN building B ON B.street = A.id WHERE A.city = ?"); //$NON-NLS-1$
}
public PreparedStatement getStreetsWayNodesPreparedStatement(Connection c) throws SQLException{
return c.prepareStatement("SELECT A.id, A.latitude, A.longitude FROM street_node A WHERE A.street = ? "); //$NON-NLS-1$
} }
public List<Street> readStreetsBuildings(PreparedStatement streetBuildingsStat, City city, List<Street> streets) throws SQLException { public List<Street> readStreetsBuildings(PreparedStatement streetBuildingsStat, City city, List<Street> streets) throws SQLException {
return readStreetsBuildings(streetBuildingsStat, city, streets, null, null); return readStreetsBuildings(streetBuildingsStat, city, streets, null, null);
} }
public PreparedStatement getStreetsWayNodesPreparedStatement(Connection c) throws SQLException{
return c.prepareStatement("SELECT A.id, A.latitude, A.longitude FROM street_node A WHERE A.street = ? "); //$NON-NLS-1$
}
public List<Street> readStreetsBuildings(PreparedStatement streetBuildingsStat, City city, List<Street> streets, public List<Street> readStreetsBuildings(PreparedStatement streetBuildingsStat, City city, List<Street> streets,
PreparedStatement waynodesStat, Map<Street, List<Node>> streetNodes) throws SQLException { PreparedStatement waynodesStat, Map<Street, List<Node>> streetNodes) throws SQLException {
Map<Long, Street> visitedStreets = new LinkedHashMap<Long, Street>(); Map<Long, Street> visitedStreets = new LinkedHashMap<Long, Street>();
@ -119,16 +115,20 @@ public class DataIndexReader {
return streets; return streets;
} }
public PreparedStatement getStreetsPreparedStatement(Connection c) throws SQLException{
return c.prepareStatement("select id, latitude, longitude , name, name_en, city from street where city = ?"); //$NON-NLS-1$
}
public List<Street> readStreets(PreparedStatement streetsStat, City city, List<Street> streets) throws SQLException{ public List<Street> readStreets(PreparedStatement streetsStat, City city, List<Street> streets) throws SQLException{
streetsStat.setLong(1, city.getId()); streetsStat.setLong(1, city.getId());
ResultSet set = streetsStat.executeQuery(); ResultSet set = streetsStat.executeQuery();
while(set.next()){ while(set.next()){
Street street = new Street(city); Street street = new Street(city);
street.setName(set.getString(IndexStreetTable.NAME.ordinal() + 1)); street.setName(set.getString(4));
street.setEnName(set.getString(IndexStreetTable.NAME_EN.ordinal() + 1)); street.setEnName(set.getString(5));
street.setLocation(set.getDouble(IndexStreetTable.LATITUDE.ordinal() + 1), street.setLocation(set.getDouble(2),
set.getDouble(IndexStreetTable.LONGITUDE.ordinal() + 1)); set.getDouble(3));
street.setId(set.getLong(IndexStreetTable.ID.ordinal() + 1)); street.setId(set.getLong(1));
streets.add(street); streets.add(street);
} }
set.close(); set.close();
@ -136,8 +136,7 @@ public class DataIndexReader {
} }
public PreparedStatement getBuildingsPreparedStatement(Connection c) throws SQLException{ public PreparedStatement getBuildingsPreparedStatement(Connection c) throws SQLException{
return c.prepareStatement(IndexConstants.generateSelectSQL(IndexBuildingTable.values(), return c.prepareStatement("select id, latitude, longitude, name, name_en, street, postcode from building where street = ?"); //$NON-NLS-1$
IndexBuildingTable.STREET.toString() +" = ? "));
} }
@ -146,11 +145,12 @@ public class DataIndexReader {
ResultSet set = buildingStat.executeQuery(); ResultSet set = buildingStat.executeQuery();
while(set.next()){ while(set.next()){
Building building = new Building(); Building building = new Building();
building.setName(set.getString(IndexBuildingTable.NAME.ordinal() + 1)); building.setName(set.getString(4));
building.setEnName(set.getString(IndexBuildingTable.NAME_EN.ordinal() + 1)); building.setEnName(set.getString(5));
building.setLocation(set.getDouble(IndexBuildingTable.LATITUDE.ordinal() + 1), building.setLocation(set.getDouble(2),
set.getDouble(IndexBuildingTable.LONGITUDE.ordinal() + 1)); set.getDouble(3));
building.setId(set.getLong(IndexBuildingTable.ID.ordinal() + 1)); building.setId(set.getLong(1));
building.setPostcode(set.getString(7));
buildings.add(building); buildings.add(building);
} }
set.close(); set.close();

125
DataExtractionOSM/src/net/osmand/data/index/DataIndexWriter.java
View file

@ -12,7 +12,6 @@ import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import com.sun.xml.internal.ws.wsdl.writer.UsingAddressing;
import net.osmand.Algoritms; import net.osmand.Algoritms;
import net.osmand.data.Amenity; import net.osmand.data.Amenity;
@ -23,11 +22,7 @@ import net.osmand.data.TransportRoute;
import net.osmand.data.TransportStop; import net.osmand.data.TransportStop;
import net.osmand.data.City.CityType; import net.osmand.data.City.CityType;
import net.osmand.data.index.IndexConstants.IndexBinaryMapRenderObject; import net.osmand.data.index.IndexConstants.IndexBinaryMapRenderObject;
import net.osmand.data.index.IndexConstants.IndexBuildingTable;
import net.osmand.data.index.IndexConstants.IndexCityTable;
import net.osmand.data.index.IndexConstants.IndexPoiTable; import net.osmand.data.index.IndexConstants.IndexPoiTable;
import net.osmand.data.index.IndexConstants.IndexStreetNodeTable;
import net.osmand.data.index.IndexConstants.IndexStreetTable;
import net.osmand.data.index.IndexConstants.IndexTransportRoute; import net.osmand.data.index.IndexConstants.IndexTransportRoute;
import net.osmand.data.index.IndexConstants.IndexTransportRouteStop; import net.osmand.data.index.IndexConstants.IndexTransportRouteStop;
import net.osmand.data.index.IndexConstants.IndexTransportStop; import net.osmand.data.index.IndexConstants.IndexTransportStop;
@ -80,59 +75,119 @@ public class DataIndexWriter {
stat.close(); stat.close();
} }
public static PreparedStatement getStreetNodeInsertPreparedStatement(Connection conn) throws SQLException {
return conn.prepareStatement("insert into street_node (id, latitude, longitude, street, way) values (?, ?, ?, ?, ?)");
}
public static void writeStreetWayNodes(PreparedStatement prepStreetNode, Map<PreparedStatement, Integer> count, Long streetId, Way way, int batchSize) public static void writeStreetWayNodes(PreparedStatement prepStreetNode, Map<PreparedStatement, Integer> count, Long streetId, Way way, int batchSize)
throws SQLException { throws SQLException {
for (Node n : way.getNodes()) { for (Node n : way.getNodes()) {
if (n == null) { if (n == null) {
continue; continue;
} }
assert IndexStreetNodeTable.values().length == 5; prepStreetNode.setLong(1, n.getId());
prepStreetNode.setLong(IndexStreetNodeTable.ID.ordinal() + 1, n.getId()); prepStreetNode.setDouble(2, n.getLatitude());
prepStreetNode.setDouble(IndexStreetNodeTable.LATITUDE.ordinal() + 1, n.getLatitude()); prepStreetNode.setDouble(3, n.getLongitude());
prepStreetNode.setDouble(IndexStreetNodeTable.LONGITUDE.ordinal() + 1, n.getLongitude()); prepStreetNode.setLong(5, way.getId());
prepStreetNode.setLong(IndexStreetNodeTable.WAY.ordinal() + 1, way.getId()); prepStreetNode.setLong(4, streetId);
prepStreetNode.setLong(IndexStreetNodeTable.STREET.ordinal() + 1, streetId);
addBatch(count, prepStreetNode, BATCH_SIZE); addBatch(count, prepStreetNode, BATCH_SIZE);
} }
} }
public static PreparedStatement getBuildingInsertPreparedStatement(Connection conn) throws SQLException {
return conn.prepareStatement("insert into building (id, latitude, longitude, name, name_en, street, postcode) values (?, ?, ?, ?, ?, ?, ?)");
}
public static void writeBuilding(PreparedStatement prepBuilding, Map<PreparedStatement, Integer> count, Long streetId, public static void writeBuilding(PreparedStatement prepBuilding, Map<PreparedStatement, Integer> count, Long streetId,
Building building, int batchSize) Building building, int batchSize)
throws SQLException { throws SQLException {
assert IndexBuildingTable.values().length == 7; prepBuilding.setLong(1, building.getId());
prepBuilding.setLong(IndexBuildingTable.ID.ordinal() + 1, building.getId()); prepBuilding.setDouble(2, building.getLocation().getLatitude());
prepBuilding.setDouble(IndexBuildingTable.LATITUDE.ordinal() + 1, building.getLocation().getLatitude()); prepBuilding.setDouble(3, building.getLocation().getLongitude());
prepBuilding.setDouble(IndexBuildingTable.LONGITUDE.ordinal() + 1, building.getLocation().getLongitude()); prepBuilding.setString(4, building.getName());
prepBuilding.setString(IndexBuildingTable.NAME.ordinal() + 1, building.getName()); prepBuilding.setString(5, building.getEnName());
prepBuilding.setString(IndexBuildingTable.NAME_EN.ordinal() + 1, building.getEnName()); prepBuilding.setLong(6, streetId);
prepBuilding.setLong(IndexBuildingTable.STREET.ordinal() + 1, streetId); prepBuilding.setString(7, building.getPostcode() == null ? null : building.getPostcode().toUpperCase());
prepBuilding.setString(IndexBuildingTable.POSTCODE.ordinal()+1, building.getPostcode() == null ? null : building.getPostcode().toUpperCase());
addBatch(count, prepBuilding); addBatch(count, prepBuilding);
} }
public static PreparedStatement getSearchStreetPreparedStatement(Connection mapConnection) throws SQLException {
return mapConnection.prepareStatement("SELECT ID FROM street WHERE ? = city AND ? = name");
}
public static PreparedStatement getSearchBuildingPreparedStatement(Connection mapConnection) throws SQLException {
return mapConnection.prepareStatement("SELECT id FROM building where ? = id");
}
public static PreparedStatement getStreeNodeSearchPreparedStatement(Connection mapConnection) throws SQLException {
return mapConnection.prepareStatement("SELECT way FROM street_node WHERE ? = way");
}
public static PreparedStatement getUpdateBuildingPostcodePreparedStatement(Connection mapConnection) throws SQLException {
return mapConnection.prepareStatement("UPDATE building SET postcode = ? WHERE id = ?");
}
public static PreparedStatement getCityInsertPreparedStatement(Connection conn) throws SQLException{
return conn.prepareStatement("insert into city (id, latitude, longitude, name, name_en, city_type) values (?, ?, ?, ?, ?, ?)");
}
public static void writeCity(PreparedStatement prepCity, Map<PreparedStatement, Integer> count, City city, int batchSize) throws SQLException { public static void writeCity(PreparedStatement prepCity, Map<PreparedStatement, Integer> count, City city, int batchSize) throws SQLException {
assert IndexCityTable.values().length == 6; prepCity.setLong(1, city.getId());
prepCity.setLong(IndexCityTable.ID.ordinal() + 1, city.getId()); prepCity.setDouble(2, city.getLocation().getLatitude());
prepCity.setDouble(IndexCityTable.LATITUDE.ordinal() + 1, city.getLocation().getLatitude()); prepCity.setDouble(3, city.getLocation().getLongitude());
prepCity.setDouble(IndexCityTable.LONGITUDE.ordinal() + 1, city.getLocation().getLongitude()); prepCity.setString(4, city.getName());
prepCity.setString(IndexCityTable.NAME.ordinal() + 1, city.getName()); prepCity.setString(5, city.getEnName());
prepCity.setString(IndexCityTable.NAME_EN.ordinal() + 1, city.getEnName()); prepCity.setString(6, CityType.valueToString(city.getType()));
prepCity.setString(IndexCityTable.CITY_TYPE.ordinal() + 1, CityType.valueToString(city.getType()));
addBatch(count, prepCity, batchSize); addBatch(count, prepCity, batchSize);
} }
public static PreparedStatement getStreetInsertPreparedStatement(Connection conn) throws SQLException{
return conn.prepareStatement("insert into street (id, latitude, longitude, name, name_en, city) values (?, ?, ?, ?, ?, ?)");
}
public static void insertStreetData(PreparedStatement addressStreetStat, long id, String name, String nameEn, double latitude,
double longitude, Long cityId) throws SQLException {
addressStreetStat.setLong(1, id);
addressStreetStat.setString(4, name);
addressStreetStat.setString(5, nameEn);
addressStreetStat.setDouble(2, latitude);
addressStreetStat.setDouble(3, longitude);
addressStreetStat.setLong(6, cityId);
}
public static void createAddressIndexStructure(Connection conn) throws SQLException{ public static void createAddressIndexStructure(Connection conn) throws SQLException{
Statement stat = conn.createStatement(); Statement stat = conn.createStatement();
stat.executeUpdate(IndexConstants.generateCreateSQL(IndexCityTable.values()));
stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexCityTable.values())); stat.executeUpdate("create table city (id bigint primary key, latitude double, longitude double, " +
stat.executeUpdate(IndexConstants.generateCreateSQL(IndexBuildingTable.values())); "name varchar(255), name_en varchar(255), city_type varchar(32))");
stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexBuildingTable.values())); stat.executeUpdate("create index city_ind on city (id, city_type)");
stat.executeUpdate(IndexConstants.generateCreateSQL(IndexStreetNodeTable.values()));
stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexStreetNodeTable.values())); stat.executeUpdate("create table street (id bigint primary key, latitude double, longitude double, " +
stat.executeUpdate(IndexConstants.generateCreateSQL(IndexStreetTable.values())); "name varchar(255), name_en varchar(255), city bigint)");
stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexStreetTable.values())); stat.executeUpdate("create index street_city on street (city)");
stat.executeUpdate("create index street_id on street (id)");
// create index on name ?
stat.executeUpdate("create table building (id bigint, latitude double, longitude double, " +
"name varchar(255), name_en varchar(255), street bigint, postcode varchar(255), primary key(street, id))");
stat.executeUpdate("create index building_postcode on building (postcode)");
stat.executeUpdate("create index building_street on building (street)");
stat.executeUpdate("create index building_id on building (id)");
stat.executeUpdate("create table street_node (id bigint, latitude double, longitude double, " +
"street bigint, way bigint)");
stat.executeUpdate("create index street_node_street on street_node (street)");
stat.executeUpdate("create index street_node_way on street_node (way)");
if(IndexCreator.usingSQLite()){ if(IndexCreator.usingSQLite()){
stat.execute("PRAGMA user_version = " + IndexConstants.ADDRESS_TABLE_VERSION); //$NON-NLS-1$ stat.execute("PRAGMA user_version = " + IndexConstants.ADDRESS_TABLE_VERSION); //$NON-NLS-1$
} }
@ -310,4 +365,6 @@ public class DataIndexWriter {
} }
} }
} }

159
DataExtractionOSM/src/net/osmand/data/index/IndexConstants.java
View file

@ -1,6 +1,5 @@
package net.osmand.data.index; package net.osmand.data.index;
import net.osmand.data.preparation.IndexCreator;
public class IndexConstants { public class IndexConstants {
@ -130,7 +129,7 @@ public class IndexConstants {
// POI index // POI index
public enum IndexPoiTable implements IndexColumn { public enum IndexPoiTable implements IndexColumn {
ID(IndexCreator.getCurrentLongType()), LATITUDE("double", true), LONGITUDE("double", true), OPENING_HOURS, NAME, NAME_EN, TYPE, SUBTYPE; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ ID("bigint"), LATITUDE("double", true), LONGITUDE("double", true), OPENING_HOURS, NAME, NAME_EN, TYPE, SUBTYPE; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
boolean index = false; boolean index = false;
String type = null; String type = null;
private IndexPoiTable(){} private IndexPoiTable(){}
@ -158,157 +157,9 @@ public class IndexConstants {
} }
} }
// Address index
public enum IndexCityTable implements IndexColumn {
ID(IndexCreator.getCurrentLongType(), true), LATITUDE("double", true), LONGITUDE("double", true), NAME, NAME_EN, CITY_TYPE(null, true); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
boolean index = false;
String type = null;
private IndexCityTable() {
}
private IndexCityTable(String type) {
this.type = type;
}
private IndexCityTable(String type, boolean index) {
this(type);
this.index = index;
}
public static String getTable() {
return "city"; //$NON-NLS-1$
}
public String getTableName() {
return getTable();
}
@Override
public String getType() {
return type;
}
@Override
public boolean isIndex() {
return index;
}
}
public enum IndexStreetTable implements IndexColumn {
ID(IndexCreator.getCurrentLongType(), true), LATITUDE("double", true), LONGITUDE("double", true), NAME(null, true), NAME_EN, CITY(IndexCreator.getCurrentLongType(), true); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
boolean index = false;
String type = null;
private IndexStreetTable() {
}
private IndexStreetTable(String type) {
this.type = type;
}
private IndexStreetTable(String type, boolean index) {
this(type);
this.index = index;
}
public static String getTable() {
return "street"; //$NON-NLS-1$
}
public String getTableName() {
return getTable();
}
@Override
public String getType() {
return type;
}
@Override
public boolean isIndex() {
return index;
}
}
public enum IndexStreetNodeTable implements IndexColumn {
ID(IndexCreator.getCurrentLongType(), true), LATITUDE("double"), LONGITUDE("double"),
STREET(IndexCreator.getCurrentLongType(), true), WAY(IndexCreator.getCurrentLongType(), true); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$
boolean index = false;
String type = null;
private IndexStreetNodeTable() {
}
private IndexStreetNodeTable(String type) {
this.type = type;
}
private IndexStreetNodeTable(String type, boolean index) {
this(type);
this.index = index;
}
public static String getTable() {
return "street_node"; //$NON-NLS-1$
}
public String getTableName() {
return getTable();
}
@Override
public String getType() {
return type;
}
@Override
public boolean isIndex() {
return index;
}
}
public enum IndexBuildingTable implements IndexColumn {
ID(IndexCreator.getCurrentLongType(), true), LATITUDE("double"), LONGITUDE("double"), NAME,
NAME_EN, STREET(IndexCreator.getCurrentLongType(), true), POSTCODE(null, true); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
boolean index = false;
String type = null;
private IndexBuildingTable() {
}
private IndexBuildingTable(String type) {
this.type = type;
}
private IndexBuildingTable(String type, boolean index) {
this(type);
this.index = index;
}
public static String getTable() {
return "building"; //$NON-NLS-1$
}
public String getTableName() {
return getTable();
}
@Override
public String getType() {
return type;
}
@Override
public boolean isIndex() {
return index;
}
}
// Transport Index // Transport Index
public enum IndexTransportStop implements IndexColumn { public enum IndexTransportStop implements IndexColumn {
ID(IndexCreator.getCurrentLongType(), true), LATITUDE("double", true), LONGITUDE("double", true), NAME, NAME_EN; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ ID("bigint", true), LATITUDE("double", true), LONGITUDE("double", true), NAME, NAME_EN; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
boolean index = false; boolean index = false;
String type = null; String type = null;
@ -345,7 +196,7 @@ public class IndexConstants {
public enum IndexTransportRouteStop implements IndexColumn { public enum IndexTransportRouteStop implements IndexColumn {
STOP(IndexCreator.getCurrentLongType(), true), ROUTE(IndexCreator.getCurrentLongType(), true), ORD("int"), DIRECTION("smallint"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ STOP("bigint", true), ROUTE("bigint", true), ORD("int"), DIRECTION("smallint"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
boolean index = false; boolean index = false;
String type = null; String type = null;
@ -381,7 +232,7 @@ public class IndexConstants {
} }
public enum IndexTransportRoute implements IndexColumn { public enum IndexTransportRoute implements IndexColumn {
ID(IndexCreator.getCurrentLongType(), true), TYPE(null, true), OPERATOR, REF(null, true), NAME, NAME_EN, DIST("int"); //$NON-NLS-1$ //$NON-NLS-2$ ID("bigint", true), TYPE(null, true), OPERATOR, REF(null, true), NAME, NAME_EN, DIST("int"); //$NON-NLS-1$ //$NON-NLS-2$
boolean index = false; boolean index = false;
String type = null; String type = null;
@ -418,7 +269,7 @@ public class IndexConstants {
public enum IndexBinaryMapRenderObject implements IndexColumn { public enum IndexBinaryMapRenderObject implements IndexColumn {
ID(IndexCreator.getCurrentLongType(), true), NAME, TYPES("BLOB"), RESTRICTIONS("BLOB"), NODES("BLOB"), HIGHWAY("INT"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ ID("bigint", true), NAME, TYPES("BLOB"), RESTRICTIONS("BLOB"), NODES("BLOB"), HIGHWAY("INT"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$
boolean index = false; boolean index = false;
String type = null; String type = null;

56
DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java
View file

@ -44,10 +44,6 @@ import net.osmand.data.index.DataIndexReader;
import net.osmand.data.index.DataIndexWriter; import net.osmand.data.index.DataIndexWriter;
import net.osmand.data.index.IndexConstants; import net.osmand.data.index.IndexConstants;
import net.osmand.data.index.IndexConstants.IndexBinaryMapRenderObject; import net.osmand.data.index.IndexConstants.IndexBinaryMapRenderObject;
import net.osmand.data.index.IndexConstants.IndexBuildingTable;
import net.osmand.data.index.IndexConstants.IndexCityTable;
import net.osmand.data.index.IndexConstants.IndexStreetNodeTable;
import net.osmand.data.index.IndexConstants.IndexStreetTable;
import net.osmand.data.index.IndexConstants.IndexTransportRoute; import net.osmand.data.index.IndexConstants.IndexTransportRoute;
import net.osmand.data.index.IndexConstants.IndexTransportRouteStop; import net.osmand.data.index.IndexConstants.IndexTransportRouteStop;
import net.osmand.data.index.IndexConstants.IndexTransportStop; import net.osmand.data.index.IndexConstants.IndexTransportStop;
@ -221,11 +217,6 @@ public class IndexCreator {
return CURRENT_DB.equals(H2_DIALECT); return CURRENT_DB.equals(H2_DIALECT);
} }
public static String getCurrentLongType(){
return usingSQLite() ? "long" : "bigint";
}
public IndexCreator(File workingDir){ public IndexCreator(File workingDir){
this.workingDir = workingDir; this.workingDir = workingDir;
@ -1111,13 +1102,8 @@ public class IndexCreator {
} }
if (foundId == null) { if (foundId == null) {
assert IndexStreetTable.values().length == 6; DataIndexWriter.insertStreetData(addressStreetStat, initId, name, Junidecode.unidecode(name),
addressStreetStat.setLong(IndexStreetTable.ID.ordinal() + 1, initId); location.getLatitude(), location.getLongitude(), city.getId());
addressStreetStat.setString(IndexStreetTable.NAME_EN.ordinal() + 1, Junidecode.unidecode(name));
addressStreetStat.setString(IndexStreetTable.NAME.ordinal() + 1, name);
addressStreetStat.setDouble(IndexStreetTable.LATITUDE.ordinal() + 1, location.getLatitude());
addressStreetStat.setDouble(IndexStreetTable.LONGITUDE.ordinal() + 1, location.getLongitude());
addressStreetStat.setLong(IndexStreetTable.CITY.ordinal() + 1, city.getId());
if(loadInMemory){ if(loadInMemory){
DataIndexWriter.addBatch(pStatements, addressStreetStat, BATCH_SIZE); DataIndexWriter.addBatch(pStatements, addressStreetStat, BATCH_SIZE);
addressStreetLocalMap.put(name+"_"+city.getId(), initId); addressStreetLocalMap.put(name+"_"+city.getId(), initId);
@ -1681,7 +1667,7 @@ public class IndexCreator {
break; break;
} }
} }
progress.startTask("Searalizing city addresses...", j + ((cities.size() - j) / 100 + 1)); progress.startTask("Serializing city addresses...", j + ((cities.size() - j) / 100 + 1));
Map<String, Set<Street>> postcodes = new TreeMap<String, Set<Street>>(); Map<String, Set<Street>> postcodes = new TreeMap<String, Set<Street>>();
boolean writeCities = true; boolean writeCities = true;
@ -1706,10 +1692,15 @@ public class IndexCreator {
if(readWayNodes){ if(readWayNodes){
streetNodes = new LinkedHashMap<Street, List<Node>>(); streetNodes = new LinkedHashMap<Street, List<Node>>();
} }
long time = System.currentTimeMillis();
reader.readStreetsBuildings(streetstat, c, streets, waynodesStat, streetNodes); reader.readStreetsBuildings(streetstat, c, streets, waynodesStat, streetNodes);
long f = System.currentTimeMillis() - time;
writer.writeCityIndex(c, streets, streetNodes); writer.writeCityIndex(c, streets, streetNodes);
int bCount = 0;
for (Street s : streets) { for (Street s : streets) {
bCount++;
for (Building b : s.getBuildings()) { for (Building b : s.getBuildings()) {
bCount++;
if (b.getPostcode() != null) { if (b.getPostcode() != null) {
if (!postcodes.containsKey(b.getPostcode())) { if (!postcodes.containsKey(b.getPostcode())) {
postcodes.put(b.getPostcode(), new LinkedHashSet<Street>(3)); postcodes.put(b.getPostcode(), new LinkedHashSet<Street>(3));
@ -1718,6 +1709,7 @@ public class IndexCreator {
} }
} }
} }
System.out.println("! " + c.getName() + " ! " + f + " " + bCount + " streets " + streets.size());
} }
writer.endCityIndexes(!writeCities); writer.endCityIndexes(!writeCities);
@ -2303,8 +2295,7 @@ public class IndexCreator {
private void processingPostcodes() throws SQLException { private void processingPostcodes() throws SQLException {
PreparedStatement pstat = mapConnection.prepareStatement("UPDATE " + IndexBuildingTable.getTable() + " SET " PreparedStatement pstat = DataIndexWriter.getUpdateBuildingPostcodePreparedStatement(mapConnection);;
+ IndexBuildingTable.POSTCODE.name() + " = ? WHERE " + IndexBuildingTable.ID.name() + " = ?");
pStatements.put(pstat, 0); pStatements.put(pstat, 0);
for (Relation r : postalCodeRelations) { for (Relation r : postalCodeRelations) {
String tag = r.getTag(OSMTagKey.POSTAL_CODE); String tag = r.getTag(OSMTagKey.POSTAL_CODE);
@ -2432,21 +2423,14 @@ public class IndexCreator {
if (indexAddress) { if (indexAddress) {
DataIndexWriter.createAddressIndexStructure(mapConnection); DataIndexWriter.createAddressIndexStructure(mapConnection);
addressCityStat = mapConnection.prepareStatement(IndexConstants.generatePrepareStatementToInsert(IndexCityTable addressCityStat = DataIndexWriter.getCityInsertPreparedStatement(mapConnection);
.getTable(), IndexCityTable.values().length)); addressStreetStat = DataIndexWriter.getStreetInsertPreparedStatement(mapConnection);
addressStreetStat = mapConnection.prepareStatement(IndexConstants.generatePrepareStatementToInsert(IndexStreetTable addressBuildingStat = DataIndexWriter.getBuildingInsertPreparedStatement(mapConnection);
.getTable(), IndexStreetTable.values().length)); addressStreetNodeStat = DataIndexWriter.getStreetNodeInsertPreparedStatement(mapConnection);
addressSearchStreetStat = mapConnection.prepareStatement("SELECT " + IndexStreetTable.ID.name() + " FROM " addressSearchStreetStat = DataIndexWriter.getSearchStreetPreparedStatement(mapConnection);
+ IndexStreetTable.getTable() + " WHERE ? = " + IndexStreetTable.CITY.name() + " AND ? =" addressSearchBuildingStat = DataIndexWriter.getSearchBuildingPreparedStatement(mapConnection);
+ IndexStreetTable.NAME.name()); addressSearchStreetNodeStat = DataIndexWriter.getStreeNodeSearchPreparedStatement(mapConnection);
addressSearchBuildingStat = mapConnection.prepareStatement("SELECT " + IndexBuildingTable.ID.name() + " FROM "
+ IndexBuildingTable.getTable() + " WHERE ? = " + IndexBuildingTable.ID.name());
addressSearchStreetNodeStat = mapConnection.prepareStatement("SELECT " + IndexStreetNodeTable.WAY.name() + " FROM "
+ IndexStreetNodeTable.getTable() + " WHERE ? = " + IndexStreetNodeTable.WAY.name());
addressBuildingStat = mapConnection.prepareStatement(IndexConstants.generatePrepareStatementToInsert(IndexBuildingTable
.getTable(), IndexBuildingTable.values().length));
addressStreetNodeStat = mapConnection.prepareStatement(IndexConstants.generatePrepareStatementToInsert(
IndexStreetNodeTable.getTable(), IndexStreetNodeTable.values().length));
pStatements.put(addressCityStat, 0); pStatements.put(addressCityStat, 0);
pStatements.put(addressStreetStat, 0); pStatements.put(addressStreetStat, 0);
pStatements.put(addressStreetNodeStat, 0); pStatements.put(addressStreetNodeStat, 0);
@ -2513,11 +2497,13 @@ public class IndexCreator {
Connection dbConn = DriverManager.getConnection("jdbc:sqlite:" + sqlitedb.getAbsolutePath()); Connection dbConn = DriverManager.getConnection("jdbc:sqlite:" + sqlitedb.getAbsolutePath());
dbConn.setAutoCommit(false); dbConn.setAutoCommit(false);
Statement st = dbConn.createStatement(); Statement st = dbConn.createStatement();
st.execute("DELETE FROM " + IndexStreetNodeTable.getTable() + " WHERE 1=1"); st.execute("DELETE FROM street_node WHERE 1=1");
st.close(); st.close();
dbConn.commit(); dbConn.commit();
st = dbConn.createStatement(); st = dbConn.createStatement();
if (usingSQLite()) {
st.execute("VACUUM"); st.execute("VACUUM");
}
st.close(); st.close();
dbConn.close(); dbConn.close();
} }

65
OsmAnd/src/net/osmand/RegionAddressRepositoryOdb.java
View file

@ -22,10 +22,6 @@ import net.osmand.data.PostCode;
import net.osmand.data.Street; import net.osmand.data.Street;
import net.osmand.data.City.CityType; import net.osmand.data.City.CityType;
import net.osmand.data.index.IndexConstants; import net.osmand.data.index.IndexConstants;
import net.osmand.data.index.IndexConstants.IndexBuildingTable;
import net.osmand.data.index.IndexConstants.IndexCityTable;
import net.osmand.data.index.IndexConstants.IndexStreetNodeTable;
import net.osmand.data.index.IndexConstants.IndexStreetTable;
import net.osmand.osm.LatLon; import net.osmand.osm.LatLon;
import net.osmand.osm.Node; import net.osmand.osm.Node;
import net.osmand.osm.Way; import net.osmand.osm.Way;
@ -300,11 +296,11 @@ public class RegionAddressRepositoryOdb implements RegionAddressRepository {
} }
StringBuilder where = new StringBuilder(80); StringBuilder where = new StringBuilder(80);
where. where.
append(IndexCityTable.CITY_TYPE.toString()).append(" not in ("). //$NON-NLS-1$ append("city_type not in ("). //$NON-NLS-1$
append('\'').append(CityType.valueToString(CityType.CITY)).append('\'').append(", "). //$NON-NLS-1$ append('\'').append(CityType.valueToString(CityType.CITY)).append('\'').append(", "). //$NON-NLS-1$
append('\'').append(CityType.valueToString(CityType.TOWN)).append('\'').append(") and "). //$NON-NLS-1$ append('\'').append(CityType.valueToString(CityType.TOWN)).append('\'').append(") and "). //$NON-NLS-1$
append(useEnglishNames ? IndexCityTable.NAME_EN.toString() : IndexCityTable.NAME.toString()).append(" LIKE '"+name+"%'"); //$NON-NLS-1$ //$NON-NLS-2$ append(useEnglishNames ? "name_en" : "name").append(" LIKE '"+name+"%'"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
Cursor query = db.query(IndexCityTable.getTable(), IndexConstants.generateColumnNames(IndexCityTable.values()), Cursor query = db.query("city", new String[]{"id","latitude", "longitude", "name", "name_en", "city_type"}, //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ //$NON-NLS-6$ //$NON-NLS-7$
where.toString(), null, null, null, null); where.toString(), null, null, null, null);
if (query.moveToFirst()) { if (query.moveToFirst()) {
List<City> hamlets = new ArrayList<City>(); List<City> hamlets = new ArrayList<City>();
@ -324,16 +320,16 @@ public class RegionAddressRepositoryOdb implements RegionAddressRepository {
public void preloadWayNodes(Street street){ public void preloadWayNodes(Street street){
if(street.getWayNodes().isEmpty()){ if(street.getWayNodes().isEmpty()){
Cursor query = db.query(IndexStreetNodeTable.getTable(), IndexConstants.generateColumnNames(IndexStreetNodeTable.values()), "? = street", //$NON-NLS-1$ Cursor query = db.query("street_node", new String[]{"id", "latitude", "longitude", "street", "way"}, "? = street", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ //$NON-NLS-6$ //$NON-NLS-7$
new String[] { street.getId() + "" }, null, null, null); //$NON-NLS-1$ new String[] { street.getId() + "" }, null, null, null); //$NON-NLS-1$
log.debug("Start loading waynodes for " + street.getName()); //$NON-NLS-1$ log.debug("Start loading waynodes for " + street.getName()); //$NON-NLS-1$
Map<Long, Way> ways = new LinkedHashMap<Long, Way>(); Map<Long, Way> ways = new LinkedHashMap<Long, Way>();
if (query.moveToFirst()) { if (query.moveToFirst()) {
do { do {
Node n = new Node(query.getDouble(IndexStreetNodeTable.LATITUDE.ordinal()), Node n = new Node(query.getDouble(1),
query.getDouble(IndexBuildingTable.LONGITUDE.ordinal()), query.getDouble(2),
query.getLong(IndexStreetNodeTable.ID.ordinal())); query.getLong(0));
long way = query.getLong(IndexStreetNodeTable.WAY.ordinal()); long way = query.getLong(4);
if(!ways.containsKey(way)){ if(!ways.containsKey(way)){
ways.put(way, new Way(way)); ways.put(way, new Way(way));
} }
@ -352,7 +348,7 @@ public class RegionAddressRepositoryOdb implements RegionAddressRepository {
public void preloadPostcodes() { public void preloadPostcodes() {
if (postCodes.isEmpty()) { if (postCodes.isEmpty()) {
// check if it possible to load postcodes // check if it possible to load postcodes
Cursor query = db.query(true, IndexBuildingTable.getTable(), new String[] { IndexBuildingTable.POSTCODE.toString() }, null, Cursor query = db.query(true, "building", new String[] { "postcode" }, null, //$NON-NLS-1$//$NON-NLS-2$
null, null, null, null, null); null, null, null, null, null);
log.debug("Start loading postcodes for "); //$NON-NLS-1$ log.debug("Start loading postcodes for "); //$NON-NLS-1$
if (query.moveToFirst()) { if (query.moveToFirst()) {
@ -370,18 +366,19 @@ public class RegionAddressRepositoryOdb implements RegionAddressRepository {
public void preloadBuildings(Street street){ public void preloadBuildings(Street street){
if (street.getBuildings().isEmpty()) { if (street.getBuildings().isEmpty()) {
Cursor query = db.query(IndexBuildingTable.getTable(), IndexConstants.generateColumnNames(IndexBuildingTable.values()), "? = street", //$NON-NLS-1$ Cursor query = db.query("building", //$NON-NLS-1$
new String[]{"id","latitude", "longitude", "name", "name_en", "street", "postcode"} //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ //$NON-NLS-6$ //$NON-NLS-7$
, "? = street", //$NON-NLS-1$
new String[] { street.getId() + "" }, null, null, null); //$NON-NLS-1$ new String[] { street.getId() + "" }, null, null, null); //$NON-NLS-1$
log.debug("Start loading buildings for " + street.getName()); //$NON-NLS-1$ log.debug("Start loading buildings for " + street.getName()); //$NON-NLS-1$
if (query.moveToFirst()) { if (query.moveToFirst()) {
do { do {
Building building = new Building(); Building building = new Building();
building.setId(query.getLong(IndexBuildingTable.ID.ordinal())); building.setId(query.getLong(0));
building.setLocation(query.getDouble(IndexBuildingTable.LATITUDE.ordinal()), query.getDouble(IndexBuildingTable.LONGITUDE building.setLocation(query.getDouble(1), query.getDouble(2));
.ordinal())); building.setName(query.getString(3));
building.setName(query.getString(IndexBuildingTable.NAME.ordinal())); building.setEnName(query.getString(4));
building.setEnName(query.getString(IndexBuildingTable.NAME_EN.ordinal())); building.setPostcode(query.getString(6));
building.setPostcode(query.getString(IndexBuildingTable.POSTCODE.ordinal()));
street.registerBuilding(building); street.registerBuilding(building);
} while (query.moveToNext()); } while (query.moveToNext());
street.sortBuildings(); street.sortBuildings();
@ -395,17 +392,18 @@ public class RegionAddressRepositoryOdb implements RegionAddressRepository {
assert o instanceof PostCode || o instanceof City; assert o instanceof PostCode || o instanceof City;
City city = (City) (o instanceof City ? o : null); City city = (City) (o instanceof City ? o : null);
PostCode post = (PostCode) (o instanceof PostCode ? o : null); PostCode post = (PostCode) (o instanceof PostCode ? o : null);
if (city != null && city.isEmptyWithStreets()) { if (city != null && city.isEmptyWithStreets()) {
log.debug("Start loading streets for " + city.getName()); //$NON-NLS-1$ log.debug("Start loading streets for " + city.getName()); //$NON-NLS-1$
Cursor query = db.query(IndexStreetTable.getTable(), IndexConstants.generateColumnNames(IndexStreetTable.values()), "? = city", //$NON-NLS-1$ Cursor query = db.query("street", new String[]{"id","latitude", "longitude", "name", "name_en", "city"}, "? = city", //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ //$NON-NLS-6$ //$NON-NLS-7$ //$NON-NLS-8$
new String[] { city.getId() + "" }, null, null, null); //$NON-NLS-1$ new String[] { city.getId() + "" }, null, null, null); //$NON-NLS-1$
if (query.moveToFirst()) { if (query.moveToFirst()) {
do { do {
Street street = new Street(city); Street street = new Street(city);
street.setId(query.getLong(IndexStreetTable.ID.ordinal())); street.setId(query.getLong(0));
street.setLocation(query.getDouble(IndexStreetTable.LATITUDE.ordinal()), query.getDouble(IndexStreetTable.LONGITUDE.ordinal())); street.setLocation(query.getDouble(1), query.getDouble(2));
street.setName(query.getString(IndexStreetTable.NAME.ordinal())); street.setName(query.getString(3));
street.setEnName(query.getString(IndexStreetTable.NAME_EN.ordinal())); street.setEnName(query.getString(4));
city.registerStreet(street, useEnglishNames); city.registerStreet(street, useEnglishNames);
} while (query.moveToNext()); } while (query.moveToNext());
} }
@ -451,14 +449,13 @@ public class RegionAddressRepositoryOdb implements RegionAddressRepository {
} }
protected City parseCityFromCursor(Cursor query){ protected City parseCityFromCursor(Cursor query){
CityType type = CityType.valueFromString(query.getString(IndexCityTable.CITY_TYPE.ordinal())); CityType type = CityType.valueFromString(query.getString(5));
if (type != null) { if (type != null) {
City city = new City(type); City city = new City(type);
city.setId(query.getLong(IndexCityTable.ID.ordinal())); city.setId(query.getLong(0));
city.setLocation(query.getDouble(IndexCityTable.LATITUDE.ordinal()), city.setLocation(query.getDouble(1), query.getDouble(2));
query.getDouble(IndexCityTable.LONGITUDE.ordinal())); city.setName(query.getString(3));
city.setName(query.getString(IndexCityTable.NAME.ordinal())); city.setEnName(query.getString(4));
city.setEnName(query.getString(IndexCityTable.NAME_EN.ordinal()));
return city; return city;
} }
return null; return null;
@ -468,11 +465,11 @@ public class RegionAddressRepositoryOdb implements RegionAddressRepository {
if (cities.isEmpty()) { if (cities.isEmpty()) {
log.debug("Start loading cities for " +getName()); //$NON-NLS-1$ log.debug("Start loading cities for " +getName()); //$NON-NLS-1$
StringBuilder where = new StringBuilder(); StringBuilder where = new StringBuilder();
where.append(IndexCityTable.CITY_TYPE.toString()).append('='). where.append("city_type="). //$NON-NLS-1$
append('\'').append(CityType.valueToString(CityType.CITY)).append('\'').append(" or "). //$NON-NLS-1$ append('\'').append(CityType.valueToString(CityType.CITY)).append('\'').append(" or "). //$NON-NLS-1$
append(IndexCityTable.CITY_TYPE.toString()).append('='). append("city_type="). //$NON-NLS-1$
append('\'').append(CityType.valueToString(CityType.TOWN)).append('\''); append('\'').append(CityType.valueToString(CityType.TOWN)).append('\'');
Cursor query = db.query(IndexCityTable.getTable(), IndexConstants.generateColumnNames(IndexCityTable.values()), Cursor query = db.query("city", new String[]{"id","latitude", "longitude", "name", "name_en", "city_type"}, //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ //$NON-NLS-6$ //$NON-NLS-7$
where.toString(), null, null, null, null); where.toString(), null, null, null, null);
if(query.moveToFirst()){ if(query.moveToFirst()){
do { do {