Android-builders-club/OsmAnd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Refactoring IndexCreator (4)

Browse source
This commit is contained in:
Victor Shcherb 2011-05-02 23:43:20 +02:00
parent a62cb50e13
commit c3f2194237
8 changed files with 432 additions and 418 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
DataExtractionOSM/src/net/osmand/data/preparation/AbstractIndexPartCreator.java
View file

@ -41,28 +41,24 @@ public class AbstractIndexPartCreator {
} }
} }
protected void addBatch(Map<PreparedStatement, Integer> count, PreparedStatement p) throws SQLException { protected void addBatch(PreparedStatement p) throws SQLException {
addBatch(count, p, BATCH_SIZE, true); addBatch(p, BATCH_SIZE, true);
} }
protected void addBatch(Map<PreparedStatement, Integer> count, PreparedStatement p, boolean commit) throws SQLException{ protected void addBatch(PreparedStatement p, boolean commit) throws SQLException{
addBatch(count, p, BATCH_SIZE, commit); addBatch(p, BATCH_SIZE, commit);
} }
protected void addBatch(Map<PreparedStatement, Integer> count, PreparedStatement p, int batchSize) throws SQLException{ protected void addBatch(PreparedStatement p, int batchSize, boolean commit) throws SQLException{
addBatch(count, p, batchSize, true);
}
protected void addBatch(Map<PreparedStatement, Integer> count, PreparedStatement p, int batchSize, boolean commit) throws SQLException{
p.addBatch(); p.addBatch();
if(count.get(p) >= batchSize){ if(pStatements.get(p) >= batchSize){
p.executeBatch(); p.executeBatch();
if(commit){ if(commit){
p.getConnection().commit(); p.getConnection().commit();
} }
count.put(p, 0); pStatements.put(p, 0);
} else { } else {
count.put(p, count.get(p) + 1); pStatements.put(p, pStatements.get(p) + 1);
} }
} }

69
DataExtractionOSM/src/net/osmand/data/preparation/IndexAddressCreator.java
View file

@ -190,7 +190,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
citiBoundaries.put(nCity, boundary); citiBoundaries.put(nCity, boundary);
cityVillageManager.registerObject(point.getLatitude(), point.getLongitude(), nCity); cityVillageManager.registerObject(point.getLatitude(), point.getLongitude(), nCity);
writeCity(addressCityStat, pStatements, nCity); writeCity(nCity);
// commit to put all cities // commit to put all cities
if (pStatements.get(addressCityStat) > 0) { if (pStatements.get(addressCityStat) > 0) {
addressCityStat.executeBatch(); addressCityStat.executeBatch();
@ -270,7 +270,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
for (Map.Entry<Entity, String> r : i.getMemberEntities().entrySet()) { for (Map.Entry<Entity, String> r : i.getMemberEntities().entrySet()) {
if ("street".equals(r.getValue())) { //$NON-NLS-1$ if ("street".equals(r.getValue())) { //$NON-NLS-1$
if (r.getKey() instanceof Way && saveAddressWays) { if (r.getKey() instanceof Way && saveAddressWays) {
writeStreetWayNodes(addressStreetNodeStat, pStatements, streetId, (Way) r.getKey()); writeStreetWayNodes(streetId, (Way) r.getKey());
if (loadInMemory) { if (loadInMemory) {
addressStreetNodeLocalSet.add(r.getKey().getId()); addressStreetNodeLocalSet.add(r.getKey().getId());
} }
@ -282,7 +282,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
if (hno != null) { if (hno != null) {
Building building = new Building(r.getKey()); Building building = new Building(r.getKey());
building.setName(hno); building.setName(hno);
writeBuilding(addressBuildingStat, pStatements, streetId, building); writeBuilding(streetId, building);
if (loadInMemory) { if (loadInMemory) {
addressBuildingLocalSet.add(r.getKey().getId()); addressBuildingLocalSet.add(r.getKey().getId());
} }
@ -308,7 +308,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
Building building = new Building(border); Building building = new Building(border);
if (building.getLocation() != null) { if (building.getLocation() != null) {
building.setName(hno); building.setName(hno);
writeBuilding(addressBuildingStat, pStatements, streetId, building); writeBuilding(streetId, building);
if (loadInMemory) { if (loadInMemory) {
addressBuildingLocalSet.add(id.getId()); addressBuildingLocalSet.add(id.getId());
} }
@ -424,7 +424,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
insertStreetData(addressStreetStat, initId, name, Junidecode.unidecode(name), insertStreetData(addressStreetStat, initId, name, Junidecode.unidecode(name),
location.getLatitude(), location.getLongitude(), city.getId()); location.getLatitude(), location.getLongitude(), city.getId());
if (loadInMemory) { if (loadInMemory) {
addBatch(pStatements, addressStreetStat, BATCH_SIZE); addBatch(addressStreetStat);
addressStreetLocalMap.put(name + "_" + city.getId(), initId); //$NON-NLS-1$ addressStreetLocalMap.put(name + "_" + city.getId(), initId); //$NON-NLS-1$
} else { } else {
addressStreetStat.execute(); addressStreetStat.execute();
@ -509,7 +509,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
if (idStreet != null) { if (idStreet != null) {
Building building = new Building(e); Building building = new Building(e);
building.setName(e.getTag(OSMTagKey.ADDR_HOUSE_NUMBER)); building.setName(e.getTag(OSMTagKey.ADDR_HOUSE_NUMBER));
writeBuilding(addressBuildingStat, pStatements, idStreet, building); writeBuilding(idStreet, building);
} }
} }
} else if (e instanceof Way /* && OSMSettings.wayForCar(e.getTag(OSMTagKey.HIGHWAY)) */ } else if (e instanceof Way /* && OSMSettings.wayForCar(e.getTag(OSMTagKey.HIGHWAY)) */
@ -537,7 +537,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
City city = getClosestCity(l); City city = getClosestCity(l);
Long idStreet = getStreetInCity(city, e.getTag(OSMTagKey.NAME), l, (e.getId() << 2) | 1); Long idStreet = getStreetInCity(city, e.getTag(OSMTagKey.NAME), l, (e.getId() << 2) | 1);
if (idStreet != null && saveAddressWays) { if (idStreet != null && saveAddressWays) {
writeStreetWayNodes(addressStreetNodeStat, pStatements, idStreet, (Way) e); writeStreetWayNodes(idStreet, (Way) e);
} }
} }
} }
@ -550,45 +550,42 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
} }
private void writeStreetWayNodes(PreparedStatement prepStreetNode, Map<PreparedStatement, Integer> count, Long streetId, Way way) private void writeStreetWayNodes(Long streetId, Way way)
throws SQLException { throws SQLException {
for (Node n : way.getNodes()) { for (Node n : way.getNodes()) {
if (n == null) { if (n == null) {
continue; continue;
} }
prepStreetNode.setLong(1, n.getId()); addressStreetNodeStat.setLong(1, n.getId());
prepStreetNode.setDouble(2, n.getLatitude()); addressStreetNodeStat.setDouble(2, n.getLatitude());
prepStreetNode.setDouble(3, n.getLongitude()); addressStreetNodeStat.setDouble(3, n.getLongitude());
prepStreetNode.setLong(5, way.getId()); addressStreetNodeStat.setLong(5, way.getId());
prepStreetNode.setLong(4, streetId); addressStreetNodeStat.setLong(4, streetId);
addBatch(count, prepStreetNode, BATCH_SIZE); addBatch(addressStreetNodeStat);
} }
} }
private void writeBuilding(PreparedStatement prepBuilding, Map<PreparedStatement, Integer> count, Long streetId, private void writeBuilding(Long streetId, Building building) throws SQLException {
Building building) addressBuildingStat.setLong(1, building.getId());
throws SQLException { addressBuildingStat.setDouble(2, building.getLocation().getLatitude());
prepBuilding.setLong(1, building.getId()); addressBuildingStat.setDouble(3, building.getLocation().getLongitude());
prepBuilding.setDouble(2, building.getLocation().getLatitude()); addressBuildingStat.setString(4, building.getName());
prepBuilding.setDouble(3, building.getLocation().getLongitude()); addressBuildingStat.setString(5, building.getEnName());
prepBuilding.setString(4, building.getName()); addressBuildingStat.setLong(6, streetId);
prepBuilding.setString(5, building.getEnName()); addressBuildingStat.setString(7, building.getPostcode() == null ? null : building.getPostcode().toUpperCase());
prepBuilding.setLong(6, streetId); addBatch(addressBuildingStat);
prepBuilding.setString(7, building.getPostcode() == null ? null : building.getPostcode().toUpperCase());
addBatch(count, prepBuilding);
} }
private void writeCity(PreparedStatement prepCity, Map<PreparedStatement, Integer> count, City city) throws SQLException { private void writeCity(City city) throws SQLException {
prepCity.setLong(1, city.getId()); addressCityStat.setLong(1, city.getId());
prepCity.setDouble(2, city.getLocation().getLatitude()); addressCityStat.setDouble(2, city.getLocation().getLatitude());
prepCity.setDouble(3, city.getLocation().getLongitude()); addressCityStat.setDouble(3, city.getLocation().getLongitude());
prepCity.setString(4, city.getName()); addressCityStat.setString(4, city.getName());
prepCity.setString(5, city.getEnName()); addressCityStat.setString(5, city.getEnName());
prepCity.setString(6, CityType.valueToString(city.getType())); addressCityStat.setString(6, CityType.valueToString(city.getType()));
addBatch(count, prepCity, BATCH_SIZE); addBatch(addressCityStat);
} }
@ -605,7 +602,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
public void writeCitiesIntoDb() throws SQLException { public void writeCitiesIntoDb() throws SQLException {
for (City c : cities.values()) { for (City c : cities.values()) {
writeCity(addressCityStat, pStatements, c); writeCity(c);
} }
// commit to put all cities // commit to put all cities
if (pStatements.get(addressCityStat) > 0) { if (pStatements.get(addressCityStat) > 0) {
@ -628,7 +625,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
for (EntityId l : r.getMemberIds()) { for (EntityId l : r.getMemberIds()) {
pstat.setString(1, tag); pstat.setString(1, tag);
pstat.setLong(2, l.getId()); pstat.setLong(2, l.getId());
addBatch(pStatements, pstat, BATCH_SIZE); addBatch(pstat);
} }
} }
if (pStatements.get(pstat) > 0) { if (pStatements.get(pstat) > 0) {

329
DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java
View file

@ -9,10 +9,7 @@ import java.io.InputStream;
import java.io.RandomAccessFile; import java.io.RandomAccessFile;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.LinkedHashMap;
import java.util.Map;
import net.osmand.Algoritms; import net.osmand.Algoritms;
import net.osmand.IProgress; import net.osmand.IProgress;
@ -23,6 +20,7 @@ import net.osmand.impl.ConsoleProgressImplementation;
import net.osmand.osm.Entity; import net.osmand.osm.Entity;
import net.osmand.osm.MapRenderingTypes; import net.osmand.osm.MapRenderingTypes;
import net.osmand.osm.Relation; import net.osmand.osm.Relation;
import net.osmand.osm.Entity.EntityId;
import net.osmand.osm.Entity.EntityType; import net.osmand.osm.Entity.EntityType;
import net.osmand.osm.io.IOsmStorageFilter; import net.osmand.osm.io.IOsmStorageFilter;
import net.osmand.osm.io.OsmBaseStorage; import net.osmand.osm.io.OsmBaseStorage;
@ -85,8 +83,6 @@ public class IndexCreator {
private Connection dbConn; private Connection dbConn;
private File dbFile; private File dbFile;
Map<PreparedStatement, Integer> pStatements = new LinkedHashMap<PreparedStatement, Integer>();
private File mapFile; private File mapFile;
private RandomAccessFile mapRAFile; private RandomAccessFile mapRAFile;
private Connection mapConnection; private Connection mapConnection;
@ -95,11 +91,6 @@ public class IndexCreator {
public IndexCreator(File workingDir) { public IndexCreator(File workingDir) {
this.workingDir = workingDir; this.workingDir = workingDir;
this.indexTransportCreator = new IndexTransportCreator();
this.indexPoiCreator = new IndexPoiCreator();
this.indexAddressCreator = new IndexAddressCreator();
this.indexMapCreator = new IndexVectorMapCreator();
this.accessor = new OsmDbAccessor(this);
} }
public void setIndexAddress(boolean indexAddress) { public void setIndexAddress(boolean indexAddress) {
@ -187,27 +178,6 @@ public class IndexCreator {
this.cityAdminLevel = cityAdminLevel; this.cityAdminLevel = cityAdminLevel;
} }
private void iterateMainEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException {
if (indexPOI) {
indexPoiCreator.iterateEntity(e, ctx);
}
if (indexTransport) {
indexTransportCreator.visitEntityMainStep(e, ctx);
}
if (indexMap) {
indexMapCreator.iterateMainEntity(e, ctx);
}
if (indexAddress) {
indexAddressCreator.iterateMainEntity(e, ctx);
}
}
public void registerCityIfNeeded(Entity e) {
indexAddressCreator.registerCityIfNeeded(e);
}
public String getRTreeMapIndexNonPackFileName() { public String getRTreeMapIndexNonPackFileName() {
return mapFile.getAbsolutePath() + ".rtree"; //$NON-NLS-1$ return mapFile.getAbsolutePath() + ".rtree"; //$NON-NLS-1$
} }
@ -224,17 +194,146 @@ public class IndexCreator {
return mapFile.getAbsolutePath() + ".prtree"; //$NON-NLS-1$ return mapFile.getAbsolutePath() + ".prtree"; //$NON-NLS-1$
} }
/* ***** END OF GETTERS/SETTERS ***** */
public void generateIndexes(File readFile, IProgress progress, IOsmStorageFilter addFilter) throws IOException, SAXException, SQLException{ public void generateIndexes(File readFile, IProgress progress, IOsmStorageFilter addFilter) throws IOException, SAXException, SQLException{
generateIndexes(readFile, progress, addFilter, null, null); generateIndexes(readFile, progress, addFilter, null, null);
} }
public void generateIndexes(File readFile, IProgress progress, IOsmStorageFilter addFilter, MapZooms mapZooms, MapRenderingTypes renderingTypes) private void iterateMainEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException {
throws IOException, SAXException, if (indexPOI) {
SQLException { indexPoiCreator.iterateEntity(e, ctx);
if(renderingTypes == null){ }
if (indexTransport) {
indexTransportCreator.visitEntityMainStep(e, ctx);
}
if (indexMap) {
indexMapCreator.iterateMainEntity(e, ctx);
}
if (indexAddress) {
indexAddressCreator.iterateMainEntity(e, ctx);
}
}
private OsmDbCreator extractOsmToNodesDB(File readFile, IProgress progress, IOsmStorageFilter addFilter) throws FileNotFoundException,
IOException, SQLException, SAXException {
boolean pbfFile = false;
InputStream stream = new BufferedInputStream(new FileInputStream(readFile), 8192 * 4);
InputStream streamFile = stream;
long st = System.currentTimeMillis();
if (readFile.getName().endsWith(".bz2")) { //$NON-NLS-1$
if (stream.read() != 'B' || stream.read() != 'Z') {
throw new RuntimeException("The source stream must start with the characters BZ if it is to be read as a BZip2 stream."); //$NON-NLS-1$
} else {
stream = new CBZip2InputStream(stream);
}
} else if (readFile.getName().endsWith(".pbf")) { //$NON-NLS-1$
pbfFile = true;
}
OsmBaseStorage storage = new OsmBaseStorage();
storage.setSupressWarnings(DataExtractionSettings.getSettings().isSupressWarningsForDuplicatedId());
if (addFilter != null) {
storage.getFilters().add(addFilter);
}
storage.getFilters().add(new IOsmStorageFilter() {
@Override
public boolean acceptEntityToLoad(OsmBaseStorage storage, EntityId entityId, Entity entity) {
indexAddressCreator.registerCityIfNeeded(entity);
// accept to allow db creator parse it
return true;
}
});
// 1. Loading osm file
OsmDbCreator dbCreator = new OsmDbCreator(this);
try {
progress.setGeneralProgress("[35 / 100]"); //$NON-NLS-1$
progress.startTask(Messages.getString("IndexCreator.LOADING_FILE") + readFile.getAbsolutePath(), -1); //$NON-NLS-1$
// 1 init database to store temporary data
dbCreator.initDatabase(dialect, dbConn);
storage.getFilters().add(dbCreator);
if (pbfFile) {
storage.parseOSMPbf(stream, progress, false);
} else {
storage.parseOSM(stream, progress, streamFile, false);
}
dbCreator.finishLoading();
dbConn.commit();
if (log.isInfoEnabled()) {
log.info("File parsed : " + (System.currentTimeMillis() - st)); //$NON-NLS-1$
}
progress.finishTask();
return dbCreator;
} finally {
if (log.isInfoEnabled()) {
log.info("File indexed : " + (System.currentTimeMillis() - st)); //$NON-NLS-1$
}
}
}
private boolean createPlainOsmDb(IProgress progress, File readFile, IOsmStorageFilter addFilter) throws SQLException, FileNotFoundException, IOException, SAXException{
// initialize db file
boolean loadFromExistingFile = dbFile != null && dialect.databaseFileExists(dbFile);
if (dbFile == null) {
dbFile = new File(workingDir, TEMP_NODES_DB);
// to save space
if (dialect.databaseFileExists(dbFile)) {
dialect.removeDatabase(dbFile);
}
}
dbConn = getDatabaseConnection(dbFile.getAbsolutePath());
int allRelations = 100000;
int allWays = 1000000;
int allNodes = 10000000;
if (!loadFromExistingFile) {
OsmDbCreator dbCreator = extractOsmToNodesDB(readFile, progress, addFilter);
if (dbCreator != null) {
allNodes = dbCreator.getAllNodes();
allWays = dbCreator.getAllWays();
allRelations = dbCreator.getAllRelations();
}
}
accessor.initDatabase(dbConn, allNodes, allWays, allRelations);
return loadFromExistingFile;
}
private void createDatabaseIndexesStructure() throws SQLException, IOException {
// 2.1 create temporary sqlite database to put temporary results to it
if (indexMap || indexAddress || indexTransport) {
mapFile = new File(workingDir, getMapFileName());
// to save space
mapFile.getParentFile().mkdirs();
File tempDBMapFile = new File(workingDir, getTempMapDBFileName());
dialect.removeDatabase(tempDBMapFile);
mapConnection = getDatabaseConnection(tempDBMapFile.getAbsolutePath());
mapConnection.setAutoCommit(false);
}
// 2.2 create rtree map
if (indexMap) {
indexMapCreator.createDatabaseStructure(mapConnection, dialect, getRTreeMapIndexNonPackFileName());
}
if (indexAddress) {
indexAddressCreator.createDatabaseStructure(mapConnection, dialect);
}
if (indexPOI) {
indexPoiCreator.createDatabaseStructure(new File(workingDir, getPoiFileName()));
}
if (indexTransport) {
indexTransportCreator.createDatabaseStructure(mapConnection, dialect, getRTreeTransportStopsFileName());
}
}
public void generateIndexes(File readFile, IProgress progress, IOsmStorageFilter addFilter, MapZooms mapZooms,
MapRenderingTypes renderingTypes) throws IOException, SAXException, SQLException {
if (renderingTypes == null) {
renderingTypes = MapRenderingTypes.getDefault(); renderingTypes = MapRenderingTypes.getDefault();
} }
if(mapZooms == null){ if (mapZooms == null) {
mapZooms = MapZooms.getDefault(); mapZooms = MapZooms.getDefault();
} }
@ -245,6 +344,12 @@ public class IndexCreator {
regionName = Algoritms.capitalizeFirstLetterAndLowercase(readFile.getName().substring(0, i)); regionName = Algoritms.capitalizeFirstLetterAndLowercase(readFile.getName().substring(0, i));
} }
} }
this.indexTransportCreator = new IndexTransportCreator();
this.indexPoiCreator = new IndexPoiCreator();
this.indexAddressCreator = new IndexAddressCreator();
this.indexMapCreator = new IndexVectorMapCreator();
this.accessor = new OsmDbAccessor();
indexMapCreator.initSettings(mapZooms, renderingTypes); indexMapCreator.initSettings(mapZooms, renderingTypes);
@ -255,38 +360,13 @@ public class IndexCreator {
normalizeDefaultSuffixes = DataExtractionSettings.getSettings().getDefaultSuffixesToNormalizeStreets(); normalizeDefaultSuffixes = DataExtractionSettings.getSettings().getDefaultSuffixesToNormalizeStreets();
normalizeSuffixes = DataExtractionSettings.getSettings().getSuffixesToNormalizeStreets(); normalizeSuffixes = DataExtractionSettings.getSettings().getSuffixesToNormalizeStreets();
} }
indexAddressCreator.initSettings(normalizeStreets, normalizeDefaultSuffixes, normalizeSuffixes, indexAddressCreator.initSettings(normalizeStreets, normalizeDefaultSuffixes, normalizeSuffixes, saveAddressWays, cityAdminLevel);
saveAddressWays, cityAdminLevel);
// Main generation method // Main generation method
try { try {
// //////////////////////////////////////////////////////////////////////// // ////////////////////////////////////////////////////////////////////////
// 1. creating nodes db to fast access for all nodes and simply import all relations, ways, nodes to it // 1. creating nodes db to fast access for all nodes and simply import all relations, ways, nodes to it
boolean loadFromPath = dbFile == null || !dialect.databaseFileExists(dbFile); boolean loadFromExistingFile = createPlainOsmDb(progress, readFile, addFilter);
if (dbFile == null) {
dbFile = new File(workingDir, TEMP_NODES_DB);
// to save space
if (dialect.databaseFileExists(dbFile)) {
dialect.removeDatabase(dbFile);
}
}
dbConn = getDatabaseConnection(dbFile.getAbsolutePath());
int allRelations = 100000;
int allWays = 1000000;
int allNodes = 10000000;
if (loadFromPath) {
progress.setGeneralProgress("[35 / 100]"); //$NON-NLS-1$
progress.startTask(Messages.getString("IndexCreator.LOADING_FILE") + readFile.getAbsolutePath(), -1); //$NON-NLS-1$
OsmDbCreator filter = extractOsmToNodesDB(readFile, progress, addFilter);
if (filter != null) {
allNodes = filter.getAllNodes();
allWays = filter.getAllWays();
allRelations = filter.getAllRelations();
}
}
accessor.initDatabase(dbConn);
// do not create temp map file and rtree files // do not create temp map file and rtree files
if (recreateOnlyBinaryFile) { if (recreateOnlyBinaryFile) {
@ -311,14 +391,14 @@ public class IndexCreator {
createDatabaseIndexesStructure(); createDatabaseIndexesStructure();
// 3. Processing all entries // 3. Processing all entries
// 3.1 write all cities
// 3.1 write all cities
if (indexAddress) { if (indexAddress) {
progress.setGeneralProgress("[40 / 100]"); //$NON-NLS-1$ progress.setGeneralProgress("[40 / 100]"); //$NON-NLS-1$
progress.startTask(Messages.getString("IndexCreator.INDEX_CITIES"), allNodes); //$NON-NLS-1$ progress.startTask(Messages.getString("IndexCreator.INDEX_CITIES"), accessor.getAllNodes()); //$NON-NLS-1$
if (!loadFromPath) { if (loadFromExistingFile) {
// load cities names // load cities names
allNodes = accessor.iterateOverEntities(progress, EntityType.NODE, allNodes, new OsmDbVisitor() { accessor.iterateOverEntities(progress, EntityType.NODE, new OsmDbVisitor() {
@Override @Override
public void iterateEntity(Entity e, OsmDbAccessorContext ctx) { public void iterateEntity(Entity e, OsmDbAccessorContext ctx) {
indexAddressCreator.registerCityIfNeeded(e); indexAddressCreator.registerCityIfNeeded(e);
@ -331,8 +411,8 @@ public class IndexCreator {
// 3.2 index address relations // 3.2 index address relations
if (indexAddress || indexMap) { if (indexAddress || indexMap) {
progress.setGeneralProgress("[30 / 100]"); //$NON-NLS-1$ progress.setGeneralProgress("[30 / 100]"); //$NON-NLS-1$
progress.startTask(Messages.getString("IndexCreator.PREINDEX_ADRESS_MAP"), allRelations); //$NON-NLS-1$ progress.startTask(Messages.getString("IndexCreator.PREINDEX_ADRESS_MAP"), accessor.getAllRelations()); //$NON-NLS-1$
allRelations = accessor.iterateOverEntities(progress, EntityType.RELATION, allRelations, new OsmDbVisitor() { accessor.iterateOverEntities(progress, EntityType.RELATION, new OsmDbVisitor() {
@Override @Override
public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException { public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException {
if (indexAddress) { if (indexAddress) {
@ -346,11 +426,11 @@ public class IndexCreator {
}); });
if (indexAddress) { if (indexAddress) {
progress.setGeneralProgress("[40 / 100]"); //$NON-NLS-1$ progress.setGeneralProgress("[40 / 100]"); //$NON-NLS-1$
progress.startTask(Messages.getString("IndexCreator.PREINDEX_ADRESS_MAP"), allWays); //$NON-NLS-1$ progress.startTask(Messages.getString("IndexCreator.PREINDEX_ADRESS_MAP"), accessor.getAllWays()); //$NON-NLS-1$
allWays = accessor.iterateOverEntities(progress, EntityType.WAY, allWays, new OsmDbVisitor() { accessor.iterateOverEntities(progress, EntityType.WAY, new OsmDbVisitor() {
@Override @Override
public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException { public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException {
indexAddressCreator.indexBoundariesRelation(e ,ctx); indexAddressCreator.indexBoundariesRelation(e, ctx);
} }
}); });
@ -362,16 +442,16 @@ public class IndexCreator {
// 3.3 MAIN iterate over all entities // 3.3 MAIN iterate over all entities
if (indexPOI || indexAddress || indexMap) { if (indexPOI || indexAddress || indexMap) {
progress.setGeneralProgress("[50 / 100]"); progress.setGeneralProgress("[50 / 100]");
progress.startTask(Messages.getString("IndexCreator.PROCESS_OSM_NODES"), allNodes); progress.startTask(Messages.getString("IndexCreator.PROCESS_OSM_NODES"), accessor.getAllNodes());
accessor.iterateOverEntities(progress, EntityType.NODE, allNodes, new OsmDbVisitor() { accessor.iterateOverEntities(progress, EntityType.NODE, new OsmDbVisitor() {
@Override @Override
public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException { public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException {
iterateMainEntity(e, ctx); iterateMainEntity(e, ctx);
} }
}); });
progress.setGeneralProgress("[70 / 100]"); progress.setGeneralProgress("[70 / 100]");
progress.startTask(Messages.getString("IndexCreator.PROCESS_OSM_WAYS"), allWays); progress.startTask(Messages.getString("IndexCreator.PROCESS_OSM_WAYS"), accessor.getAllWays());
accessor.iterateOverEntities(progress, EntityType.WAY, allWays, new OsmDbVisitor() { accessor.iterateOverEntities(progress, EntityType.WAY, new OsmDbVisitor() {
@Override @Override
public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException { public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException {
iterateMainEntity(e, ctx); iterateMainEntity(e, ctx);
@ -379,8 +459,8 @@ public class IndexCreator {
}); });
} }
progress.setGeneralProgress("[85 / 100]"); progress.setGeneralProgress("[85 / 100]");
progress.startTask(Messages.getString("IndexCreator.PROCESS_OSM_REL"), allRelations); progress.startTask(Messages.getString("IndexCreator.PROCESS_OSM_REL"), accessor.getAllRelations());
accessor.iterateOverEntities(progress, EntityType.RELATION, allRelations, new OsmDbVisitor() { accessor.iterateOverEntities(progress, EntityType.RELATION, new OsmDbVisitor() {
@Override @Override
public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException { public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException {
iterateMainEntity(e, ctx); iterateMainEntity(e, ctx);
@ -388,7 +468,7 @@ public class IndexCreator {
}); });
// 3.4 combine all low level ways and simplify them // 3.4 combine all low level ways and simplify them
if(indexMap){ if (indexMap) {
progress.setGeneralProgress("[90 / 100]"); progress.setGeneralProgress("[90 / 100]");
progress.startTask(Messages.getString("IndexCreator.INDEX_LO_LEVEL_WAYS"), indexMapCreator.getLowLevelWays()); progress.startTask(Messages.getString("IndexCreator.INDEX_LO_LEVEL_WAYS"), indexMapCreator.getLowLevelWays());
indexMapCreator.processingLowLevelWays(progress); indexMapCreator.processingLowLevelWays(progress);
@ -462,10 +542,10 @@ public class IndexCreator {
indexPoiCreator.commitAndClosePoiFile(lastModifiedDate); indexPoiCreator.commitAndClosePoiFile(lastModifiedDate);
indexAddressCreator.closeAllPreparedStatements(); indexAddressCreator.closeAllPreparedStatements();
indexTransportCreator.commitAndCloseFiles(getRTreeTransportStopsFileName(), indexTransportCreator.commitAndCloseFiles(getRTreeTransportStopsFileName(), getRTreeTransportStopsPackFileName(),
getRTreeTransportStopsPackFileName(), deleteDatabaseIndexes); deleteDatabaseIndexes);
indexMapCreator.commitAndCloseFiles(getRTreeMapIndexNonPackFileName(), indexMapCreator.commitAndCloseFiles(getRTreeMapIndexNonPackFileName(), getRTreeMapIndexPackFileName(),
getRTreeMapIndexPackFileName(), deleteDatabaseIndexes); deleteDatabaseIndexes);
if (mapConnection != null) { if (mapConnection != null) {
mapConnection.commit(); mapConnection.commit();
@ -478,7 +558,6 @@ public class IndexCreator {
} }
} }
// do not delete first db connection // do not delete first db connection
if (dbConn != null) { if (dbConn != null) {
if (DBDialect.H2 == dialect) { if (DBDialect.H2 == dialect) {
@ -503,88 +582,6 @@ public class IndexCreator {
} }
private OsmDbCreator extractOsmToNodesDB(File readFile, IProgress progress, IOsmStorageFilter addFilter) throws FileNotFoundException,
IOException, SQLException, SAXException {
boolean pbfFile = false;
InputStream stream = new BufferedInputStream(new FileInputStream(readFile), 8192*4);;
InputStream streamFile = stream;
long st = System.currentTimeMillis();
if (readFile.getName().endsWith(".bz2")) { //$NON-NLS-1$
if (stream.read() != 'B' || stream.read() != 'Z') {
throw new RuntimeException("The source stream must start with the characters BZ if it is to be read as a BZip2 stream."); //$NON-NLS-1$
} else {
stream = new CBZip2InputStream(stream);
}
} else if (readFile.getName().endsWith(".pbf")) { //$NON-NLS-1$
pbfFile = true;
}
OsmBaseStorage storage = new OsmBaseStorage();
storage.setSupressWarnings(DataExtractionSettings.getSettings().isSupressWarningsForDuplicatedId());
if (addFilter != null) {
storage.getFilters().add(addFilter);
}
// 1. Loading osm file
OsmDbCreator filter = new OsmDbCreator(this);
try {
// 1 init database to store temporary data
filter.initDatabase(dialect, dbConn);
storage.getFilters().add(filter);
if (pbfFile) {
storage.parseOSMPbf(stream, progress, false);
} else {
storage.parseOSM(stream, progress, streamFile, false);
}
filter.finishLoading();
dbConn.commit();
if (log.isInfoEnabled()) {
log.info("File parsed : " + (System.currentTimeMillis() - st)); //$NON-NLS-1$
}
progress.finishTask();
return filter;
} finally {
if (log.isInfoEnabled()) {
log.info("File indexed : " + (System.currentTimeMillis() - st)); //$NON-NLS-1$
}
}
}
private void createDatabaseIndexesStructure() throws SQLException, IOException {
// 2.1 create temporary sqlite database to put temporary results to it
if (indexMap || indexAddress || indexTransport) {
mapFile = new File(workingDir, getMapFileName());
// to save space
mapFile.getParentFile().mkdirs();
File tempDBMapFile = new File(workingDir, getTempMapDBFileName());
dialect.removeDatabase(tempDBMapFile);
mapConnection = getDatabaseConnection(tempDBMapFile.getAbsolutePath());
mapConnection.setAutoCommit(false);
}
// 2.2 create rtree map
if (indexMap) {
indexMapCreator.createDatabaseStructure(mapConnection, dialect, getRTreeMapIndexNonPackFileName());
}
if (indexAddress) {
indexAddressCreator.createDatabaseStructure(mapConnection, dialect);
}
if (indexPOI) {
indexPoiCreator.createDatabaseStructure(new File(workingDir, getPoiFileName()));
}
if (indexTransport) {
indexTransportCreator.createTransportIndexStructure(mapConnection, dialect, getRTreeTransportStopsFileName());
}
}
public static void main(String[] args) throws IOException, SAXException, SQLException { public static void main(String[] args) throws IOException, SAXException, SQLException {
long time = System.currentTimeMillis(); long time = System.currentTimeMillis();

70
DataExtractionOSM/src/net/osmand/data/preparation/IndexPoiCreator.java
View file

@ -9,7 +9,6 @@ import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Map;
import net.osmand.Algoritms; import net.osmand.Algoritms;
import net.osmand.data.Amenity; import net.osmand.data.Amenity;
@ -47,13 +46,14 @@ public class IndexPoiCreator extends AbstractIndexPartCreator {
if (a.getLocation() != null) { if (a.getLocation() != null) {
// do not convert english name // do not convert english name
// convertEnglishName(a); // convertEnglishName(a);
insertAmenityIntoPoi(pStatements, a); insertAmenityIntoPoi(a);
} }
} }
} }
} }
public void commitAndClosePoiFile(Long lastModifiedDate) throws SQLException { public void commitAndClosePoiFile(Long lastModifiedDate) throws SQLException {
closeAllPreparedStatements();
if (poiConnection != null) { if (poiConnection != null) {
poiConnection.commit(); poiConnection.commit();
poiConnection.close(); poiConnection.close();
@ -62,34 +62,16 @@ public class IndexPoiCreator extends AbstractIndexPartCreator {
poiIndexFile.setLastModified(lastModifiedDate); poiIndexFile.setLastModified(lastModifiedDate);
} }
} }
closeAllPreparedStatements();
} }
public void insertAmenityIntoPoi( Map<PreparedStatement, Integer> map, Amenity amenity) throws SQLException { private void checkEntity(Entity e){
assert IndexConstants.POI_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
poiPreparedStatement.setLong(1, amenity.getId());
poiPreparedStatement.setInt(2, MapUtils.get31TileNumberX(amenity.getLocation().getLongitude()));
poiPreparedStatement.setInt(3, MapUtils.get31TileNumberY(amenity.getLocation().getLatitude()));
poiPreparedStatement.setString(4, amenity.getEnName());
poiPreparedStatement.setString(5, amenity.getName());
poiPreparedStatement.setString(6, AmenityType.valueToString(amenity.getType()));
poiPreparedStatement.setString(7, amenity.getSubType());
poiPreparedStatement.setString(8, amenity.getOpeningHours());
poiPreparedStatement.setString(9, amenity.getSite());
poiPreparedStatement.setString(10, amenity.getPhone());
addBatch(map, poiPreparedStatement, BATCH_SIZE);
}
public void checkEntity(Entity e){
String name = e.getTag(OSMTagKey.NAME); String name = e.getTag(OSMTagKey.NAME);
if (name == null){ if (name == null){
String msg = ""; String msg = "";
Collection<String> keys = e.getTagKeySet(); Collection<String> keys = e.getTagKeySet();
int cnt = 0; int cnt = 0;
for (Iterator iter = keys.iterator(); iter.hasNext();) { for (Iterator<String> iter = keys.iterator(); iter.hasNext();) {
String key = (String) iter.next(); String key = iter.next();
if (key.startsWith("name:") && key.length() <= 8) { if (key.startsWith("name:") && key.length() <= 8) {
// ignore specialties like name:botanical // ignore specialties like name:botanical
if (cnt == 0) if (cnt == 0)
@ -105,23 +87,35 @@ public class IndexPoiCreator extends AbstractIndexPartCreator {
} }
} }
private void insertAmenityIntoPoi(Amenity amenity) throws SQLException {
assert IndexConstants.POI_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
poiPreparedStatement.setLong(1, amenity.getId());
poiPreparedStatement.setInt(2, MapUtils.get31TileNumberX(amenity.getLocation().getLongitude()));
poiPreparedStatement.setInt(3, MapUtils.get31TileNumberY(amenity.getLocation().getLatitude()));
poiPreparedStatement.setString(4, amenity.getEnName());
poiPreparedStatement.setString(5, amenity.getName());
poiPreparedStatement.setString(6, AmenityType.valueToString(amenity.getType()));
poiPreparedStatement.setString(7, amenity.getSubType());
poiPreparedStatement.setString(8, amenity.getOpeningHours());
poiPreparedStatement.setString(9, amenity.getSite());
poiPreparedStatement.setString(10, amenity.getPhone());
addBatch(poiPreparedStatement);
}
public void createDatabaseStructure(File poiIndexFile) throws SQLException { public void createDatabaseStructure(File poiIndexFile) throws SQLException {
this.poiIndexFile = poiIndexFile; this.poiIndexFile = poiIndexFile;
// to save space // delete previous file to save space
if (poiIndexFile.exists()) { if (poiIndexFile.exists()) {
Algoritms.removeAllFiles(poiIndexFile); Algoritms.removeAllFiles(poiIndexFile);
} }
poiIndexFile.getParentFile().mkdirs(); poiIndexFile.getParentFile().mkdirs();
// creating nodes db to fast access for all nodes // creating connection
poiConnection = DBDialect.SQLITE.getDatabaseConnection(poiIndexFile.getAbsolutePath(), log); poiConnection = DBDialect.SQLITE.getDatabaseConnection(poiIndexFile.getAbsolutePath(), log);
createPoiIndexStructure(poiConnection);
poiPreparedStatement = createStatementAmenityInsert(poiConnection);
pStatements.put(poiPreparedStatement, 0);
poiConnection.setAutoCommit(false);
}
public void createPoiIndexStructure(Connection conn) throws SQLException{ // create database structure
Statement stat = conn.createStatement(); Statement stat = poiConnection.createStatement();
stat.executeUpdate("create table " + IndexConstants.POI_TABLE + //$NON-NLS-1$ stat.executeUpdate("create table " + IndexConstants.POI_TABLE + //$NON-NLS-1$
"(id bigint, x int, y int, name_en varchar(255), name varchar(255), " + "(id bigint, x int, y int, name_en varchar(255), name varchar(255), " +
"type varchar(255), subtype varchar(255), opening_hours varchar(255), phone varchar(255), site varchar(255)," + "type varchar(255), subtype varchar(255), opening_hours varchar(255), phone varchar(255), site varchar(255)," +
@ -130,11 +124,15 @@ public class IndexPoiCreator extends AbstractIndexPartCreator {
stat.executeUpdate("create index poi_id on poi (id, type, subtype)"); stat.executeUpdate("create index poi_id on poi (id, type, subtype)");
stat.execute("PRAGMA user_version = " + IndexConstants.POI_TABLE_VERSION); //$NON-NLS-1$ stat.execute("PRAGMA user_version = " + IndexConstants.POI_TABLE_VERSION); //$NON-NLS-1$
stat.close(); stat.close();
}
public PreparedStatement createStatementAmenityInsert(Connection conn) throws SQLException{ // create prepared statment
return conn.prepareStatement("INSERT INTO " + IndexConstants.POI_TABLE + "(id, x, y, name_en, name, type, subtype, opening_hours, site, phone) " + //$NON-NLS-1$//$NON-NLS-2$ poiPreparedStatement = poiConnection
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); .prepareStatement("INSERT INTO " + IndexConstants.POI_TABLE + "(id, x, y, name_en, name, type, subtype, opening_hours, site, phone) " + //$NON-NLS-1$//$NON-NLS-2$
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
pStatements.put(poiPreparedStatement, 0);
poiConnection.setAutoCommit(false);
} }
} }

190
DataExtractionOSM/src/net/osmand/data/preparation/IndexTransportCreator.java
View file

@ -73,7 +73,72 @@ public class IndexTransportCreator extends AbstractIndexPartCreator {
transportStopsTree = new RTree(rtreeTransportStopFile); transportStopsTree = new RTree(rtreeTransportStopFile);
} }
public void createTransportIndexStructure(Connection conn, DBDialect dialect, String rtreeStopsFileName) throws SQLException, IOException{ public void writeBinaryTransportTree(rtree.Node parent, RTree r, BinaryMapIndexWriter writer,
PreparedStatement selectTransportStop, PreparedStatement selectTransportRouteStop,
Map<Long, Long> transportRoutes, Map<String, Integer> stringTable) throws IOException, RTreeException, SQLException {
Element[] e = parent.getAllElements();
List<Long> routes = null;
for (int i = 0; i < parent.getTotalElements(); i++) {
Rect re = e[i].getRect();
if (e[i].getElementType() == rtree.Node.LEAF_NODE) {
long id = ((LeafElement) e[i]).getPtr();
selectTransportStop.setLong(1, id);
selectTransportRouteStop.setLong(1, id);
ResultSet rs = selectTransportStop.executeQuery();
if (rs.next()) {
int x24 = (int) MapUtils.getTileNumberX(24, rs.getDouble(3));
int y24 = (int) MapUtils.getTileNumberY(24, rs.getDouble(2));
String name = rs.getString(4);
String nameEn = rs.getString(5);
if (nameEn != null && nameEn.equals(Junidecode.unidecode(name))) {
nameEn = null;
}
ResultSet rset = selectTransportRouteStop.executeQuery();
if (routes == null) {
routes = new ArrayList<Long>();
} else {
routes.clear();
}
while (rset.next()) {
Long route = transportRoutes.get(rset.getLong(1));
if (route == null) {
log.error("Something goes wrong with transport route id = " + rset.getLong(1)); //$NON-NLS-1$
} else {
routes.add(route);
}
}
rset.close();
writer.writeTransportStop(id, x24, y24, name, nameEn, stringTable, routes);
} else {
log.error("Something goes wrong with transport id = " + id); //$NON-NLS-1$
}
} else {
long ptr = ((NonLeafElement) e[i]).getPtr();
rtree.Node ns = r.getReadNode(ptr);
writer.startTransportTreeElement(re.getMinX(), re.getMaxX(), re.getMinY(), re.getMaxY());
writeBinaryTransportTree(ns, r, writer, selectTransportStop, selectTransportRouteStop, transportRoutes, stringTable);
writer.endWriteTransportTreeElement();
}
}
}
public void packRTree(String rtreeTransportStopsFileName, String rtreeTransportStopsPackFileName) throws IOException {
transportStopsTree = packRtreeFile(transportStopsTree, rtreeTransportStopsFileName, rtreeTransportStopsPackFileName);
}
public void visitEntityMainStep(Entity e, OsmDbAccessorContext ctx) throws SQLException {
if (e instanceof Relation && e.getTag(OSMTagKey.ROUTE) != null) {
ctx.loadEntityData(e, true);
TransportRoute route = indexTransportRoute((Relation) e);
if (route != null) {
insertTransportIntoIndex(route);
}
}
}
public void createDatabaseStructure(Connection conn, DBDialect dialect, String rtreeStopsFileName) throws SQLException, IOException{
Statement stat = conn.createStatement(); Statement stat = conn.createStatement();
stat.executeUpdate("create table transport_route (id bigint primary key, type varchar(255), operator varchar(255)," + stat.executeUpdate("create table transport_route (id bigint primary key, type varchar(255), operator varchar(255)," +
@ -111,37 +176,18 @@ public class IndexTransportCreator extends AbstractIndexPartCreator {
} }
public void packRTree(String rtreeTransportStopsFileName, String rtreeTransportStopsPackFileName) throws IOException { private void insertTransportIntoIndex(TransportRoute route) throws SQLException {
transportStopsTree = packRtreeFile(transportStopsTree, rtreeTransportStopsFileName, rtreeTransportStopsPackFileName); transRouteStat.setLong(1, route.getId());
} transRouteStat.setString(2, route.getType());
transRouteStat.setString(3, route.getOperator());
transRouteStat.setString(4, route.getRef());
transRouteStat.setString(5, route.getName());
transRouteStat.setString(6, route.getEnName());
transRouteStat.setInt(7, route.getAvgBothDistance());
addBatch(transRouteStat);
public void visitEntityMainStep(Entity e, OsmDbAccessorContext ctx) throws SQLException { writeRouteStops(route, route.getForwardStops(), true);
if (e instanceof Relation && e.getTag(OSMTagKey.ROUTE) != null) { writeRouteStops(route, route.getBackwardStops(), false);
ctx.loadEntityData(e, true);
TransportRoute route = indexTransportRoute((Relation) e);
if (route != null) {
insertTransportIntoIndex(transRouteStat, transRouteStopsStat, transStopsStat, transportStopsTree,
visitedStops, route, pStatements, BATCH_SIZE);
}
}
}
private void insertTransportIntoIndex(PreparedStatement prepRoute, PreparedStatement prepRouteStops,
PreparedStatement prepStops, RTree transportStopsTree,
Set<Long> writtenStops, TransportRoute route, Map<PreparedStatement, Integer> statements,
int batchSize) throws SQLException {
prepRoute.setLong(1, route.getId());
prepRoute.setString(2, route.getType());
prepRoute.setString(3, route.getOperator());
prepRoute.setString(4, route.getRef());
prepRoute.setString(5, route.getName());
prepRoute.setString(6, route.getEnName());
prepRoute.setInt(7, route.getAvgBothDistance());
addBatch(statements, prepRoute);
writeRouteStops(transportStopsTree, prepRouteStops, prepStops, statements, writtenStops, route, route.getForwardStops(), true);
writeRouteStops(transportStopsTree, prepRouteStops, prepStops, statements, writtenStops, route, route.getBackwardStops(), false);
} }
@ -152,19 +198,18 @@ public class IndexTransportCreator extends AbstractIndexPartCreator {
return conn.prepareStatement("insert into transport_route_stop(route, stop, direction, ord) values(?, ?, ?, ?)"); return conn.prepareStatement("insert into transport_route_stop(route, stop, direction, ord) values(?, ?, ?, ?)");
} }
private void writeRouteStops(RTree transportStopsTree, PreparedStatement prepRouteStops, PreparedStatement prepStops, Map<PreparedStatement, Integer> count, private void writeRouteStops(TransportRoute r, List<TransportStop> stops, boolean direction) throws SQLException {
Set<Long> writtenStops, TransportRoute r, List<TransportStop> stops, boolean direction) throws SQLException {
int i = 0; int i = 0;
for(TransportStop s : stops){ for(TransportStop s : stops){
if (!writtenStops.contains(s.getId())) { if (!visitedStops.contains(s.getId())) {
prepStops.setLong(1, s.getId()); transStopsStat.setLong(1, s.getId());
prepStops.setDouble(2, s.getLocation().getLatitude()); transStopsStat.setDouble(2, s.getLocation().getLatitude());
prepStops.setDouble(3, s.getLocation().getLongitude()); transStopsStat.setDouble(3, s.getLocation().getLongitude());
prepStops.setString(4, s.getName()); transStopsStat.setString(4, s.getName());
prepStops.setString(5, s.getEnName()); transStopsStat.setString(5, s.getEnName());
int x = (int) MapUtils.getTileNumberX(24, s.getLocation().getLongitude()); int x = (int) MapUtils.getTileNumberX(24, s.getLocation().getLongitude());
int y = (int) MapUtils.getTileNumberY(24, s.getLocation().getLatitude()); int y = (int) MapUtils.getTileNumberY(24, s.getLocation().getLatitude());
addBatch(count, prepStops); addBatch(transStopsStat);
try { try {
transportStopsTree.insert(new LeafElement(new Rect(x, y, x, y), s.getId())); transportStopsTree.insert(new LeafElement(new Rect(x, y, x, y), s.getId()));
} catch (RTreeInsertException e) { } catch (RTreeInsertException e) {
@ -172,13 +217,13 @@ public class IndexTransportCreator extends AbstractIndexPartCreator {
} catch (IllegalValueException e) { } catch (IllegalValueException e) {
throw new IllegalArgumentException(e); throw new IllegalArgumentException(e);
} }
writtenStops.add(s.getId()); visitedStops.add(s.getId());
} }
prepRouteStops.setLong(1, r.getId()); transRouteStopsStat.setLong(1, r.getId());
prepRouteStops.setLong(2, s.getId()); transRouteStopsStat.setLong(2, s.getId());
prepRouteStops.setInt(3, direction ? 1 : 0); transRouteStopsStat.setInt(3, direction ? 1 : 0);
prepRouteStops.setInt(4, i++); transRouteStopsStat.setInt(4, i++);
addBatch(count, prepRouteStops); addBatch(transRouteStopsStat);
} }
} }
@ -282,8 +327,7 @@ public class IndexTransportCreator extends AbstractIndexPartCreator {
throw new IllegalStateException(e); throw new IllegalStateException(e);
} }
} }
private Rect calcBounds(rtree.Node n) {
public Rect calcBounds(rtree.Node n) {
Rect r = null; Rect r = null;
Element[] e = n.getAllElements(); Element[] e = n.getAllElements();
for (int i = 0; i < n.getTotalElements(); i++) { for (int i = 0; i < n.getTotalElements(); i++) {
@ -340,57 +384,9 @@ public class IndexTransportCreator extends AbstractIndexPartCreator {
} }
public void writeBinaryTransportTree(rtree.Node parent, RTree r, BinaryMapIndexWriter writer,
PreparedStatement selectTransportStop, PreparedStatement selectTransportRouteStop,
Map<Long, Long> transportRoutes, Map<String, Integer> stringTable) throws IOException, RTreeException, SQLException {
Element[] e = parent.getAllElements();
List<Long> routes = null;
for (int i = 0; i < parent.getTotalElements(); i++) {
Rect re = e[i].getRect();
if (e[i].getElementType() == rtree.Node.LEAF_NODE) {
long id = ((LeafElement) e[i]).getPtr();
selectTransportStop.setLong(1, id);
selectTransportRouteStop.setLong(1, id);
ResultSet rs = selectTransportStop.executeQuery();
if (rs.next()) {
int x24 = (int) MapUtils.getTileNumberX(24, rs.getDouble(3));
int y24 = (int) MapUtils.getTileNumberY(24, rs.getDouble(2));
String name = rs.getString(4);
String nameEn = rs.getString(5);
if (nameEn != null && nameEn.equals(Junidecode.unidecode(name))) {
nameEn = null;
}
ResultSet rset = selectTransportRouteStop.executeQuery();
if (routes == null) {
routes = new ArrayList<Long>();
} else {
routes.clear();
}
while (rset.next()) {
Long route = transportRoutes.get(rset.getLong(1));
if (route == null) {
log.error("Something goes wrong with transport route id = " + rset.getLong(1)); //$NON-NLS-1$
} else {
routes.add(route);
}
}
rset.close();
writer.writeTransportStop(id, x24, y24, name, nameEn, stringTable, routes);
} else {
log.error("Something goes wrong with transport id = " + id); //$NON-NLS-1$
}
} else {
long ptr = ((NonLeafElement) e[i]).getPtr();
rtree.Node ns = r.getReadNode(ptr);
writer.startTransportTreeElement(re.getMinX(), re.getMaxX(), re.getMinY(), re.getMaxY());
writeBinaryTransportTree(ns, r, writer, selectTransportStop, selectTransportRouteStop, transportRoutes, stringTable);
writer.endWriteTransportTreeElement();
}
}
}
public TransportRoute indexTransportRoute(Relation rel) { private TransportRoute indexTransportRoute(Relation rel) {
String ref = rel.getTag(OSMTagKey.REF); String ref = rel.getTag(OSMTagKey.REF);
String route = rel.getTag(OSMTagKey.ROUTE); String route = rel.getTag(OSMTagKey.ROUTE);
String operator = rel.getTag(OSMTagKey.OPERATOR); String operator = rel.getTag(OSMTagKey.OPERATOR);

108
DataExtractionOSM/src/net/osmand/data/preparation/IndexVectorMapCreator.java
View file

@ -71,42 +71,11 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator {
} }
public void indexMapRelationsAndMultiPolygons(Entity e, OsmDbAccessorContext ctx) throws SQLException { public void indexMapRelationsAndMultiPolygons(Entity e, OsmDbAccessorContext ctx) throws SQLException {
if (e instanceof Relation && "restriction".equals(e.getTag(OSMTagKey.TYPE))) { //$NON-NLS-1$ indexHighwayRestrictions(e, ctx);
String val = e.getTag("restriction"); //$NON-NLS-1$ indexMultiPolygon(e, ctx);
if (val != null) { }
byte type = -1;
if ("no_right_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$ private void indexMultiPolygon(Entity e, OsmDbAccessorContext ctx) throws SQLException {
type = MapRenderingTypes.RESTRICTION_NO_RIGHT_TURN;
} else if ("no_left_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_NO_LEFT_TURN;
} else if ("no_u_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_NO_U_TURN;
} else if ("no_straight_on".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_NO_STRAIGHT_ON;
} else if ("only_right_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_ONLY_RIGHT_TURN;
} else if ("only_left_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_ONLY_LEFT_TURN;
} else if ("only_straight_on".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_ONLY_STRAIGHT_ON;
}
if (type != -1) {
ctx.loadEntityData(e, true);
Collection<EntityId> fromL = ((Relation) e).getMemberIds("from"); //$NON-NLS-1$
Collection<EntityId> toL = ((Relation) e).getMemberIds("to"); //$NON-NLS-1$
if (!fromL.isEmpty() && !toL.isEmpty()) {
EntityId from = fromL.iterator().next();
EntityId to = toL.iterator().next();
if (from.getType() == EntityType.WAY) {
if (!highwayRestrictions.containsKey(from.getId())) {
highwayRestrictions.put(from.getId(), new ArrayList<Long>(4));
}
highwayRestrictions.get(from.getId()).add((to.getId() << 3) | (long) type);
}
}
}
}
}
if (e instanceof Relation && "multipolygon".equals(e.getTag(OSMTagKey.TYPE))) { //$NON-NLS-1$ if (e instanceof Relation && "multipolygon".equals(e.getTag(OSMTagKey.TYPE))) { //$NON-NLS-1$
ctx.loadEntityData(e, true); ctx.loadEntityData(e, true);
Map<Entity, String> entities = ((Relation) e).getMemberEntities(); Map<Entity, String> entities = ((Relation) e).getMemberEntities();
@ -184,7 +153,47 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator {
} }
} }
}
private void indexHighwayRestrictions(Entity e, OsmDbAccessorContext ctx) throws SQLException {
if (e instanceof Relation && "restriction".equals(e.getTag(OSMTagKey.TYPE))) { //$NON-NLS-1$
String val = e.getTag("restriction"); //$NON-NLS-1$
if (val != null) {
byte type = -1;
if ("no_right_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_NO_RIGHT_TURN;
} else if ("no_left_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_NO_LEFT_TURN;
} else if ("no_u_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_NO_U_TURN;
} else if ("no_straight_on".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_NO_STRAIGHT_ON;
} else if ("only_right_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_ONLY_RIGHT_TURN;
} else if ("only_left_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_ONLY_LEFT_TURN;
} else if ("only_straight_on".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_ONLY_STRAIGHT_ON;
}
if (type != -1) {
ctx.loadEntityData(e, true);
Collection<EntityId> fromL = ((Relation) e).getMemberIds("from"); //$NON-NLS-1$
Collection<EntityId> toL = ((Relation) e).getMemberIds("to"); //$NON-NLS-1$
if (!fromL.isEmpty() && !toL.isEmpty()) {
EntityId from = fromL.iterator().next();
EntityId to = toL.iterator().next();
if (from.getType() == EntityType.WAY) {
if (!highwayRestrictions.containsKey(from.getId())) {
highwayRestrictions.put(from.getId(), new ArrayList<Long>(4));
}
highwayRestrictions.get(from.getId()).add((to.getId() << 3) | (long) type);
}
}
}
}
}
} }
private void putMultipolygonType(Map<Long, Set<Integer>> multiPolygonsWays, long baseId, int mtType, boolean inverse) { private void putMultipolygonType(Map<Long, Set<Integer>> multiPolygonsWays, long baseId, int mtType, boolean inverse) {
@ -220,7 +229,7 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator {
} }
public void combineMultiPolygons(Way w, List<List<Way>> completedRings, List<List<Way>> incompletedRings) { private void combineMultiPolygons(Way w, List<List<Way>> completedRings, List<List<Way>> incompletedRings) {
long lId = w.getEntityIds().get(w.getEntityIds().size() - 1).getId().longValue(); long lId = w.getEntityIds().get(w.getEntityIds().size() - 1).getId().longValue();
long fId = w.getEntityIds().get(0).getId().longValue(); long fId = w.getEntityIds().get(0).getId().longValue();
if (fId == lId) { if (fId == lId) {
@ -263,6 +272,8 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator {
} }
} }
private void writeBinaryEntityToMapDatabase(Entity e, long baseId, boolean inverse, int level) throws SQLException { private void writeBinaryEntityToMapDatabase(Entity e, long baseId, boolean inverse, int level) throws SQLException {
int type = renderingTypes.encodeEntityWithType(e, mapZooms.getLevel(level).getMaxZoom(), false, typeUse); int type = renderingTypes.encodeEntityWithType(e, mapZooms.getLevel(level).getMaxZoom(), false, typeUse);
Map<Long, Set<Integer>> multiPolygonsWays = this.multiPolygonsWays[level]; Map<Long, Set<Integer>> multiPolygonsWays = this.multiPolygonsWays[level];
@ -329,13 +340,12 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator {
} }
if (!skip) { if (!skip) {
insertBinaryMapRenderObjectIndex(pStatements, mapBinaryStat, rtree, e, eName, id, type, typeUse, insertBinaryMapRenderObjectIndex(rtree, e, eName, id, type, typeUse, highwayAttributes, restrictionsUse, inverse, point, true);
highwayAttributes, restrictionsUse, inverse, point, true);
} }
} }
protected long encodeTypesToOneLong(int mainType) { private long encodeTypesToOneLong(int mainType) {
long i = 0; long i = 0;
int ind = 0; int ind = 0;
int sh = 0; int sh = 0;
@ -409,7 +419,7 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator {
return way; return way;
} else { } else {
lowLevelWays ++; lowLevelWays ++;
insertLowLevelMapBinaryObject(pStatements, mapLowLevelBinaryStat, level, longType, id, way.getNodes(), name); insertLowLevelMapBinaryObject(level, longType, id, way.getNodes(), name);
return null; return null;
} }
@ -485,7 +495,7 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator {
startNode = fs.getLong(2); startNode = fs.getLong(2);
visitedWays.add(lid); visitedWays.add(lid);
loadNodes(fs.getBytes(4), list); loadNodes(fs.getBytes(4), list);
ArrayList li = new ArrayList<Float>(list); ArrayList<Float> li = new ArrayList<Float>(list);
// remove first lat/lon point // remove first lat/lon point
wayNodes.remove(0); wayNodes.remove(0);
wayNodes.remove(0); wayNodes.remove(0);
@ -535,7 +545,7 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator {
MapAlgorithms.simplifyDouglasPeucker(wNodes, zoom - 1 + 8, 3, newWs); MapAlgorithms.simplifyDouglasPeucker(wNodes, zoom - 1 + 8, 3, newWs);
int type = decodeTypesFromOneLong(ltype); int type = decodeTypesFromOneLong(ltype);
insertBinaryMapRenderObjectIndex(pStatements, mapBinaryStat, mapTree[level], newWs, name, insertBinaryMapRenderObjectIndex(mapTree[level], newWs, name,
id, type, typeUse, 0, restrictionsUse, false, false, false); id, type, typeUse, 0, restrictionsUse, false, false, false);
} }
@ -568,6 +578,7 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator {
} }
@SuppressWarnings("unchecked")
public void initSettings(MapZooms mapZooms, MapRenderingTypes renderingTypes) { public void initSettings(MapZooms mapZooms, MapRenderingTypes renderingTypes) {
this.mapZooms = mapZooms; this.mapZooms = mapZooms;
this.renderingTypes = renderingTypes; this.renderingTypes = renderingTypes;
@ -723,8 +734,7 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator {
return conn.prepareStatement("insert into low_level_map_objects(id, start_node, end_node, name, nodes, type, level) values(?, ?, ?, ?, ?, ?, ?)"); return conn.prepareStatement("insert into low_level_map_objects(id, start_node, end_node, name, nodes, type, level) values(?, ?, ?, ?, ?, ?, ?)");
} }
private void insertLowLevelMapBinaryObject(Map<PreparedStatement, Integer> statements, private void insertLowLevelMapBinaryObject(int level,long types, long id, List<Node> nodes, String name) throws SQLException{
PreparedStatement mapLowLevelBinaryStat, int level,long types, long id, List<Node> nodes, String name) throws SQLException{
boolean first = true; boolean first = true;
long firstId = -1; long firstId = -1;
long lastId = -1; long lastId = -1;
@ -756,10 +766,10 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator {
mapLowLevelBinaryStat.setLong(6, types); mapLowLevelBinaryStat.setLong(6, types);
mapLowLevelBinaryStat.setShort(7, (short) level); mapLowLevelBinaryStat.setShort(7, (short) level);
addBatch(statements, mapLowLevelBinaryStat); addBatch(mapLowLevelBinaryStat);
} }
private void insertBinaryMapRenderObjectIndex(Map<PreparedStatement, Integer> statements,
PreparedStatement mapBinaryStat, RTree mapTree, Entity e, String name, private void insertBinaryMapRenderObjectIndex(RTree mapTree, Entity e, String name,
long id, int type, List<Integer> typeUse, int highwayAttributes, List<Long> restrictions, long id, int type, List<Integer> typeUse, int highwayAttributes, List<Long> restrictions,
boolean inversePath, boolean writeAsPoint, boolean commit) throws SQLException { boolean inversePath, boolean writeAsPoint, boolean commit) throws SQLException {
if(e instanceof Relation){ if(e instanceof Relation){
@ -824,7 +834,7 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator {
mapBinaryStat.setBytes(5, bnodes.toByteArray()); mapBinaryStat.setBytes(5, bnodes.toByteArray());
mapBinaryStat.setInt(6, highwayAttributes); mapBinaryStat.setInt(6, highwayAttributes);
addBatch(statements, mapBinaryStat, commit); addBatch(mapBinaryStat, commit);
try { try {
mapTree.insert(new LeafElement(new Rect(minX, minY, maxX, maxY), id)); mapTree.insert(new LeafElement(new Rect(minX, minY, maxX, maxY), id));
} catch (RTreeInsertException e1) { } catch (RTreeInsertException e1) {

34
DataExtractionOSM/src/net/osmand/data/preparation/OsmDbAccessor.java
View file

@ -18,24 +18,28 @@ import net.osmand.osm.Entity.EntityId;
import net.osmand.osm.Entity.EntityType; import net.osmand.osm.Entity.EntityType;
public class OsmDbAccessor implements OsmDbAccessorContext { public class OsmDbAccessor implements OsmDbAccessorContext {
private final IndexCreator indexCreator;
private PreparedStatement pselectNode; private PreparedStatement pselectNode;
private PreparedStatement pselectWay; private PreparedStatement pselectWay;
private PreparedStatement pselectRelation; private PreparedStatement pselectRelation;
private PreparedStatement pselectTags; private PreparedStatement pselectTags;
private int allRelations ;
private int allWays;
private int allNodes;
private Connection dbConn; private Connection dbConn;
public interface OsmDbVisitor { public interface OsmDbVisitor {
public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException; public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException;
} }
public OsmDbAccessor(IndexCreator indexCreator){ public OsmDbAccessor(){
this.indexCreator = indexCreator;
} }
public void initDatabase(Connection dbConn) throws SQLException { public void initDatabase(Connection dbConn, int allNodes, int allWays, int allRelations) throws SQLException {
this.dbConn = dbConn; this.dbConn = dbConn;
this.allNodes = allNodes;
this.allWays = allWays;
this.allRelations = allRelations;
pselectNode = dbConn.prepareStatement("select n.latitude, n.longitude, t.skeys, t.value from node n left join tags t on n.id = t.id and t.type = 0 where n.id = ?"); //$NON-NLS-1$ pselectNode = dbConn.prepareStatement("select n.latitude, n.longitude, t.skeys, t.value from node n left join tags t on n.id = t.id and t.type = 0 where n.id = ?"); //$NON-NLS-1$
pselectWay = dbConn.prepareStatement("select w.node, w.ord, t.skeys, t.value, n.latitude, n.longitude " + //$NON-NLS-1$ pselectWay = dbConn.prepareStatement("select w.node, w.ord, t.skeys, t.value, n.latitude, n.longitude " + //$NON-NLS-1$
"from ways w left join tags t on w.id = t.id and t.type = 1 and w.ord = 0 inner join node n on w.node = n.id " + //$NON-NLS-1$ "from ways w left join tags t on w.id = t.id and t.type = 1 and w.ord = 0 inner join node n on w.node = n.id " + //$NON-NLS-1$
@ -46,6 +50,18 @@ public class OsmDbAccessor implements OsmDbAccessorContext {
pselectTags = dbConn.prepareStatement("select skeys, value from tags where id = ? and type = ?"); //$NON-NLS-1$ pselectTags = dbConn.prepareStatement("select skeys, value from tags where id = ? and type = ?"); //$NON-NLS-1$
} }
public int getAllNodes() {
return allNodes;
}
public int getAllRelations() {
return allRelations;
}
public int getAllWays() {
return allWays;
}
public void loadEntityData(Entity e, boolean loadTags) throws SQLException { public void loadEntityData(Entity e, boolean loadTags) throws SQLException {
if (e instanceof Node || (e instanceof Way && !((Way) e).getNodes().isEmpty())) { if (e instanceof Node || (e instanceof Way && !((Way) e).getNodes().isEmpty())) {
// do not load tags for nodes inside way // do not load tags for nodes inside way
@ -153,8 +169,7 @@ public class OsmDbAccessor implements OsmDbAccessorContext {
} }
public int iterateOverEntities(IProgress progress, EntityType type, int allCount, public int iterateOverEntities(IProgress progress, EntityType type, OsmDbVisitor visitor) throws SQLException {
OsmDbVisitor visitor) throws SQLException {
Statement statement = dbConn.createStatement(); Statement statement = dbConn.createStatement();
String select; String select;
int count = 0; int count = 0;
@ -220,6 +235,13 @@ public class OsmDbAccessor implements OsmDbAccessorContext {
visitor.iterateEntity(prevEntity, this); visitor.iterateEntity(prevEntity, this);
} }
rs.close(); rs.close();
if(EntityType.NODE == type){
allNodes = count;
} else if(EntityType.WAY == type){
allWays = count;
} else if(EntityType.RELATION == type){
allRelations = count;
}
return count; return count;
} }

2
DataExtractionOSM/src/net/osmand/data/preparation/OsmDbCreator.java
View file

@ -99,8 +99,6 @@ public class OsmDbCreator implements IOsmStorageFilter {
@Override @Override
public boolean acceptEntityToLoad(OsmBaseStorage storage, EntityId entityId, Entity e) { public boolean acceptEntityToLoad(OsmBaseStorage storage, EntityId entityId, Entity e) {
// Register all city labelbs
indexCreator.registerCityIfNeeded(e);
// put all nodes into temporary db to get only required nodes after loading all data // put all nodes into temporary db to get only required nodes after loading all data
try { try {
if (e instanceof Node) { if (e instanceof Node) {