Refactoring IndexCreator (4)

This commit is contained in:
Victor Shcherb 2011-05-02 23:43:20 +02:00
parent a62cb50e13
commit c3f2194237
8 changed files with 432 additions and 418 deletions

View file

@ -41,28 +41,24 @@ public class AbstractIndexPartCreator {
}
}
protected void addBatch(Map<PreparedStatement, Integer> count, PreparedStatement p) throws SQLException {
addBatch(count, p, BATCH_SIZE, true);
protected void addBatch(PreparedStatement p) throws SQLException {
addBatch(p, BATCH_SIZE, true);
}
protected void addBatch(Map<PreparedStatement, Integer> count, PreparedStatement p, boolean commit) throws SQLException{
addBatch(count, p, BATCH_SIZE, commit);
protected void addBatch(PreparedStatement p, boolean commit) throws SQLException{
addBatch(p, BATCH_SIZE, commit);
}
protected void addBatch(Map<PreparedStatement, Integer> count, PreparedStatement p, int batchSize) throws SQLException{
addBatch(count, p, batchSize, true);
}
protected void addBatch(Map<PreparedStatement, Integer> count, PreparedStatement p, int batchSize, boolean commit) throws SQLException{
protected void addBatch(PreparedStatement p, int batchSize, boolean commit) throws SQLException{
p.addBatch();
if(count.get(p) >= batchSize){
if(pStatements.get(p) >= batchSize){
p.executeBatch();
if(commit){
p.getConnection().commit();
}
count.put(p, 0);
pStatements.put(p, 0);
} else {
count.put(p, count.get(p) + 1);
pStatements.put(p, pStatements.get(p) + 1);
}
}

View file

@ -190,7 +190,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
citiBoundaries.put(nCity, boundary);
cityVillageManager.registerObject(point.getLatitude(), point.getLongitude(), nCity);
writeCity(addressCityStat, pStatements, nCity);
writeCity(nCity);
// commit to put all cities
if (pStatements.get(addressCityStat) > 0) {
addressCityStat.executeBatch();
@ -270,7 +270,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
for (Map.Entry<Entity, String> r : i.getMemberEntities().entrySet()) {
if ("street".equals(r.getValue())) { //$NON-NLS-1$
if (r.getKey() instanceof Way && saveAddressWays) {
writeStreetWayNodes(addressStreetNodeStat, pStatements, streetId, (Way) r.getKey());
writeStreetWayNodes(streetId, (Way) r.getKey());
if (loadInMemory) {
addressStreetNodeLocalSet.add(r.getKey().getId());
}
@ -282,7 +282,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
if (hno != null) {
Building building = new Building(r.getKey());
building.setName(hno);
writeBuilding(addressBuildingStat, pStatements, streetId, building);
writeBuilding(streetId, building);
if (loadInMemory) {
addressBuildingLocalSet.add(r.getKey().getId());
}
@ -308,7 +308,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
Building building = new Building(border);
if (building.getLocation() != null) {
building.setName(hno);
writeBuilding(addressBuildingStat, pStatements, streetId, building);
writeBuilding(streetId, building);
if (loadInMemory) {
addressBuildingLocalSet.add(id.getId());
}
@ -424,7 +424,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
insertStreetData(addressStreetStat, initId, name, Junidecode.unidecode(name),
location.getLatitude(), location.getLongitude(), city.getId());
if (loadInMemory) {
addBatch(pStatements, addressStreetStat, BATCH_SIZE);
addBatch(addressStreetStat);
addressStreetLocalMap.put(name + "_" + city.getId(), initId); //$NON-NLS-1$
} else {
addressStreetStat.execute();
@ -509,7 +509,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
if (idStreet != null) {
Building building = new Building(e);
building.setName(e.getTag(OSMTagKey.ADDR_HOUSE_NUMBER));
writeBuilding(addressBuildingStat, pStatements, idStreet, building);
writeBuilding(idStreet, building);
}
}
} else if (e instanceof Way /* && OSMSettings.wayForCar(e.getTag(OSMTagKey.HIGHWAY)) */
@ -537,7 +537,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
City city = getClosestCity(l);
Long idStreet = getStreetInCity(city, e.getTag(OSMTagKey.NAME), l, (e.getId() << 2) | 1);
if (idStreet != null && saveAddressWays) {
writeStreetWayNodes(addressStreetNodeStat, pStatements, idStreet, (Way) e);
writeStreetWayNodes(idStreet, (Way) e);
}
}
}
@ -550,45 +550,42 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
}
private void writeStreetWayNodes(PreparedStatement prepStreetNode, Map<PreparedStatement, Integer> count, Long streetId, Way way)
private void writeStreetWayNodes(Long streetId, Way way)
throws SQLException {
for (Node n : way.getNodes()) {
if (n == null) {
continue;
}
prepStreetNode.setLong(1, n.getId());
prepStreetNode.setDouble(2, n.getLatitude());
prepStreetNode.setDouble(3, n.getLongitude());
prepStreetNode.setLong(5, way.getId());
prepStreetNode.setLong(4, streetId);
addBatch(count, prepStreetNode, BATCH_SIZE);
addressStreetNodeStat.setLong(1, n.getId());
addressStreetNodeStat.setDouble(2, n.getLatitude());
addressStreetNodeStat.setDouble(3, n.getLongitude());
addressStreetNodeStat.setLong(5, way.getId());
addressStreetNodeStat.setLong(4, streetId);
addBatch(addressStreetNodeStat);
}
}
private void writeBuilding(PreparedStatement prepBuilding, Map<PreparedStatement, Integer> count, Long streetId,
Building building)
throws SQLException {
prepBuilding.setLong(1, building.getId());
prepBuilding.setDouble(2, building.getLocation().getLatitude());
prepBuilding.setDouble(3, building.getLocation().getLongitude());
prepBuilding.setString(4, building.getName());
prepBuilding.setString(5, building.getEnName());
prepBuilding.setLong(6, streetId);
prepBuilding.setString(7, building.getPostcode() == null ? null : building.getPostcode().toUpperCase());
addBatch(count, prepBuilding);
private void writeBuilding(Long streetId, Building building) throws SQLException {
addressBuildingStat.setLong(1, building.getId());
addressBuildingStat.setDouble(2, building.getLocation().getLatitude());
addressBuildingStat.setDouble(3, building.getLocation().getLongitude());
addressBuildingStat.setString(4, building.getName());
addressBuildingStat.setString(5, building.getEnName());
addressBuildingStat.setLong(6, streetId);
addressBuildingStat.setString(7, building.getPostcode() == null ? null : building.getPostcode().toUpperCase());
addBatch(addressBuildingStat);
}
private void writeCity(PreparedStatement prepCity, Map<PreparedStatement, Integer> count, City city) throws SQLException {
prepCity.setLong(1, city.getId());
prepCity.setDouble(2, city.getLocation().getLatitude());
prepCity.setDouble(3, city.getLocation().getLongitude());
prepCity.setString(4, city.getName());
prepCity.setString(5, city.getEnName());
prepCity.setString(6, CityType.valueToString(city.getType()));
addBatch(count, prepCity, BATCH_SIZE);
private void writeCity(City city) throws SQLException {
addressCityStat.setLong(1, city.getId());
addressCityStat.setDouble(2, city.getLocation().getLatitude());
addressCityStat.setDouble(3, city.getLocation().getLongitude());
addressCityStat.setString(4, city.getName());
addressCityStat.setString(5, city.getEnName());
addressCityStat.setString(6, CityType.valueToString(city.getType()));
addBatch(addressCityStat);
}
@ -605,7 +602,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
public void writeCitiesIntoDb() throws SQLException {
for (City c : cities.values()) {
writeCity(addressCityStat, pStatements, c);
writeCity(c);
}
// commit to put all cities
if (pStatements.get(addressCityStat) > 0) {
@ -628,7 +625,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
for (EntityId l : r.getMemberIds()) {
pstat.setString(1, tag);
pstat.setLong(2, l.getId());
addBatch(pStatements, pstat, BATCH_SIZE);
addBatch(pstat);
}
}
if (pStatements.get(pstat) > 0) {

View file

@ -9,10 +9,7 @@ import java.io.InputStream;
import java.io.RandomAccessFile;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.LinkedHashMap;
import java.util.Map;
import net.osmand.Algoritms;
import net.osmand.IProgress;
@ -23,6 +20,7 @@ import net.osmand.impl.ConsoleProgressImplementation;
import net.osmand.osm.Entity;
import net.osmand.osm.MapRenderingTypes;
import net.osmand.osm.Relation;
import net.osmand.osm.Entity.EntityId;
import net.osmand.osm.Entity.EntityType;
import net.osmand.osm.io.IOsmStorageFilter;
import net.osmand.osm.io.OsmBaseStorage;
@ -85,8 +83,6 @@ public class IndexCreator {
private Connection dbConn;
private File dbFile;
Map<PreparedStatement, Integer> pStatements = new LinkedHashMap<PreparedStatement, Integer>();
private File mapFile;
private RandomAccessFile mapRAFile;
private Connection mapConnection;
@ -95,11 +91,6 @@ public class IndexCreator {
public IndexCreator(File workingDir) {
this.workingDir = workingDir;
this.indexTransportCreator = new IndexTransportCreator();
this.indexPoiCreator = new IndexPoiCreator();
this.indexAddressCreator = new IndexAddressCreator();
this.indexMapCreator = new IndexVectorMapCreator();
this.accessor = new OsmDbAccessor(this);
}
public void setIndexAddress(boolean indexAddress) {
@ -187,27 +178,6 @@ public class IndexCreator {
this.cityAdminLevel = cityAdminLevel;
}
private void iterateMainEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException {
if (indexPOI) {
indexPoiCreator.iterateEntity(e, ctx);
}
if (indexTransport) {
indexTransportCreator.visitEntityMainStep(e, ctx);
}
if (indexMap) {
indexMapCreator.iterateMainEntity(e, ctx);
}
if (indexAddress) {
indexAddressCreator.iterateMainEntity(e, ctx);
}
}
public void registerCityIfNeeded(Entity e) {
indexAddressCreator.registerCityIfNeeded(e);
}
public String getRTreeMapIndexNonPackFileName() {
return mapFile.getAbsolutePath() + ".rtree"; //$NON-NLS-1$
}
@ -224,13 +194,142 @@ public class IndexCreator {
return mapFile.getAbsolutePath() + ".prtree"; //$NON-NLS-1$
}
/* ***** END OF GETTERS/SETTERS ***** */
public void generateIndexes(File readFile, IProgress progress, IOsmStorageFilter addFilter) throws IOException, SAXException, SQLException{
generateIndexes(readFile, progress, addFilter, null, null);
}
public void generateIndexes(File readFile, IProgress progress, IOsmStorageFilter addFilter, MapZooms mapZooms, MapRenderingTypes renderingTypes)
throws IOException, SAXException,
SQLException {
private void iterateMainEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException {
if (indexPOI) {
indexPoiCreator.iterateEntity(e, ctx);
}
if (indexTransport) {
indexTransportCreator.visitEntityMainStep(e, ctx);
}
if (indexMap) {
indexMapCreator.iterateMainEntity(e, ctx);
}
if (indexAddress) {
indexAddressCreator.iterateMainEntity(e, ctx);
}
}
private OsmDbCreator extractOsmToNodesDB(File readFile, IProgress progress, IOsmStorageFilter addFilter) throws FileNotFoundException,
IOException, SQLException, SAXException {
boolean pbfFile = false;
InputStream stream = new BufferedInputStream(new FileInputStream(readFile), 8192 * 4);
InputStream streamFile = stream;
long st = System.currentTimeMillis();
if (readFile.getName().endsWith(".bz2")) { //$NON-NLS-1$
if (stream.read() != 'B' || stream.read() != 'Z') {
throw new RuntimeException("The source stream must start with the characters BZ if it is to be read as a BZip2 stream."); //$NON-NLS-1$
} else {
stream = new CBZip2InputStream(stream);
}
} else if (readFile.getName().endsWith(".pbf")) { //$NON-NLS-1$
pbfFile = true;
}
OsmBaseStorage storage = new OsmBaseStorage();
storage.setSupressWarnings(DataExtractionSettings.getSettings().isSupressWarningsForDuplicatedId());
if (addFilter != null) {
storage.getFilters().add(addFilter);
}
storage.getFilters().add(new IOsmStorageFilter() {
@Override
public boolean acceptEntityToLoad(OsmBaseStorage storage, EntityId entityId, Entity entity) {
indexAddressCreator.registerCityIfNeeded(entity);
// accept to allow db creator parse it
return true;
}
});
// 1. Loading osm file
OsmDbCreator dbCreator = new OsmDbCreator(this);
try {
progress.setGeneralProgress("[35 / 100]"); //$NON-NLS-1$
progress.startTask(Messages.getString("IndexCreator.LOADING_FILE") + readFile.getAbsolutePath(), -1); //$NON-NLS-1$
// 1 init database to store temporary data
dbCreator.initDatabase(dialect, dbConn);
storage.getFilters().add(dbCreator);
if (pbfFile) {
storage.parseOSMPbf(stream, progress, false);
} else {
storage.parseOSM(stream, progress, streamFile, false);
}
dbCreator.finishLoading();
dbConn.commit();
if (log.isInfoEnabled()) {
log.info("File parsed : " + (System.currentTimeMillis() - st)); //$NON-NLS-1$
}
progress.finishTask();
return dbCreator;
} finally {
if (log.isInfoEnabled()) {
log.info("File indexed : " + (System.currentTimeMillis() - st)); //$NON-NLS-1$
}
}
}
private boolean createPlainOsmDb(IProgress progress, File readFile, IOsmStorageFilter addFilter) throws SQLException, FileNotFoundException, IOException, SAXException{
// initialize db file
boolean loadFromExistingFile = dbFile != null && dialect.databaseFileExists(dbFile);
if (dbFile == null) {
dbFile = new File(workingDir, TEMP_NODES_DB);
// to save space
if (dialect.databaseFileExists(dbFile)) {
dialect.removeDatabase(dbFile);
}
}
dbConn = getDatabaseConnection(dbFile.getAbsolutePath());
int allRelations = 100000;
int allWays = 1000000;
int allNodes = 10000000;
if (!loadFromExistingFile) {
OsmDbCreator dbCreator = extractOsmToNodesDB(readFile, progress, addFilter);
if (dbCreator != null) {
allNodes = dbCreator.getAllNodes();
allWays = dbCreator.getAllWays();
allRelations = dbCreator.getAllRelations();
}
}
accessor.initDatabase(dbConn, allNodes, allWays, allRelations);
return loadFromExistingFile;
}
private void createDatabaseIndexesStructure() throws SQLException, IOException {
// 2.1 create temporary sqlite database to put temporary results to it
if (indexMap || indexAddress || indexTransport) {
mapFile = new File(workingDir, getMapFileName());
// to save space
mapFile.getParentFile().mkdirs();
File tempDBMapFile = new File(workingDir, getTempMapDBFileName());
dialect.removeDatabase(tempDBMapFile);
mapConnection = getDatabaseConnection(tempDBMapFile.getAbsolutePath());
mapConnection.setAutoCommit(false);
}
// 2.2 create rtree map
if (indexMap) {
indexMapCreator.createDatabaseStructure(mapConnection, dialect, getRTreeMapIndexNonPackFileName());
}
if (indexAddress) {
indexAddressCreator.createDatabaseStructure(mapConnection, dialect);
}
if (indexPOI) {
indexPoiCreator.createDatabaseStructure(new File(workingDir, getPoiFileName()));
}
if (indexTransport) {
indexTransportCreator.createDatabaseStructure(mapConnection, dialect, getRTreeTransportStopsFileName());
}
}
public void generateIndexes(File readFile, IProgress progress, IOsmStorageFilter addFilter, MapZooms mapZooms,
MapRenderingTypes renderingTypes) throws IOException, SAXException, SQLException {
if (renderingTypes == null) {
renderingTypes = MapRenderingTypes.getDefault();
}
@ -245,6 +344,12 @@ public class IndexCreator {
regionName = Algoritms.capitalizeFirstLetterAndLowercase(readFile.getName().substring(0, i));
}
}
this.indexTransportCreator = new IndexTransportCreator();
this.indexPoiCreator = new IndexPoiCreator();
this.indexAddressCreator = new IndexAddressCreator();
this.indexMapCreator = new IndexVectorMapCreator();
this.accessor = new OsmDbAccessor();
indexMapCreator.initSettings(mapZooms, renderingTypes);
@ -255,38 +360,13 @@ public class IndexCreator {
normalizeDefaultSuffixes = DataExtractionSettings.getSettings().getDefaultSuffixesToNormalizeStreets();
normalizeSuffixes = DataExtractionSettings.getSettings().getSuffixesToNormalizeStreets();
}
indexAddressCreator.initSettings(normalizeStreets, normalizeDefaultSuffixes, normalizeSuffixes,
saveAddressWays, cityAdminLevel);
indexAddressCreator.initSettings(normalizeStreets, normalizeDefaultSuffixes, normalizeSuffixes, saveAddressWays, cityAdminLevel);
// Main generation method
try {
// ////////////////////////////////////////////////////////////////////////
// 1. creating nodes db to fast access for all nodes and simply import all relations, ways, nodes to it
boolean loadFromPath = dbFile == null || !dialect.databaseFileExists(dbFile);
if (dbFile == null) {
dbFile = new File(workingDir, TEMP_NODES_DB);
// to save space
if (dialect.databaseFileExists(dbFile)) {
dialect.removeDatabase(dbFile);
}
}
dbConn = getDatabaseConnection(dbFile.getAbsolutePath());
int allRelations = 100000;
int allWays = 1000000;
int allNodes = 10000000;
if (loadFromPath) {
progress.setGeneralProgress("[35 / 100]"); //$NON-NLS-1$
progress.startTask(Messages.getString("IndexCreator.LOADING_FILE") + readFile.getAbsolutePath(), -1); //$NON-NLS-1$
OsmDbCreator filter = extractOsmToNodesDB(readFile, progress, addFilter);
if (filter != null) {
allNodes = filter.getAllNodes();
allWays = filter.getAllWays();
allRelations = filter.getAllRelations();
}
}
accessor.initDatabase(dbConn);
boolean loadFromExistingFile = createPlainOsmDb(progress, readFile, addFilter);
// do not create temp map file and rtree files
if (recreateOnlyBinaryFile) {
@ -311,14 +391,14 @@ public class IndexCreator {
createDatabaseIndexesStructure();
// 3. Processing all entries
// 3.1 write all cities
// 3.1 write all cities
if (indexAddress) {
progress.setGeneralProgress("[40 / 100]"); //$NON-NLS-1$
progress.startTask(Messages.getString("IndexCreator.INDEX_CITIES"), allNodes); //$NON-NLS-1$
if (!loadFromPath) {
progress.startTask(Messages.getString("IndexCreator.INDEX_CITIES"), accessor.getAllNodes()); //$NON-NLS-1$
if (loadFromExistingFile) {
// load cities names
allNodes = accessor.iterateOverEntities(progress, EntityType.NODE, allNodes, new OsmDbVisitor() {
accessor.iterateOverEntities(progress, EntityType.NODE, new OsmDbVisitor() {
@Override
public void iterateEntity(Entity e, OsmDbAccessorContext ctx) {
indexAddressCreator.registerCityIfNeeded(e);
@ -331,8 +411,8 @@ public class IndexCreator {
// 3.2 index address relations
if (indexAddress || indexMap) {
progress.setGeneralProgress("[30 / 100]"); //$NON-NLS-1$
progress.startTask(Messages.getString("IndexCreator.PREINDEX_ADRESS_MAP"), allRelations); //$NON-NLS-1$
allRelations = accessor.iterateOverEntities(progress, EntityType.RELATION, allRelations, new OsmDbVisitor() {
progress.startTask(Messages.getString("IndexCreator.PREINDEX_ADRESS_MAP"), accessor.getAllRelations()); //$NON-NLS-1$
accessor.iterateOverEntities(progress, EntityType.RELATION, new OsmDbVisitor() {
@Override
public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException {
if (indexAddress) {
@ -346,8 +426,8 @@ public class IndexCreator {
});
if (indexAddress) {
progress.setGeneralProgress("[40 / 100]"); //$NON-NLS-1$
progress.startTask(Messages.getString("IndexCreator.PREINDEX_ADRESS_MAP"), allWays); //$NON-NLS-1$
allWays = accessor.iterateOverEntities(progress, EntityType.WAY, allWays, new OsmDbVisitor() {
progress.startTask(Messages.getString("IndexCreator.PREINDEX_ADRESS_MAP"), accessor.getAllWays()); //$NON-NLS-1$
accessor.iterateOverEntities(progress, EntityType.WAY, new OsmDbVisitor() {
@Override
public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException {
indexAddressCreator.indexBoundariesRelation(e, ctx);
@ -362,16 +442,16 @@ public class IndexCreator {
// 3.3 MAIN iterate over all entities
if (indexPOI || indexAddress || indexMap) {
progress.setGeneralProgress("[50 / 100]");
progress.startTask(Messages.getString("IndexCreator.PROCESS_OSM_NODES"), allNodes);
accessor.iterateOverEntities(progress, EntityType.NODE, allNodes, new OsmDbVisitor() {
progress.startTask(Messages.getString("IndexCreator.PROCESS_OSM_NODES"), accessor.getAllNodes());
accessor.iterateOverEntities(progress, EntityType.NODE, new OsmDbVisitor() {
@Override
public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException {
iterateMainEntity(e, ctx);
}
});
progress.setGeneralProgress("[70 / 100]");
progress.startTask(Messages.getString("IndexCreator.PROCESS_OSM_WAYS"), allWays);
accessor.iterateOverEntities(progress, EntityType.WAY, allWays, new OsmDbVisitor() {
progress.startTask(Messages.getString("IndexCreator.PROCESS_OSM_WAYS"), accessor.getAllWays());
accessor.iterateOverEntities(progress, EntityType.WAY, new OsmDbVisitor() {
@Override
public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException {
iterateMainEntity(e, ctx);
@ -379,8 +459,8 @@ public class IndexCreator {
});
}
progress.setGeneralProgress("[85 / 100]");
progress.startTask(Messages.getString("IndexCreator.PROCESS_OSM_REL"), allRelations);
accessor.iterateOverEntities(progress, EntityType.RELATION, allRelations, new OsmDbVisitor() {
progress.startTask(Messages.getString("IndexCreator.PROCESS_OSM_REL"), accessor.getAllRelations());
accessor.iterateOverEntities(progress, EntityType.RELATION, new OsmDbVisitor() {
@Override
public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException {
iterateMainEntity(e, ctx);
@ -462,10 +542,10 @@ public class IndexCreator {
indexPoiCreator.commitAndClosePoiFile(lastModifiedDate);
indexAddressCreator.closeAllPreparedStatements();
indexTransportCreator.commitAndCloseFiles(getRTreeTransportStopsFileName(),
getRTreeTransportStopsPackFileName(), deleteDatabaseIndexes);
indexMapCreator.commitAndCloseFiles(getRTreeMapIndexNonPackFileName(),
getRTreeMapIndexPackFileName(), deleteDatabaseIndexes);
indexTransportCreator.commitAndCloseFiles(getRTreeTransportStopsFileName(), getRTreeTransportStopsPackFileName(),
deleteDatabaseIndexes);
indexMapCreator.commitAndCloseFiles(getRTreeMapIndexNonPackFileName(), getRTreeMapIndexPackFileName(),
deleteDatabaseIndexes);
if (mapConnection != null) {
mapConnection.commit();
@ -478,7 +558,6 @@ public class IndexCreator {
}
}
// do not delete first db connection
if (dbConn != null) {
if (DBDialect.H2 == dialect) {
@ -503,88 +582,6 @@ public class IndexCreator {
}
private OsmDbCreator extractOsmToNodesDB(File readFile, IProgress progress, IOsmStorageFilter addFilter) throws FileNotFoundException,
IOException, SQLException, SAXException {
boolean pbfFile = false;
InputStream stream = new BufferedInputStream(new FileInputStream(readFile), 8192*4);;
InputStream streamFile = stream;
long st = System.currentTimeMillis();
if (readFile.getName().endsWith(".bz2")) { //$NON-NLS-1$
if (stream.read() != 'B' || stream.read() != 'Z') {
throw new RuntimeException("The source stream must start with the characters BZ if it is to be read as a BZip2 stream."); //$NON-NLS-1$
} else {
stream = new CBZip2InputStream(stream);
}
} else if (readFile.getName().endsWith(".pbf")) { //$NON-NLS-1$
pbfFile = true;
}
OsmBaseStorage storage = new OsmBaseStorage();
storage.setSupressWarnings(DataExtractionSettings.getSettings().isSupressWarningsForDuplicatedId());
if (addFilter != null) {
storage.getFilters().add(addFilter);
}
// 1. Loading osm file
OsmDbCreator filter = new OsmDbCreator(this);
try {
// 1 init database to store temporary data
filter.initDatabase(dialect, dbConn);
storage.getFilters().add(filter);
if (pbfFile) {
storage.parseOSMPbf(stream, progress, false);
} else {
storage.parseOSM(stream, progress, streamFile, false);
}
filter.finishLoading();
dbConn.commit();
if (log.isInfoEnabled()) {
log.info("File parsed : " + (System.currentTimeMillis() - st)); //$NON-NLS-1$
}
progress.finishTask();
return filter;
} finally {
if (log.isInfoEnabled()) {
log.info("File indexed : " + (System.currentTimeMillis() - st)); //$NON-NLS-1$
}
}
}
private void createDatabaseIndexesStructure() throws SQLException, IOException {
// 2.1 create temporary sqlite database to put temporary results to it
if (indexMap || indexAddress || indexTransport) {
mapFile = new File(workingDir, getMapFileName());
// to save space
mapFile.getParentFile().mkdirs();
File tempDBMapFile = new File(workingDir, getTempMapDBFileName());
dialect.removeDatabase(tempDBMapFile);
mapConnection = getDatabaseConnection(tempDBMapFile.getAbsolutePath());
mapConnection.setAutoCommit(false);
}
// 2.2 create rtree map
if (indexMap) {
indexMapCreator.createDatabaseStructure(mapConnection, dialect, getRTreeMapIndexNonPackFileName());
}
if (indexAddress) {
indexAddressCreator.createDatabaseStructure(mapConnection, dialect);
}
if (indexPOI) {
indexPoiCreator.createDatabaseStructure(new File(workingDir, getPoiFileName()));
}
if (indexTransport) {
indexTransportCreator.createTransportIndexStructure(mapConnection, dialect, getRTreeTransportStopsFileName());
}
}
public static void main(String[] args) throws IOException, SAXException, SQLException {
long time = System.currentTimeMillis();

View file

@ -9,7 +9,6 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import net.osmand.Algoritms;
import net.osmand.data.Amenity;
@ -47,13 +46,14 @@ public class IndexPoiCreator extends AbstractIndexPartCreator {
if (a.getLocation() != null) {
// do not convert english name
// convertEnglishName(a);
insertAmenityIntoPoi(pStatements, a);
insertAmenityIntoPoi(a);
}
}
}
}
public void commitAndClosePoiFile(Long lastModifiedDate) throws SQLException {
closeAllPreparedStatements();
if (poiConnection != null) {
poiConnection.commit();
poiConnection.close();
@ -62,34 +62,16 @@ public class IndexPoiCreator extends AbstractIndexPartCreator {
poiIndexFile.setLastModified(lastModifiedDate);
}
}
closeAllPreparedStatements();
}
public void insertAmenityIntoPoi( Map<PreparedStatement, Integer> map, Amenity amenity) throws SQLException {
assert IndexConstants.POI_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
poiPreparedStatement.setLong(1, amenity.getId());
poiPreparedStatement.setInt(2, MapUtils.get31TileNumberX(amenity.getLocation().getLongitude()));
poiPreparedStatement.setInt(3, MapUtils.get31TileNumberY(amenity.getLocation().getLatitude()));
poiPreparedStatement.setString(4, amenity.getEnName());
poiPreparedStatement.setString(5, amenity.getName());
poiPreparedStatement.setString(6, AmenityType.valueToString(amenity.getType()));
poiPreparedStatement.setString(7, amenity.getSubType());
poiPreparedStatement.setString(8, amenity.getOpeningHours());
poiPreparedStatement.setString(9, amenity.getSite());
poiPreparedStatement.setString(10, amenity.getPhone());
addBatch(map, poiPreparedStatement, BATCH_SIZE);
}
public void checkEntity(Entity e){
private void checkEntity(Entity e){
String name = e.getTag(OSMTagKey.NAME);
if (name == null){
String msg = "";
Collection<String> keys = e.getTagKeySet();
int cnt = 0;
for (Iterator iter = keys.iterator(); iter.hasNext();) {
String key = (String) iter.next();
for (Iterator<String> iter = keys.iterator(); iter.hasNext();) {
String key = iter.next();
if (key.startsWith("name:") && key.length() <= 8) {
// ignore specialties like name:botanical
if (cnt == 0)
@ -105,23 +87,35 @@ public class IndexPoiCreator extends AbstractIndexPartCreator {
}
}
private void insertAmenityIntoPoi(Amenity amenity) throws SQLException {
assert IndexConstants.POI_TABLE != null : "use constants here to show table usage "; //$NON-NLS-1$
poiPreparedStatement.setLong(1, amenity.getId());
poiPreparedStatement.setInt(2, MapUtils.get31TileNumberX(amenity.getLocation().getLongitude()));
poiPreparedStatement.setInt(3, MapUtils.get31TileNumberY(amenity.getLocation().getLatitude()));
poiPreparedStatement.setString(4, amenity.getEnName());
poiPreparedStatement.setString(5, amenity.getName());
poiPreparedStatement.setString(6, AmenityType.valueToString(amenity.getType()));
poiPreparedStatement.setString(7, amenity.getSubType());
poiPreparedStatement.setString(8, amenity.getOpeningHours());
poiPreparedStatement.setString(9, amenity.getSite());
poiPreparedStatement.setString(10, amenity.getPhone());
addBatch(poiPreparedStatement);
}
public void createDatabaseStructure(File poiIndexFile) throws SQLException {
this.poiIndexFile = poiIndexFile;
// to save space
// delete previous file to save space
if (poiIndexFile.exists()) {
Algoritms.removeAllFiles(poiIndexFile);
}
poiIndexFile.getParentFile().mkdirs();
// creating nodes db to fast access for all nodes
// creating connection
poiConnection = DBDialect.SQLITE.getDatabaseConnection(poiIndexFile.getAbsolutePath(), log);
createPoiIndexStructure(poiConnection);
poiPreparedStatement = createStatementAmenityInsert(poiConnection);
pStatements.put(poiPreparedStatement, 0);
poiConnection.setAutoCommit(false);
}
public void createPoiIndexStructure(Connection conn) throws SQLException{
Statement stat = conn.createStatement();
// create database structure
Statement stat = poiConnection.createStatement();
stat.executeUpdate("create table " + IndexConstants.POI_TABLE + //$NON-NLS-1$
"(id bigint, x int, y int, name_en varchar(255), name varchar(255), " +
"type varchar(255), subtype varchar(255), opening_hours varchar(255), phone varchar(255), site varchar(255)," +
@ -130,11 +124,15 @@ public class IndexPoiCreator extends AbstractIndexPartCreator {
stat.executeUpdate("create index poi_id on poi (id, type, subtype)");
stat.execute("PRAGMA user_version = " + IndexConstants.POI_TABLE_VERSION); //$NON-NLS-1$
stat.close();
}
public PreparedStatement createStatementAmenityInsert(Connection conn) throws SQLException{
return conn.prepareStatement("INSERT INTO " + IndexConstants.POI_TABLE + "(id, x, y, name_en, name, type, subtype, opening_hours, site, phone) " + //$NON-NLS-1$//$NON-NLS-2$
// create prepared statment
poiPreparedStatement = poiConnection
.prepareStatement("INSERT INTO " + IndexConstants.POI_TABLE + "(id, x, y, name_en, name, type, subtype, opening_hours, site, phone) " + //$NON-NLS-1$//$NON-NLS-2$
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
pStatements.put(poiPreparedStatement, 0);
poiConnection.setAutoCommit(false);
}
}

View file

@ -73,7 +73,72 @@ public class IndexTransportCreator extends AbstractIndexPartCreator {
transportStopsTree = new RTree(rtreeTransportStopFile);
}
public void createTransportIndexStructure(Connection conn, DBDialect dialect, String rtreeStopsFileName) throws SQLException, IOException{
public void writeBinaryTransportTree(rtree.Node parent, RTree r, BinaryMapIndexWriter writer,
PreparedStatement selectTransportStop, PreparedStatement selectTransportRouteStop,
Map<Long, Long> transportRoutes, Map<String, Integer> stringTable) throws IOException, RTreeException, SQLException {
Element[] e = parent.getAllElements();
List<Long> routes = null;
for (int i = 0; i < parent.getTotalElements(); i++) {
Rect re = e[i].getRect();
if (e[i].getElementType() == rtree.Node.LEAF_NODE) {
long id = ((LeafElement) e[i]).getPtr();
selectTransportStop.setLong(1, id);
selectTransportRouteStop.setLong(1, id);
ResultSet rs = selectTransportStop.executeQuery();
if (rs.next()) {
int x24 = (int) MapUtils.getTileNumberX(24, rs.getDouble(3));
int y24 = (int) MapUtils.getTileNumberY(24, rs.getDouble(2));
String name = rs.getString(4);
String nameEn = rs.getString(5);
if (nameEn != null && nameEn.equals(Junidecode.unidecode(name))) {
nameEn = null;
}
ResultSet rset = selectTransportRouteStop.executeQuery();
if (routes == null) {
routes = new ArrayList<Long>();
} else {
routes.clear();
}
while (rset.next()) {
Long route = transportRoutes.get(rset.getLong(1));
if (route == null) {
log.error("Something goes wrong with transport route id = " + rset.getLong(1)); //$NON-NLS-1$
} else {
routes.add(route);
}
}
rset.close();
writer.writeTransportStop(id, x24, y24, name, nameEn, stringTable, routes);
} else {
log.error("Something goes wrong with transport id = " + id); //$NON-NLS-1$
}
} else {
long ptr = ((NonLeafElement) e[i]).getPtr();
rtree.Node ns = r.getReadNode(ptr);
writer.startTransportTreeElement(re.getMinX(), re.getMaxX(), re.getMinY(), re.getMaxY());
writeBinaryTransportTree(ns, r, writer, selectTransportStop, selectTransportRouteStop, transportRoutes, stringTable);
writer.endWriteTransportTreeElement();
}
}
}
public void packRTree(String rtreeTransportStopsFileName, String rtreeTransportStopsPackFileName) throws IOException {
transportStopsTree = packRtreeFile(transportStopsTree, rtreeTransportStopsFileName, rtreeTransportStopsPackFileName);
}
public void visitEntityMainStep(Entity e, OsmDbAccessorContext ctx) throws SQLException {
if (e instanceof Relation && e.getTag(OSMTagKey.ROUTE) != null) {
ctx.loadEntityData(e, true);
TransportRoute route = indexTransportRoute((Relation) e);
if (route != null) {
insertTransportIntoIndex(route);
}
}
}
public void createDatabaseStructure(Connection conn, DBDialect dialect, String rtreeStopsFileName) throws SQLException, IOException{
Statement stat = conn.createStatement();
stat.executeUpdate("create table transport_route (id bigint primary key, type varchar(255), operator varchar(255)," +
@ -111,37 +176,18 @@ public class IndexTransportCreator extends AbstractIndexPartCreator {
}
public void packRTree(String rtreeTransportStopsFileName, String rtreeTransportStopsPackFileName) throws IOException {
transportStopsTree = packRtreeFile(transportStopsTree, rtreeTransportStopsFileName, rtreeTransportStopsPackFileName);
}
private void insertTransportIntoIndex(TransportRoute route) throws SQLException {
transRouteStat.setLong(1, route.getId());
transRouteStat.setString(2, route.getType());
transRouteStat.setString(3, route.getOperator());
transRouteStat.setString(4, route.getRef());
transRouteStat.setString(5, route.getName());
transRouteStat.setString(6, route.getEnName());
transRouteStat.setInt(7, route.getAvgBothDistance());
addBatch(transRouteStat);
public void visitEntityMainStep(Entity e, OsmDbAccessorContext ctx) throws SQLException {
if (e instanceof Relation && e.getTag(OSMTagKey.ROUTE) != null) {
ctx.loadEntityData(e, true);
TransportRoute route = indexTransportRoute((Relation) e);
if (route != null) {
insertTransportIntoIndex(transRouteStat, transRouteStopsStat, transStopsStat, transportStopsTree,
visitedStops, route, pStatements, BATCH_SIZE);
}
}
}
private void insertTransportIntoIndex(PreparedStatement prepRoute, PreparedStatement prepRouteStops,
PreparedStatement prepStops, RTree transportStopsTree,
Set<Long> writtenStops, TransportRoute route, Map<PreparedStatement, Integer> statements,
int batchSize) throws SQLException {
prepRoute.setLong(1, route.getId());
prepRoute.setString(2, route.getType());
prepRoute.setString(3, route.getOperator());
prepRoute.setString(4, route.getRef());
prepRoute.setString(5, route.getName());
prepRoute.setString(6, route.getEnName());
prepRoute.setInt(7, route.getAvgBothDistance());
addBatch(statements, prepRoute);
writeRouteStops(transportStopsTree, prepRouteStops, prepStops, statements, writtenStops, route, route.getForwardStops(), true);
writeRouteStops(transportStopsTree, prepRouteStops, prepStops, statements, writtenStops, route, route.getBackwardStops(), false);
writeRouteStops(route, route.getForwardStops(), true);
writeRouteStops(route, route.getBackwardStops(), false);
}
@ -152,19 +198,18 @@ public class IndexTransportCreator extends AbstractIndexPartCreator {
return conn.prepareStatement("insert into transport_route_stop(route, stop, direction, ord) values(?, ?, ?, ?)");
}
private void writeRouteStops(RTree transportStopsTree, PreparedStatement prepRouteStops, PreparedStatement prepStops, Map<PreparedStatement, Integer> count,
Set<Long> writtenStops, TransportRoute r, List<TransportStop> stops, boolean direction) throws SQLException {
private void writeRouteStops(TransportRoute r, List<TransportStop> stops, boolean direction) throws SQLException {
int i = 0;
for(TransportStop s : stops){
if (!writtenStops.contains(s.getId())) {
prepStops.setLong(1, s.getId());
prepStops.setDouble(2, s.getLocation().getLatitude());
prepStops.setDouble(3, s.getLocation().getLongitude());
prepStops.setString(4, s.getName());
prepStops.setString(5, s.getEnName());
if (!visitedStops.contains(s.getId())) {
transStopsStat.setLong(1, s.getId());
transStopsStat.setDouble(2, s.getLocation().getLatitude());
transStopsStat.setDouble(3, s.getLocation().getLongitude());
transStopsStat.setString(4, s.getName());
transStopsStat.setString(5, s.getEnName());
int x = (int) MapUtils.getTileNumberX(24, s.getLocation().getLongitude());
int y = (int) MapUtils.getTileNumberY(24, s.getLocation().getLatitude());
addBatch(count, prepStops);
addBatch(transStopsStat);
try {
transportStopsTree.insert(new LeafElement(new Rect(x, y, x, y), s.getId()));
} catch (RTreeInsertException e) {
@ -172,13 +217,13 @@ public class IndexTransportCreator extends AbstractIndexPartCreator {
} catch (IllegalValueException e) {
throw new IllegalArgumentException(e);
}
writtenStops.add(s.getId());
visitedStops.add(s.getId());
}
prepRouteStops.setLong(1, r.getId());
prepRouteStops.setLong(2, s.getId());
prepRouteStops.setInt(3, direction ? 1 : 0);
prepRouteStops.setInt(4, i++);
addBatch(count, prepRouteStops);
transRouteStopsStat.setLong(1, r.getId());
transRouteStopsStat.setLong(2, s.getId());
transRouteStopsStat.setInt(3, direction ? 1 : 0);
transRouteStopsStat.setInt(4, i++);
addBatch(transRouteStopsStat);
}
}
@ -282,8 +327,7 @@ public class IndexTransportCreator extends AbstractIndexPartCreator {
throw new IllegalStateException(e);
}
}
public Rect calcBounds(rtree.Node n) {
private Rect calcBounds(rtree.Node n) {
Rect r = null;
Element[] e = n.getAllElements();
for (int i = 0; i < n.getTotalElements(); i++) {
@ -340,57 +384,9 @@ public class IndexTransportCreator extends AbstractIndexPartCreator {
}
public void writeBinaryTransportTree(rtree.Node parent, RTree r, BinaryMapIndexWriter writer,
PreparedStatement selectTransportStop, PreparedStatement selectTransportRouteStop,
Map<Long, Long> transportRoutes, Map<String, Integer> stringTable) throws IOException, RTreeException, SQLException {
Element[] e = parent.getAllElements();
List<Long> routes = null;
for (int i = 0; i < parent.getTotalElements(); i++) {
Rect re = e[i].getRect();
if (e[i].getElementType() == rtree.Node.LEAF_NODE) {
long id = ((LeafElement) e[i]).getPtr();
selectTransportStop.setLong(1, id);
selectTransportRouteStop.setLong(1, id);
ResultSet rs = selectTransportStop.executeQuery();
if (rs.next()) {
int x24 = (int) MapUtils.getTileNumberX(24, rs.getDouble(3));
int y24 = (int) MapUtils.getTileNumberY(24, rs.getDouble(2));
String name = rs.getString(4);
String nameEn = rs.getString(5);
if (nameEn != null && nameEn.equals(Junidecode.unidecode(name))) {
nameEn = null;
}
ResultSet rset = selectTransportRouteStop.executeQuery();
if (routes == null) {
routes = new ArrayList<Long>();
} else {
routes.clear();
}
while (rset.next()) {
Long route = transportRoutes.get(rset.getLong(1));
if (route == null) {
log.error("Something goes wrong with transport route id = " + rset.getLong(1)); //$NON-NLS-1$
} else {
routes.add(route);
}
}
rset.close();
writer.writeTransportStop(id, x24, y24, name, nameEn, stringTable, routes);
} else {
log.error("Something goes wrong with transport id = " + id); //$NON-NLS-1$
}
} else {
long ptr = ((NonLeafElement) e[i]).getPtr();
rtree.Node ns = r.getReadNode(ptr);
writer.startTransportTreeElement(re.getMinX(), re.getMaxX(), re.getMinY(), re.getMaxY());
writeBinaryTransportTree(ns, r, writer, selectTransportStop, selectTransportRouteStop, transportRoutes, stringTable);
writer.endWriteTransportTreeElement();
}
}
}
public TransportRoute indexTransportRoute(Relation rel) {
private TransportRoute indexTransportRoute(Relation rel) {
String ref = rel.getTag(OSMTagKey.REF);
String route = rel.getTag(OSMTagKey.ROUTE);
String operator = rel.getTag(OSMTagKey.OPERATOR);

View file

@ -71,42 +71,11 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator {
}
public void indexMapRelationsAndMultiPolygons(Entity e, OsmDbAccessorContext ctx) throws SQLException {
if (e instanceof Relation && "restriction".equals(e.getTag(OSMTagKey.TYPE))) { //$NON-NLS-1$
String val = e.getTag("restriction"); //$NON-NLS-1$
if (val != null) {
byte type = -1;
if ("no_right_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_NO_RIGHT_TURN;
} else if ("no_left_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_NO_LEFT_TURN;
} else if ("no_u_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_NO_U_TURN;
} else if ("no_straight_on".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_NO_STRAIGHT_ON;
} else if ("only_right_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_ONLY_RIGHT_TURN;
} else if ("only_left_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_ONLY_LEFT_TURN;
} else if ("only_straight_on".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_ONLY_STRAIGHT_ON;
}
if (type != -1) {
ctx.loadEntityData(e, true);
Collection<EntityId> fromL = ((Relation) e).getMemberIds("from"); //$NON-NLS-1$
Collection<EntityId> toL = ((Relation) e).getMemberIds("to"); //$NON-NLS-1$
if (!fromL.isEmpty() && !toL.isEmpty()) {
EntityId from = fromL.iterator().next();
EntityId to = toL.iterator().next();
if (from.getType() == EntityType.WAY) {
if (!highwayRestrictions.containsKey(from.getId())) {
highwayRestrictions.put(from.getId(), new ArrayList<Long>(4));
}
highwayRestrictions.get(from.getId()).add((to.getId() << 3) | (long) type);
}
}
}
}
indexHighwayRestrictions(e, ctx);
indexMultiPolygon(e, ctx);
}
private void indexMultiPolygon(Entity e, OsmDbAccessorContext ctx) throws SQLException {
if (e instanceof Relation && "multipolygon".equals(e.getTag(OSMTagKey.TYPE))) { //$NON-NLS-1$
ctx.loadEntityData(e, true);
Map<Entity, String> entities = ((Relation) e).getMemberEntities();
@ -184,7 +153,47 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator {
}
}
}
private void indexHighwayRestrictions(Entity e, OsmDbAccessorContext ctx) throws SQLException {
if (e instanceof Relation && "restriction".equals(e.getTag(OSMTagKey.TYPE))) { //$NON-NLS-1$
String val = e.getTag("restriction"); //$NON-NLS-1$
if (val != null) {
byte type = -1;
if ("no_right_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_NO_RIGHT_TURN;
} else if ("no_left_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_NO_LEFT_TURN;
} else if ("no_u_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_NO_U_TURN;
} else if ("no_straight_on".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_NO_STRAIGHT_ON;
} else if ("only_right_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_ONLY_RIGHT_TURN;
} else if ("only_left_turn".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_ONLY_LEFT_TURN;
} else if ("only_straight_on".equalsIgnoreCase(val)) { //$NON-NLS-1$
type = MapRenderingTypes.RESTRICTION_ONLY_STRAIGHT_ON;
}
if (type != -1) {
ctx.loadEntityData(e, true);
Collection<EntityId> fromL = ((Relation) e).getMemberIds("from"); //$NON-NLS-1$
Collection<EntityId> toL = ((Relation) e).getMemberIds("to"); //$NON-NLS-1$
if (!fromL.isEmpty() && !toL.isEmpty()) {
EntityId from = fromL.iterator().next();
EntityId to = toL.iterator().next();
if (from.getType() == EntityType.WAY) {
if (!highwayRestrictions.containsKey(from.getId())) {
highwayRestrictions.put(from.getId(), new ArrayList<Long>(4));
}
highwayRestrictions.get(from.getId()).add((to.getId() << 3) | (long) type);
}
}
}
}
}
}
private void putMultipolygonType(Map<Long, Set<Integer>> multiPolygonsWays, long baseId, int mtType, boolean inverse) {
@ -220,7 +229,7 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator {
}
public void combineMultiPolygons(Way w, List<List<Way>> completedRings, List<List<Way>> incompletedRings) {
private void combineMultiPolygons(Way w, List<List<Way>> completedRings, List<List<Way>> incompletedRings) {
long lId = w.getEntityIds().get(w.getEntityIds().size() - 1).getId().longValue();
long fId = w.getEntityIds().get(0).getId().longValue();
if (fId == lId) {
@ -263,6 +272,8 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator {
}
}
private void writeBinaryEntityToMapDatabase(Entity e, long baseId, boolean inverse, int level) throws SQLException {
int type = renderingTypes.encodeEntityWithType(e, mapZooms.getLevel(level).getMaxZoom(), false, typeUse);
Map<Long, Set<Integer>> multiPolygonsWays = this.multiPolygonsWays[level];
@ -329,13 +340,12 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator {
}
if (!skip) {
insertBinaryMapRenderObjectIndex(pStatements, mapBinaryStat, rtree, e, eName, id, type, typeUse,
highwayAttributes, restrictionsUse, inverse, point, true);
insertBinaryMapRenderObjectIndex(rtree, e, eName, id, type, typeUse, highwayAttributes, restrictionsUse, inverse, point, true);
}
}
protected long encodeTypesToOneLong(int mainType) {
private long encodeTypesToOneLong(int mainType) {
long i = 0;
int ind = 0;
int sh = 0;
@ -409,7 +419,7 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator {
return way;
} else {
lowLevelWays ++;
insertLowLevelMapBinaryObject(pStatements, mapLowLevelBinaryStat, level, longType, id, way.getNodes(), name);
insertLowLevelMapBinaryObject(level, longType, id, way.getNodes(), name);
return null;
}
@ -485,7 +495,7 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator {
startNode = fs.getLong(2);
visitedWays.add(lid);
loadNodes(fs.getBytes(4), list);
ArrayList li = new ArrayList<Float>(list);
ArrayList<Float> li = new ArrayList<Float>(list);
// remove first lat/lon point
wayNodes.remove(0);
wayNodes.remove(0);
@ -535,7 +545,7 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator {
MapAlgorithms.simplifyDouglasPeucker(wNodes, zoom - 1 + 8, 3, newWs);
int type = decodeTypesFromOneLong(ltype);
insertBinaryMapRenderObjectIndex(pStatements, mapBinaryStat, mapTree[level], newWs, name,
insertBinaryMapRenderObjectIndex(mapTree[level], newWs, name,
id, type, typeUse, 0, restrictionsUse, false, false, false);
}
@ -568,6 +578,7 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator {
}
@SuppressWarnings("unchecked")
public void initSettings(MapZooms mapZooms, MapRenderingTypes renderingTypes) {
this.mapZooms = mapZooms;
this.renderingTypes = renderingTypes;
@ -723,8 +734,7 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator {
return conn.prepareStatement("insert into low_level_map_objects(id, start_node, end_node, name, nodes, type, level) values(?, ?, ?, ?, ?, ?, ?)");
}
private void insertLowLevelMapBinaryObject(Map<PreparedStatement, Integer> statements,
PreparedStatement mapLowLevelBinaryStat, int level,long types, long id, List<Node> nodes, String name) throws SQLException{
private void insertLowLevelMapBinaryObject(int level,long types, long id, List<Node> nodes, String name) throws SQLException{
boolean first = true;
long firstId = -1;
long lastId = -1;
@ -756,10 +766,10 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator {
mapLowLevelBinaryStat.setLong(6, types);
mapLowLevelBinaryStat.setShort(7, (short) level);
addBatch(statements, mapLowLevelBinaryStat);
addBatch(mapLowLevelBinaryStat);
}
private void insertBinaryMapRenderObjectIndex(Map<PreparedStatement, Integer> statements,
PreparedStatement mapBinaryStat, RTree mapTree, Entity e, String name,
private void insertBinaryMapRenderObjectIndex(RTree mapTree, Entity e, String name,
long id, int type, List<Integer> typeUse, int highwayAttributes, List<Long> restrictions,
boolean inversePath, boolean writeAsPoint, boolean commit) throws SQLException {
if(e instanceof Relation){
@ -824,7 +834,7 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator {
mapBinaryStat.setBytes(5, bnodes.toByteArray());
mapBinaryStat.setInt(6, highwayAttributes);
addBatch(statements, mapBinaryStat, commit);
addBatch(mapBinaryStat, commit);
try {
mapTree.insert(new LeafElement(new Rect(minX, minY, maxX, maxY), id));
} catch (RTreeInsertException e1) {

View file

@ -18,24 +18,28 @@ import net.osmand.osm.Entity.EntityId;
import net.osmand.osm.Entity.EntityType;
public class OsmDbAccessor implements OsmDbAccessorContext {
private final IndexCreator indexCreator;
private PreparedStatement pselectNode;
private PreparedStatement pselectWay;
private PreparedStatement pselectRelation;
private PreparedStatement pselectTags;
private int allRelations ;
private int allWays;
private int allNodes;
private Connection dbConn;
public interface OsmDbVisitor {
public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException;
}
public OsmDbAccessor(IndexCreator indexCreator){
this.indexCreator = indexCreator;
public OsmDbAccessor(){
}
public void initDatabase(Connection dbConn) throws SQLException {
public void initDatabase(Connection dbConn, int allNodes, int allWays, int allRelations) throws SQLException {
this.dbConn = dbConn;
this.allNodes = allNodes;
this.allWays = allWays;
this.allRelations = allRelations;
pselectNode = dbConn.prepareStatement("select n.latitude, n.longitude, t.skeys, t.value from node n left join tags t on n.id = t.id and t.type = 0 where n.id = ?"); //$NON-NLS-1$
pselectWay = dbConn.prepareStatement("select w.node, w.ord, t.skeys, t.value, n.latitude, n.longitude " + //$NON-NLS-1$
"from ways w left join tags t on w.id = t.id and t.type = 1 and w.ord = 0 inner join node n on w.node = n.id " + //$NON-NLS-1$
@ -46,6 +50,18 @@ public class OsmDbAccessor implements OsmDbAccessorContext {
pselectTags = dbConn.prepareStatement("select skeys, value from tags where id = ? and type = ?"); //$NON-NLS-1$
}
public int getAllNodes() {
return allNodes;
}
public int getAllRelations() {
return allRelations;
}
public int getAllWays() {
return allWays;
}
public void loadEntityData(Entity e, boolean loadTags) throws SQLException {
if (e instanceof Node || (e instanceof Way && !((Way) e).getNodes().isEmpty())) {
// do not load tags for nodes inside way
@ -153,8 +169,7 @@ public class OsmDbAccessor implements OsmDbAccessorContext {
}
public int iterateOverEntities(IProgress progress, EntityType type, int allCount,
OsmDbVisitor visitor) throws SQLException {
public int iterateOverEntities(IProgress progress, EntityType type, OsmDbVisitor visitor) throws SQLException {
Statement statement = dbConn.createStatement();
String select;
int count = 0;
@ -220,6 +235,13 @@ public class OsmDbAccessor implements OsmDbAccessorContext {
visitor.iterateEntity(prevEntity, this);
}
rs.close();
if(EntityType.NODE == type){
allNodes = count;
} else if(EntityType.WAY == type){
allWays = count;
} else if(EntityType.RELATION == type){
allRelations = count;
}
return count;
}

View file

@ -99,8 +99,6 @@ public class OsmDbCreator implements IOsmStorageFilter {
@Override
public boolean acceptEntityToLoad(OsmBaseStorage storage, EntityId entityId, Entity e) {
// Register all city labelbs
indexCreator.registerCityIfNeeded(e);
// put all nodes into temporary db to get only required nodes after loading all data
try {
if (e instanceof Node) {