implement new indexer mechanism
git-svn-id: https://osmand.googlecode.com/svn/trunk@448 e29c36b1-1cfa-d876-8d93-3434fc2bb7b8
This commit is contained in:
Victor Shcherb
parent
b538993237
commit
d8e20d3499
5 changed files with 406 additions and 210 deletions
|
|
@ -8,6 +8,7 @@ import java.sql.PreparedStatement;
|
|||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
|
@ -82,13 +83,13 @@ public class DataIndexWriter {
|
|||
try {
|
||||
createPoiIndexStructure(conn);
|
||||
PreparedStatement prep = createStatementAmenityInsert(conn);
|
||||
Map<PreparedStatement, Integer> map = new LinkedHashMap<PreparedStatement, Integer>();
|
||||
map.put(prep, 0);
|
||||
conn.setAutoCommit(false);
|
||||
int currentCount = 0;
|
||||
for (Amenity a : region.getAmenityManager().getAllObjects()) {
|
||||
currentCount = insertAmenityIntoPoi(prep, a, currentCount, BATCH_SIZE);
|
||||
|
||||
insertAmenityIntoPoi(prep, map, a, BATCH_SIZE);
|
||||
}
|
||||
if(currentCount > 0){
|
||||
if(map.get(prep) > 0){
|
||||
prep.executeBatch();
|
||||
}
|
||||
prep.close();
|
||||
|
|
@ -103,7 +104,7 @@ public class DataIndexWriter {
|
|||
return this;
|
||||
}
|
||||
|
||||
public static int insertAmenityIntoPoi(PreparedStatement prep, Amenity amenity, int currentInBatch, int batchSize) throws SQLException {
|
||||
public static void insertAmenityIntoPoi(PreparedStatement prep, Map<PreparedStatement, Integer> map, Amenity amenity, int batchSize) throws SQLException {
|
||||
prep.setLong(IndexPoiTable.ID.ordinal() + 1, amenity.getId());
|
||||
prep.setDouble(IndexPoiTable.LATITUDE.ordinal() + 1, amenity.getLocation().getLatitude());
|
||||
prep.setDouble(IndexPoiTable.LONGITUDE.ordinal() + 1, amenity.getLocation().getLongitude());
|
||||
|
|
@ -112,13 +113,7 @@ public class DataIndexWriter {
|
|||
prep.setString(IndexPoiTable.TYPE.ordinal() + 1, AmenityType.valueToString(amenity.getType()));
|
||||
prep.setString(IndexPoiTable.SUBTYPE.ordinal() + 1, amenity.getSubType());
|
||||
prep.setString(IndexPoiTable.OPENING_HOURS.ordinal() + 1 , amenity.getOpeningHours());
|
||||
prep.addBatch();
|
||||
currentInBatch++;
|
||||
if(currentInBatch >= batchSize){
|
||||
prep.executeBatch();
|
||||
currentInBatch = 0;
|
||||
}
|
||||
return currentInBatch;
|
||||
addBatch(map, prep, batchSize);
|
||||
}
|
||||
|
||||
public static PreparedStatement createStatementAmenityInsert(Connection conn) throws SQLException{
|
||||
|
|
@ -278,18 +273,7 @@ public class DataIndexWriter {
|
|||
}
|
||||
Connection conn = DriverManager.getConnection("jdbc:sqlite:"+file.getAbsolutePath()); //$NON-NLS-1$
|
||||
try {
|
||||
Statement stat = conn.createStatement();
|
||||
|
||||
|
||||
|
||||
stat.executeUpdate(IndexConstants.generateCreateSQL(IndexTransportRoute.values()));
|
||||
stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexTransportRoute.values()));
|
||||
stat.executeUpdate(IndexConstants.generateCreateSQL(IndexTransportRouteStop.values()));
|
||||
stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexTransportRouteStop.values()));
|
||||
stat.executeUpdate(IndexConstants.generateCreateSQL(IndexTransportStop.values()));
|
||||
stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexTransportStop.values()));
|
||||
stat.execute("PRAGMA user_version = " + IndexConstants.TRANSPORT_TABLE_VERSION); //$NON-NLS-1$
|
||||
stat.close();
|
||||
createTransportIndexStructure(conn);
|
||||
|
||||
PreparedStatement prepRoute = conn.prepareStatement(
|
||||
IndexConstants.generatePrepareStatementToInsert(IndexTransportRoute.getTable(), IndexTransportRoute.values().length));
|
||||
|
|
@ -307,18 +291,7 @@ public class DataIndexWriter {
|
|||
Set<Long> writtenStops = new LinkedHashSet<Long>();
|
||||
for(String t : region.getTransportRoutes().keySet()){
|
||||
for(TransportRoute r : region.getTransportRoutes().get(t)) {
|
||||
assert IndexTransportRoute.values().length == 7;
|
||||
prepRoute.setLong(IndexTransportRoute.ID.ordinal() + 1, r.getId());
|
||||
prepRoute.setString(IndexTransportRoute.TYPE.ordinal() + 1, r.getType());
|
||||
prepRoute.setString(IndexTransportRoute.OPERATOR.ordinal() + 1, r.getOperator());
|
||||
prepRoute.setString(IndexTransportRoute.REF.ordinal() + 1, r.getRef());
|
||||
prepRoute.setString(IndexTransportRoute.NAME.ordinal() + 1, r.getName());
|
||||
prepRoute.setString(IndexTransportRoute.NAME_EN.ordinal() + 1, r.getEnName());
|
||||
prepRoute.setInt(IndexTransportRoute.DIST.ordinal() + 1, r.getAvgBothDistance());
|
||||
addBatch(count, prepRoute);
|
||||
|
||||
writeRouteStops(prepRouteStops, prepStops, count, writtenStops, r, r.getForwardStops(), true);
|
||||
writeRouteStops(prepRouteStops, prepStops, count, writtenStops, r, r.getBackwardStops(), false);
|
||||
insertTransportIntoIndex(prepRoute, prepRouteStops, prepStops, writtenStops, r, count, BATCH_SIZE);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -339,7 +312,7 @@ public class DataIndexWriter {
|
|||
return this;
|
||||
}
|
||||
|
||||
private void writeRouteStops(PreparedStatement prepRouteStops, PreparedStatement prepStops, Map<PreparedStatement, Integer> count,
|
||||
private static void writeRouteStops(PreparedStatement prepRouteStops, PreparedStatement prepStops, Map<PreparedStatement, Integer> count,
|
||||
Set<Long> writtenStops, TransportRoute r, List<TransportStop> stops, boolean direction) throws SQLException {
|
||||
int i = 0;
|
||||
for(TransportStop s : stops){
|
||||
|
|
@ -363,11 +336,44 @@ public class DataIndexWriter {
|
|||
}
|
||||
|
||||
|
||||
public static void insertTransportIntoIndex(PreparedStatement prepRoute, PreparedStatement prepRouteStops,
|
||||
PreparedStatement prepStops, Set<Long> writtenStops, TransportRoute route, Map<PreparedStatement, Integer> statements,
|
||||
int batchSize) throws SQLException {
|
||||
assert IndexTransportRoute.values().length == 7;
|
||||
prepRoute.setLong(IndexTransportRoute.ID.ordinal() + 1, route.getId());
|
||||
prepRoute.setString(IndexTransportRoute.TYPE.ordinal() + 1, route.getType());
|
||||
prepRoute.setString(IndexTransportRoute.OPERATOR.ordinal() + 1, route.getOperator());
|
||||
prepRoute.setString(IndexTransportRoute.REF.ordinal() + 1, route.getRef());
|
||||
prepRoute.setString(IndexTransportRoute.NAME.ordinal() + 1, route.getName());
|
||||
prepRoute.setString(IndexTransportRoute.NAME_EN.ordinal() + 1, route.getEnName());
|
||||
prepRoute.setInt(IndexTransportRoute.DIST.ordinal() + 1, route.getAvgBothDistance());
|
||||
addBatch(statements, prepRoute);
|
||||
|
||||
writeRouteStops(prepRouteStops, prepStops, statements, writtenStops, route, route.getForwardStops(), true);
|
||||
writeRouteStops(prepRouteStops, prepStops, statements, writtenStops, route, route.getBackwardStops(), false);
|
||||
|
||||
}
|
||||
|
||||
public static void createTransportIndexStructure(Connection conn) throws SQLException{
|
||||
Statement stat = conn.createStatement();
|
||||
stat.executeUpdate(IndexConstants.generateCreateSQL(IndexTransportRoute.values()));
|
||||
stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexTransportRoute.values()));
|
||||
stat.executeUpdate(IndexConstants.generateCreateSQL(IndexTransportRouteStop.values()));
|
||||
stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexTransportRouteStop.values()));
|
||||
stat.executeUpdate(IndexConstants.generateCreateSQL(IndexTransportStop.values()));
|
||||
stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexTransportStop.values()));
|
||||
stat.execute("PRAGMA user_version = " + IndexConstants.TRANSPORT_TABLE_VERSION); //$NON-NLS-1$
|
||||
stat.close();
|
||||
}
|
||||
|
||||
|
||||
private void addBatch(Map<PreparedStatement, Integer> count, PreparedStatement p) throws SQLException{
|
||||
private static void addBatch(Map<PreparedStatement, Integer> count, PreparedStatement p) throws SQLException{
|
||||
addBatch(count, p, BATCH_SIZE);
|
||||
}
|
||||
|
||||
private static void addBatch(Map<PreparedStatement, Integer> count, PreparedStatement p, int batchSize) throws SQLException{
|
||||
p.addBatch();
|
||||
if(count.get(p) >= BATCH_SIZE){
|
||||
if(count.get(p) >= batchSize){
|
||||
p.executeBatch();
|
||||
count.put(p, 0);
|
||||
} else {
|
||||
|
|
@ -376,4 +382,6 @@ public class DataIndexWriter {
|
|||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,8 +11,13 @@ import java.sql.ResultSet;
|
|||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.Map.Entry;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
|
|
@ -23,8 +28,13 @@ import org.xml.sax.SAXException;
|
|||
import com.osmand.Algoritms;
|
||||
import com.osmand.IProgress;
|
||||
import com.osmand.data.Amenity;
|
||||
import com.osmand.data.TransportRoute;
|
||||
import com.osmand.data.TransportStop;
|
||||
import com.osmand.data.index.DataIndexWriter;
|
||||
import com.osmand.data.index.IndexConstants;
|
||||
import com.osmand.data.index.IndexConstants.IndexTransportRoute;
|
||||
import com.osmand.data.index.IndexConstants.IndexTransportRouteStop;
|
||||
import com.osmand.data.index.IndexConstants.IndexTransportStop;
|
||||
import com.osmand.impl.ConsoleProgressImplementation;
|
||||
import com.osmand.osm.Entity;
|
||||
import com.osmand.osm.Node;
|
||||
|
|
@ -32,6 +42,7 @@ import com.osmand.osm.Relation;
|
|||
import com.osmand.osm.Way;
|
||||
import com.osmand.osm.Entity.EntityId;
|
||||
import com.osmand.osm.Entity.EntityType;
|
||||
import com.osmand.osm.OSMSettings.OSMTagKey;
|
||||
import com.osmand.osm.io.IOsmStorageFilter;
|
||||
import com.osmand.osm.io.OsmBaseStorage;
|
||||
import com.osmand.swing.DataExtractionSettings;
|
||||
|
|
@ -43,21 +54,24 @@ import com.osmand.swing.DataExtractionSettings;
|
|||
*
|
||||
*
|
||||
*/
|
||||
public class NewDataExtraction {
|
||||
public class IndexCreator {
|
||||
private static final Log log = LogFactory.getLog(DataExtraction.class);
|
||||
|
||||
public static final int BATCH_SIZE = 5000;
|
||||
public static final String TEMP_NODES_DB = "nodes"+IndexConstants.MAP_INDEX_EXT;
|
||||
|
||||
|
||||
private final boolean normalizeStreets;
|
||||
private final boolean indexAddress;
|
||||
private final boolean indexPOI;
|
||||
private final boolean indexTransport;
|
||||
private final boolean indexMap;
|
||||
private final boolean saveAddressWays;
|
||||
private File workingDir = null;
|
||||
|
||||
private boolean indexMap;
|
||||
private boolean indexPOI;
|
||||
private boolean indexTransport;
|
||||
|
||||
private boolean indexAddress;
|
||||
private boolean normalizeStreets;
|
||||
private boolean saveAddressWays;
|
||||
|
||||
private String regionName;
|
||||
|
||||
private String transportFileName = null;
|
||||
private String poiFileName = null;
|
||||
private String addressFileName = null;
|
||||
|
|
@ -73,28 +87,49 @@ public class NewDataExtraction {
|
|||
private Connection dbConn;
|
||||
private File dbFile;
|
||||
|
||||
Map<PreparedStatement, Integer> pStatements = new LinkedHashMap<PreparedStatement, Integer>();
|
||||
|
||||
private Connection poiConnection;
|
||||
private File poiIndexFile;
|
||||
private PreparedStatement poiPreparedStatement;
|
||||
private int poiInBatch = 0;
|
||||
|
||||
private String regionName;
|
||||
|
||||
Set<Long> visitedStops = new HashSet<Long>();
|
||||
private File transportIndexFile;
|
||||
private Connection transportConnection;
|
||||
private PreparedStatement transRouteStat;
|
||||
private PreparedStatement transRouteStopsStat;
|
||||
private PreparedStatement transStopsStat;
|
||||
|
||||
|
||||
|
||||
|
||||
public NewDataExtraction(boolean indexAddress, boolean indexPOI, boolean indexTransport,
|
||||
boolean indexMap, boolean saveAddressWays, boolean normalizeStreets,
|
||||
File workingDir){
|
||||
this.indexAddress = indexAddress;
|
||||
this.indexPOI = indexPOI;
|
||||
this.indexTransport = indexTransport;
|
||||
this.indexMap = indexMap;
|
||||
this.saveAddressWays = saveAddressWays;
|
||||
this.normalizeStreets = normalizeStreets;
|
||||
public IndexCreator(File workingDir){
|
||||
this.workingDir = workingDir;
|
||||
}
|
||||
|
||||
|
||||
public void setIndexAddress(boolean indexAddress) {
|
||||
this.indexAddress = indexAddress;
|
||||
}
|
||||
|
||||
public void setIndexMap(boolean indexMap) {
|
||||
this.indexMap = indexMap;
|
||||
}
|
||||
|
||||
public void setIndexPOI(boolean indexPOI) {
|
||||
this.indexPOI = indexPOI;
|
||||
}
|
||||
|
||||
public void setIndexTransport(boolean indexTransport) {
|
||||
this.indexTransport = indexTransport;
|
||||
}
|
||||
|
||||
public void setSaveAddressWays(boolean saveAddressWays) {
|
||||
this.saveAddressWays = saveAddressWays;
|
||||
}
|
||||
|
||||
public void setNormalizeStreets(boolean normalizeStreets) {
|
||||
this.normalizeStreets = normalizeStreets;
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -248,146 +283,174 @@ public class NewDataExtraction {
|
|||
this.regionName = regionName;
|
||||
}
|
||||
|
||||
public void generateIndexes(String path, IProgress progress, IOsmStorageFilter addFilter) throws IOException, SAXException, SQLException{
|
||||
File f = new File(path);
|
||||
InputStream stream = new FileInputStream(f);
|
||||
int i = f.getName().indexOf('.');
|
||||
if(regionName == null){
|
||||
regionName = Algoritms.capitalizeFirstLetterAndLowercase(f.getName().substring(0, i));
|
||||
}
|
||||
public void generateIndexes(String path, IProgress progress, IOsmStorageFilter addFilter) throws IOException, SAXException,
|
||||
SQLException {
|
||||
File f = new File(path);
|
||||
InputStream stream = new FileInputStream(f);
|
||||
int i = f.getName().indexOf('.');
|
||||
if (regionName == null) {
|
||||
regionName = Algoritms.capitalizeFirstLetterAndLowercase(f.getName().substring(0, i));
|
||||
}
|
||||
|
||||
InputStream streamFile = stream;
|
||||
long st = System.currentTimeMillis();
|
||||
if (path.endsWith(".bz2")) {
|
||||
if (stream.read() != 'B' || stream.read() != 'Z') {
|
||||
throw new RuntimeException("The source stream must start with the characters BZ if it is to be read as a BZip2 stream.");
|
||||
} else {
|
||||
stream = new CBZip2InputStream(stream);
|
||||
}
|
||||
}
|
||||
|
||||
if(progress != null){
|
||||
progress.startTask("Loading file " + path, -1);
|
||||
}
|
||||
OsmBaseStorage storage = new OsmBaseStorage();
|
||||
storage.setSupressWarnings(DataExtractionSettings.getSettings().isSupressWarningsForDuplicatedId());
|
||||
if (addFilter != null) {
|
||||
storage.getFilters().add(addFilter);
|
||||
}
|
||||
|
||||
|
||||
try {
|
||||
Class.forName("org.sqlite.JDBC");
|
||||
} catch (ClassNotFoundException e) {
|
||||
log.error("Illegal configuration", e);
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
if(indexMap){
|
||||
dbFile = new File(workingDir, getMapFileName());
|
||||
InputStream streamFile = stream;
|
||||
long st = System.currentTimeMillis();
|
||||
if (path.endsWith(".bz2")) {
|
||||
if (stream.read() != 'B' || stream.read() != 'Z') {
|
||||
throw new RuntimeException("The source stream must start with the characters BZ if it is to be read as a BZip2 stream.");
|
||||
} else {
|
||||
dbFile = new File(workingDir, TEMP_NODES_DB);
|
||||
stream = new CBZip2InputStream(stream);
|
||||
}
|
||||
}
|
||||
|
||||
if (progress != null) {
|
||||
progress.startTask("Loading file " + path, -1);
|
||||
}
|
||||
OsmBaseStorage storage = new OsmBaseStorage();
|
||||
storage.setSupressWarnings(DataExtractionSettings.getSettings().isSupressWarningsForDuplicatedId());
|
||||
if (addFilter != null) {
|
||||
storage.getFilters().add(addFilter);
|
||||
}
|
||||
|
||||
try {
|
||||
Class.forName("org.sqlite.JDBC");
|
||||
} catch (ClassNotFoundException e) {
|
||||
log.error("Illegal configuration", e);
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
if (indexMap) {
|
||||
dbFile = new File(workingDir, getMapFileName());
|
||||
} else {
|
||||
dbFile = new File(workingDir, TEMP_NODES_DB);
|
||||
}
|
||||
// to save space
|
||||
if (dbFile.exists()) {
|
||||
dbFile.delete();
|
||||
}
|
||||
// creating nodes db to fast access for all nodes
|
||||
dbConn = DriverManager.getConnection("jdbc:sqlite:" + dbFile.getAbsolutePath());
|
||||
|
||||
// 1. Loading osm file
|
||||
NewDataExtractionOsmFilter filter = new NewDataExtractionOsmFilter();
|
||||
try {
|
||||
// 1 init database to store temporary data
|
||||
progress.setGeneralProgress("[40 of 100]");
|
||||
|
||||
filter.initDatabase();
|
||||
storage.getFilters().add(filter);
|
||||
storage.parseOSM(stream, progress, streamFile, false);
|
||||
filter.finishLoading();
|
||||
|
||||
if (log.isInfoEnabled()) {
|
||||
log.info("File parsed : " + (System.currentTimeMillis() - st));
|
||||
}
|
||||
progress.finishTask();
|
||||
|
||||
} finally {
|
||||
if (log.isInfoEnabled()) {
|
||||
log.info("File indexed : " + (System.currentTimeMillis() - st));
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Processing all entries
|
||||
progress.setGeneralProgress("[90 of 100]");
|
||||
|
||||
pselectNode = dbConn.prepareStatement("select * from node where id = ?");
|
||||
pselectWay = dbConn.prepareStatement("select * from ways where id = ?");
|
||||
pselectRelation = dbConn.prepareStatement("select * from relations where id = ?");
|
||||
pselectTags = dbConn.prepareStatement("select key, value from tags where id = ? and type = ?");
|
||||
|
||||
if (indexPOI) {
|
||||
poiIndexFile = new File(workingDir, getPoiFileName());
|
||||
// to save space
|
||||
if(dbFile.exists()){
|
||||
dbFile.delete();
|
||||
if (poiIndexFile.exists()) {
|
||||
poiIndexFile.delete();
|
||||
}
|
||||
poiIndexFile.getParentFile().mkdirs();
|
||||
// creating nodes db to fast access for all nodes
|
||||
dbConn = DriverManager.getConnection("jdbc:sqlite:" + dbFile.getAbsolutePath());
|
||||
poiConnection = DriverManager.getConnection("jdbc:sqlite:" + poiIndexFile.getAbsolutePath());
|
||||
poiConnection.setAutoCommit(false);
|
||||
DataIndexWriter.createPoiIndexStructure(poiConnection);
|
||||
poiPreparedStatement = DataIndexWriter.createStatementAmenityInsert(poiConnection);
|
||||
pStatements.put(poiPreparedStatement, 0);
|
||||
}
|
||||
|
||||
// 1. Loading osm file
|
||||
NewDataExtractionOsmFilter filter = new NewDataExtractionOsmFilter();
|
||||
try {
|
||||
// 1 init database to store temporary data
|
||||
progress.setGeneralProgress("[40 of 100]");
|
||||
if (indexTransport) {
|
||||
transportIndexFile = new File(workingDir, getTransportFileName());
|
||||
// to save space
|
||||
if (transportIndexFile.exists()) {
|
||||
transportIndexFile.delete();
|
||||
}
|
||||
transportIndexFile.getParentFile().mkdirs();
|
||||
// creating nodes db to fast access for all nodes
|
||||
transportConnection = DriverManager.getConnection("jdbc:sqlite:" + transportIndexFile.getAbsolutePath());
|
||||
|
||||
filter.initDatabase();
|
||||
storage.getFilters().add(filter);
|
||||
storage.parseOSM(stream, progress, streamFile, false);
|
||||
filter.finishLoading();
|
||||
DataIndexWriter.createTransportIndexStructure(transportConnection);
|
||||
transRouteStat = transportConnection.prepareStatement(IndexConstants.generatePrepareStatementToInsert(
|
||||
IndexTransportRoute.getTable(), IndexTransportRoute.values().length));
|
||||
transRouteStopsStat = transportConnection.prepareStatement(IndexConstants.generatePrepareStatementToInsert(
|
||||
IndexTransportRouteStop.getTable(), IndexTransportRouteStop.values().length));
|
||||
transStopsStat = transportConnection.prepareStatement(IndexConstants.generatePrepareStatementToInsert(IndexTransportStop
|
||||
.getTable(), IndexTransportStop.values().length));
|
||||
pStatements.put(transRouteStat, 0);
|
||||
pStatements.put(transRouteStopsStat, 0);
|
||||
pStatements.put(transStopsStat, 0);
|
||||
transportConnection.setAutoCommit(false);
|
||||
|
||||
if (log.isInfoEnabled()) {
|
||||
log.info("File parsed : " + (System.currentTimeMillis() - st));
|
||||
}
|
||||
progress.finishTask();
|
||||
}
|
||||
|
||||
iterateOverAllEntities(progress, filter);
|
||||
|
||||
} finally {
|
||||
if (log.isInfoEnabled()) {
|
||||
log.info("File indexed : " + (System.currentTimeMillis() - st));
|
||||
try {
|
||||
if (pselectNode != null) {
|
||||
pselectNode.close();
|
||||
}
|
||||
if (pselectWay != null) {
|
||||
pselectWay.close();
|
||||
}
|
||||
if (pselectRelation != null) {
|
||||
pselectRelation.close();
|
||||
}
|
||||
if (pselectTags != null) {
|
||||
pselectTags.close();
|
||||
}
|
||||
for (PreparedStatement p : pStatements.keySet()) {
|
||||
if (pStatements.get(p) > 0) {
|
||||
p.executeBatch();
|
||||
p.close();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// 2. Processing all entries
|
||||
progress.setGeneralProgress("[90 of 100]");
|
||||
|
||||
pselectNode = dbConn.prepareStatement("select * from node where id = ?");
|
||||
pselectWay = dbConn.prepareStatement("select * from ways where id = ?");
|
||||
pselectRelation = dbConn.prepareStatement("select * from relations where id = ?");
|
||||
pselectTags = dbConn.prepareStatement("select key, value from tags where id = ? and type = ?");
|
||||
|
||||
if(indexPOI){
|
||||
poiIndexFile = new File(workingDir, getPoiFileName());
|
||||
// to save space
|
||||
if(poiIndexFile.exists()){
|
||||
poiIndexFile.delete();
|
||||
if (poiConnection != null) {
|
||||
poiConnection.commit();
|
||||
poiConnection.close();
|
||||
if (lastModifiedDate != null) {
|
||||
poiIndexFile.setLastModified(lastModifiedDate);
|
||||
}
|
||||
}
|
||||
if (transportConnection != null) {
|
||||
transportConnection.commit();
|
||||
transportConnection.close();
|
||||
if (lastModifiedDate != null) {
|
||||
transportIndexFile.setLastModified(lastModifiedDate);
|
||||
}
|
||||
poiIndexFile.getParentFile().mkdirs();
|
||||
// creating nodes db to fast access for all nodes
|
||||
poiConnection = DriverManager.getConnection("jdbc:sqlite:" + poiIndexFile.getAbsolutePath());
|
||||
poiConnection.setAutoCommit(false);
|
||||
DataIndexWriter.createPoiIndexStructure(poiConnection);
|
||||
poiPreparedStatement = DataIndexWriter.createStatementAmenityInsert(poiConnection);
|
||||
poiInBatch = 0;
|
||||
}
|
||||
|
||||
|
||||
iterateOverAllEntities(progress, filter);
|
||||
|
||||
try {
|
||||
if(pselectNode != null){
|
||||
pselectNode.close();
|
||||
}
|
||||
if(pselectWay != null){
|
||||
pselectWay.close();
|
||||
}
|
||||
if(pselectRelation != null){
|
||||
pselectRelation.close();
|
||||
}
|
||||
if(pselectTags != null){
|
||||
pselectTags.close();
|
||||
}
|
||||
if(poiConnection != null){
|
||||
if(poiPreparedStatement != null){
|
||||
if(poiInBatch > 0){
|
||||
poiPreparedStatement.executeBatch();
|
||||
}
|
||||
poiPreparedStatement.close();
|
||||
}
|
||||
poiConnection.commit();
|
||||
poiConnection.close();
|
||||
if(lastModifiedDate != null){
|
||||
poiIndexFile.setLastModified(lastModifiedDate);
|
||||
}
|
||||
|
||||
}
|
||||
dbConn.close();
|
||||
} catch (SQLException e) {
|
||||
}
|
||||
dbConn.close();
|
||||
} catch (SQLException e) {
|
||||
}
|
||||
}
|
||||
|
||||
public void loadEntityData(Entity e) throws SQLException {
|
||||
public void loadEntityData(Entity e, boolean loadTags) throws SQLException {
|
||||
if(e instanceof Node){
|
||||
return;
|
||||
}
|
||||
Map<EntityId, Entity> map = new LinkedHashMap<EntityId, Entity>();
|
||||
ArrayList<EntityId> ids = new ArrayList<EntityId>();
|
||||
if(e instanceof Relation){
|
||||
pselectRelation.setLong(1, e.getId());
|
||||
if (pselectRelation.execute()) {
|
||||
ResultSet rs = pselectNode.getResultSet();
|
||||
ResultSet rs = pselectRelation.getResultSet();
|
||||
while (rs.next()) {
|
||||
((Relation) e).addMember(rs.getLong(1), EntityType.values()[rs.getByte(2)], rs.getString(3));
|
||||
((Relation) e).addMember(rs.getLong(2), EntityType.values()[rs.getByte(3)], rs.getString(4));
|
||||
}
|
||||
rs.close();
|
||||
}
|
||||
|
|
@ -401,6 +464,7 @@ public class NewDataExtraction {
|
|||
rs.close();
|
||||
}
|
||||
}
|
||||
Collection<EntityId> ids = e instanceof Relation? ((Relation)e).getMemberIds() : ((Way)e).getEntityIds();
|
||||
for (EntityId i : ids) {
|
||||
if (i.getType() == EntityType.NODE) {
|
||||
pselectNode.setLong(1, i.getId());
|
||||
|
|
@ -421,6 +485,7 @@ public class NewDataExtraction {
|
|||
way.addNode(rs.getLong(2));
|
||||
}
|
||||
rs.close();
|
||||
loadEntityData(way, false);
|
||||
}
|
||||
} else if (i.getType() == EntityType.RELATION) {
|
||||
pselectRelation.setLong(1, i.getId());
|
||||
|
|
@ -436,6 +501,11 @@ public class NewDataExtraction {
|
|||
}
|
||||
}
|
||||
}
|
||||
if(loadTags){
|
||||
for(Map.Entry<EntityId, Entity> es : map.entrySet()){
|
||||
loadEntityTags(es.getKey().getType(), es.getValue());
|
||||
}
|
||||
}
|
||||
e.initializeLinks(map);
|
||||
}
|
||||
|
||||
|
|
@ -455,7 +525,10 @@ public class NewDataExtraction {
|
|||
this.mapFileName = mapFileName;
|
||||
}
|
||||
public String getMapFileName() {
|
||||
return getRegionName() + IndexConstants.MAP_INDEX_EXT;
|
||||
if(mapFileName == null){
|
||||
return getRegionName() + IndexConstants.MAP_INDEX_EXT;
|
||||
}
|
||||
return mapFileName;
|
||||
}
|
||||
|
||||
public String getTransportFileName() {
|
||||
|
|
@ -537,13 +610,139 @@ public class NewDataExtraction {
|
|||
}
|
||||
|
||||
|
||||
private void iterateEntity(Entity e) throws SQLException {
|
||||
if (indexPOI && Amenity.isAmenity(e)) {
|
||||
loadEntityData(e);
|
||||
if(poiPreparedStatement != null){
|
||||
poiInBatch = DataIndexWriter.insertAmenityIntoPoi(poiPreparedStatement, new Amenity(e), poiInBatch, BATCH_SIZE);
|
||||
private static Set<String> acceptedRoutes = new HashSet<String>();
|
||||
static {
|
||||
acceptedRoutes.add("bus");
|
||||
acceptedRoutes.add("trolleybus");
|
||||
acceptedRoutes.add("share_taxi");
|
||||
|
||||
acceptedRoutes.add("subway");
|
||||
acceptedRoutes.add("train");
|
||||
|
||||
acceptedRoutes.add("tram");
|
||||
|
||||
acceptedRoutes.add("ferry");
|
||||
}
|
||||
|
||||
private TransportRoute indexTransportRoute(Relation rel) {
|
||||
String ref = rel.getTag(OSMTagKey.REF);
|
||||
String route = rel.getTag(OSMTagKey.ROUTE);
|
||||
String operator = rel.getTag(OSMTagKey.OPERATOR);
|
||||
if (route == null || ref == null) {
|
||||
return null;
|
||||
}
|
||||
if (!acceptedRoutes.contains(route)) {
|
||||
return null;
|
||||
}
|
||||
TransportRoute r = new TransportRoute(rel, ref);
|
||||
r.setOperator(operator);
|
||||
r.setType(route);
|
||||
|
||||
if (operator != null) {
|
||||
route = operator + " : " + route;
|
||||
}
|
||||
|
||||
final Map<TransportStop, Integer> forwardStops = new LinkedHashMap<TransportStop, Integer>();
|
||||
final Map<TransportStop, Integer> backwardStops = new LinkedHashMap<TransportStop, Integer>();
|
||||
int currentStop = 0;
|
||||
int forwardStop = 0;
|
||||
int backwardStop = 0;
|
||||
for (Entry<Entity, String> e : rel.getMemberEntities().entrySet()) {
|
||||
if (e.getValue().contains("stop")) {
|
||||
if (e.getKey() instanceof Node) {
|
||||
TransportStop stop = new TransportStop(e.getKey());
|
||||
boolean forward = e.getValue().contains("forward");
|
||||
boolean backward = e.getValue().contains("backward");
|
||||
currentStop++;
|
||||
if (forward || !backward) {
|
||||
forwardStop++;
|
||||
}
|
||||
if (backward) {
|
||||
backwardStop++;
|
||||
}
|
||||
boolean common = !forward && !backward;
|
||||
int index = -1;
|
||||
int i = e.getValue().length() - 1;
|
||||
int accum = 1;
|
||||
while (i >= 0 && Character.isDigit(e.getValue().charAt(i))) {
|
||||
if (index < 0) {
|
||||
index = 0;
|
||||
}
|
||||
index = accum * Character.getNumericValue(e.getValue().charAt(i)) + index;
|
||||
accum *= 10;
|
||||
i--;
|
||||
}
|
||||
if (index < 0) {
|
||||
index = forward ? forwardStop : (backward ? backwardStop : currentStop);
|
||||
}
|
||||
if (forward || common) {
|
||||
forwardStops.put(stop, index);
|
||||
r.getForwardStops().add(stop);
|
||||
}
|
||||
if (backward || common) {
|
||||
if (common) {
|
||||
// put with negative index
|
||||
backwardStops.put(stop, -index);
|
||||
} else {
|
||||
backwardStops.put(stop, index);
|
||||
}
|
||||
|
||||
r.getBackwardStops().add(stop);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
} else if (e.getKey() instanceof Way) {
|
||||
r.addWay((Way) e.getKey());
|
||||
}
|
||||
}
|
||||
if (forwardStops.isEmpty() && backwardStops.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
Collections.sort(r.getForwardStops(), new Comparator<TransportStop>() {
|
||||
@Override
|
||||
public int compare(TransportStop o1, TransportStop o2) {
|
||||
return forwardStops.get(o1) - forwardStops.get(o2);
|
||||
}
|
||||
});
|
||||
// all common stops are with negative index (reeval them)
|
||||
for (TransportStop s : new ArrayList<TransportStop>(backwardStops.keySet())) {
|
||||
if (backwardStops.get(s) < 0) {
|
||||
backwardStops.put(s, backwardStops.size() + backwardStops.get(s) - 1);
|
||||
}
|
||||
}
|
||||
Collections.sort(r.getBackwardStops(), new Comparator<TransportStop>() {
|
||||
@Override
|
||||
public int compare(TransportStop o1, TransportStop o2) {
|
||||
return backwardStops.get(o1) - backwardStops.get(o2);
|
||||
}
|
||||
});
|
||||
|
||||
return r;
|
||||
|
||||
}
|
||||
|
||||
private void iterateEntity(Entity e) throws SQLException {
|
||||
if (indexPOI && Amenity.isAmenity(e)) {
|
||||
loadEntityData(e, false);
|
||||
if(poiPreparedStatement != null){
|
||||
Amenity a = new Amenity(e);
|
||||
if(a.getLocation() != null){
|
||||
DataIndexWriter.insertAmenityIntoPoi(poiPreparedStatement, pStatements, a, BATCH_SIZE);
|
||||
}
|
||||
}
|
||||
}
|
||||
if(indexTransport){
|
||||
if(e instanceof Relation && e.getTag(OSMTagKey.ROUTE) != null){
|
||||
loadEntityData(e, true);
|
||||
TransportRoute route = indexTransportRoute((Relation) e);
|
||||
if(route != null){
|
||||
DataIndexWriter.insertTransportIntoIndex(transRouteStat, transRouteStopsStat, transStopsStat, visitedStops, route, pStatements,
|
||||
BATCH_SIZE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* if (e instanceof Node && e.getTag(OSMTagKey.PLACE) != null) {
|
||||
places.add((Node) e);
|
||||
processed = true;
|
||||
|
|
@ -578,31 +777,17 @@ public class NewDataExtraction {
|
|||
}
|
||||
|
||||
}
|
||||
}
|
||||
if(indexTransport){
|
||||
if(e instanceof Relation && e.getTag(OSMTagKey.ROUTE) != null){
|
||||
transport.add((Relation) e);
|
||||
processed = true;
|
||||
}
|
||||
if(e instanceof Node && "bus_stop".equals(e.getTag(OSMTagKey.HIGHWAY))){
|
||||
// load all stops in order to get their names
|
||||
processed = true;
|
||||
}
|
||||
} */
|
||||
|
||||
if (e instanceof Node && e.getTag(OSMTagKey.RAILWAY) != null) {
|
||||
if ("tram_stop".equals(e.getTag(OSMTagKey.RAILWAY)) || "station".equals(e.getTag(OSMTagKey.RAILWAY))) {
|
||||
// load all stops in order to get their names
|
||||
processed = true;
|
||||
}
|
||||
}
|
||||
}*/
|
||||
}
|
||||
|
||||
|
||||
// TODO transliteration !!!
|
||||
public static void main(String[] args) throws IOException, SAXException, SQLException {
|
||||
NewDataExtraction extr = new NewDataExtraction(false, true, false,false, false, false, new File("E:/temp"));
|
||||
extr.setPoiFileName("1.odb");
|
||||
extr.generateIndexes("e:\\Temp\\ams_poi.osm", new ConsoleProgressImplementation(4), null);
|
||||
IndexCreator extr = new IndexCreator(new File("e:/Information/OSM maps/osmand/"));
|
||||
extr.setIndexPOI(true);
|
||||
extr.setIndexTransport(true);
|
||||
extr.generateIndexes("e:/Information/OSM maps/belarus osm/minsk.osm", new ConsoleProgressImplementation(4), null);
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -70,6 +70,9 @@ public class Relation extends Entity {
|
|||
}
|
||||
|
||||
public Map<Entity, String> getMemberEntities() {
|
||||
if(memberEntities == null){
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
return memberEntities;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -76,7 +76,7 @@ public class AHSupermarketResolver {
|
|||
}
|
||||
|
||||
|
||||
private static void selfTest() throws IOException {
|
||||
public static void selfTest() throws IOException {
|
||||
final String json = "[{city:'Eindhoven',lat:51.443278,format:'TOGO',lng:5.480161,sunday:true,street:'Neckerspoel',hours:[{F:'0630',U:'2300',D:'09-08-2010'},{F:'0630',U:'2300',D:'10-08-2010'},{F:'0630',U:'2300',D:'11-08-2010'},{F:'0630',U:'2300',D:'12-08-2010'},{F:'0630',U:'2330',D:'13-08-2010'},{F:'0700',U:'2330',D:'14-08-2010'},{F:'0800',U:'2300',D:'15-08-2010'},{F:'0630',U:'2300',D:'16-08-2010'},{F:'0630',U:'2300',D:'17-08-2010'},{F:'0630',U:'2300',D:'18-08-2010'},{F:'0630',U:'2300',D:'19-08-2010'},{F:'0630',U:'2330',D:'20-08-2010'},{F:'0700',U:'2330',D:'21-08-2010'},{F:'0800',U:'2300',D:'22-08-2010'}],no:5816,phone:'040-2376060',zip:'5611 AD',housenr:'10'},{city:'Amsterdam',lat:52.346837,format:'TOGO',lng:4.918422,sunday:true,street:'Julianaplein',hours:[{F:'0630',U:'2359',D:'09-08-2010'},{F:'0630',U:'2359',D:'10-08-2010'},{F:'0630',U:'2359',D:'11-08-2010'},{F:'0630',U:'2359',D:'12-08-2010'},{F:'0630',U:'2359',D:'13-08-2010'},{F:'0700',U:'2359',D:'14-08-2010'},{F:'0900',U:'2359',D:'15-08-2010'},{F:'0630',U:'2359',D:'16-08-2010'},{F:'0630',U:'2359',D:'17-08-2010'},{F:'0630',U:'2359',D:'18-08-2010'},{F:'0630',U:'2359',D:'19-08-2010'},{F:'0630',U:'2359',D:'20-08-2010'},{F:'0700',U:'2359',D:'21-08-2010'},{F:'0900',U:'2359',D:'22-08-2010'}],no:5817,phone:'020-4689944',zip:'1097 DN',housenr:'1'}]";
|
||||
AHSupermarketResolver resolver = new AHSupermarketResolver() {
|
||||
protected InputStream openStream() throws IOException {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<classpath>
|
||||
<classpathentry kind="src" path="src"/>
|
||||
<classpathentry excluding="**/MapPanel*|com/osmand/LogUtil.java|com/osmand/osm/io/OSMStorageWriter.java|com/osmand/DataExtraction.java|com/osmand/swing/|com/osmand/data/preparation/DataExtraction.java|com/osmand/data/preparation/DataIndexBuilder.java|com/osmand/osm/io/OsmStorageWriter.java|test/|com/osmand/ExceptionHandler.java|com/osmand/osm/util/|com/osmand/data/index/IndexBatchCreator.java|com/osmand/data/preparation/NewDataExtraction.java" kind="src" path="use"/>
|
||||
<classpathentry excluding="**/MapPanel*|com/osmand/LogUtil.java|com/osmand/osm/io/OSMStorageWriter.java|com/osmand/DataExtraction.java|com/osmand/swing/|com/osmand/data/preparation/DataExtraction.java|com/osmand/data/preparation/DataIndexBuilder.java|com/osmand/osm/io/OsmStorageWriter.java|test/|com/osmand/ExceptionHandler.java|com/osmand/osm/util/|com/osmand/data/index/IndexBatchCreator.java|com/osmand/data/preparation/IndexCreator.java" kind="src" path="use"/>
|
||||
<classpathentry kind="src" path="gen"/>
|
||||
<classpathentry kind="con" path="com.android.ide.eclipse.adt.ANDROID_FRAMEWORK"/>
|
||||
<classpathentry kind="lib" path="lib/bzip2-20090327.jar"/>
|
||||
|
|
|
|||
Loading…
Reference in a new issue