implement new indexer mechanism

git-svn-id: https://osmand.googlecode.com/svn/trunk@448 e29c36b1-1cfa-d876-8d93-3434fc2bb7b8

DataExtractionOSM/src/com/osmand/data/index/DataIndexWriter.java

@@ -8,6 +8,7 @@ import java.sql.PreparedStatement;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
@@ -82,13 +83,13 @@ public class DataIndexWriter {
 		try {
 			createPoiIndexStructure(conn);
 			PreparedStatement prep = createStatementAmenityInsert(conn);
+			Map<PreparedStatement, Integer> map = new LinkedHashMap<PreparedStatement, Integer>();
+			map.put(prep, 0);
 	        conn.setAutoCommit(false);
-	        int currentCount = 0;
 			for (Amenity a : region.getAmenityManager().getAllObjects()) {
-				currentCount = insertAmenityIntoPoi(prep, a, currentCount, BATCH_SIZE);
+				insertAmenityIntoPoi(prep, map, a, BATCH_SIZE);
-				
 			}
-			if(currentCount > 0){
+			if(map.get(prep) > 0){
 				prep.executeBatch();
 			}
 			prep.close();
@@ -103,7 +104,7 @@ public class DataIndexWriter {
 		return this;
 	}
 
-	public static int insertAmenityIntoPoi(PreparedStatement prep, Amenity amenity, int currentInBatch, int batchSize) throws SQLException {
+	public static void insertAmenityIntoPoi(PreparedStatement prep, Map<PreparedStatement, Integer> map, Amenity amenity, int batchSize) throws SQLException {
 		prep.setLong(IndexPoiTable.ID.ordinal() + 1, amenity.getId());
 		prep.setDouble(IndexPoiTable.LATITUDE.ordinal() + 1, amenity.getLocation().getLatitude());
 		prep.setDouble(IndexPoiTable.LONGITUDE.ordinal() + 1, amenity.getLocation().getLongitude());
@@ -112,13 +113,7 @@ public class DataIndexWriter {
 		prep.setString(IndexPoiTable.TYPE.ordinal() + 1, AmenityType.valueToString(amenity.getType()));
 		prep.setString(IndexPoiTable.SUBTYPE.ordinal() + 1, amenity.getSubType());
 		prep.setString(IndexPoiTable.OPENING_HOURS.ordinal() + 1 , amenity.getOpeningHours());
-		prep.addBatch();
+		addBatch(map, prep, batchSize);
-		currentInBatch++;
-		if(currentInBatch >= batchSize){
-			prep.executeBatch();
-			currentInBatch = 0;
-		}
-		return currentInBatch;
 	}
 	
 	public static PreparedStatement createStatementAmenityInsert(Connection conn) throws SQLException{
@@ -278,18 +273,7 @@ public class DataIndexWriter {
 		}
         Connection conn = DriverManager.getConnection("jdbc:sqlite:"+file.getAbsolutePath()); //$NON-NLS-1$
 		try {
-			Statement stat = conn.createStatement();
+			createTransportIndexStructure(conn);
-			
-
-			
-	        stat.executeUpdate(IndexConstants.generateCreateSQL(IndexTransportRoute.values()));
-	        stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexTransportRoute.values()));
-	        stat.executeUpdate(IndexConstants.generateCreateSQL(IndexTransportRouteStop.values()));
-	        stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexTransportRouteStop.values()));
-	        stat.executeUpdate(IndexConstants.generateCreateSQL(IndexTransportStop.values()));
-	        stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexTransportStop.values()));
-	        stat.execute("PRAGMA user_version = " + IndexConstants.TRANSPORT_TABLE_VERSION); //$NON-NLS-1$
-	        stat.close();
 	        
 	        PreparedStatement prepRoute = conn.prepareStatement(
 	            IndexConstants.generatePrepareStatementToInsert(IndexTransportRoute.getTable(), IndexTransportRoute.values().length));
@@ -307,18 +291,7 @@ public class DataIndexWriter {
 	        Set<Long> writtenStops = new LinkedHashSet<Long>();
 	        for(String t : region.getTransportRoutes().keySet()){
 	        	for(TransportRoute r : region.getTransportRoutes().get(t)) {
-	        		assert IndexTransportRoute.values().length == 7;
+	        		insertTransportIntoIndex(prepRoute, prepRouteStops, prepStops, writtenStops, r, count, BATCH_SIZE);
-	        		prepRoute.setLong(IndexTransportRoute.ID.ordinal() + 1, r.getId());
-	        		prepRoute.setString(IndexTransportRoute.TYPE.ordinal() + 1, r.getType());
-					prepRoute.setString(IndexTransportRoute.OPERATOR.ordinal() + 1, r.getOperator());
-					prepRoute.setString(IndexTransportRoute.REF.ordinal() + 1, r.getRef());
-	        		prepRoute.setString(IndexTransportRoute.NAME.ordinal() + 1, r.getName());
-					prepRoute.setString(IndexTransportRoute.NAME_EN.ordinal() + 1, r.getEnName());
-					prepRoute.setInt(IndexTransportRoute.DIST.ordinal() + 1, r.getAvgBothDistance());
-					addBatch(count, prepRoute);
-					
-					writeRouteStops(prepRouteStops, prepStops, count, writtenStops, r, r.getForwardStops(), true);
-					writeRouteStops(prepRouteStops, prepStops, count, writtenStops, r, r.getBackwardStops(), false);
 	        	}
 	        }
 			
@@ -339,7 +312,7 @@ public class DataIndexWriter {
 		return this;
 	}
 
-	private void writeRouteStops(PreparedStatement prepRouteStops, PreparedStatement prepStops, Map<PreparedStatement, Integer> count,
+	private static void writeRouteStops(PreparedStatement prepRouteStops, PreparedStatement prepStops, Map<PreparedStatement, Integer> count,
 			Set<Long> writtenStops, TransportRoute r, List<TransportStop> stops, boolean direction) throws SQLException {
 		int i = 0;
 		for(TransportStop s : stops){
@@ -363,11 +336,44 @@ public class DataIndexWriter {
 	}
 	
 	
+	public static void insertTransportIntoIndex(PreparedStatement prepRoute, PreparedStatement prepRouteStops,
+			PreparedStatement prepStops, Set<Long> writtenStops, TransportRoute route, Map<PreparedStatement, Integer> statements,
+			int batchSize) throws SQLException {
+		assert IndexTransportRoute.values().length == 7;
+		prepRoute.setLong(IndexTransportRoute.ID.ordinal() + 1, route.getId());
+		prepRoute.setString(IndexTransportRoute.TYPE.ordinal() + 1, route.getType());
+		prepRoute.setString(IndexTransportRoute.OPERATOR.ordinal() + 1, route.getOperator());
+		prepRoute.setString(IndexTransportRoute.REF.ordinal() + 1, route.getRef());
+		prepRoute.setString(IndexTransportRoute.NAME.ordinal() + 1, route.getName());
+		prepRoute.setString(IndexTransportRoute.NAME_EN.ordinal() + 1, route.getEnName());
+		prepRoute.setInt(IndexTransportRoute.DIST.ordinal() + 1, route.getAvgBothDistance());
+		addBatch(statements, prepRoute);
+		
+		writeRouteStops(prepRouteStops, prepStops, statements, writtenStops, route, route.getForwardStops(), true);
+		writeRouteStops(prepRouteStops, prepStops, statements, writtenStops, route, route.getBackwardStops(), false);
+		
+	}
+	
+	public static void createTransportIndexStructure(Connection conn) throws SQLException{
+		Statement stat = conn.createStatement();
+        stat.executeUpdate(IndexConstants.generateCreateSQL(IndexTransportRoute.values()));
+        stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexTransportRoute.values()));
+        stat.executeUpdate(IndexConstants.generateCreateSQL(IndexTransportRouteStop.values()));
+        stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexTransportRouteStop.values()));
+        stat.executeUpdate(IndexConstants.generateCreateSQL(IndexTransportStop.values()));
+        stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexTransportStop.values()));
+        stat.execute("PRAGMA user_version = " + IndexConstants.TRANSPORT_TABLE_VERSION); //$NON-NLS-1$
+        stat.close();
+	}
 	
 
-	private void addBatch(Map<PreparedStatement, Integer> count, PreparedStatement p) throws SQLException{
+	private static void addBatch(Map<PreparedStatement, Integer> count, PreparedStatement p) throws SQLException{
+		addBatch(count, p, BATCH_SIZE);
+	}
+	
+	private static void addBatch(Map<PreparedStatement, Integer> count, PreparedStatement p, int batchSize) throws SQLException{
 		p.addBatch();
-		if(count.get(p) >= BATCH_SIZE){
+		if(count.get(p) >= batchSize){
 			p.executeBatch();
 			count.put(p, 0);
 		} else {
@@ -376,4 +382,6 @@ public class DataIndexWriter {
 	}
 
 	
+	
+	
 }

DataExtractionOSM/src/com/osmand/data/preparation/IndexCreator.java

@@ -11,8 +11,13 @@ import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.Map;
+import java.util.Set;
 import java.util.Map.Entry;
 
 import org.apache.commons.logging.Log;
@@ -23,8 +28,13 @@ import org.xml.sax.SAXException;
 import com.osmand.Algoritms;
 import com.osmand.IProgress;
 import com.osmand.data.Amenity;
+import com.osmand.data.TransportRoute;
+import com.osmand.data.TransportStop;
 import com.osmand.data.index.DataIndexWriter;
 import com.osmand.data.index.IndexConstants;
+import com.osmand.data.index.IndexConstants.IndexTransportRoute;
+import com.osmand.data.index.IndexConstants.IndexTransportRouteStop;
+import com.osmand.data.index.IndexConstants.IndexTransportStop;
 import com.osmand.impl.ConsoleProgressImplementation;
 import com.osmand.osm.Entity;
 import com.osmand.osm.Node;
@@ -32,6 +42,7 @@ import com.osmand.osm.Relation;
 import com.osmand.osm.Way;
 import com.osmand.osm.Entity.EntityId;
 import com.osmand.osm.Entity.EntityType;
+import com.osmand.osm.OSMSettings.OSMTagKey;
 import com.osmand.osm.io.IOsmStorageFilter;
 import com.osmand.osm.io.OsmBaseStorage;
 import com.osmand.swing.DataExtractionSettings;
@@ -43,21 +54,24 @@ import com.osmand.swing.DataExtractionSettings;
  *  
  *
  */
-public class NewDataExtraction {
+public class IndexCreator {
 	private static final Log log = LogFactory.getLog(DataExtraction.class);
 
 	public static final int BATCH_SIZE = 5000;
 	public static final String TEMP_NODES_DB = "nodes"+IndexConstants.MAP_INDEX_EXT;
 	
-	
-	private final boolean normalizeStreets;
-	private final boolean indexAddress;
-	private final boolean indexPOI;
-	private final boolean indexTransport;
-	private final boolean indexMap;
-	private final boolean saveAddressWays;
 	private File workingDir = null;
 	
+	private boolean indexMap;
+	private boolean indexPOI;
+	private boolean indexTransport;
+	
+	private boolean indexAddress;
+	private boolean normalizeStreets;
+	private boolean saveAddressWays;
+
+	private String regionName;
+	
 	private String transportFileName = null;
 	private String poiFileName = null;
 	private String addressFileName = null;
@@ -73,28 +87,49 @@ public class NewDataExtraction {
 	private Connection dbConn;
 	private File dbFile;
 	
+	Map<PreparedStatement, Integer> pStatements = new LinkedHashMap<PreparedStatement, Integer>();
 	
 	private Connection poiConnection;
 	private File poiIndexFile;
 	private PreparedStatement poiPreparedStatement;
-	private int poiInBatch = 0;
 	
-	private String regionName;
+	
+	Set<Long> visitedStops = new HashSet<Long>();
+	private File transportIndexFile;
+	private Connection transportConnection;
+	private PreparedStatement transRouteStat;
+	private PreparedStatement transRouteStopsStat;
+	private PreparedStatement transStopsStat;
 	
 	
 	
-	
+	public IndexCreator(File workingDir){
-	public NewDataExtraction(boolean indexAddress, boolean indexPOI, boolean indexTransport,
-			boolean indexMap, boolean saveAddressWays, boolean normalizeStreets, 
-			File workingDir){
-		this.indexAddress = indexAddress;
-		this.indexPOI = indexPOI;
-		this.indexTransport = indexTransport;
-		this.indexMap = indexMap;
-		this.saveAddressWays = saveAddressWays;
-		this.normalizeStreets = normalizeStreets;
 		this.workingDir = workingDir;
+	}
 	
+	
+	public void setIndexAddress(boolean indexAddress) {
+		this.indexAddress = indexAddress;
+	}
+	
+	public void setIndexMap(boolean indexMap) {
+		this.indexMap = indexMap;
+	}
+	
+	public void setIndexPOI(boolean indexPOI) {
+		this.indexPOI = indexPOI;
+	}
+	
+	public void setIndexTransport(boolean indexTransport) {
+		this.indexTransport = indexTransport;
+	}
+	
+	public void setSaveAddressWays(boolean saveAddressWays) {
+		this.saveAddressWays = saveAddressWays;
+	}
+	
+	public void setNormalizeStreets(boolean normalizeStreets) {
+		this.normalizeStreets = normalizeStreets;
 	}
 
 	
@@ -248,146 +283,174 @@ public class NewDataExtraction {
 		this.regionName = regionName;
 	}
 	
-	public void generateIndexes(String path, IProgress progress, IOsmStorageFilter addFilter) throws IOException, SAXException, SQLException{
+	public void generateIndexes(String path, IProgress progress, IOsmStorageFilter addFilter) throws IOException, SAXException,
-			File f = new File(path);
+			SQLException {
-			InputStream stream = new FileInputStream(f);
+		File f = new File(path);
-			int i = f.getName().indexOf('.');
+		InputStream stream = new FileInputStream(f);
-			if(regionName == null){
+		int i = f.getName().indexOf('.');
-				regionName = Algoritms.capitalizeFirstLetterAndLowercase(f.getName().substring(0, i));
+		if (regionName == null) {
-			}
+			regionName = Algoritms.capitalizeFirstLetterAndLowercase(f.getName().substring(0, i));
+		}
 
-			InputStream streamFile = stream;
+		InputStream streamFile = stream;
-			long st = System.currentTimeMillis();
+		long st = System.currentTimeMillis();
-			if (path.endsWith(".bz2")) {
+		if (path.endsWith(".bz2")) {
-				if (stream.read() != 'B' || stream.read() != 'Z') {
+			if (stream.read() != 'B' || stream.read() != 'Z') {
-					throw new RuntimeException("The source stream must start with the characters BZ if it is to be read as a BZip2 stream.");
+				throw new RuntimeException("The source stream must start with the characters BZ if it is to be read as a BZip2 stream.");
-				} else {
-					stream = new CBZip2InputStream(stream);
-				}
-			}
-			
-			if(progress != null){
-				progress.startTask("Loading file " + path, -1);
-			}
-	        OsmBaseStorage storage = new OsmBaseStorage();
-	        storage.setSupressWarnings(DataExtractionSettings.getSettings().isSupressWarningsForDuplicatedId());
-	        if (addFilter != null) {
-				storage.getFilters().add(addFilter);
-			}
-
-	        
-	        try {
-				Class.forName("org.sqlite.JDBC");
-			} catch (ClassNotFoundException e) {
-				log.error("Illegal configuration", e);
-				throw new IllegalStateException(e);
-			}
-			if(indexMap){
-				dbFile = new File(workingDir, getMapFileName()); 
 			} else {
-				dbFile = new File(workingDir, TEMP_NODES_DB);
+				stream = new CBZip2InputStream(stream);
 			}
+		}
+
+		if (progress != null) {
+			progress.startTask("Loading file " + path, -1);
+		}
+		OsmBaseStorage storage = new OsmBaseStorage();
+		storage.setSupressWarnings(DataExtractionSettings.getSettings().isSupressWarningsForDuplicatedId());
+		if (addFilter != null) {
+			storage.getFilters().add(addFilter);
+		}
+
+		try {
+			Class.forName("org.sqlite.JDBC");
+		} catch (ClassNotFoundException e) {
+			log.error("Illegal configuration", e);
+			throw new IllegalStateException(e);
+		}
+		if (indexMap) {
+			dbFile = new File(workingDir, getMapFileName());
+		} else {
+			dbFile = new File(workingDir, TEMP_NODES_DB);
+		}
+		// to save space
+		if (dbFile.exists()) {
+			dbFile.delete();
+		}
+		// creating nodes db to fast access for all nodes
+		dbConn = DriverManager.getConnection("jdbc:sqlite:" + dbFile.getAbsolutePath());
+
+		// 1. Loading osm file
+		NewDataExtractionOsmFilter filter = new NewDataExtractionOsmFilter();
+		try {
+			// 1 init database to store temporary data
+			progress.setGeneralProgress("[40 of 100]");
+
+			filter.initDatabase();
+			storage.getFilters().add(filter);
+			storage.parseOSM(stream, progress, streamFile, false);
+			filter.finishLoading();
+
+			if (log.isInfoEnabled()) {
+				log.info("File parsed : " + (System.currentTimeMillis() - st));
+			}
+			progress.finishTask();
+
+		} finally {
+			if (log.isInfoEnabled()) {
+				log.info("File indexed : " + (System.currentTimeMillis() - st));
+			}
+		}
+
+		// 2. Processing all entries
+		progress.setGeneralProgress("[90 of 100]");
+
+		pselectNode = dbConn.prepareStatement("select * from node where id = ?");
+		pselectWay = dbConn.prepareStatement("select * from ways where id = ?");
+		pselectRelation = dbConn.prepareStatement("select * from relations where id = ?");
+		pselectTags = dbConn.prepareStatement("select key, value from tags where id = ? and type = ?");
+
+		if (indexPOI) {
+			poiIndexFile = new File(workingDir, getPoiFileName());
 			// to save space
-			if(dbFile.exists()){
+			if (poiIndexFile.exists()) {
-				dbFile.delete();
+				poiIndexFile.delete();
 			}
+			poiIndexFile.getParentFile().mkdirs();
 			// creating nodes db to fast access for all nodes
-			dbConn = DriverManager.getConnection("jdbc:sqlite:" + dbFile.getAbsolutePath());
+			poiConnection = DriverManager.getConnection("jdbc:sqlite:" + poiIndexFile.getAbsolutePath());
+			poiConnection.setAutoCommit(false);
+			DataIndexWriter.createPoiIndexStructure(poiConnection);
+			poiPreparedStatement = DataIndexWriter.createStatementAmenityInsert(poiConnection);
+			pStatements.put(poiPreparedStatement, 0);
+		}
 
-	        // 1. Loading osm file
+		if (indexTransport) {
-			NewDataExtractionOsmFilter filter = new NewDataExtractionOsmFilter();
+			transportIndexFile = new File(workingDir, getTransportFileName());
-			try {
+			// to save space
-				// 1 init database to store temporary data
+			if (transportIndexFile.exists()) {
-				progress.setGeneralProgress("[40 of 100]");
+				transportIndexFile.delete();
+			}
+			transportIndexFile.getParentFile().mkdirs();
+			// creating nodes db to fast access for all nodes
+			transportConnection = DriverManager.getConnection("jdbc:sqlite:" + transportIndexFile.getAbsolutePath());
 
-				filter.initDatabase();
+			DataIndexWriter.createTransportIndexStructure(transportConnection);
-				storage.getFilters().add(filter);
+			transRouteStat = transportConnection.prepareStatement(IndexConstants.generatePrepareStatementToInsert(
-				storage.parseOSM(stream, progress, streamFile, false);
+					IndexTransportRoute.getTable(), IndexTransportRoute.values().length));
-				filter.finishLoading();
+			transRouteStopsStat = transportConnection.prepareStatement(IndexConstants.generatePrepareStatementToInsert(
+					IndexTransportRouteStop.getTable(), IndexTransportRouteStop.values().length));
+			transStopsStat = transportConnection.prepareStatement(IndexConstants.generatePrepareStatementToInsert(IndexTransportStop
+					.getTable(), IndexTransportStop.values().length));
+			pStatements.put(transRouteStat, 0);
+			pStatements.put(transRouteStopsStat, 0);
+			pStatements.put(transStopsStat, 0);
+			transportConnection.setAutoCommit(false);
 
-				if (log.isInfoEnabled()) {
+		}
-					log.info("File parsed : " + (System.currentTimeMillis() - st));
-				}
-				progress.finishTask();
 
+		iterateOverAllEntities(progress, filter);
 
-			} finally {
+		try {
-				if (log.isInfoEnabled()) {
+			if (pselectNode != null) {
-					log.info("File indexed : " + (System.currentTimeMillis() - st));
+				pselectNode.close();
+			}
+			if (pselectWay != null) {
+				pselectWay.close();
+			}
+			if (pselectRelation != null) {
+				pselectRelation.close();
+			}
+			if (pselectTags != null) {
+				pselectTags.close();
+			}
+			for (PreparedStatement p : pStatements.keySet()) {
+				if (pStatements.get(p) > 0) {
+					p.executeBatch();
+					p.close();
 				}
 			}
 
-	
+			if (poiConnection != null) {
-	        // 2. Processing all entries
+				poiConnection.commit();
-			progress.setGeneralProgress("[90 of 100]");
+				poiConnection.close();
-
+				if (lastModifiedDate != null) {
-			pselectNode = dbConn.prepareStatement("select * from node where id = ?");
+					poiIndexFile.setLastModified(lastModifiedDate);
-			pselectWay = dbConn.prepareStatement("select * from ways where id = ?");
+				}
-			pselectRelation = dbConn.prepareStatement("select * from relations where id = ?");
+			}
-			pselectTags = dbConn.prepareStatement("select key, value from tags where id = ? and type = ?");
+			if (transportConnection != null) {
-			
+				transportConnection.commit();
-			if(indexPOI){
+				transportConnection.close();
-				poiIndexFile = new File(workingDir, getPoiFileName());
+				if (lastModifiedDate != null) {
-				// to save space
+					transportIndexFile.setLastModified(lastModifiedDate);
-				if(poiIndexFile.exists()){
-					poiIndexFile.delete();
 				}
-				poiIndexFile.getParentFile().mkdirs();
-				// creating nodes db to fast access for all nodes
-				poiConnection = DriverManager.getConnection("jdbc:sqlite:" + poiIndexFile.getAbsolutePath());
-				poiConnection.setAutoCommit(false);
-				DataIndexWriter.createPoiIndexStructure(poiConnection);
-				poiPreparedStatement = DataIndexWriter.createStatementAmenityInsert(poiConnection);
-				poiInBatch = 0;
 			}
 			
-
+			dbConn.close();
-			iterateOverAllEntities(progress, filter);
+		} catch (SQLException e) {
-			
+		}
-			try {
-				if(pselectNode != null){
-					pselectNode.close();
-				}
-				if(pselectWay != null){
-					pselectWay.close();
-				}
-				if(pselectRelation != null){
-					pselectRelation.close();
-				}
-				if(pselectTags != null){
-					pselectTags.close();
-				}
-				if(poiConnection != null){
-					if(poiPreparedStatement != null){
-						if(poiInBatch > 0){
-							poiPreparedStatement.executeBatch();
-						}
-						poiPreparedStatement.close();
-					}
-					poiConnection.commit();
-					poiConnection.close();
-					if(lastModifiedDate != null){
-						poiIndexFile.setLastModified(lastModifiedDate);
-					}
-					
-				}
-				dbConn.close();
-			} catch (SQLException e) {
-			}
 	}
 	
-	public void loadEntityData(Entity e) throws SQLException {
+	public void loadEntityData(Entity e, boolean loadTags) throws SQLException {
 		if(e instanceof Node){
 			return;
 		}
 		Map<EntityId, Entity> map = new LinkedHashMap<EntityId, Entity>();
-		ArrayList<EntityId> ids = new ArrayList<EntityId>();
 		if(e instanceof Relation){
 			pselectRelation.setLong(1, e.getId());
 			if (pselectRelation.execute()) {
-				ResultSet rs = pselectNode.getResultSet();
+				ResultSet rs = pselectRelation.getResultSet();
 				while (rs.next()) {
-					((Relation) e).addMember(rs.getLong(1), EntityType.values()[rs.getByte(2)], rs.getString(3));
+					((Relation) e).addMember(rs.getLong(2), EntityType.values()[rs.getByte(3)], rs.getString(4));
 				}
 				rs.close();
 			}
@@ -401,6 +464,7 @@ public class NewDataExtraction {
 				rs.close();
 			}
 		}
+		Collection<EntityId> ids = e instanceof Relation? ((Relation)e).getMemberIds() : ((Way)e).getEntityIds();
 		for (EntityId i : ids) {
 			if (i.getType() == EntityType.NODE) {
 				pselectNode.setLong(1, i.getId());
@@ -421,6 +485,7 @@ public class NewDataExtraction {
 						way.addNode(rs.getLong(2));
 					}
 					rs.close();
+					loadEntityData(way, false);
 				}
 			} else if (i.getType() == EntityType.RELATION) {
 				pselectRelation.setLong(1, i.getId());
@@ -436,6 +501,11 @@ public class NewDataExtraction {
 				}
 			}
 		}
+		if(loadTags){
+			for(Map.Entry<EntityId, Entity> es : map.entrySet()){
+				loadEntityTags(es.getKey().getType(), es.getValue());
+			}
+		}
 		e.initializeLinks(map);
 	}
 	
@@ -455,7 +525,10 @@ public class NewDataExtraction {
 		this.mapFileName = mapFileName;
 	}
 	public String getMapFileName() {
-		return getRegionName() + IndexConstants.MAP_INDEX_EXT;
+		if(mapFileName == null){
+			return getRegionName() + IndexConstants.MAP_INDEX_EXT;
+		}
+		return mapFileName;
 	}
 	
 	public String getTransportFileName() {
@@ -537,13 +610,139 @@ public class NewDataExtraction {
 	}
 	
 	
-	private void iterateEntity(Entity e) throws SQLException {
+	private static Set<String> acceptedRoutes = new HashSet<String>();
-		if (indexPOI && Amenity.isAmenity(e)) {
+	static {
-			loadEntityData(e);
+		acceptedRoutes.add("bus");
-			if(poiPreparedStatement != null){
+		acceptedRoutes.add("trolleybus");
-				poiInBatch = DataIndexWriter.insertAmenityIntoPoi(poiPreparedStatement, new Amenity(e), poiInBatch, BATCH_SIZE);
+		acceptedRoutes.add("share_taxi");
+		
+		acceptedRoutes.add("subway");
+		acceptedRoutes.add("train");
+		
+		acceptedRoutes.add("tram");
+		
+		acceptedRoutes.add("ferry");
+	}
+	
+	private TransportRoute indexTransportRoute(Relation rel) {
+		String ref = rel.getTag(OSMTagKey.REF);
+		String route = rel.getTag(OSMTagKey.ROUTE);
+		String operator = rel.getTag(OSMTagKey.OPERATOR);
+		if (route == null || ref == null) {
+			return null;
+		}
+		if (!acceptedRoutes.contains(route)) {
+			return null;
+		}
+		TransportRoute r = new TransportRoute(rel, ref);
+		r.setOperator(operator);
+		r.setType(route);
+
+		if (operator != null) {
+			route = operator + " : " + route;
+		}
+
+		final Map<TransportStop, Integer> forwardStops = new LinkedHashMap<TransportStop, Integer>();
+		final Map<TransportStop, Integer> backwardStops = new LinkedHashMap<TransportStop, Integer>();
+		int currentStop = 0;
+		int forwardStop = 0;
+		int backwardStop = 0;
+		for (Entry<Entity, String> e : rel.getMemberEntities().entrySet()) {
+			if (e.getValue().contains("stop")) {
+				if (e.getKey() instanceof Node) {
+					TransportStop stop = new TransportStop(e.getKey());
+					boolean forward = e.getValue().contains("forward");
+					boolean backward = e.getValue().contains("backward");
+					currentStop++;
+					if (forward || !backward) {
+						forwardStop++;
+					}
+					if (backward) {
+						backwardStop++;
+					}
+					boolean common = !forward && !backward;
+					int index = -1;
+					int i = e.getValue().length() - 1;
+					int accum = 1;
+					while (i >= 0 && Character.isDigit(e.getValue().charAt(i))) {
+						if (index < 0) {
+							index = 0;
+						}
+						index = accum * Character.getNumericValue(e.getValue().charAt(i)) + index;
+						accum *= 10;
+						i--;
+					}
+					if (index < 0) {
+						index = forward ? forwardStop : (backward ? backwardStop : currentStop);
+					}
+					if (forward || common) {
+						forwardStops.put(stop, index);
+						r.getForwardStops().add(stop);
+					}
+					if (backward || common) {
+						if (common) {
+							// put with negative index
+							backwardStops.put(stop, -index);
+						} else {
+							backwardStops.put(stop, index);
+						}
+
+						r.getBackwardStops().add(stop);
+					}
+
+				}
+
+			} else if (e.getKey() instanceof Way) {
+				r.addWay((Way) e.getKey());
 			}
 		}
+		if (forwardStops.isEmpty() && backwardStops.isEmpty()) {
+			return null;
+		}
+		Collections.sort(r.getForwardStops(), new Comparator<TransportStop>() {
+			@Override
+			public int compare(TransportStop o1, TransportStop o2) {
+				return forwardStops.get(o1) - forwardStops.get(o2);
+			}
+		});
+		// all common stops are with negative index (reeval them)
+		for (TransportStop s : new ArrayList<TransportStop>(backwardStops.keySet())) {
+			if (backwardStops.get(s) < 0) {
+				backwardStops.put(s, backwardStops.size() + backwardStops.get(s) - 1);
+			}
+		}
+		Collections.sort(r.getBackwardStops(), new Comparator<TransportStop>() {
+			@Override
+			public int compare(TransportStop o1, TransportStop o2) {
+				return backwardStops.get(o1) - backwardStops.get(o2);
+			}
+		});
+		
+		return r;
+
+	}
+
+	private void iterateEntity(Entity e) throws SQLException {
+		if (indexPOI && Amenity.isAmenity(e)) {
+			loadEntityData(e, false);
+			if(poiPreparedStatement != null){
+				Amenity a = new Amenity(e);
+				if(a.getLocation() != null){
+					DataIndexWriter.insertAmenityIntoPoi(poiPreparedStatement, pStatements, a, BATCH_SIZE);
+				}
+			}
+		}
+		if(indexTransport){
+			if(e instanceof Relation && e.getTag(OSMTagKey.ROUTE) != null){
+				loadEntityData(e, true);
+				TransportRoute route = indexTransportRoute((Relation) e);
+				if(route != null){
+					DataIndexWriter.insertTransportIntoIndex(transRouteStat, transRouteStopsStat, transStopsStat, visitedStops, route, pStatements,
+							BATCH_SIZE);
+				}
+			}
+		}
+		
 /*		if (e instanceof Node && e.getTag(OSMTagKey.PLACE) != null) {
 			places.add((Node) e);
 			processed = true;
@@ -578,31 +777,17 @@ public class NewDataExtraction {
 				}
 
 			}
-		}
+		} */
-		if(indexTransport){
-			if(e instanceof Relation && e.getTag(OSMTagKey.ROUTE) != null){
-				transport.add((Relation) e);
-				processed = true;
-			}
-			if(e instanceof Node && "bus_stop".equals(e.getTag(OSMTagKey.HIGHWAY))){
-				// load all stops in order to get their names
-				processed = true;
-			}
 		
-			if (e instanceof Node && e.getTag(OSMTagKey.RAILWAY) != null) {
-				if ("tram_stop".equals(e.getTag(OSMTagKey.RAILWAY)) || "station".equals(e.getTag(OSMTagKey.RAILWAY))) {
-					// load all stops in order to get their names
-					processed = true;
-				}
-			}
-		}*/
 	}
 	
 
+	// TODO transliteration !!!
 	public static void main(String[] args) throws IOException, SAXException, SQLException {
-		NewDataExtraction extr = new NewDataExtraction(false, true, false,false, false, false, new File("E:/temp"));
+		IndexCreator extr = new IndexCreator(new File("e:/Information/OSM maps/osmand/"));
-		extr.setPoiFileName("1.odb");
+		extr.setIndexPOI(true);
-		extr.generateIndexes("e:\\Temp\\ams_poi.osm", new ConsoleProgressImplementation(4), null);
+		extr.setIndexTransport(true);
+		extr.generateIndexes("e:/Information/OSM maps/belarus osm/minsk.osm", new ConsoleProgressImplementation(4), null);
 		
 	}
 }

DataExtractionOSM/src/com/osmand/osm/Relation.java

@@ -70,6 +70,9 @@ public class Relation extends Entity {
 	}
 	
 	public Map<Entity, String> getMemberEntities() {
+		if(memberEntities == null){
+			return Collections.emptyMap();
+		}
 		return memberEntities;
 	}
 	

DataExtractionOSM/src/com/osmand/osm/util/AHSupermarketResolver.java

@@ -76,7 +76,7 @@ public class AHSupermarketResolver {
 	}
 	
 	
-	private static void selfTest() throws IOException {
+	public static void selfTest() throws IOException {
 		final String json = "[{city:'Eindhoven',lat:51.443278,format:'TOGO',lng:5.480161,sunday:true,street:'Neckerspoel',hours:[{F:'0630',U:'2300',D:'09-08-2010'},{F:'0630',U:'2300',D:'10-08-2010'},{F:'0630',U:'2300',D:'11-08-2010'},{F:'0630',U:'2300',D:'12-08-2010'},{F:'0630',U:'2330',D:'13-08-2010'},{F:'0700',U:'2330',D:'14-08-2010'},{F:'0800',U:'2300',D:'15-08-2010'},{F:'0630',U:'2300',D:'16-08-2010'},{F:'0630',U:'2300',D:'17-08-2010'},{F:'0630',U:'2300',D:'18-08-2010'},{F:'0630',U:'2300',D:'19-08-2010'},{F:'0630',U:'2330',D:'20-08-2010'},{F:'0700',U:'2330',D:'21-08-2010'},{F:'0800',U:'2300',D:'22-08-2010'}],no:5816,phone:'040-2376060',zip:'5611 AD',housenr:'10'},{city:'Amsterdam',lat:52.346837,format:'TOGO',lng:4.918422,sunday:true,street:'Julianaplein',hours:[{F:'0630',U:'2359',D:'09-08-2010'},{F:'0630',U:'2359',D:'10-08-2010'},{F:'0630',U:'2359',D:'11-08-2010'},{F:'0630',U:'2359',D:'12-08-2010'},{F:'0630',U:'2359',D:'13-08-2010'},{F:'0700',U:'2359',D:'14-08-2010'},{F:'0900',U:'2359',D:'15-08-2010'},{F:'0630',U:'2359',D:'16-08-2010'},{F:'0630',U:'2359',D:'17-08-2010'},{F:'0630',U:'2359',D:'18-08-2010'},{F:'0630',U:'2359',D:'19-08-2010'},{F:'0630',U:'2359',D:'20-08-2010'},{F:'0700',U:'2359',D:'21-08-2010'},{F:'0900',U:'2359',D:'22-08-2010'}],no:5817,phone:'020-4689944',zip:'1097 DN',housenr:'1'}]";
 		AHSupermarketResolver resolver = new AHSupermarketResolver() {
 			protected InputStream openStream() throws IOException {

OsmAnd/.classpath

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <classpath>
 	<classpathentry kind="src" path="src"/>
-	<classpathentry excluding="**/MapPanel*|com/osmand/LogUtil.java|com/osmand/osm/io/OSMStorageWriter.java|com/osmand/DataExtraction.java|com/osmand/swing/|com/osmand/data/preparation/DataExtraction.java|com/osmand/data/preparation/DataIndexBuilder.java|com/osmand/osm/io/OsmStorageWriter.java|test/|com/osmand/ExceptionHandler.java|com/osmand/osm/util/|com/osmand/data/index/IndexBatchCreator.java|com/osmand/data/preparation/NewDataExtraction.java" kind="src" path="use"/>
+	<classpathentry excluding="**/MapPanel*|com/osmand/LogUtil.java|com/osmand/osm/io/OSMStorageWriter.java|com/osmand/DataExtraction.java|com/osmand/swing/|com/osmand/data/preparation/DataExtraction.java|com/osmand/data/preparation/DataIndexBuilder.java|com/osmand/osm/io/OsmStorageWriter.java|test/|com/osmand/ExceptionHandler.java|com/osmand/osm/util/|com/osmand/data/index/IndexBatchCreator.java|com/osmand/data/preparation/IndexCreator.java" kind="src" path="use"/>
 	<classpathentry kind="src" path="gen"/>
 	<classpathentry kind="con" path="com.android.ide.eclipse.adt.ANDROID_FRAMEWORK"/>
 	<classpathentry kind="lib" path="lib/bzip2-20090327.jar"/>