add address index to binary format

git-svn-id: https://osmand.googlecode.com/svn/trunk@620 e29c36b1-1cfa-d876-8d93-3434fc2bb7b8

DataExtractionOSM/src/net/osmand/ToDoConstants.java

@@ -7,15 +7,12 @@ package net.osmand;
  */
 public class ToDoConstants {
 	
-	// TODO max 100
+	// TODO max 101
-	// FOR 0.4 beta RELEASE
-	// Profile vector rendering
 	
 
 	// 	Outside base 0.4 release
 	//   69. Add phone and site information to POI (enable call to POI and open site)
 	//   86. Allow to add/edit custom tags to POI objects (Issue)
-	//   91. Invent binary format (minimize disk space, maximize speed) 
 	//   92. Replace poi index with standard map index and unify POI categories
 	//   94. Revise index to decrease their size (especially address) - replace to float lat/lon and remove for POI 
 	//		 remove en_names from POI (possibly from address)
@@ -42,6 +39,7 @@ public class ToDoConstants {
 	// DONE ANDROID :
 	//   99. Implement better file downloader for big files
 	//  100. Show impoted gpx points (as favorites), sort the by distance
+	//   91. Invent binary format (minimize disk space, maximize speed)
 	
 	// DONE SWING
 

DataExtractionOSM/src/net/osmand/binary/BinaryMapIndexWriter.java

@@ -8,7 +8,14 @@ import java.util.Map;
 import java.util.Stack;
 
 import net.osmand.Algoritms;
+import net.osmand.binary.OsmandOdb.StreetIndex;
+import net.osmand.data.Building;
+import net.osmand.data.City;
+import net.osmand.data.MapObject;
+import net.osmand.data.Street;
 import net.osmand.data.index.IndexConstants;
+import net.osmand.osm.LatLon;
+import net.osmand.osm.MapUtils;
 
 import com.google.protobuf.CodedOutputStream;
 import com.google.protobuf.WireFormat;
@@ -39,6 +46,9 @@ public class BinaryMapIndexWriter {
 	private Stack<Long> stackBaseIds = new Stack<Long>();
 	private Stack<Map<String, Integer>> stackStringTable = new Stack<Map<String, Integer>>();
 	
+	// needed for address index
+	private MapObject cityOrPostcode = null;
+	
 	// internal constants to track state of index writing
 	private Stack<Integer> state = new Stack<Integer>();
 	private Stack<Long> stackSizes = new Stack<Long>();
@@ -48,6 +58,10 @@ public class BinaryMapIndexWriter {
 	private final static int MAP_ROOT_LEVEL_INIT = 3;
 	private final static int MAP_TREE = 4;
 	
+	private final static int ADDRESS_INDEX_INIT = 5;
+	private final static int CITY_INDEX_INIT = 6;
+	private final static int POSTCODES_INDEX_INIT = 7;
+
 	public BinaryMapIndexWriter(final RandomAccessFile raf) throws IOException{
 		this.raf = raf;
 		codedOutStream = CodedOutputStream.newInstance(new OutputStream() {
@@ -88,22 +102,19 @@ public class BinaryMapIndexWriter {
 	}
 	
 	public void startWriteMapIndex() throws IOException{
-		assert state.peek() == OSMAND_STRUCTURE_INIT;
+		pushState(MAP_INDEX_INIT, OSMAND_STRUCTURE_INIT);
-		state.push(MAP_INDEX_INIT);
 		codedOutStream.writeTag(OsmandOdb.OsmAndStructure.MAPINDEX_FIELD_NUMBER, WireFormat.WIRETYPE_FIXED32_LENGTH_DELIMITED);
 		preserveInt32Size();
 	}
 	
 	
 	public void endWriteMapIndex() throws IOException{
-		Integer st = state.pop();
+		popState(MAP_INDEX_INIT);
-		assert st == MAP_INDEX_INIT;
 		writeInt32Size();
 	}
 		
 	public void startWriteMapLevelIndex(int minZoom, int maxZoom, int leftX, int rightX, int topY, int bottomY) throws IOException{
-		assert state.peek() == MAP_INDEX_INIT;
+		pushState(MAP_ROOT_LEVEL_INIT, MAP_INDEX_INIT);
-		state.push(MAP_ROOT_LEVEL_INIT);
 		
 		codedOutStream.writeTag(OsmandOdb.OsmAndMapIndex.LEVELS_FIELD_NUMBER, WireFormat.WIRETYPE_FIXED32_LENGTH_DELIMITED);
 		preserveInt32Size();
@@ -119,8 +130,7 @@ public class BinaryMapIndexWriter {
 	}
 	
 	public void endWriteMapLevelIndex() throws IOException{
-		assert state.peek() == MAP_ROOT_LEVEL_INIT;
+		popState(MAP_ROOT_LEVEL_INIT);
-		state.pop();
 		stackBounds.pop();
 		writeInt32Size();
 	}
@@ -130,14 +140,14 @@ public class BinaryMapIndexWriter {
 	}
 	
 	public void startMapTreeElement(long baseId, int leftX, int rightX, int topY, int bottomY) throws IOException{
-		assert state.peek() == MAP_ROOT_LEVEL_INIT || state.peek() == MAP_TREE;
+		checkPeekState(MAP_ROOT_LEVEL_INIT, MAP_TREE);
 		if(state.peek() == MAP_ROOT_LEVEL_INIT){
 			codedOutStream.writeTag(OsmandOdb.MapRootLevel.ROOT_FIELD_NUMBER, WireFormat.WIRETYPE_FIXED32_LENGTH_DELIMITED);
 		} else {
 			codedOutStream.writeTag(OsmandOdb.MapTree.SUBTREES_FIELD_NUMBER, WireFormat.WIRETYPE_FIXED32_LENGTH_DELIMITED);
 		}
-		preserveInt32Size();
 		state.push(MAP_TREE);
+		preserveInt32Size();
 		
 		
 		Bounds bounds = stackBounds.peek();
@@ -150,9 +160,9 @@ public class BinaryMapIndexWriter {
 		stackStringTable.push(null);
 	}
 	
+
 	public void endWriteMapTreeElement() throws IOException{
-		assert state.peek() == MAP_TREE;
+		popState(MAP_TREE);
-		state.pop();
 		
 		stackBounds.pop();
 		Long l = stackBaseIds.pop();
@@ -319,6 +329,108 @@ public class BinaryMapIndexWriter {
 		}
 	}
 	
+	public void startWriteAddressIndex(String name) throws IOException {
+		pushState(OSMAND_STRUCTURE_INIT, ADDRESS_INDEX_INIT);
+		codedOutStream.writeTag(OsmandOdb.OsmAndStructure.ADDRESSINDEX_FIELD_NUMBER, WireFormat.WIRETYPE_FIXED32_LENGTH_DELIMITED);
+		preserveInt32Size();
+		
+		codedOutStream.writeString(OsmandOdb.OsmAndAddressIndex.NAME_FIELD_NUMBER, name);
+		
+	}
+	
+	public void endWriteAddressIndex() throws IOException {
+		popState(ADDRESS_INDEX_INIT);
+		writeInt32Size();
+	}
+	
+	
+	public void startWriteCityIndex(City city) throws IOException {
+		pushState(CITY_INDEX_INIT, ADDRESS_INDEX_INIT);
+		codedOutStream.writeTag(OsmandOdb.OsmAndStructure.ADDRESSINDEX_FIELD_NUMBER, WireFormat.WIRETYPE_FIXED32_LENGTH_DELIMITED);
+		preserveInt32Size();
+		
+		codedOutStream.writeUInt32(OsmandOdb.CityIndex.CITY_TYPE_FIELD_NUMBER, city.getType().ordinal());
+		codedOutStream.writeUInt64(OsmandOdb.CityIndex.ID_FIELD_NUMBER, city.getId());
+		codedOutStream.writeString(OsmandOdb.CityIndex.NAME_FIELD_NUMBER, city.getName());
+		if(city.getEnName() != null){
+			codedOutStream.writeString(OsmandOdb.CityIndex.NAME_EN_FIELD_NUMBER, city.getEnName());
+		}
+		codedOutStream.writeFixed32(OsmandOdb.CityIndex.X_FIELD_NUMBER, MapUtils.get31TileNumberX(city.getLocation().getLongitude()));
+		codedOutStream.writeFixed32(OsmandOdb.CityIndex.Y_FIELD_NUMBER, MapUtils.get31TileNumberY(city.getLocation().getLatitude()));
+		cityOrPostcode = city;
+	}
+	
+	public void writeStreetAndBuildings(Street street) throws IOException {
+		checkPeekState(CITY_INDEX_INIT, POSTCODES_INDEX_INIT);
+		StreetIndex.Builder streetBuilder = OsmandOdb.StreetIndex.newBuilder();
+		streetBuilder.setName(street.getName());
+		if(street.getEnName() != null){
+			streetBuilder.setNameEn(street.getEnName());
+		}
+		streetBuilder.setId(street.getId());
+		
+		
+		LatLon location = cityOrPostcode.getLocation();
+		int cx = MapUtils.get31TileNumberX(location.getLongitude());
+		int cy = MapUtils.get31TileNumberY(location.getLatitude());
+		int sx = MapUtils.get31TileNumberX(street.getLocation().getLongitude());
+		int sy = MapUtils.get31TileNumberY(street.getLocation().getLatitude());
+		streetBuilder.setX((sx - cx) >> 7);
+		streetBuilder.setY((sy - cy) >> 7);
+		
+		for(Building b : street.getBuildings()){
+			OsmandOdb.BuildingIndex.Builder bbuilder= OsmandOdb.BuildingIndex.newBuilder();
+			int bx = MapUtils.get31TileNumberX(b.getLocation().getLongitude());
+			int by = MapUtils.get31TileNumberY(b.getLocation().getLatitude());
+			bbuilder.setX((bx - sx) >> 7);
+			bbuilder.setY((by - sy) >> 7);
+			bbuilder.setId(b.getId());
+			bbuilder.setName(b.getName());
+			if(b.getEnName() != null){
+				bbuilder.setNameEn(b.getEnName());
+			}
+			streetBuilder.addBuildings(bbuilder.build());
+		}
+		
+		
+		if(state.peek() == CITY_INDEX_INIT){
+			codedOutStream.writeMessage(OsmandOdb.CityIndex.STREETS_FIELD_NUMBER, streetBuilder.build());
+		} else {
+			throw new UnsupportedOperationException();
+		}
+		
+		
+		
+	}
+	
+	public void endWriteCityIndex() throws IOException {
+		popState(CITY_INDEX_INIT);
+		writeInt32Size();
+		cityOrPostcode = null;
+	}
+	 
+	private void pushState(int push, int peek){
+		if(state.peek() != peek){
+			throw new IllegalStateException("expected " + peek+ " != "+ state.peek());
+		}
+		state.push(push);
+	}
+	
+	private void checkPeekState(int... states) {
+		for(int i=0;i<states.length; i++){
+			if(states[i] == state.peek()){
+				return;
+			}
+		}
+		throw new IllegalStateException("Note expected state : " + state.peek());
+	}
+	
+	private void popState(int state){
+		Integer st = this.state.pop();
+		if(st != state){
+			throw new IllegalStateException("expected " + state + " != "+ st);
+		}
+	}
 	
 	public void close() throws IOException{
 		assert state.peek() == OSMAND_STRUCTURE_INIT;

DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java

@@ -1484,14 +1484,15 @@ public class IndexCreator {
 		
 	}
 	
-	public void writeBinaryMapIndex() throws IOException, SQLException {
+	public void writeBinaryAddressIndex(BinaryMapIndexWriter writer) throws IOException, SQLException {
+		
+	}
+	
+	public void writeBinaryMapIndex(BinaryMapIndexWriter writer) throws IOException, SQLException {
 		try {
 			assert IndexConstants.IndexBinaryMapRenderObject.values().length == 6;
 			PreparedStatement selectData = mapConnection.prepareStatement("SELECT * FROM " + IndexBinaryMapRenderObject.getTable() + " WHERE id = ?");
 			
-			
-			
-			BinaryMapIndexWriter writer = new BinaryMapIndexWriter(mapRAFile);
 			writer.startWriteMapIndex();
 			
 			for (int i = 0; i < MAP_ZOOMS.length - 1; i++) {
@@ -1897,12 +1898,21 @@ public class IndexCreator {
 				pStatements.remove(mapBinaryStat);
 				mapConnection.commit();
 				
+				log.info("Finish packing RTree files");
+			}
+			
+			if(indexMap || indexAddress){
 				if(mapFile.exists()){
 					mapFile.delete();
 				}
 				mapRAFile = new RandomAccessFile(mapFile, "rw");
-				log.info("Finish packing RTree files");
+				BinaryMapIndexWriter writer = new BinaryMapIndexWriter(mapRAFile);
-				writeBinaryMapIndex();
+				if(indexMap){
+					writeBinaryMapIndex(writer);
+				}
+				if(indexAddress){
+					writeBinaryAddressIndex(writer);
+				}
 				log.info("Finish writing binary file");
 			}
 
@@ -2020,7 +2030,6 @@ public class IndexCreator {
 //		 creator.setNodesDBFile(new File("e:/Information/OSM maps/osmand/belarus_nodes.tmp.odb"));
 //		 creator.generateIndexes(new File("e:/Information/OSM maps/belarus osm/belarus.osm.bz2"), new ConsoleProgressImplementation(3), null);
 		 
-		 creator.generateIndexes(new File("e:/Information/OSM maps/osm_map/zimbabwe.osm.bz2"), new ConsoleProgressImplementation(3), null);
 		 
 		 
 //		 creator.generateIndexes(new File("e:/Information/OSM maps/belarus osm/forest.osm"), new ConsoleProgressImplementation(3), null);

DataExtractionOSM/src/osmand_odb.proto

@@ -12,8 +12,11 @@ message OsmAndStructure {
    required uint32 version = 1;
    // encoded as fixed32 length delimited
    repeated OsmAndMapIndex mapIndex = 2;
+   // encoded as fixed32 length delimited  
+   repeated OsmAndAddressIndex addressIndex = 3;
 } 
 
+
 message OsmAndMapIndex {
    // encoded as fixed32 length delimited 
    repeated MapRootLevel levels = 1;  
@@ -70,3 +73,57 @@ message MapData {
   optional int32 highwayMeta = 6; 
 }
 
+
+/// ADDRESS INFORMATION TEST -----
+
+message OsmAndAddressIndex {
+	
+	required string name = 1;
+	optional string name_en = 2;
+	
+	// encoded as fixed32 length delimited
+	repeated CityIndex cityIndex= 5; // cities and towns 
+
+	repeated PostcodeIndex postcodes= 6;
+	
+	repeated CityIndex villages = 7; // suburbs and villages
+}
+
+message CityIndex {
+    required uint32 city_type = 1; // 0-5 enum CityType
+	required string name = 2;
+	optional string name_en = 3;
+	optional uint64 id = 4;
+	
+	required fixed32 x = 5; // x tile of 31 zoom
+	required fixed32 y = 6; // y tile of 31 zoom
+	
+	repeated StreetIndex streets = 7; 
+}
+
+message PostcodeIndex {
+    required string postcode = 1;
+    repeated StreetIndex streets = 5;
+}
+
+message StreetIndex {
+	required string name = 1;
+	optional string name_en = 2;
+	
+	optional uint64 id = 6;
+	
+	required sint32 x = 3; // delta encoded to parent 24 zoom
+	required sint32 y = 4; // delta encoded to parent 24 zoom
+	
+	repeated BuildingIndex buildings = 5;
+} 
+
+message BuildingIndex {
+	required string name = 1;
+	optional string name_en = 2;
+	optional uint64 id = 5;
+	
+	required sint32 x = 3; // delta encoded to street 24 zoom
+	required sint32 y = 4; // delta encoded to street 24 zoom
+	
+}