Read/write poi

DataExtractionOSM/src/net/osmand/binary/BinaryInspector.java

@@ -16,6 +16,7 @@ import java.util.Set;
 import net.osmand.binary.BinaryMapAddressReaderAdapter.AddressRegion;
 import net.osmand.binary.BinaryMapIndexReader.MapIndex;
 import net.osmand.binary.BinaryMapIndexReader.MapRoot;
+import net.osmand.binary.BinaryMapPoiReaderAdapter.PoiRegion;
 import net.osmand.binary.BinaryMapTransportReaderAdapter.TransportIndex;
 import net.osmand.data.City;
 import net.osmand.data.Street;
@@ -33,7 +34,7 @@ public class BinaryInspector {
 		inspector(args);
 		// test cases show info
 		//inspector(new String[]{"/home/victor/projects/OsmAnd/data/osm-gen/saved/Belarus-newzooms-new-rt.obf"});
-		//inspector(new String[]{"-v","C:\\Users\\tpd\\osmand\\Slovakia.obf "});
+		// inspector(new String[]{""});
 		
 		
 		// test case extract parts
@@ -238,6 +239,9 @@ public class BinaryInspector {
 					} else if (part instanceof TransportIndex) {
 						ous.writeTag(OsmandOdb.OsmAndStructure.TRANSPORTINDEX_FIELD_NUMBER, WireFormat.WIRETYPE_FIXED32_LENGTH_DELIMITED);
 						map = "Transport";
+					} else if (part instanceof PoiRegion) {
+						ous.writeTag(OsmandOdb.OsmAndStructure.POIINDEX_FIELD_NUMBER, WireFormat.WIRETYPE_FIXED32_LENGTH_DELIMITED);
+						map = "POI";
 					} else {
 						throw new UnsupportedOperationException();
 					}
@@ -306,6 +310,8 @@ public class BinaryInspector {
 					partname = "Map";
 				} else if(p instanceof TransportIndex){
 					partname = "Transport";
+				} else if(p instanceof PoiRegion){
+					partname = "Poi";
 				} else if(p instanceof AddressRegion){
 					partname = "Address";
 				}

DataExtractionOSM/src/net/osmand/binary/BinaryMapIndexReader.java

@@ -18,6 +18,7 @@ import net.osmand.LogUtil;
 import net.osmand.ResultMatcher;
 import net.osmand.StringMatcher;
 import net.osmand.binary.BinaryMapAddressReaderAdapter.AddressRegion;
+import net.osmand.binary.BinaryMapPoiReaderAdapter.PoiRegion;
 import net.osmand.binary.BinaryMapTransportReaderAdapter.TransportIndex;
 import net.osmand.data.Building;
 import net.osmand.data.City;
@@ -44,6 +45,7 @@ public class BinaryMapIndexReader {
 	private final RandomAccessFile raf;
 	private int version;
 	private List<MapIndex> mapIndexes = new ArrayList<MapIndex>();
+	private List<PoiRegion> poiIndexes = new ArrayList<PoiRegion>();
 	private List<AddressRegion> addressIndexes = new ArrayList<AddressRegion>();
 	private List<TransportIndex> transportIndexes = new ArrayList<TransportIndex>();
 	private List<BinaryIndexPart> indexes = new ArrayList<BinaryIndexPart>();
@@ -51,6 +53,7 @@ public class BinaryMapIndexReader {
 	protected CodedInputStreamRAF codedIS;
 	
 	private final BinaryMapTransportReaderAdapter transportAdapter;
+	private final BinaryMapPoiReaderAdapter poiAdapter;
 	private final BinaryMapAddressReaderAdapter addressAdapter;
 	
 	public BinaryMapIndexReader(final RandomAccessFile raf) throws IOException {
@@ -64,9 +67,11 @@ public class BinaryMapIndexReader {
 		if(!readOnlyMapData){
 			transportAdapter = new BinaryMapTransportReaderAdapter(this);
 			addressAdapter = new BinaryMapAddressReaderAdapter(this);
+			poiAdapter = new BinaryMapPoiReaderAdapter(this);
 		} else {
 			transportAdapter = null;
 			addressAdapter = null;
+			poiAdapter = null;
 		}
 		init();
 	}
@@ -125,6 +130,19 @@ public class BinaryMapIndexReader {
 				}
 				codedIS.seek(ind.filePointer + ind.length);
 				break;
+			case OsmandOdb.OsmAndStructure.POIINDEX_FIELD_NUMBER:
+				PoiRegion poiInd = new PoiRegion();
+				poiInd.length = readInt();
+				poiInd.filePointer = codedIS.getTotalBytesRead();
+				if (poiAdapter != null) {
+					oldLimit = codedIS.pushLimit(poiInd.length);
+					poiAdapter.readPoiIndex(poiInd);
+					codedIS.popLimit(oldLimit);
+					poiIndexes.add(poiInd);
+					indexes.add(poiInd);
+				}
+				codedIS.seek(poiInd.filePointer + poiInd.length);
+				break;
 			case OsmandOdb.OsmAndStructure.VERSIONCONFIRM_FIELD_NUMBER :
 				int cversion = codedIS.readUInt32();
 				calculateCenterPointForRegions();
@@ -895,6 +913,10 @@ public class BinaryMapIndexReader {
 	protected List<AddressRegion> getAddressIndexes() {
 		return addressIndexes;
 	}
+	
+	protected List<PoiRegion> getPoiIndexes() {
+		return poiIndexes;
+	}
 
 	
 	public static SearchRequest<BinaryMapDataObject> buildSearchRequest(int sleft, int sright, int stop, int sbottom, int zoom){
@@ -1066,8 +1088,6 @@ public class BinaryMapIndexReader {
 			return true;
 		}
 		
-		
-		
 	}
 	
 	
@@ -1116,57 +1136,26 @@ public class BinaryMapIndexReader {
 
 	
 	public static void main(String[] args) throws IOException {
-		RandomAccessFile raf = new RandomAccessFile(new File("/home/victor/projects/OsmAnd/data/osm-gen/Ru-mow.obf"), "r");
+		RandomAccessFile raf = new RandomAccessFile(new File("/home/victor/projects/OsmAnd/data/osm-gen/POI/Test-Ru.poi.obf"), "r");
 		BinaryMapIndexReader reader = new BinaryMapIndexReader(raf);
 		System.out.println("VERSION " + reader.getVersion()); //$NON-NLS-1$
 		long time = System.currentTimeMillis();
-		
-		System.out.println(reader.mapIndexes.get(0).encodingRules);
-		
-		// test search
-//		int sleft = MapUtils.get31TileNumberX(27.596);
-//		int sright = MapUtils.get31TileNumberX(27.599);
-//		int stop = MapUtils.get31TileNumberY(53.921);
-//		int sbottom = MapUtils.get31TileNumberY(53.919);
-//		System.out.println("SEARCH " + sleft + " " + sright + " " + stop + " " + sbottom);
-//
-//		for (BinaryMapDataObject obj : reader.searchMapIndex(buildSearchRequest(sleft, sright, stop, sbottom, 8))) {
-//			if (obj.getName() != null) {
-//				System.out.println(" " + obj.getName());
-//			}
-//		}
-		
-		// test address index search
-		String reg = reader.getRegionNames().get(0);
-		List<City> cs = reader.getCities(reg, null);
-		for(City c : cs){
-			int buildings = 0;
-			reader.preloadStreets(c, null);
-			for(Street s : c.getStreets()){
-				if(s.getName().toLowerCase().startsWith("заре")){
-					System.out.println("  " + s.getName());
-				} else if(s.getName().toLowerCase().contains("инстит")){
-					System.out.println("  " + s.getName());
-				}
-//				reader.preloadBuildings(s);
-//				buildings += s.getBuildings().size();
-			}
-			System.out.println(c.getName() + " " + c.getLocation() + " " + c.getStreets().size() + " " + buildings + " " + c.getEnName());
+
+		testMapSearch(reader);
+		testAddressSearch(reader);
+		testTransportSearch(reader);
+
+		PoiRegion poiRegion = reader.getPoiIndexes().get(0);
+		for (int i = 0; i < poiRegion.categories.size(); i++) {
+			System.out.println(poiRegion.categories.get(i));
+			System.out.println(" " + poiRegion.subcategories.get(i));
 		}
-//		List<PostCode> postcodes = reader.getPostcodes(reg);
-//		for(PostCode c : postcodes){
-//			reader.preloadStreets(c);
-//			System.out.println(c.getName());
-//		}
-//		List<City> villages = reader.getVillages(reg, new StringMatcher() {
-//			
-//			@Override
-//			public boolean matches(String name) {
-//				return false;
-//			}
-//		}, true);
-//		System.out.println("Villages " + villages.size());
-		
+
+		System.out.println("MEMORY " + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())); //$NON-NLS-1$
+		System.out.println("Time " + (System.currentTimeMillis() - time)); //$NON-NLS-1$
+	}
+
+	private static void testTransportSearch(BinaryMapIndexReader reader) {
 		// test transport
 //		for(TransportIndex i : reader.transportIndexes){
 //			System.out.println(i.left + " " + i.right + " " + i.top + " " + i.bottom);
@@ -1202,9 +1191,49 @@ public class BinaryMapIndexReader {
 //				}
 //			}
 //		}
-		
-		System.out.println("MEMORY " + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())); //$NON-NLS-1$
-		System.out.println("Time " + (System.currentTimeMillis() - time)); //$NON-NLS-1$
+	}
+
+	private static void testAddressSearch(BinaryMapIndexReader reader) throws IOException {
+		// test address index search
+//		String reg = reader.getRegionNames().get(0);
+//		List<City> cs = reader.getCities(reg, null);
+//		for(City c : cs){
+//			int buildings = 0;
+//			reader.preloadStreets(c, null);
+//			for(Street s : c.getStreets()){
+//				reader.preloadBuildings(s);
+//				buildings += s.getBuildings().size();
+//			}
+//			System.out.println(c.getName() + " " + c.getLocation() + " " + c.getStreets().size() + " " + buildings + " " + c.getEnName());
+//		}
+//		List<PostCode> postcodes = reader.getPostcodes(reg);
+//		for(PostCode c : postcodes){
+//			reader.preloadStreets(c);
+//			System.out.println(c.getName());
+//		}
+//		List<City> villages = reader.getVillages(reg, new StringMatcher() {
+//			
+//			@Override
+//			public boolean matches(String name) {
+//				return false;
+//			}
+//		}, true);
+//		System.out.println("Villages " + villages.size());
+	}
+
+	private static void testMapSearch(BinaryMapIndexReader reader) throws IOException {
+//		System.out.println(reader.mapIndexes.get(0).encodingRules);
+//		int sleft = MapUtils.get31TileNumberX(27.596);
+//		int sright = MapUtils.get31TileNumberX(27.599);
+//		int stop = MapUtils.get31TileNumberY(53.921);
+//		int sbottom = MapUtils.get31TileNumberY(53.919);
+//		System.out.println("SEARCH " + sleft + " " + sright + " " + stop + " " + sbottom);
+//
+//		for (BinaryMapDataObject obj : reader.searchMapIndex(buildSearchRequest(sleft, sright, stop, sbottom, 8))) {
+//			if (obj.getName() != null) {
+//				System.out.println(" " + obj.getName());
+//			}
+//		}
 	}
 	
 }

DataExtractionOSM/src/net/osmand/binary/BinaryMapPoiReaderAdapter.java

@@ -0,0 +1,85 @@
+package net.osmand.binary;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+
+import com.google.protobuf.CodedInputStreamRAF;
+import com.google.protobuf.WireFormat;
+
+public class BinaryMapPoiReaderAdapter {
+	
+	public static class PoiRegion extends BinaryIndexPart {
+
+		List<String> categories = new ArrayList<String>();
+		List<List<String>> subcategories = new ArrayList<List<String>>();
+	}
+	
+	private CodedInputStreamRAF codedIS;
+	private final BinaryMapIndexReader map;
+	
+	protected BinaryMapPoiReaderAdapter(BinaryMapIndexReader map){
+		this.codedIS = map.codedIS;
+		this.map = map;
+	}
+
+	private void skipUnknownField(int t) throws IOException {
+		map.skipUnknownField(t);
+	}
+	
+	private int readInt() throws IOException {
+		return map.readInt();
+	}
+	
+
+	protected void readPoiIndex(PoiRegion region) throws IOException {
+		while(true){
+			int t = codedIS.readTag();
+			int tag = WireFormat.getTagFieldNumber(t);
+			switch (tag) {
+			case 0:
+				return;
+			case OsmandOdb.OsmAndPoiIndex.NAME_FIELD_NUMBER :
+				region.name = codedIS.readString();
+				break;
+			case OsmandOdb.OsmAndPoiIndex.CATEGORIESTABLE_FIELD_NUMBER :
+				int length = codedIS.readRawVarint32();
+				
+				int oldLimit = codedIS.pushLimit(length);
+				readCategory(region);
+				codedIS.popLimit(oldLimit);
+				break;
+			case OsmandOdb.OsmAndPoiIndex.BOXES_FIELD_NUMBER :
+				codedIS.skipRawBytes(codedIS.getBytesUntilLimit());
+				return;
+			default:
+				skipUnknownField(t);
+				break;
+			}
+		}
+	}
+	
+	private void readCategory(PoiRegion region) throws IOException {
+		while(true){
+			int t = codedIS.readTag();
+			int tag = WireFormat.getTagFieldNumber(t);
+			switch (tag) {
+			case 0:
+				return;
+			case OsmandOdb.OsmAndCategoryTable.CATEGORY_FIELD_NUMBER :
+				region.categories.add(codedIS.readString());
+				region.subcategories.add(new ArrayList<String>());
+				break;
+			case OsmandOdb.OsmAndCategoryTable.SUBCATEGORIES_FIELD_NUMBER :
+				region.subcategories.get(region.subcategories.size() - 1).add(codedIS.readString());
+				break;
+			default:
+				skipUnknownField(t);
+				break;
+			}
+		}
+	}
+	
+
+}

DataExtractionOSM/src/net/osmand/data/preparation/IndexPoiCreator.java

@@ -11,12 +11,14 @@ import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
+import java.text.Collator;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.TreeMap;
 
 import net.osmand.Algoritms;
 import net.osmand.IProgress;
@@ -182,14 +184,15 @@ public class IndexPoiCreator extends AbstractIndexPartCreator {
 			closePreparedStatements(poiPreparedStatement);
 		}
 		poiConnection.commit();
-
+		Collator collator = Collator.getInstance();
+		collator.setStrength(Collator.PRIMARY);
 		ResultSet rs = poiConnection.createStatement().executeQuery("SELECT DISTINCT type, subtype FROM poi");
 		Map<String, Map<String, Integer>> categories = new LinkedHashMap<String, Map<String, Integer>>();
 		while (rs.next()) {
 			String category = rs.getString(1);
 			String subcategory = rs.getString(2).trim();
 			if (!categories.containsKey(category)) {
-				categories.put(category, new LinkedHashMap<String, Integer>());
+				categories.put(category, new TreeMap<String, Integer>(collator));
 			}
 			if (subcategory.contains(";") || subcategory.contains(",")) {
 				String[] split = subcategory.split(",|;");