Address data refactoring
This commit is contained in:
Victor Shcherb
parent
88ee964a64
commit
06f3a81ee5
4 changed files with 309 additions and 64 deletions
|
|
@ -178,10 +178,10 @@ public class BinaryMapAddressReaderAdapter {
|
|||
int x = 0;
|
||||
int y = 0;
|
||||
int streetInd = 0;
|
||||
boolean englishNameMatched = false;
|
||||
while(true){
|
||||
int t = codedIS.readTag();
|
||||
int tag = WireFormat.getTagFieldNumber(t);
|
||||
boolean englishNameMatched = false;
|
||||
switch (tag) {
|
||||
case 0:
|
||||
return c;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,171 @@
|
|||
package net.osmand.binary;
|
||||
|
||||
import gnu.trove.list.array.TIntArrayList;
|
||||
import gnu.trove.set.hash.TIntHashSet;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.RandomAccessFile;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import net.osmand.ResultMatcher;
|
||||
import net.osmand.binary.BinaryMapIndexReader.MapIndex;
|
||||
import net.osmand.binary.BinaryMapIndexReader.SearchFilter;
|
||||
import net.osmand.binary.BinaryMapIndexReader.SearchRequest;
|
||||
import net.osmand.binary.BinaryMapIndexReader.TagValuePair;
|
||||
import net.osmand.osm.MapRenderingTypes;
|
||||
import net.osmand.osm.MapUtils;
|
||||
|
||||
public class BinaryMapIndexFilter {
|
||||
|
||||
private final BinaryMapIndexReader reader;
|
||||
|
||||
public BinaryMapIndexFilter(File file) throws IOException{
|
||||
reader = new BinaryMapIndexReader(new RandomAccessFile(file, "r"));
|
||||
}
|
||||
|
||||
|
||||
private static class Stat {
|
||||
int pointCount = 0;
|
||||
int totalCount = 0;
|
||||
int wayCount = 0;
|
||||
int polygonCount = 0;
|
||||
int polygonBigSize = 0;
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return " ways " + wayCount + " polygons " + polygonCount + " points " + pointCount + " total " + totalCount +"\n"+
|
||||
" polygons big size " + polygonBigSize;
|
||||
}
|
||||
}
|
||||
|
||||
private double calculateArea(BinaryMapDataObject o, int zoom){
|
||||
double sum = 0;
|
||||
for(int i=0; i< o.getPointsLength(); i++){
|
||||
double x = MapUtils.getTileNumberX(zoom + 8, MapUtils.get31LongitudeX(o.getPoint31XTile(i)));
|
||||
int prev = i == 0 ? o.getPointsLength() - 1 : i -1;
|
||||
int next = i == o.getPointsLength() - 1 ? 0 : i + 1;
|
||||
double y1 = MapUtils.getTileNumberY(zoom + 8, MapUtils.get31LatitudeY(o.getPoint31YTile(prev)));
|
||||
double y2 = MapUtils.getTileNumberY(zoom + 8, MapUtils.get31LatitudeY(o.getPoint31YTile(next)));
|
||||
sum += x * (y1 - y2);
|
||||
}
|
||||
return Math.abs(sum);
|
||||
}
|
||||
|
||||
private double calculateLength(BinaryMapDataObject o, int zoom){
|
||||
double sum = 0;
|
||||
for(int i=1; i< o.getPointsLength(); i++){
|
||||
double x = MapUtils.getTileNumberX(zoom + 8, MapUtils.get31LongitudeX(o.getPoint31XTile(i)));
|
||||
double y = MapUtils.getTileNumberY(zoom + 8, MapUtils.get31LatitudeY(o.getPoint31YTile(i)));
|
||||
double x2 = MapUtils.getTileNumberX(zoom + 8, MapUtils.get31LongitudeX(o.getPoint31XTile(i - 1)));
|
||||
double y2 = MapUtils.getTileNumberY(zoom + 8, MapUtils.get31LatitudeY(o.getPoint31YTile(i - 1)));
|
||||
sum += Math.sqrt((x - x2) * (x - x2) + (y - y2) * (y - y2));
|
||||
}
|
||||
return Math.abs(sum);
|
||||
}
|
||||
|
||||
private int tilesCovers(BinaryMapDataObject o, int zoom, TIntHashSet set){
|
||||
set.clear();
|
||||
for(int i=0; i< o.getPointsLength(); i++){
|
||||
int x = (int) MapUtils.getTileNumberX(zoom, MapUtils.get31LongitudeX(o.getPoint31XTile(i)));
|
||||
int y = (int) MapUtils.getTileNumberY(zoom, MapUtils.get31LatitudeY(o.getPoint31YTile(i)));
|
||||
int val = ((x << 16) | y);
|
||||
set.add(val);
|
||||
}
|
||||
return set.size();
|
||||
}
|
||||
|
||||
private Stat process(final int zoom) throws IOException {
|
||||
final Stat stat = new Stat();
|
||||
final Map<TagValuePair, Integer> map = new LinkedHashMap<TagValuePair, Integer>();
|
||||
SearchFilter sf = new SearchFilter() {
|
||||
@Override
|
||||
public boolean accept(TIntArrayList types, MapIndex index) {
|
||||
boolean polygon = false;
|
||||
boolean polyline = false;
|
||||
for (int j = 0; j < types.size(); j++) {
|
||||
int wholeType = types.get(j);
|
||||
TagValuePair pair = index.decodeType(wholeType);
|
||||
if (pair != null) {
|
||||
int t = wholeType & 3;
|
||||
if (t == MapRenderingTypes.POINT_TYPE) {
|
||||
stat.pointCount++;
|
||||
} else if (t == MapRenderingTypes.POLYLINE_TYPE) {
|
||||
stat.wayCount++;
|
||||
polyline = true;
|
||||
} else {
|
||||
polygon = true;
|
||||
stat.polygonCount++;
|
||||
if (!map.containsKey(pair)) {
|
||||
map.put(pair, 0);
|
||||
}
|
||||
map.put(pair, map.get(pair) + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
stat.totalCount++;
|
||||
return polyline;
|
||||
}
|
||||
};
|
||||
ResultMatcher<BinaryMapDataObject> matcher = new ResultMatcher<BinaryMapDataObject>() {
|
||||
TIntHashSet set = new TIntHashSet();
|
||||
@Override
|
||||
public boolean isCancelled() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean publish(BinaryMapDataObject object) {
|
||||
// double area = calculateArea(object, zoom);
|
||||
double len = calculateLength(object, zoom);
|
||||
if(/*tilesCovers(object, zoom, set) >= 2 && */ len > 100){
|
||||
stat.polygonBigSize ++;
|
||||
if(stat.polygonBigSize % 10000 == 0){
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
};
|
||||
SearchRequest<BinaryMapDataObject> req = BinaryMapIndexReader.buildSearchRequest(0, Integer.MAX_VALUE, 0, Integer.MAX_VALUE, zoom,
|
||||
sf, matcher);
|
||||
List<BinaryMapDataObject> result = reader.searchMapIndex(req);
|
||||
|
||||
ArrayList<TagValuePair> list = new ArrayList<TagValuePair>(map.keySet());
|
||||
Collections.sort(list, new Comparator<TagValuePair>() {
|
||||
@Override
|
||||
public int compare(TagValuePair o1, TagValuePair o2) {
|
||||
return -map.get(o1) + map.get(o2);
|
||||
}
|
||||
|
||||
});
|
||||
for(TagValuePair tp : list){
|
||||
Integer i = map.get(tp);
|
||||
if(i > 10){
|
||||
// System.out.println(tp.toString() + " " + i);
|
||||
}
|
||||
}
|
||||
|
||||
for(BinaryMapDataObject obj : result){
|
||||
System.out.println("id " + (obj.getId() >> 3) + " " + calculateArea(obj, zoom));
|
||||
}
|
||||
return stat;
|
||||
}
|
||||
|
||||
public static void main(String[] iargs) throws IOException {
|
||||
BinaryMapIndexFilter filter = new BinaryMapIndexFilter(new File("/home/victor/projects/OsmAnd/data/osmand_index/Netherlands_europe.obf"));
|
||||
for (int i = 10; i <= 14; i++) {
|
||||
Stat st = filter.process(i);
|
||||
System.out.println(i + " zoom -> " + st);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -10,6 +10,8 @@ import java.io.IOException;
|
|||
import java.io.RandomAccessFile;
|
||||
import java.text.Collator;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashMap;
|
||||
|
|
@ -1093,20 +1095,13 @@ public class BinaryMapIndexReader {
|
|||
return poiIndexes;
|
||||
}
|
||||
|
||||
public static SearchRequest<BinaryMapDataObject> buildSearchRequest(int sleft, int sright, int stop, int sbottom, int zoom,
|
||||
SearchFilter searchFilter, ResultMatcher<BinaryMapDataObject> matcher) {
|
||||
SearchRequest<BinaryMapDataObject> request = new SearchRequest<BinaryMapDataObject>();
|
||||
request.left = sleft;
|
||||
request.right = sright;
|
||||
request.top = stop;
|
||||
request.bottom = sbottom;
|
||||
request.zoom = zoom;
|
||||
request.resultMatcher = matcher;
|
||||
request.searchFilter = searchFilter;
|
||||
return request;
|
||||
}
|
||||
|
||||
public static SearchRequest<BinaryMapDataObject> buildSearchRequest(int sleft, int sright, int stop, int sbottom, int zoom, SearchFilter searchFilter){
|
||||
return buildSearchRequest(sleft, sright, stop, sbottom, zoom, searchFilter, null);
|
||||
}
|
||||
|
||||
public static SearchRequest<BinaryMapDataObject> buildSearchRequest(int sleft, int sright, int stop, int sbottom, int zoom, SearchFilter searchFilter,
|
||||
ResultMatcher<BinaryMapDataObject> resultMatcher){
|
||||
SearchRequest<BinaryMapDataObject> request = new SearchRequest<BinaryMapDataObject>();
|
||||
request.left = sleft;
|
||||
request.right = sright;
|
||||
|
|
@ -1114,6 +1109,7 @@ public class BinaryMapIndexReader {
|
|||
request.bottom = sbottom;
|
||||
request.zoom = zoom;
|
||||
request.searchFilter = searchFilter;
|
||||
request.resultMatcher = resultMatcher;
|
||||
return request;
|
||||
}
|
||||
|
||||
|
|
@ -1393,8 +1389,8 @@ public class BinaryMapIndexReader {
|
|||
|
||||
|
||||
private static boolean testMapSearch = false;
|
||||
private static boolean testAddressSearch = false;
|
||||
private static boolean testPoiSearch = true;
|
||||
private static boolean testAddressSearch = true;
|
||||
private static boolean testPoiSearch = false;
|
||||
private static boolean testTransportSearch = false;
|
||||
|
||||
private static void println(String s){
|
||||
|
|
@ -1402,7 +1398,7 @@ public class BinaryMapIndexReader {
|
|||
}
|
||||
|
||||
public static void main(String[] args) throws IOException {
|
||||
RandomAccessFile raf = new RandomAccessFile(new File("/home/victor/projects/OsmAnd/data/osmand_index/Parkcharge_me.obf"), "r");
|
||||
RandomAccessFile raf = new RandomAccessFile(new File("/home/victor/projects/OsmAnd/data/osmand_index/Netherlands.obf"), "r");
|
||||
BinaryMapIndexReader reader = new BinaryMapIndexReader(raf);
|
||||
println("VERSION " + reader.getVersion()); //$NON-NLS-1$
|
||||
long time = System.currentTimeMillis();
|
||||
|
|
@ -1490,14 +1486,25 @@ public class BinaryMapIndexReader {
|
|||
}
|
||||
}
|
||||
|
||||
private static void updateFrequence(Map<String, Integer> street , String key){
|
||||
if(!street.containsKey(key)){
|
||||
street.put(key, 1);
|
||||
} else {
|
||||
street.put(key, street.get(key) + 1);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private static void testAddressSearch(BinaryMapIndexReader reader) throws IOException {
|
||||
// test address index search
|
||||
String reg = reader.getRegionNames().get(0);
|
||||
final Map<String, Integer> streetFreq = new LinkedHashMap<String, Integer>();
|
||||
List<City> cs = reader.getCities(reg, null);
|
||||
for(City c : cs){
|
||||
int buildings = 0;
|
||||
reader.preloadStreets(c, null);
|
||||
for(Street s : c.getStreets()){
|
||||
updateFrequence(streetFreq, s.getName());
|
||||
reader.preloadBuildings(s, buildAddressRequest((ResultMatcher<Building>) null));
|
||||
buildings += s.getBuildings().size();
|
||||
}
|
||||
|
|
@ -1508,14 +1515,41 @@ public class BinaryMapIndexReader {
|
|||
reader.preloadStreets(c, buildAddressRequest((ResultMatcher<Street>) null));
|
||||
println(c.getName());
|
||||
}
|
||||
// System.out.println(c.getName() + " " + c.getLocation() + " " + c.getStreets().size() + " " + buildings + " " + c.getEnName());
|
||||
// List<PostCode> postcodes = reader.getPostcodes(reg, buildAddressRequest((ResultMatcher<MapObject>) null), null);
|
||||
// for(PostCode c : postcodes){
|
||||
// reader.preloadStreets(c, buildAddressRequest((ResultMatcher<Street>) null));
|
||||
// System.out.println(c.getName());
|
||||
// }
|
||||
// int[] count = new int[1];
|
||||
List<City> villages = reader.getVillages(reg, buildAddressRequest((ResultMatcher<MapObject>) null), new StringMatcher() {
|
||||
|
||||
@Override
|
||||
public boolean matches(String name) {
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
}, true);
|
||||
println("Villages " + villages.size());
|
||||
for(City v : villages) {
|
||||
reader.preloadStreets(v, null);
|
||||
for(Street s : v.getStreets()){
|
||||
updateFrequence(streetFreq, s.getName());
|
||||
}
|
||||
}
|
||||
System.out.println("Villages " + villages.size());
|
||||
|
||||
List<String> sorted = new ArrayList<String>(streetFreq.keySet());
|
||||
Collections.sort(sorted, new Comparator<String>() {
|
||||
@Override
|
||||
public int compare(String o1, String o2) {
|
||||
return - streetFreq.get(o1) + streetFreq.get(o2);
|
||||
}
|
||||
});
|
||||
System.out.println(streetFreq.size());
|
||||
for(String s : sorted) {
|
||||
System.out.println(s + " " + streetFreq.get(s));
|
||||
if(streetFreq.get(s) < 10){
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void testMapSearch(BinaryMapIndexReader reader) throws IOException {
|
||||
|
|
|
|||
|
|
@ -12,15 +12,21 @@ option java_package = "net.osmand.binary";
|
|||
// The main difference that size of that messages is not var int and is always fixed int size
|
||||
message OsmAndStructure {
|
||||
required uint32 version = 1;
|
||||
|
||||
// first version of map index was 3.
|
||||
// encoded as fixed32 length delimited
|
||||
repeated OsmAndMapIndex mapIndex = 2;
|
||||
// encoded as fixed32 length delimited
|
||||
repeated OsmAndAddressIndex addressIndex = 3;
|
||||
repeated OsmAndAddressIndex addressIndex = 7;
|
||||
|
||||
// encoded as fixed32 length delimited
|
||||
repeated OsmAndTransportIndex transportIndex = 4;
|
||||
|
||||
// encoded as fixed32 length delimited
|
||||
repeated OsmAndPoiIndex poiIndex = 5;
|
||||
|
||||
// first version of map index was 2.
|
||||
// encoded as fixed32 length delimited
|
||||
repeated OsmAndMapIndex mapIndex = 6;
|
||||
|
||||
// last field should version again (to check consistency)
|
||||
required uint32 versionConfirm = 32;
|
||||
}
|
||||
|
|
@ -112,68 +118,81 @@ message MapData {
|
|||
}
|
||||
|
||||
/// Address messages
|
||||
|
||||
message OsmAndAddressIndex {
|
||||
|
||||
required string name = 1;
|
||||
optional string name_en = 2;
|
||||
|
||||
// encoded as fixed32 length delimited
|
||||
optional CitiesIndex cities = 5; // cities and towns
|
||||
optional OsmAndAddressNameIndexData nameIndex = 3;
|
||||
|
||||
// encoded as fixed32 length delimited
|
||||
optional PostcodesIndex postcodes = 6;
|
||||
|
||||
// encoded as fixed32 length delimited
|
||||
optional CitiesIndex villages = 7; // suburbs and villages
|
||||
// 1) cities and towns - type 1, 2) postcodes - type 2, 3) villages - type 3
|
||||
repeated CitiesIndex cities = 5;
|
||||
}
|
||||
|
||||
message OsmAndAddressNameIndexData {
|
||||
// shift to data starting from first AddressNameIndexData message
|
||||
// encoded as fixed32 length delimited
|
||||
required IndexedStringTable table = 4;
|
||||
|
||||
repeated AddressNameIndexData data = 5;
|
||||
}
|
||||
|
||||
message AddressNameIndexData {
|
||||
|
||||
repeated AddressNameIndexDataAtom = 4;
|
||||
}
|
||||
|
||||
message AddressNameIndexDataAtom {
|
||||
required string name = 1;
|
||||
optional string nameEn = 2;
|
||||
// 1 - city, 2 - postcode, 3 - village, 4 - street
|
||||
required uint32 type = 3;
|
||||
|
||||
// shift from start AddressNameIndexDataAtom to ...Index
|
||||
repeated uint32 shiftToIndex;
|
||||
// optional used in case of type=street
|
||||
repeated string shiftToCityIndex;
|
||||
}
|
||||
|
||||
|
||||
message CitiesIndex {
|
||||
// cities and towns - type 1, postcodes - type 2, villages - type 3
|
||||
required uint32 type = 2;
|
||||
|
||||
// the list cities and blocks are synchronized by order (so even empty block will be written)
|
||||
repeated CityIndex cities = 5;
|
||||
|
||||
// encoded as fixed32 length delimited
|
||||
repeated CityIndex cities = 1;
|
||||
repeated CityBlockIndex blocks = 7;
|
||||
}
|
||||
|
||||
message PostcodesIndex {
|
||||
// encoded as fixed32 length delimited
|
||||
repeated PostcodeIndex postcodes = 1;
|
||||
}
|
||||
|
||||
message CityIndex {
|
||||
// should be written the first
|
||||
required uint32 city_type = 1; // 0-5 enum CityType
|
||||
// should be written the first (for city), for postcode is optional
|
||||
optional uint32 city_type = 1; // 0-5 enum CityType
|
||||
|
||||
required string name = 2;
|
||||
optional string name_en = 3;
|
||||
optional uint64 id = 4;
|
||||
|
||||
required fixed32 x = 5; // x tile of 31 zoom
|
||||
required fixed32 y = 6; // y tile of 31 zoom
|
||||
required uint32 x = 5; // x tile of 24 zoom
|
||||
required uint32 y = 6; // y tile of 24 zoom
|
||||
|
||||
optional InteresectedStreets intersections = 14;
|
||||
// shift from start CityIndex to cityBlockIndex
|
||||
optional fixed32 shiftToCityBlockIndex = 10;
|
||||
|
||||
}
|
||||
|
||||
message CityBlockIndex {
|
||||
|
||||
// buildings that doesn't belong to the street
|
||||
repeated BuildingIndex buildings = 16;
|
||||
|
||||
repeated StreetIndex streets = 18;
|
||||
}
|
||||
|
||||
message InteresectedStreets {
|
||||
repeated StreetIntersection intersections = 5;
|
||||
}
|
||||
|
||||
message StreetIntersection {
|
||||
required uint32 intersectedStreet1 = 2; // index of street in parent list
|
||||
required uint32 intersectedStreet2 = 3; // index of street in parent list
|
||||
required sint32 intersectedX = 4; // delta encoded to parent x 24 zoom
|
||||
required sint32 intersectedY = 5; // delta encoded to parent y 24 zoom
|
||||
}
|
||||
|
||||
|
||||
|
||||
message PostcodeIndex {
|
||||
required string postcode = 1;
|
||||
required fixed32 x = 2; // x tile of 31 zoom
|
||||
required fixed32 y = 3; // y tile of 31 zoom
|
||||
repeated StreetIndex streets = 5;
|
||||
}
|
||||
|
||||
message StreetIndex {
|
||||
required string name = 1;
|
||||
optional string name_en = 2;
|
||||
|
|
@ -183,19 +202,40 @@ message StreetIndex {
|
|||
|
||||
optional uint64 id = 6;
|
||||
|
||||
repeated BuildingIndex buildings = 12;
|
||||
|
||||
repeated BuildingIndex buildings = 18;
|
||||
repeated StreetIntersection intersections = 5;
|
||||
}
|
||||
|
||||
message StreetIntersection {
|
||||
required string name = 2;
|
||||
optional string name_en = 3;
|
||||
|
||||
required sint32 intersectedX = 4; // delta encoded to parent street x 24 zoom
|
||||
required sint32 intersectedY = 5; // delta encoded to parent street y 24 zoom
|
||||
}
|
||||
|
||||
message BuildingIndex {
|
||||
required string name = 1;
|
||||
optional string name_en = 2;
|
||||
optional uint64 id = 5;
|
||||
optional string postcode = 6;
|
||||
|
||||
required sint32 x = 3; // delta encoded to street 24 zoom
|
||||
required sint32 y = 4; // delta encoded to street 24 zoom
|
||||
// used for interpolation
|
||||
optional string name2 = 3;
|
||||
optional string name_en2 = 4;
|
||||
|
||||
// special values 0 means - nothing, -1 - all, -2 - even, -3 - odd, -4, alphabetic
|
||||
// positive - just positive interpolation
|
||||
optional sint32 interpolation = 5;
|
||||
|
||||
required sint32 x = 7; // delta encoded to parent 24 zoom
|
||||
required sint32 y = 8; // delta encoded to parent 24 zoom
|
||||
|
||||
// used for interpolation
|
||||
optional sint32 x2 = 9; // delta encoded to parent 24 zoom
|
||||
optional sint32 y2 = 10; // delta encoded to parent 24 zoom
|
||||
|
||||
optional uint64 id = 13;
|
||||
optional string postcode = 14;
|
||||
}
|
||||
|
||||
// transport messages
|
||||
|
|
|
|||
Loading…
Reference in a new issue