Android-builders-club/OsmAnd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Modify binary POI data structure

Browse source
This commit is contained in:
Victor Shcherb 2011-09-26 09:13:20 +02:00
parent 8db4893db8
commit c5179ada99
7 changed files with 285 additions and 239 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
DataExtractionOSM/build_proto.xml
View file

@ -1,5 +1,5 @@
<!-- build JAR libraty -->
<project name="OsmPDB" default="build" basedir=".">
<project name="DataExtractionOsm" default="build" basedir=".">
<target name="build">
<exec dir="." executable="/usr/bin/protoc">
<arg value="src/osmand_odb.proto" />

2
DataExtractionOSM/src/net/osmand/binary/BinaryMapIndexWriter.java
View file

@ -817,7 +817,6 @@ public class BinaryMapIndexWriter {
int j = types.get(i);
builder.addCategories(j);
}
if(!Algoritms.isEmpty(name)){
builder.setName(name);
}
@ -825,6 +824,7 @@ public class BinaryMapIndexWriter {
builder.setNameEn(nameEn);
}
builder.setId(id);
if(!Algoritms.isEmpty(openingHours)){
builder.setOpeningHours(openingHours);
}

20
DataExtractionOSM/src/net/osmand/binary/BinaryMapPoiReaderAdapter.java
View file

@ -270,9 +270,14 @@ public class BinaryMapPoiReaderAdapter {
Amenity am = null;
int x = 0;
int y = 0;
AmenityType amenityType = null;
while(true){
int t = codedIS.readTag();
int tag = WireFormat.getTagFieldNumber(t);
if(amenityType == null && (tag > OsmandOdb.OsmAndPoiBoxDataAtom.CATEGORIES_FIELD_NUMBER || tag == 0)) {
codedIS.skipRawBytes(codedIS.getBytesUntilLimit());
return null;
}
switch (tag) {
case 0:
if(Algoritms.isEmpty(am.getEnName())){
@ -294,7 +299,6 @@ public class BinaryMapPoiReaderAdapter {
am.setLocation(MapUtils.get31LatitudeY(y), MapUtils.get31LongitudeX(x));
break;
case OsmandOdb.OsmAndPoiBoxDataAtom.CATEGORIES_FIELD_NUMBER :
// TODO support many amenities type !!
int cat = codedIS.readUInt32();
int subcatId = cat >> SHIFT_BITS_CATEGORY;
int catId = cat & CATEGORY_MASK;
@ -307,12 +311,16 @@ public class BinaryMapPoiReaderAdapter {
subtype = subcats.get(subcatId);
}
}
if (req.poiTypeFilter != null && !req.poiTypeFilter.accept(type, subtype)) {
codedIS.skipRawBytes(codedIS.getBytesUntilLimit());
return null;
if (req.poiTypeFilter == null || req.poiTypeFilter.accept(type, subtype)) {
if (amenityType == null) {
amenityType = type;
am.setSubType(subtype);
am.setType(amenityType);
} else {
am.setSubType(am.getSubType() + ";" + subtype);
}
}
am.setSubType(subtype);
am.setType(type);
break;
case OsmandOdb.OsmAndPoiBoxDataAtom.ID_FIELD_NUMBER :
am.setId(codedIS.readUInt64());

252
DataExtractionOSM/src/net/osmand/binary/OsmandOdb.java
View file

@ -14306,29 +14306,8 @@ public final class OsmandOdb {
public boolean hasDy() { return hasDy; }
public int getDy() { return dy_; }
// optional uint64 id = 4;
public static final int ID_FIELD_NUMBER = 4;
private boolean hasId;
private long id_ = 0L;
public boolean hasId() { return hasId; }
public long getId() { return id_; }
// optional string name = 5;
public static final int NAME_FIELD_NUMBER = 5;
private boolean hasName;
private java.lang.String name_ = "";
public boolean hasName() { return hasName; }
public java.lang.String getName() { return name_; }
// optional string nameEn = 6;
public static final int NAMEEN_FIELD_NUMBER = 6;
private boolean hasNameEn;
private java.lang.String nameEn_ = "";
public boolean hasNameEn() { return hasNameEn; }
public java.lang.String getNameEn() { return nameEn_; }
// repeated uint32 categories = 7;
public static final int CATEGORIES_FIELD_NUMBER = 7;
// repeated uint32 categories = 4;
public static final int CATEGORIES_FIELD_NUMBER = 4;
private java.util.List<java.lang.Integer> categories_ =
java.util.Collections.emptyList();
public java.util.List<java.lang.Integer> getCategoriesList() {
@ -14339,6 +14318,27 @@ public final class OsmandOdb {
return categories_.get(index);
}
// optional string name = 6;
public static final int NAME_FIELD_NUMBER = 6;
private boolean hasName;
private java.lang.String name_ = "";
public boolean hasName() { return hasName; }
public java.lang.String getName() { return name_; }
// optional string nameEn = 7;
public static final int NAMEEN_FIELD_NUMBER = 7;
private boolean hasNameEn;
private java.lang.String nameEn_ = "";
public boolean hasNameEn() { return hasNameEn; }
public java.lang.String getNameEn() { return nameEn_; }
// optional uint64 id = 8;
public static final int ID_FIELD_NUMBER = 8;
private boolean hasId;
private long id_ = 0L;
public boolean hasId() { return hasId; }
public long getId() { return id_; }
// optional string openingHours = 10;
public static final int OPENINGHOURS_FIELD_NUMBER = 10;
private boolean hasOpeningHours;
@ -14384,17 +14384,17 @@ public final class OsmandOdb {
if (hasDy()) {
output.writeSInt32(3, getDy());
}
if (hasId()) {
output.writeUInt64(4, getId());
for (int element : getCategoriesList()) {
output.writeUInt32(4, element);
}
if (hasName()) {
output.writeString(5, getName());
output.writeString(6, getName());
}
if (hasNameEn()) {
output.writeString(6, getNameEn());
output.writeString(7, getNameEn());
}
for (int element : getCategoriesList()) {
output.writeUInt32(7, element);
if (hasId()) {
output.writeUInt64(8, getId());
}
if (hasOpeningHours()) {
output.writeString(10, getOpeningHours());
@ -14425,18 +14425,6 @@ public final class OsmandOdb {
size += com.google.protobuf.CodedOutputStream
.computeSInt32Size(3, getDy());
}
if (hasId()) {
size += com.google.protobuf.CodedOutputStream
.computeUInt64Size(4, getId());
}
if (hasName()) {
size += com.google.protobuf.CodedOutputStream
.computeStringSize(5, getName());
}
if (hasNameEn()) {
size += com.google.protobuf.CodedOutputStream
.computeStringSize(6, getNameEn());
}
{
int dataSize = 0;
for (int element : getCategoriesList()) {
@ -14446,6 +14434,18 @@ public final class OsmandOdb {
size += dataSize;
size += 1 * getCategoriesList().size();
}
if (hasName()) {
size += com.google.protobuf.CodedOutputStream
.computeStringSize(6, getName());
}
if (hasNameEn()) {
size += com.google.protobuf.CodedOutputStream
.computeStringSize(7, getNameEn());
}
if (hasId()) {
size += com.google.protobuf.CodedOutputStream
.computeUInt64Size(8, getId());
}
if (hasOpeningHours()) {
size += com.google.protobuf.CodedOutputStream
.computeStringSize(10, getOpeningHours());
@ -14630,8 +14630,11 @@ public final class OsmandOdb {
if (other.hasDy()) {
setDy(other.getDy());
}
if (other.hasId()) {
setId(other.getId());
if (!other.categories_.isEmpty()) {
if (result.categories_.isEmpty()) {
result.categories_ = new java.util.ArrayList<java.lang.Integer>();
}
result.categories_.addAll(other.categories_);
}
if (other.hasName()) {
setName(other.getName());
@ -14639,11 +14642,8 @@ public final class OsmandOdb {
if (other.hasNameEn()) {
setNameEn(other.getNameEn());
}
if (!other.categories_.isEmpty()) {
if (result.categories_.isEmpty()) {
result.categories_ = new java.util.ArrayList<java.lang.Integer>();
}
result.categories_.addAll(other.categories_);
if (other.hasId()) {
setId(other.getId());
}
if (other.hasOpeningHours()) {
setOpeningHours(other.getOpeningHours());
@ -14691,22 +14691,10 @@ public final class OsmandOdb {
break;
}
case 32: {
setId(input.readUInt64());
break;
}
case 42: {
setName(input.readString());
break;
}
case 50: {
setNameEn(input.readString());
break;
}
case 56: {
addCategories(input.readUInt32());
break;
}
case 58: {
case 34: {
int length = input.readRawVarint32();
int limit = input.pushLimit(length);
while (input.getBytesUntilLimit() > 0) {
@ -14715,6 +14703,18 @@ public final class OsmandOdb {
input.popLimit(limit);
break;
}
case 50: {
setName(input.readString());
break;
}
case 58: {
setNameEn(input.readString());
break;
}
case 64: {
setId(input.readUInt64());
break;
}
case 82: {
setOpeningHours(input.readString());
break;
@ -14772,67 +14772,7 @@ public final class OsmandOdb {
return this;
}
// optional uint64 id = 4;
public boolean hasId() {
return result.hasId();
}
public long getId() {
return result.getId();
}
public Builder setId(long value) {
result.hasId = true;
result.id_ = value;
return this;
}
public Builder clearId() {
result.hasId = false;
result.id_ = 0L;
return this;
}
// optional string name = 5;
public boolean hasName() {
return result.hasName();
}
public java.lang.String getName() {
return result.getName();
}
public Builder setName(java.lang.String value) {
if (value == null) {
throw new NullPointerException();
}
result.hasName = true;
result.name_ = value;
return this;
}
public Builder clearName() {
result.hasName = false;
result.name_ = getDefaultInstance().getName();
return this;
}
// optional string nameEn = 6;
public boolean hasNameEn() {
return result.hasNameEn();
}
public java.lang.String getNameEn() {
return result.getNameEn();
}
public Builder setNameEn(java.lang.String value) {
if (value == null) {
throw new NullPointerException();
}
result.hasNameEn = true;
result.nameEn_ = value;
return this;
}
public Builder clearNameEn() {
result.hasNameEn = false;
result.nameEn_ = getDefaultInstance().getNameEn();
return this;
}
// repeated uint32 categories = 7;
// repeated uint32 categories = 4;
public java.util.List<java.lang.Integer> getCategoriesList() {
return java.util.Collections.unmodifiableList(result.categories_);
}
@ -14866,6 +14806,66 @@ public final class OsmandOdb {
return this;
}
// optional string name = 6;
public boolean hasName() {
return result.hasName();
}
public java.lang.String getName() {
return result.getName();
}
public Builder setName(java.lang.String value) {
if (value == null) {
throw new NullPointerException();
}
result.hasName = true;
result.name_ = value;
return this;
}
public Builder clearName() {
result.hasName = false;
result.name_ = getDefaultInstance().getName();
return this;
}
// optional string nameEn = 7;
public boolean hasNameEn() {
return result.hasNameEn();
}
public java.lang.String getNameEn() {
return result.getNameEn();
}
public Builder setNameEn(java.lang.String value) {
if (value == null) {
throw new NullPointerException();
}
result.hasNameEn = true;
result.nameEn_ = value;
return this;
}
public Builder clearNameEn() {
result.hasNameEn = false;
result.nameEn_ = getDefaultInstance().getNameEn();
return this;
}
// optional uint64 id = 8;
public boolean hasId() {
return result.hasId();
}
public long getId() {
return result.getId();
}
public Builder setId(long value) {
result.hasId = true;
result.id_ = value;
return this;
}
public Builder clearId() {
result.hasId = false;
result.id_ = 0L;
return this;
}
// optional string openingHours = 10;
public boolean hasOpeningHours() {
return result.hasOpeningHours();
@ -15207,9 +15207,9 @@ public final class OsmandOdb {
"\r\"^\n\020OsmAndPoiBoxData\022\014\n\004zoom\030\001 \001(\r\022\t\n\001x" +
"\030\002 \001(\r\022\t\n\001y\030\003 \001(\r\022&\n\007poiData\030\005 \003(\0132\025.Osm" +
"AndPoiBoxDataAtom\"\255\001\n\024OsmAndPoiBoxDataAt" +
"om\022\n\n\002dx\030\002 \002(\021\022\n\n\002dy\030\003 \002(\021\022\n\n\002id\030\004 \001(\004\022\014" +
"\n\004name\030\005 \001(\t\022\016\n\006nameEn\030\006 \001(\t\022\022\n\ncategori" +
"es\030\007 \003(\r\022\024\n\014openingHours\030\n \001(\t\022\014\n\004site\030\013" +
"om\022\n\n\002dx\030\002 \002(\021\022\n\n\002dy\030\003 \002(\021\022\022\n\ncategories" +
"\030\004 \003(\r\022\014\n\004name\030\006 \001(\t\022\016\n\006nameEn\030\007 \001(\t\022\n\n\002" +
"id\030\010 \001(\004\022\024\n\014openingHours\030\n \001(\t\022\014\n\004site\030\013" +
" \001(\t\022\r\n\005phone\030\014 \001(\t\022\014\n\004note\030\r \001(\tB\023\n\021net",
".osmand.binary"
};
@ -15455,7 +15455,7 @@ public final class OsmandOdb {
internal_static_OsmAndPoiBoxDataAtom_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_OsmAndPoiBoxDataAtom_descriptor,
new java.lang.String[] { "Dx", "Dy", "Id", "Name", "NameEn", "Categories", "OpeningHours", "Site", "Phone", "Note", },
new java.lang.String[] { "Dx", "Dy", "Categories", "Name", "NameEn", "Id", "OpeningHours", "Site", "Phone", "Note", },
net.osmand.binary.OsmandOdb.OsmAndPoiBoxDataAtom.class,
net.osmand.binary.OsmandOdb.OsmAndPoiBoxDataAtom.Builder.class);
return null;

15
DataExtractionOSM/src/net/osmand/data/index/IndexBatchCreator.java
View file

@ -516,10 +516,11 @@ public class IndexBatchCreator {
if(zoomWaySmoothness != null){
indexCreator.setZoomWaySmothness(zoomWaySmoothness);
}
if (indexPOI) {
uploadIndex(new File(indexDirFiles, poiFileName), alreadyUploadedFiles);
}
if (indexMap || indexAddress || indexTransport) {
// Do not upload poi files any more
// if (indexPOI) {
// uploadIndex(new File(indexDirFiles, poiFileName), alreadyUploadedFiles);
// }
if (indexMap || indexAddress || indexTransport || indexPOI) {
uploadIndex(new File(indexDirFiles, mapFileName), alreadyUploadedFiles);
}
} catch (Exception e) {
@ -575,6 +576,7 @@ public class IndexBatchCreator {
boolean addr = indexAddress;
boolean trans = indexTransport;
boolean map = indexMap;
boolean poi = indexPOI;
RandomAccessFile raf = null;
if (fileName.endsWith(IndexConstants.BINARY_MAP_INDEX_EXT)) {
try {
@ -583,6 +585,7 @@ public class IndexBatchCreator {
trans = reader.hasTransportData();
map = reader.containsMapData();
addr = reader.containsAddressData();
poi = reader.containsPoiData();
reader.close();
} catch (Exception e) {
log.info("Exception", e);
@ -604,6 +607,10 @@ public class IndexBatchCreator {
summary = "Transport" + (fir ? "" : ", ") + summary;
fir = false;
}
if (poi) {
summary = "POI" + (fir ? "" : ", ") + summary;
fir = false;
}
if (map) {
summary = "Map" + (fir ? "" : ", ") + summary;
fir = false;

223
DataExtractionOSM/src/net/osmand/data/preparation/IndexPoiCreator.java
View file

@ -18,7 +18,9 @@ import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import net.osmand.Algoritms;
import net.osmand.IProgress;
@ -31,6 +33,7 @@ import net.osmand.impl.ConsoleProgressImplementation;
import net.osmand.osm.Entity;
import net.osmand.osm.MapUtils;
import net.osmand.osm.OSMSettings.OSMTagKey;
import net.sf.junidecode.Junidecode;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -180,7 +183,7 @@ public class IndexPoiCreator extends AbstractIndexPartCreator {
types.add((subcatInd << BinaryMapPoiReaderAdapter.SHIFT_BITS_CATEGORY) | catInd);
}
}
public void writeBinaryPoiIndex(BinaryMapIndexWriter writer, String regionName, IProgress progress) throws SQLException, IOException {
if (poiPreparedStatement != null) {
closePreparedStatements(poiPreparedStatement);
@ -188,62 +191,79 @@ public class IndexPoiCreator extends AbstractIndexPartCreator {
poiConnection.commit();
Collator collator = Collator.getInstance();
collator.setStrength(Collator.PRIMARY);
ResultSet rs = poiConnection.createStatement().executeQuery("SELECT DISTINCT type, subtype FROM poi");
Map<String, Map<String, Integer>> categories = new LinkedHashMap<String, Map<String, Integer>>();
while (rs.next()) {
String category = rs.getString(1);
String subcategory = rs.getString(2).trim();
if (!categories.containsKey(category)) {
categories.put(category, new TreeMap<String, Integer>(collator));
}
if (subcategory.contains(";") || subcategory.contains(",")) {
String[] split = subcategory.split(",|;");
for (String sub : split) {
categories.get(category).put(sub.trim(), 0);
}
} else {
categories.get(category).put(subcategory.trim(), 0);
}
}
Statement stat = rs.getStatement();
rs.close();
stat.close();
// 1. write header
rs = poiConnection.createStatement().executeQuery("SELECT max(x), min(x), max(y), min(y) FROM poi");
rs.next();
int right31 = rs.getInt(1);
int left31 = rs.getInt(2);
int bottom31 = rs.getInt(3);
int top31 = rs.getInt(4);
rs.close();
long startFpPoiIndex = writer.startWritePOIIndex(regionName, left31, right31, bottom31, top31);
// 2. write categories table
Map<String, Integer> catIndexes = writer.writePOICategoriesTable(categories);
// 3. write boxes
String selectZm = (31 - ZOOM_TO_SAVE_END) + "";
rs = poiConnection.createStatement().executeQuery("SELECT DISTINCT x>>" + selectZm + ", y>>" + selectZm + " from poi");
Tree<Long> rootZoomsTree = new Tree<Long>();
// 0. process all entities
ResultSet rs = poiConnection.createStatement().executeQuery("SELECT x,y,name,name_en,type,subtype from poi");
int zoomToStart = ZOOM_TO_SAVE_START;
Tree<PoiBox> rootZoomsTree = new Tree<PoiBox>();
rootZoomsTree.setNode(new PoiBox());
int minX = Integer.MAX_VALUE;
int maxX = 0;
int minY = Integer.MAX_VALUE;
int maxY = 0;
int count = 0;
while (rs.next()) {
int x = rs.getInt(1);
int y = rs.getInt(2);
Tree<Long> prevTree = rootZoomsTree;
for (int i = zoomToStart; i <= ZOOM_TO_SAVE_END; i++) {
int shift = ZOOM_TO_SAVE_END - i;
long l = (((long) x >> shift) << 31) | ((long) y >> shift);
minX = Math.min(x, minX);
maxX = Math.max(x, maxX);
minY = Math.min(y, minY);
maxY = Math.max(y, maxY);
if(count++ > 10000){
count = 0;
log.info("proccess 10000 entities");
}
Tree<Long> subtree = prevTree.getSubtreeByNode(l);
String name = rs.getString(3);
String nameEn = rs.getString(4);
String type = rs.getString(5);
String subtype = rs.getString(6);
Tree<PoiBox> prevTree = rootZoomsTree;
rootZoomsTree.getNode().addCategory(type, subtype);
rootZoomsTree.getNode().addNamePrefix(name, nameEn);
for (int i = zoomToStart; i <= ZOOM_TO_SAVE_END; i++) {
int xs = x >> (31 - i);
int ys = y >> (31 - i);
Tree<PoiBox> subtree = null;
for (Tree<PoiBox> sub : prevTree.getSubtrees()) {
if (sub.getNode().x == xs && sub.getNode().y == ys && sub.getNode().zoom == i) {
subtree = sub;
break;
}
}
if (subtree == null) {
subtree = new Tree<Long>();
subtree.setNode(l);
subtree = new Tree<PoiBox>();
PoiBox poiBox = new PoiBox();
subtree.setNode(poiBox);
poiBox.x = xs;
poiBox.y = ys;
poiBox.zoom = i;
prevTree.addSubTree(subtree);
}
subtree.getNode().addCategory(type, subtype);
subtree.getNode().addNamePrefix(name, nameEn);
prevTree = subtree;
}
}
log.info("Poi processing finishied");
// Finish process all entities
// 1. write header
int right31 = maxX;
int left31 = minX;
int bottom31 = maxY;
int top31 = minY;
long startFpPoiIndex = writer.startWritePOIIndex(regionName, left31, right31, bottom31, top31);
// 2. write categories table
Map<String, Map<String, Integer>> categories = rootZoomsTree.node.categories;
Map<String, Integer> catIndexes = writer.writePOICategoriesTable(categories);
// 3. write boxes
log.info("Poi box processing finishied");
int level = 0;
for (; level < (ZOOM_TO_SAVE_END - zoomToStart); level++) {
int subtrees = rootZoomsTree.getSubTreesOnLevel(level);
@ -258,28 +278,21 @@ public class IndexPoiCreator extends AbstractIndexPartCreator {
}
// 3.2 write tree using stack
PreparedStatement prepareStatement = poiConnection.prepareStatement("SELECT DISTINCT type, subtype FROM poi WHERE x >= ? AND x < ? AND y >= ? AND y < ?");
Map<Long, Long> fpToWriteSeeks = new LinkedHashMap<Long, Long>();
for (Tree<Long> subs : rootZoomsTree.getSubtrees()) {
writePoiBoxes(writer, subs, zoomToStart, fpToWriteSeeks, prepareStatement,
categories, catIndexes);
Map<PoiBox, Long> fpToWriteSeeks = new LinkedHashMap<PoiBox, Long>();
for (Tree<PoiBox> subs : rootZoomsTree.getSubtrees()) {
writePoiBoxes(writer, subs, fpToWriteSeeks, categories, catIndexes);
}
stat = rs.getStatement();
rs.close();
stat.close();
prepareStatement.close();
// 4. write poi data
prepareStatement = poiConnection
// not so effictive probably better to load in memory one time
PreparedStatement prepareStatement = poiConnection
.prepareStatement("SELECT id, x, y, name_en, name, type, subtype, opening_hours, site, phone from poi "
+ "where x >= ? AND x < ? AND y >= ? AND y < ?");
TIntArrayList types = new TIntArrayList();
for (Map.Entry<Long, Long> entry : fpToWriteSeeks.entrySet()) {
long l = entry.getKey();
int z = ZOOM_TO_SAVE_END;
int x = (int) (l >> 31);
int y = (int) (l & ((1 << 31) - 1));
for (Map.Entry<PoiBox, Long> entry : fpToWriteSeeks.entrySet()) {
int z = entry.getKey().zoom;
int x = entry.getKey().x;
int y = entry.getKey().y;
writer.startWritePoiData(z, x, y, startFpPoiIndex, entry.getValue());
prepareStatement.setInt(1, x << (31 - z));
@ -318,38 +331,65 @@ public class IndexPoiCreator extends AbstractIndexPartCreator {
}
private void writePoiBoxes(BinaryMapIndexWriter writer, Tree<Long> tree, int zoom, Map<Long, Long> fpToWriteSeeks,
PreparedStatement categoriesGet, Map<String, Map<String, Integer>> categories, Map<String, Integer> catIndexes) throws IOException, SQLException {
long l = tree.getNode();
int x = (int) (l >> 31);
int y = (int) (l & ((1 << 31) - 1));
private void writePoiBoxes(BinaryMapIndexWriter writer, Tree<PoiBox> tree, Map<PoiBox, Long> fpToWriteSeeks,
Map<String, Map<String, Integer>> categories, Map<String, Integer> catIndexes) throws IOException, SQLException {
int x = tree.getNode().x;
int y = tree.getNode().y;
int zoom = tree.getNode().zoom;
boolean end = zoom == ZOOM_TO_SAVE_END;
long fp = writer.startWritePoiBox(zoom, x, y, end);
if(zoom >= ZOOM_TO_WRITE_CATEGORIES_START && zoom <= ZOOM_TO_WRITE_CATEGORIES_END){
categoriesGet.setInt(1, x << (31 - zoom));
categoriesGet.setInt(2, (x + 1) << (31 - zoom));
categoriesGet.setInt(3, y << (31 - zoom));
categoriesGet.setInt(4, (y + 1) << (31 - zoom));
ResultSet rs = categoriesGet.executeQuery();
TIntArrayList types = new TIntArrayList();
while(rs.next()){
String cat = rs.getString(1);
String subcat = rs.getString(2);
buildTypeIds(cat, subcat, categories, catIndexes, types);
for(Map.Entry<String, Map<String, Integer>> cats : tree.getNode().categories.entrySet()) {
for(String subcat : cats.getValue().keySet()){
String cat = cats.getKey();
buildTypeIds(cat, subcat, categories, catIndexes, types);
}
}
writer.writePOICategories(types);
rs.close();
}
if (!end) {
for (Tree<Long> subTree : tree.getSubtrees()) {
writePoiBoxes(writer, subTree, zoom + 1, fpToWriteSeeks, categoriesGet, categories, catIndexes);
for (Tree<PoiBox> subTree : tree.getSubtrees()) {
writePoiBoxes(writer, subTree, fpToWriteSeeks, categories, catIndexes);
}
} else {
fpToWriteSeeks.put(l, fp);
fpToWriteSeeks.put(tree.getNode(), fp);
}
writer.endWritePoiBox();
}
private static class PoiBox {
int x;
int y;
int zoom;
Map<String, Map<String, Integer>> categories = new LinkedHashMap<String, Map<String, Integer>>();
Set<String> startsName = new TreeSet<String>();
private void addCategory(String cat, String subCat){
if(!categories.containsKey(cat)){
categories.put(cat, new TreeMap<String, Integer>());
}
if (subCat.contains(";") || subCat.contains(",")) {
String[] split = subCat.split(",|;");
for (String sub : split) {
categories.get(cat).put(sub.trim(), 0);
}
} else {
categories.get(cat).put(subCat.trim(), 0);
}
categories.get(cat).put(subCat, 0);
}
public void addNamePrefix(String name, String nameEn) {
if(Algoritms.isEmpty(nameEn)){
nameEn = Junidecode.unidecode(name);
}
// TODO split by 3 characters and save here
}
}
private static class Tree<T> {
@ -411,28 +451,19 @@ public class IndexPoiCreator extends AbstractIndexPartCreator {
}
}
public Tree<T> getSubtreeByNode(T node) {
if (subtrees == null) {
return null;
}
for (Tree<T> s : subtrees) {
if (node.equals(s.getNode())) {
return s;
}
}
return null;
}
}
public static void main(String[] args) throws SQLException, FileNotFoundException, IOException {
// TODO support multiple reading amenity types!
// TODO support multiple reading amenity types! +/-
// TODO support proper POI editing
// TODO support string trigramms
// TODO support cancelling poi search request! Do it in another thread
long time = System.currentTimeMillis();
IndexPoiCreator poiCreator = new IndexPoiCreator();
String fileSqlte = "/home/victor/projects/OsmAnd/data/osm-gen/POI/Ru-mow.poi.odb";
String outFile = "/home/victor/projects/OsmAnd/data/osm-gen/POI/Test-Ru.poi.obf";
// String fileSqlte = "/home/victor/projects/OsmAnd/data/osm-gen/POI/Netherlands_europe.poi.odb";
// String outFile = "/home/victor/projects/OsmAnd/data/osm-gen/POI/Netherlands.poi.obf";
// String fileSqlte = "/home/victor/projects/OsmAnd/data/osm-gen/POI/Ru-mow.poi.odb";
// String outFile = "/home/victor/projects/OsmAnd/data/osm-gen/POI/Ru-mow.poi.obf";
String fileSqlte = "/home/victor/projects/OsmAnd/data/osm-gen/POI/Netherlands_europe.poi.odb";
String outFile = "/home/victor/projects/OsmAnd/data/osm-gen/POI/Netherlands.poi.obf";
poiCreator.poiConnection = (Connection) DBDialect.SQLITE.getDatabaseConnection(
fileSqlte, log);

10
DataExtractionOSM/src/osmand_odb.proto
View file

@ -337,11 +337,11 @@ message OsmAndPoiBoxDataAtom {
required sint32 dx = 2; // delta encoded to OsmAndPoiBox on 24 zoom
required sint32 dy = 3; // delta encoded to OsmAndPoiBox on 24 zoom
optional uint64 id = 4;
optional string name = 5;
optional string nameEn = 6;
repeated uint32 categories = 7;
repeated uint32 categories = 4;
optional string name = 6;
optional string nameEn = 7;
optional uint64 id = 8;
optional string openingHours = 10;
optional string site = 11;