Android-builders-club/OsmAnd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Partial map write implementation

Browse source
This commit is contained in:
Victor Shcherb 2012-02-16 00:21:07 +01:00
parent 5caf563aa1
commit 8ee3f3f05b
5 changed files with 172 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
DataExtractionOSM/src/net/osmand/binary/BinaryInspector.java
View file

@ -249,7 +249,7 @@ public class BinaryInspector {
byte[] BUFFER_TO_READ = new byte[BUFFER_SIZE];
ous.writeInt32(OsmandOdb.OsmAndStructure.VERSION_FIELD_NUMBER, version);
ous.writeInt32(OsmandOdb.OsmAndStructure.DATECREATED_FIELD_NUMBER, System.currentTimeMillis());
ous.writeInt64(OsmandOdb.OsmAndStructure.DATECREATED_FIELD_NUMBER, System.currentTimeMillis());
for (int k = 0; k < indexes.length; k++) {

153
DataExtractionOSM/src/net/osmand/data/preparation/BinaryMapIndexWriter.java
View file

@ -9,6 +9,8 @@ import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
@ -23,6 +25,9 @@ import net.osmand.Algoritms;
import net.osmand.binary.BinaryMapIndexReader;
import net.osmand.binary.OsmandOdb;
import net.osmand.binary.OsmandOdb.CityIndex;
import net.osmand.binary.OsmandOdb.OsmAndMapIndex.MapDataBox;
import net.osmand.binary.OsmandOdb.OsmAndMapIndex.MapEncodingRule;
import net.osmand.binary.OsmandOdb.OsmAndMapIndex.MapRootLevel;
import net.osmand.binary.OsmandOdb.OsmAndPoiBoxDataAtom;
import net.osmand.binary.OsmandOdb.OsmAndPoiNameIndex;
import net.osmand.binary.OsmandOdb.OsmAndPoiNameIndexDataAtom;
@ -178,12 +183,12 @@ public class BinaryMapIndexWriter {
codedOutStream.writeTag(OsmandOdb.OsmAndMapIndex.LEVELS_FIELD_NUMBER, WireFormat.WIRETYPE_FIXED32_LENGTH_DELIMITED);
preserveInt32Size();
codedOutStream.writeInt32(OsmandOdb.MapRootLevel.MAXZOOM_FIELD_NUMBER, maxZoom);
codedOutStream.writeInt32(OsmandOdb.MapRootLevel.MINZOOM_FIELD_NUMBER, minZoom);
codedOutStream.writeInt32(OsmandOdb.MapRootLevel.LEFT_FIELD_NUMBER, leftX);
codedOutStream.writeInt32(OsmandOdb.MapRootLevel.RIGHT_FIELD_NUMBER, rightX);
codedOutStream.writeInt32(OsmandOdb.MapRootLevel.TOP_FIELD_NUMBER, topY);
codedOutStream.writeInt32(OsmandOdb.MapRootLevel.BOTTOM_FIELD_NUMBER, bottomY);
codedOutStream.writeInt32(OsmandOdb.OsmAndMapIndex.MapRootLevel.MAXZOOM_FIELD_NUMBER, maxZoom);
codedOutStream.writeInt32(OsmandOdb.OsmAndMapIndex.MapRootLevel.MINZOOM_FIELD_NUMBER, minZoom);
codedOutStream.writeInt32(OsmandOdb.OsmAndMapIndex.MapRootLevel.LEFT_FIELD_NUMBER, leftX);
codedOutStream.writeInt32(OsmandOdb.OsmAndMapIndex.MapRootLevel.RIGHT_FIELD_NUMBER, rightX);
codedOutStream.writeInt32(OsmandOdb.OsmAndMapIndex.MapRootLevel.TOP_FIELD_NUMBER, topY);
codedOutStream.writeInt32(OsmandOdb.OsmAndMapIndex.MapRootLevel.BOTTOM_FIELD_NUMBER, bottomY);
stackBounds.push(new Bounds(leftX, rightX, topY, bottomY));
}
@ -194,60 +199,73 @@ public class BinaryMapIndexWriter {
writeInt32Size();
}
public void writeMapEncodingRules(Map<String, MapRulType> types) throws IOException{
public void writeMapEncodingRules(Map<String, MapRulType> types) throws IOException {
checkPeekState(MAP_INDEX_INIT);
long fp = getFilePointer();
MapEncodingRule.Builder builder = OsmandOdb.MapEncodingRule.newBuilder();
for(String tag : types.keySet()){
MapRulType rule = types.get(tag);
int type = rule.getType(null);
int subType = rule.getSubType(null);
Collection<String> valuesSet = types.get(tag).getValuesSet();
if(type != 0 && subType != 0){
builder.setTag(tag).setValue("").setType(type).setSubtype(subType).setMinZoom(rule.getMinZoom(null));
if (valuesSet.isEmpty()) {
codedOutStream.writeMessage(OsmandOdb.OsmAndMapIndex.RULES_FIELD_NUMBER, builder.build());
}
builder = OsmandOdb.MapEncodingRule.newBuilder();
ArrayList<MapRulType> out = new ArrayList<MapRulType>();
int highestTargetId = types.size();
// 1. prepare map rule type to write
for (MapRulType t : types.values()) {
if (!t.isMapIndexed() || t.getTargetTagValue() != null || t.getFreq() == 0) {
t.setTargetId(highestTargetId++);
} else {
out.add(t);
}
for(String val : valuesSet){
type = rule.getType(val);
subType = rule.getSubType(val);
builder.setTag(tag).setValue(val).setType(type).setSubtype(subType).setMinZoom(rule.getMinZoom(null));
codedOutStream.writeMessage(OsmandOdb.OsmAndMapIndex.RULES_FIELD_NUMBER, builder.build());
builder = OsmandOdb.MapEncodingRule.newBuilder();
}
// 2. sort by frequency and assign ids
Collections.sort(out, new Comparator<MapRulType>() {
@Override
public int compare(MapRulType o1, MapRulType o2) {
return o1.getFreq() - o2.getFreq();
}
});
for (int i = 0; i < out.size(); i++) {
MapEncodingRule.Builder builder = OsmandOdb.OsmAndMapIndex.MapEncodingRule.newBuilder();
MapRulType rule = out.get(i);
rule.setTargetId(i);
builder.setTag(rule.getTag());
if (rule.getValue() != null) {
builder.setValue(rule.getValue());
}
builder.setMinZoom(rule.getMinzoom());
if (rule.isAdditional()) {
builder.setType(1);
}
codedOutStream.writeMessage(OsmandOdb.OsmAndMapIndex.RULES_FIELD_NUMBER, builder.build());
}
long newfp = getFilePointer();
System.out.println("RENDERING SCHEMA takes " + (newfp - fp));
}
public void startMapTreeElement(int leftX, int rightX, int topY, int bottomY) throws IOException{
startMapTreeElement(-1L, leftX, rightX, topY, bottomY);
}
public void startMapTreeElement(long baseId, int leftX, int rightX, int topY, int bottomY) throws IOException{
public BinaryFileReference startMapTreeElement(int leftX, int rightX, int topY, int bottomY, boolean containsLeaf) throws IOException{
checkPeekState(MAP_ROOT_LEVEL_INIT, MAP_TREE);
if(state.peek() == MAP_ROOT_LEVEL_INIT){
codedOutStream.writeTag(OsmandOdb.MapRootLevel.ROOT_FIELD_NUMBER, WireFormat.WIRETYPE_FIXED32_LENGTH_DELIMITED);
codedOutStream.writeTag(MapRootLevel.BOXES_FIELD_NUMBER, WireFormat.WIRETYPE_FIXED32_LENGTH_DELIMITED);
} else {
codedOutStream.writeTag(OsmandOdb.MapTree.SUBTREES_FIELD_NUMBER, WireFormat.WIRETYPE_FIXED32_LENGTH_DELIMITED);
codedOutStream.writeTag(MapDataBox.BOXES_FIELD_NUMBER, WireFormat.WIRETYPE_FIXED32_LENGTH_DELIMITED);
}
state.push(MAP_TREE);
preserveInt32Size();
long fp = getFilePointer();
Bounds bounds = stackBounds.peek();
codedOutStream.writeSInt32(OsmandOdb.MapTree.LEFT_FIELD_NUMBER, leftX - bounds.leftX);
codedOutStream.writeSInt32(OsmandOdb.MapTree.RIGHT_FIELD_NUMBER, rightX - bounds.rightX);
codedOutStream.writeSInt32(OsmandOdb.MapTree.TOP_FIELD_NUMBER, topY - bounds.topY);
codedOutStream.writeSInt32(OsmandOdb.MapTree.BOTTOM_FIELD_NUMBER, bottomY - bounds.bottomY);
codedOutStream.writeSInt32(MapDataBox.LEFT_FIELD_NUMBER, leftX - bounds.leftX);
codedOutStream.writeSInt32(MapDataBox.RIGHT_FIELD_NUMBER, rightX - bounds.rightX);
codedOutStream.writeSInt32(MapDataBox.TOP_FIELD_NUMBER, topY - bounds.topY);
codedOutStream.writeSInt32(MapDataBox.BOTTOM_FIELD_NUMBER, bottomY - bounds.bottomY);
stackBounds.push(new Bounds(leftX, rightX, topY, bottomY));
stackBaseIds.push(baseId);
stackStringTable.push(null);
BinaryFileReference ref = null;
if(containsLeaf){
ref = BinaryFileReference.createShiftReference(getFilePointer(), fp);
}
return ref;
}
@ -255,32 +273,33 @@ public class BinaryMapIndexWriter {
popState(MAP_TREE);
stackBounds.pop();
Long l = stackBaseIds.pop();
if(l != -1){
codedOutStream.writeTag(OsmandOdb.MapTree.BASEID_FIELD_NUMBER, WireFormat.FieldType.UINT64.getWireType());
codedOutStream.writeUInt64NoTag(l);
}
Map<String, Integer> map = stackStringTable.peek();
if(map != null){
int i = 0;
int size = 0;
for(String s : map.keySet()){
Integer integer = map.get(s);
if(integer != i){
throw new IllegalStateException();
}
i++;
size += CodedOutputStream.computeStringSize(OsmandOdb.StringTable.S_FIELD_NUMBER, s);
}
codedOutStream.writeTag(OsmandOdb.MapTree.STRINGTABLE_FIELD_NUMBER, WireFormat.FieldType.MESSAGE.getWireType());
STRING_TABLE_SIZE += CodedOutputStream.computeTagSize(OsmandOdb.MapTree.STRINGTABLE_FIELD_NUMBER) +
CodedOutputStream.computeRawVarint32Size(size) + size;
codedOutStream.writeRawVarint32(size);
for(String s : map.keySet()){
codedOutStream.writeString(OsmandOdb.StringTable.S_FIELD_NUMBER, s);
}
}
// TODO move to box
// Long l = stackBaseIds.pop();
// if(l != -1){
// codedOutStream.writeTag(OsmandOdb.MapTree.BASEID_FIELD_NUMBER, WireFormat.FieldType.UINT64.getWireType());
// codedOutStream.writeUInt64NoTag(l);
// }
// if(map != null){
//
// int i = 0;
// int size = 0;
// for(String s : map.keySet()){
// Integer integer = map.get(s);
// if(integer != i){
// throw new IllegalStateException();
// }
// i++;
// size += CodedOutputStream.computeStringSize(OsmandOdb.StringTable.S_FIELD_NUMBER, s);
// }
// codedOutStream.writeTag(OsmandOdb.MapTree.STRINGTABLE_FIELD_NUMBER, WireFormat.FieldType.MESSAGE.getWireType());
// STRING_TABLE_SIZE += CodedOutputStream.computeTagSize(OsmandOdb.MapTree.STRINGTABLE_FIELD_NUMBER) +
// CodedOutputStream.computeRawVarint32Size(size) + size;
// codedOutStream.writeRawVarint32(size);
// for(String s : map.keySet()){
// codedOutStream.writeString(OsmandOdb.StringTable.S_FIELD_NUMBER, s);
// }
// }
writeInt32Size();
}
@ -961,7 +980,7 @@ public class BinaryMapIndexWriter {
if (end) {
codedOutStream.writeTag(OsmandOdb.OsmAndPoiBox.SHIFTTODATA_FIELD_NUMBER, WireFormat.WIRETYPE_FIXED32_LENGTH_DELIMITED);
BinaryFileReference shift = BinaryFileReference.createShiftReference(codedOutStream.getWrittenBytes(), startPoiIndex.getStartPointer());
BinaryFileReference shift = BinaryFileReference.createShiftReference(getFilePointer(), startPoiIndex.getStartPointer());
codedOutStream.writeFixed32NoTag(0);
return shift;
}

68
DataExtractionOSM/src/net/osmand/data/preparation/IndexVectorMapCreator.java
View file

@ -1,6 +1,7 @@
package net.osmand.data.preparation;
import gnu.trove.list.array.TIntArrayList;
import gnu.trove.map.hash.TLongObjectHashMap;
import java.io.ByteArrayOutputStream;
import java.io.File;
@ -598,10 +599,12 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator {
closePreparedStatements(mapBinaryStat, mapLowLevelBinaryStat);
mapConnection.commit();
try {
PreparedStatement selectData = mapConnection.prepareStatement("SELECT nodes, types, name, highway, restrictions FROM binary_map_objects WHERE id = ?"); //$NON-NLS-1$
writer.startWriteMapIndex(regionName);
// write map encoding rules
writer.writeMapEncodingRules(renderingTypes.getEncodingRuleTypes());
// write map levels and map index
TLongObjectHashMap<BinaryFileReference> bounds = new TLongObjectHashMap<BinaryFileReference>();
for (int i = 0; i < mapZooms.size(); i++) {
RTree rtree = mapTree[i];
long rootIndex = rtree.getFileHdr().getRootIndex();
@ -612,10 +615,10 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator {
writer.startWriteMapLevelIndex(mapZooms.getLevel(i).getMinZoom(), mapZooms.getLevel(i).getMaxZoom(), rootBounds
.getMinX(), rootBounds.getMaxX(), rootBounds.getMinY(), rootBounds.getMaxY());
if (last) {
writer.startMapTreeElement(rootBounds.getMinX(), rootBounds.getMaxX(), rootBounds.getMinY(), rootBounds.getMaxY());
BinaryFileReference ref = writer.startMapTreeElement(rootBounds.getMinX(), rootBounds.getMaxX(), rootBounds.getMinY(), rootBounds.getMaxY(), true);
bounds.put(rootIndex, ref);
}
writeBinaryMapTree(root, rtree, writer, selectData);
writeBinaryMapTree(root, rtree, writer, bounds);
if (last) {
writer.endWriteMapTreeElement();
}
@ -623,8 +626,21 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator {
writer.endWriteMapLevelIndex();
}
}
// write map data blocks
PreparedStatement selectData = mapConnection.prepareStatement("SELECT nodes, types, name FROM binary_map_objects WHERE id = ?"); //$NON-NLS-1$
for (int i = 0; i < mapZooms.size(); i++) {
RTree rtree = mapTree[i];
long rootIndex = rtree.getFileHdr().getRootIndex();
rtree.Node root = rtree.getReadNode(rootIndex);
Rect rootBounds = calcBounds(root);
if (rootBounds != null) {
writeBinaryMapBlock(root, rtree, writer, selectData, bounds);
}
}
selectData.close();
writer.writeMapEncodingRules(renderingTypes.getEncodingRuleTypes());
writer.endWriteMapIndex();
writer.flush();
} catch (RTreeException e) {
@ -643,30 +659,52 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator {
return (id >> (MAP_LEVELS_POWER)) + (id & 1);
}
public void writeBinaryMapTree(rtree.Node parent, RTree r, BinaryMapIndexWriter writer, PreparedStatement selectData) throws IOException, RTreeException, SQLException {
public void writeBinaryMapBlock(rtree.Node parent, RTree r, BinaryMapIndexWriter writer, PreparedStatement selectData, TLongObjectHashMap<BinaryFileReference> bounds) throws IOException, RTreeException, SQLException {
Element[] e = parent.getAllElements();
for (int i = 0; i < parent.getTotalElements(); i++) {
Rect re = e[i].getRect();
if (e[i].getElementType() == rtree.Node.LEAF_NODE) {
BinaryFileReference ref = bounds.get(parent.getNodeIndex());
// TODO create block
long id = ((LeafElement) e[i]).getPtr();
selectData.setLong(1, id);
ResultSet rs = selectData.executeQuery();
if (rs.next()) {
// mapConnection.prepareStatement("SELECT nodes, types, name, highway, restrictions FROM binary_map_objects WHERE id = ?");
// mapConnection.prepareStatement("SELECT nodes, types, name FROM binary_map_objects WHERE id = ?");
writer.writeMapData(convertGeneratedIdToObfWrite(id), rs.getBytes(1),
rs.getBytes(2), rs.getString(3),
rs.getInt(4), rs.getBytes(5));
rs.getBytes(2), rs.getString(3));
} else {
logMapDataWarn.error("Something goes wrong with id = " + id); //$NON-NLS-1$
}
} else {
long ptr = ((NonLeafElement) e[i]).getPtr();
rtree.Node ns = r.getReadNode(ptr);
writer.startMapTreeElement(re.getMinX(), re.getMaxX(), re.getMinY(), re.getMaxY());
writeBinaryMapTree(ns, r, writer, selectData);
writeBinaryMapBlock(ns, r, writer, selectData, bounds);
writer.endWriteMapTreeElement();
}
}
}
public void writeBinaryMapTree(rtree.Node parent, RTree r, BinaryMapIndexWriter writer, TLongObjectHashMap<BinaryFileReference> bounds)
throws IOException, RTreeException {
Element[] e = parent.getAllElements();
boolean containsLeaf = false;
for (int i = 0; i < parent.getTotalElements(); i++) {
if (e[i].getElementType() == rtree.Node.LEAF_NODE) {
containsLeaf = true;
}
}
for (int i = 0; i < parent.getTotalElements(); i++) {
Rect re = e[i].getRect();
if (e[i].getElementType() != rtree.Node.LEAF_NODE) {
long ptr = ((NonLeafElement) e[i]).getPtr();
rtree.Node ns = r.getReadNode(ptr);
BinaryFileReference ref = writer.startMapTreeElement(re.getMinX(), re.getMaxX(), re.getMinY(), re.getMaxY(), containsLeaf);
if (ref != null) {
bounds.put(ns.getNodeIndex(), ref);
}
writeBinaryMapTree(ns, r, writer, bounds);
writer.endWriteMapTreeElement();
}
}

25
DataExtractionOSM/src/net/osmand/osm/MapRenderingTypes.java
View file

@ -387,6 +387,7 @@ public class MapRenderingTypes {
int id;
int freq;
int targetId;
String poiPrefix;
AmenityType poiCategory;
@ -420,6 +421,18 @@ public class MapRenderingTypes {
return tag;
}
public int getTargetId() {
return targetId;
}
public void setTargetId(int targetId) {
this.targetId = targetId;
}
public MapRulType getTargetTagValue() {
return targetTagValue;
}
public String getValue() {
return value;
}
@ -428,6 +441,18 @@ public class MapRenderingTypes {
return minzoom;
}
public boolean isAdditional() {
return additional;
}
public int getFreq() {
return freq;
}
public int updateFreq(){
return ++freq;
}
}
// TODO Move to Routing Attributes and finalize

12
DataExtractionOSM/src/osmand_odb.proto
View file

@ -72,19 +72,21 @@ message OsmAndMapIndex {
required string name = 2;
// encoded as fixed32 length delimited
repeated MapEncodingRule rules = 4;
message MapEncodingRule {
required string tag = 3;
optional string value = 5;
// it is optional because natural order in block of rules
// is the same as id
// is the same as id (id is 1-based and order)
optional uint32 id = 7;
// min zoom combination exists
optional uint32 minZoom = 9;
// additional - 1
// "additional" flag is encoded as %2 == 1
optional uint32 type = 10;
}
// encoded as fixed32 length delimited
repeated MapRootLevel levels = 5;
message MapRootLevel {
required int32 maxZoom = 1;
@ -103,8 +105,8 @@ message OsmAndMapIndex {
required sint32 top = 3; // delta encoded
required sint32 bottom = 4; // delta encoded
//shift to message MapDataBlock
optional uint32 shiftToMapData = 5;
//shift from mapdatabox start to message MapDataBlock
optional fixed32 shiftToMapData = 5;
repeated MapDataBox boxes = 7;
}