fix some issues

git-svn-id: https://osmand.googlecode.com/svn/trunk@596 e29c36b1-1cfa-d876-8d93-3434fc2bb7b8
This commit is contained in:
Victor Shcherb 2010-10-27 20:04:09 +00:00
parent 37606d7f0d
commit c9982c0b97
2 changed files with 145 additions and 113 deletions

View file

@ -1,20 +1,19 @@
package net.osmand.binary;
import java.io.IOException;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Stack;
import net.osmand.Algoritms;
import net.osmand.data.index.IndexConstants;
import com.google.protobuf.CodedOutputStream;
import com.google.protobuf.WireFormat;
public class BinaryIndexWriter {
public class BinaryMapIndexWriter {
private final OutputStream out;
private RandomAccessFile raf;
private CodedOutputStream codedOutStream;
private static class Bounds {
@ -39,55 +38,45 @@ public class BinaryIndexWriter {
// internal constants to track state of index writing
private Stack<Integer> state = new Stack<Integer>();
private Stack<Long> stackSizes = new Stack<Long>();
private final static int OSMAND_STRUCTURE_INIT = 1;
private final static int MAP_INDEX_INIT = 2;
private final static int MAP_ROOT_LEVEL_INIT = 3;
private final static int MAP_TREE = 4;
public BinaryIndexWriter(OutputStream out) throws IOException{
this.out = out;
codedOutStream = CodedOutputStream.newInstance(out);
codedOutStream.writeInt32(OsmandOdb.OsmAndStructure.VERSION_FIELD_NUMBER, IndexConstants.BINARY_MAP_VERSION);
public BinaryMapIndexWriter(RandomAccessFile raf, CodedOutputStream codedOutputStream) throws IOException{
this.raf = raf;
codedOutStream = codedOutputStream;
state.push(OSMAND_STRUCTURE_INIT);
}
// message MapTree {
// required sint32 left = 1; // delta encoded
// required sint32 right = 2; // delta encoded
// required sint32 top = 3; // delta encoded
// required sint32 bottom = 4; // delta encoded
//
// optional StringTable stringTable = 5;
// optional uint64 baseId = 6;
//
// repeated MapTree subtrees = 7;
// repeated MapData leafs = 8;
//
// }
// /// Simple messages
// message MapData {
// required bytes coordinates = 1; // array of delta x,y uin32 could be read by codedinputstream
// required bytes types = 2; // array of fixed int16
//
// required sint64 id = 3; // delta encoded
// optional uint32 stringId = 4;
//
// repeated sint64 restrictions = 5; // delta encoded 3 bytes for type and other for id
// optional int32 highwayMeta = 6;
// }
private void preserveInt32Size() throws IOException {
stackSizes.push(raf.getFilePointer());
codedOutStream.writeFixed32NoTag(0);
}
private void writeInt32Size() throws IOException{
long filePointer = raf.getFilePointer();
Long old = stackSizes.pop();
int length = (int) (filePointer - old);
raf.seek(old);
raf.writeInt(length);
raf.seek(filePointer);
}
public void startWriteMapIndex() throws IOException{
assert state.peek() == OSMAND_STRUCTURE_INIT;
state.push(MAP_INDEX_INIT);
codedOutStream.writeTag(OsmandOdb.OsmAndStructure.MAPINDEX_FIELD_NUMBER, WireFormat.FieldType.MESSAGE.getWireType());
// TODO write size of map index
codedOutStream.writeFixed32NoTag(0);
preserveInt32Size();
}
public void endWriteMapIndex() throws IOException{
Integer st = state.pop();
assert st == MAP_INDEX_INIT;
codedOutStream.writeRawVarint32(0);
writeInt32Size();
}
public void startWriteMapLevelIndex(int maxZoom, int minZoom, int leftX, int rightX, int topY, int bottomY) throws IOException{
@ -95,8 +84,7 @@ public class BinaryIndexWriter {
state.push(MAP_ROOT_LEVEL_INIT);
codedOutStream.writeTag(OsmandOdb.OsmAndMapIndex.LEVELS_FIELD_NUMBER, WireFormat.FieldType.MESSAGE.getWireType());
// TODO write size of level map index
codedOutStream.writeFixed32NoTag(0);
preserveInt32Size();
codedOutStream.writeInt32(OsmandOdb.MapRootLevel.MAXZOOM_FIELD_NUMBER, maxZoom);
codedOutStream.writeInt32(OsmandOdb.MapRootLevel.MINZOOM_FIELD_NUMBER, minZoom);
@ -111,9 +99,8 @@ public class BinaryIndexWriter {
public void endWriteMapLevelIndex() throws IOException{
assert state.peek() == MAP_ROOT_LEVEL_INIT;
state.pop();
stackBounds.pop();
codedOutStream.writeRawVarint32(0);
writeInt32Size();
}
public void startMapTreeElement(int leftX, int rightX, int topY, int bottomY) throws IOException{
@ -123,8 +110,7 @@ public class BinaryIndexWriter {
} else {
codedOutStream.writeTag(OsmandOdb.MapTree.SUBTREES_FIELD_NUMBER, WireFormat.FieldType.MESSAGE.getWireType());
}
// TODO write size of level map index
codedOutStream.writeFixed32NoTag(0);
preserveInt32Size();
state.push(MAP_TREE);
@ -134,7 +120,7 @@ public class BinaryIndexWriter {
codedOutStream.writeSInt32(OsmandOdb.MapTree.TOP_FIELD_NUMBER, topY - bounds.topY);
codedOutStream.writeSInt32(OsmandOdb.MapTree.BOTTOM_FIELD_NUMBER, bottomY - bounds.bottomY);
stackBounds.push(new Bounds(leftX, rightX, topY, bottomY));
stackBaseIds.push(0L);
stackBaseIds.push(-1L);
stackStringTable.push(null);
}
@ -144,47 +130,97 @@ public class BinaryIndexWriter {
stackBounds.pop();
Long l = stackBaseIds.pop();
if(l != 0){
if(l != -1){
codedOutStream.writeTag(OsmandOdb.MapTree.BASEID_FIELD_NUMBER, WireFormat.FieldType.UINT64.getWireType());
codedOutStream.writeUInt64NoTag(l);
}
Map<String, Integer> map = stackStringTable.peek();
if(map != null){
codedOutStream.writeTag(OsmandOdb.MapTree.STRINGTABLE_FIELD_NUMBER, WireFormat.FieldType.MESSAGE.getWireType());
// TODO write size
codedOutStream.writeFixed32NoTag(0);
int i = 0;
int size = 0;
for(String s : map.keySet()){
Integer integer = map.get(s);
if(integer != i){
throw new IllegalStateException();
}
i++;
codedOutStream.writeTag(OsmandOdb.StringTable.S_FIELD_NUMBER, WireFormat.FieldType.MESSAGE.getWireType());
codedOutStream.writeStringNoTag(s);
size += CodedOutputStream.computeStringSize(OsmandOdb.StringTable.S_FIELD_NUMBER, s);
}
codedOutStream.writeTag(OsmandOdb.MapTree.STRINGTABLE_FIELD_NUMBER, WireFormat.FieldType.MESSAGE.getWireType());
codedOutStream.writeRawVarint32(size);
for(String s : map.keySet()){
codedOutStream.writeString(OsmandOdb.StringTable.S_FIELD_NUMBER, s);
}
}
writeInt32Size();
}
public void writeMapData(long id, byte[] nodes, byte[] types, String name, int highwayAttributes, byte[] restrictions) throws IOException{
assert state.peek() == MAP_TREE;
codedOutStream.writeTag(OsmandOdb.MapTree.LEAFS_FIELD_NUMBER, WireFormat.FieldType.MESSAGE.getWireType());
// TODO write size of map data !!! here
Bounds bounds = stackBounds.peek();
codedOutStream.writeTag(OsmandOdb.MapData.COORDINATES_FIELD_NUMBER, WireFormat.FieldType.BYTES.getWireType());
int size = 0;
for(int i=0; i< nodes.length / 8; i++){
int x = Algoritms.parseIntFromBytes(nodes, i*8) - bounds.leftX;
int y = Algoritms.parseIntFromBytes(nodes, i*8 + 4) - bounds.topY;
size += CodedOutputStream.computeInt32SizeNoTag(x);
size += CodedOutputStream.computeInt32SizeNoTag(y);
if(stackBaseIds.peek() == -1){
stackBaseIds.pop();
stackBaseIds.push(id);
}
codedOutStream.writeRawVarint32(size);
// calculate size
int allSize = CodedOutputStream.computeTagSize(OsmandOdb.MapData.COORDINATES_FIELD_NUMBER);
int sizeCoordinates = 0;
for(int i=0; i< nodes.length / 8; i++){
int x = Algoritms.parseIntFromBytes(nodes, i*8) - bounds.leftX;
int y = Algoritms.parseIntFromBytes(nodes, i*8 + 4) - bounds.topY;
sizeCoordinates += CodedOutputStream.computeInt32SizeNoTag(x);
sizeCoordinates += CodedOutputStream.computeInt32SizeNoTag(y);
}
allSize += sizeCoordinates;
allSize += CodedOutputStream.computeTagSize(OsmandOdb.MapData.TYPES_FIELD_NUMBER);
allSize += CodedOutputStream.computeRawVarint32Size(types.length);
allSize += types.length;
allSize += CodedOutputStream.computeSInt64Size(OsmandOdb.MapData.ID_FIELD_NUMBER, id - stackBaseIds.peek());
int nameId = 0;
if(name != null){
if(stackStringTable.peek() == null) {
stackStringTable.pop();
stackStringTable.push(new LinkedHashMap<String, Integer>());
}
Map<String, Integer> map = stackStringTable.peek();
if(map.containsKey(name)) {
nameId = map.get(name);
} else {
nameId = map.size();
map.put(name, nameId);
}
allSize += CodedOutputStream.computeUInt32Size(OsmandOdb.MapData.STRINGID_FIELD_NUMBER, nameId);
}
int restrictionsSize = 0;
if(restrictions.length > 0){
allSize += CodedOutputStream.computeTagSize(OsmandOdb.MapData.RESTRICTIONS_FIELD_NUMBER);
for (int i = 0; i < restrictions.length / 8; i++) {
long l = Algoritms.parseLongFromBytes(restrictions, i * 8) - stackBaseIds.peek();
restrictionsSize += CodedOutputStream.computeSInt64SizeNoTag(l);
}
allSize += CodedOutputStream.computeRawVarint32Size(restrictionsSize);
allSize += restrictionsSize;
}
if(highwayAttributes != 0){
allSize += CodedOutputStream.computeInt32Size(OsmandOdb.MapData.HIGHWAYMETA_FIELD_NUMBER, highwayAttributes);
}
// writing data
codedOutStream.writeTag(OsmandOdb.MapTree.LEAFS_FIELD_NUMBER, WireFormat.FieldType.MESSAGE.getWireType());
codedOutStream.writeRawVarint32(allSize);
codedOutStream.writeTag(OsmandOdb.MapData.COORDINATES_FIELD_NUMBER, WireFormat.FieldType.BYTES.getWireType());
codedOutStream.writeRawVarint32(sizeCoordinates);
for (int i = 0; i < nodes.length / 8; i++) {
int x = Algoritms.parseIntFromBytes(nodes, i * 8) - bounds.leftX;
int y = Algoritms.parseIntFromBytes(nodes, i * 8 + 4) - bounds.topY;
codedOutStream.writeInt32NoTag(x);
codedOutStream.writeInt32NoTag(y);
}
@ -193,47 +229,29 @@ public class BinaryIndexWriter {
codedOutStream.writeRawVarint32(types.length);
codedOutStream.writeRawBytes(types);
if(stackBaseIds.peek() == 0){
stackBaseIds.pop();
stackBaseIds.push(id);
}
codedOutStream.writeTag(OsmandOdb.MapData.ID_FIELD_NUMBER, WireFormat.FieldType.SINT64.getWireType());
codedOutStream.writeSInt64NoTag(id - stackBaseIds.peek());
codedOutStream.writeSInt64(OsmandOdb.MapData.ID_FIELD_NUMBER, id - stackBaseIds.peek());
if(name != null){
if(stackStringTable.peek() == null) {
stackStringTable.pop();
stackStringTable.push(new LinkedHashMap<String, Integer>());
}
Map<String, Integer> map = stackStringTable.peek();
int s;
if(map.containsKey(name)) {
s = map.get(name);
} else {
s = map.size();
map.put(name, s);
}
codedOutStream.writeTag(OsmandOdb.MapData.STRINGID_FIELD_NUMBER, WireFormat.FieldType.UINT32.getWireType());
codedOutStream.writeUInt32NoTag(s);
codedOutStream.writeUInt32(OsmandOdb.MapData.STRINGID_FIELD_NUMBER, nameId);
}
if(restrictions.length > 0){
// TODO restrictions delta?
codedOutStream.writeTag(OsmandOdb.MapData.RESTRICTIONS_FIELD_NUMBER, WireFormat.FieldType.BYTES.getWireType());
codedOutStream.writeRawVarint32(restrictions.length);
codedOutStream.writeRawBytes(restrictions);
codedOutStream.writeRawVarint32(restrictionsSize);
for (int i = 0; i < restrictions.length / 8; i++) {
long l = Algoritms.parseLongFromBytes(restrictions, i * 8) - stackBaseIds.peek();
codedOutStream.writeSInt64NoTag(l);
}
}
if(highwayAttributes != 0){
codedOutStream.writeTag(OsmandOdb.MapData.HIGHWAYMETA_FIELD_NUMBER, WireFormat.FieldType.UINT32.getWireType());
codedOutStream.writeRawVarint32(highwayAttributes);
codedOutStream.writeInt32(OsmandOdb.MapData.HIGHWAYMETA_FIELD_NUMBER, highwayAttributes);
}
}
public void close() throws IOException{
assert state.peek() == OSMAND_STRUCTURE_INIT;
codedOutStream.flush();
out.close();
}
}

View file

@ -4,9 +4,9 @@ import java.awt.geom.Line2D;
import java.awt.geom.Point2D;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.sql.Connection;
import java.sql.DriverManager;
@ -28,7 +28,8 @@ import java.util.Map.Entry;
import net.osmand.Algoritms;
import net.osmand.IProgress;
import net.osmand.binary.BinaryIndexWriter;
import net.osmand.binary.BinaryMapIndexWriter;
import net.osmand.binary.OsmandOdb;
import net.osmand.data.Amenity;
import net.osmand.data.Building;
import net.osmand.data.City;
@ -77,6 +78,8 @@ import rtree.RTree;
import rtree.RTreeException;
import rtree.Rect;
import com.google.protobuf.CodedOutputStream;
/**
* http://wiki.openstreetmap.org/wiki/OSM_tags_for_routing#Is_inside.2Foutside
@ -148,13 +151,12 @@ public class IndexCreator {
private PreparedStatement addressStreetNodeStat;
private File mapFile;
private RandomAccessFile mapRAFile;
private Connection mapConnection;
private PreparedStatement mapBinaryStat;
private static final int[] MAP_ZOOMS = new int[]{6, 9, 14, 22};
private RTree[] mapTree = null;
private File binaryMapFile;
// MEMORY map : save it in memory while that is allowed
private Map<Long, Set<Integer>> multiPolygonsWays0 = new LinkedHashMap<Long, Set<Integer>>();
private Map<Long, Set<Integer>> multiPolygonsWays1 = new LinkedHashMap<Long, Set<Integer>>();
@ -467,19 +469,13 @@ public class IndexCreator {
}
public String getMapFileName() {
if(mapFileName == null){
return getRegionName() + IndexConstants.MAP_INDEX_EXT;
return getRegionName() + IndexConstants.BINARY_MAP_INDEX_EXT;
}
return mapFileName;
}
public void setBinaryMapFileName(String mapFileName) {
this.binaryMapFileName = mapFileName;
}
public String getBinaryMapFileName(){
if(binaryMapFileName == null){
return getRegionName() + IndexConstants.BINARY_MAP_INDEX_EXT;
}
return binaryMapFileName;
public String getTempMapDBFileName(){
return getMapFileName() + ".tmp";
}
public String getTransportFileName() {
@ -1073,6 +1069,7 @@ public class IndexCreator {
type = MapRenderingTypes.RESTRICTION_ONLY_STRAIGHT_ON;
}
if(type != -1){
loadEntityData(e, true);
Collection<EntityId> fromL = ((Relation) e).getMemberIds("from");
Collection<EntityId> toL = ((Relation) e).getMemberIds("to");
if(!fromL.isEmpty() && !toL.isEmpty()) {
@ -1451,15 +1448,21 @@ public class IndexCreator {
}
public void writeBinaryData() throws IOException, SQLException {
public void writeBinaryMapIndex() throws IOException, SQLException {
try {
assert IndexConstants.IndexBinaryMapRenderObject.values().length == 6;
PreparedStatement selectData = mapConnection.prepareStatement("SELECT * FROM " + IndexBinaryMapRenderObject.getTable() + " WHERE id = ?");
binaryMapFile = new File(workingDir, getBinaryMapFileName());
FileOutputStream fout = new FileOutputStream(binaryMapFile);
BinaryIndexWriter writer = new BinaryIndexWriter(fout);
CodedOutputStream codedOutStream = CodedOutputStream.newInstance(new OutputStream() {
@Override
public void write(int b) throws IOException {
mapRAFile.write(b);
}
});
codedOutStream.writeInt32(OsmandOdb.OsmAndStructure.VERSION_FIELD_NUMBER, IndexConstants.BINARY_MAP_VERSION);
BinaryMapIndexWriter writer = new BinaryMapIndexWriter(mapRAFile, codedOutStream);
writer.startWriteMapIndex();
for (int i = 0; i < MAP_ZOOMS.length - 1; i++) {
@ -1483,7 +1486,7 @@ public class IndexCreator {
}
}
public void writeBinaryMapTree(rtree.Node parent, RTree r, BinaryIndexWriter writer, PreparedStatement selectData) throws IOException, RTreeException, SQLException {
public void writeBinaryMapTree(rtree.Node parent, RTree r, BinaryMapIndexWriter writer, PreparedStatement selectData) throws IOException, RTreeException, SQLException {
Element[] e = parent.getAllElements();
for (int i = 0; i < parent.getTotalElements(); i++) {
Rect re = e[i].getRect();
@ -1644,11 +1647,12 @@ public class IndexCreator {
if (indexMap) {
mapFile = new File(workingDir, getMapFileName());
// to save space
if (mapFile.exists()) {
mapFile.delete();
}
mapFile.getParentFile().mkdirs();
mapConnection = DriverManager.getConnection("jdbc:sqlite:" + mapFile.getAbsolutePath());
File tempDBMapFile = new File(workingDir, getTempMapDBFileName());
if(tempDBMapFile.exists()){
tempDBMapFile.delete();
}
mapConnection = DriverManager.getConnection("jdbc:sqlite:" + tempDBMapFile.getAbsolutePath());
DataIndexWriter.createMapIndexStructure(mapConnection);
mapBinaryStat = DataIndexWriter.createStatementMapBinaryInsert(mapConnection);
@ -1847,8 +1851,11 @@ public class IndexCreator {
pStatements.remove(mapBinaryStat);
mapConnection.commit();
// TODO !!! create binary output stream in order to close it properly (finally)
writeBinaryData();
if(mapFile.exists()){
mapFile.delete();
}
mapRAFile = new RandomAccessFile(mapFile, "rw");
writeBinaryMapIndex();
}
} finally {
@ -1888,14 +1895,21 @@ public class IndexCreator {
transportIndexFile.setLastModified(lastModifiedDate);
}
}
if(mapRAFile != null){
mapRAFile.close();
}
if (mapConnection != null) {
mapConnection.commit();
mapConnection.close();
mapConnection = null;
if (lastModifiedDate != null) {
if (lastModifiedDate != null && mapFile.exists()) {
mapFile.setLastModified(lastModifiedDate);
}
File tempDBFile = new File(getTempMapDBFileName());
if(tempDBFile.exists()){
tempDBFile.delete();
}
}
if (addressConnection != null) {