Android-builders-club/OsmAnd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

fix reader

Browse source 
git-svn-id: https://osmand.googlecode.com/svn/trunk@597 e29c36b1-1cfa-d876-8d93-3434fc2bb7b8
This commit is contained in:
Victor Shcherb 2010-10-28 07:52:19 +00:00
parent c9982c0b97
commit c6993e8810
8 changed files with 294 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
DataExtractionOSM/.classpath
View file

@ -7,6 +7,6 @@
<classpathentry kind="lib" path="lib/commons-logging-1.1.1.jar"/>
<classpathentry kind="lib" path="lib/junidecode-0.1.jar"/>
<classpathentry kind="lib" path="lib/json-20090211.jar"/>
<classpathentry kind="lib" path="lib/osm-pbf.jar"/>
<classpathentry kind="lib" path="lib/osm-pbf.jar" sourcepath="/OsmPdb"/>
<classpathentry kind="output" path="bin"/>
</classpath>

BIN
DataExtractionOSM/build.zip
View file

Binary file not shown.

BIN
DataExtractionOSM/lib/osm-pbf.jar
View file

Binary file not shown.

212
DataExtractionOSM/src/net/osmand/binary/BinaryMapIndexReader.java Normal file
View file

@ -0,0 +1,212 @@
package net.osmand.binary;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.ArrayList;
import java.util.List;
import com.google.protobuf.CodedInputStreamRAF;
import com.google.protobuf.WireFormat;
public class BinaryMapIndexReader {
private final RandomAccessFile raf;
private int version;
private List<MapRoot> mapIndexes = new ArrayList<MapRoot>();
private CodedInputStreamRAF codedIS;
public class MapRoot {
int minZoom = 0;
int maxZoom = 0;
int left = 0;
int right = 0;
int top = 0;
int bottom = 0;
List<MapTree> trees = new ArrayList<MapTree>();
}
public class MapTree {
int filePointer = 0;
int length = 0;
int left = 0;
int right = 0;
int top = 0;
int bottom = 0;
List<String> stringTable = null;
List<MapTree> subTrees = null;
List<MapObject> children = null;
}
public class MapObject {
}
public BinaryMapIndexReader(final RandomAccessFile raf) throws IOException {
this.raf = raf;
codedIS = CodedInputStreamRAF.newInstance(raf, 256);
init();
}
private void init() throws IOException {
while(true){
int tag = WireFormat.getTagFieldNumber(codedIS.readTag());
switch (tag) {
case 0:
return;
case OsmandOdb.OsmAndStructure.VERSION_FIELD_NUMBER :
version = codedIS.readUInt32();
break;
case OsmandOdb.OsmAndStructure.MAPINDEX_FIELD_NUMBER:
int length = readInt();
int filePointer = codedIS.getTotalBytesRead();
int oldLimit = codedIS.pushLimit(length);
readMapIndex();
codedIS.popLimit(oldLimit);
codedIS.seek(filePointer + length);
break;
default:
// TODO skip unknown fields
return;
}
}
}
private void readMapIndex() throws IOException {
while(true){
int tag = WireFormat.getTagFieldNumber(codedIS.readTag());
switch (tag) {
case 0:
return;
case OsmandOdb.OsmAndMapIndex.LEVELS_FIELD_NUMBER :
int length = readInt();
int filePointer = codedIS.getTotalBytesRead();
int oldLimit = codedIS.pushLimit(length);
MapRoot mapRoot = readMapLevel();
mapIndexes.add(mapRoot);
codedIS.popLimit(oldLimit);
codedIS.seek(filePointer + length);
break;
default:
// TODO skip unknown fields
return;
}
}
}
private MapRoot readMapLevel() throws IOException {
MapRoot root = new MapRoot();
while(true){
int tag = WireFormat.getTagFieldNumber(codedIS.readTag());
switch (tag) {
case 0:
return root;
case OsmandOdb.MapRootLevel.BOTTOM_FIELD_NUMBER :
root.bottom = codedIS.readInt32();
break;
case OsmandOdb.MapRootLevel.LEFT_FIELD_NUMBER :
root.left = codedIS.readInt32();
break;
case OsmandOdb.MapRootLevel.RIGHT_FIELD_NUMBER :
root.right = codedIS.readInt32();
break;
case OsmandOdb.MapRootLevel.TOP_FIELD_NUMBER :
root.top = codedIS.readInt32();
break;
case OsmandOdb.MapRootLevel.MAXZOOM_FIELD_NUMBER :
root.maxZoom = codedIS.readInt32();
break;
case OsmandOdb.MapRootLevel.MINZOOM_FIELD_NUMBER :
root.minZoom = codedIS.readInt32();
break;
case OsmandOdb.MapRootLevel.ROOT_FIELD_NUMBER :
MapTree r = new MapTree();
// left, ... already initialized
r.length = readInt();
r.filePointer = codedIS.getTotalBytesRead();
int oldLimit = codedIS.pushLimit(r.length);
readMapTreeBounds(r, root.left, root.right, root.top, root.bottom);
root.trees.add(r);
codedIS.popLimit(oldLimit);
codedIS.seek(r.filePointer + r.length);
break;
default:
// TODO skip unknown fields
return root;
}
}
}
private void readMapTreeBounds(MapTree tree, int aleft, int aright, int atop, int abottom) throws IOException {
while(true){
int tag = WireFormat.getTagFieldNumber(codedIS.readTag());
switch (tag) {
case 0:
return;
case OsmandOdb.MapTree.BOTTOM_FIELD_NUMBER :
tree.bottom = codedIS.readSInt32() + abottom;
break;
case OsmandOdb.MapTree.LEFT_FIELD_NUMBER :
tree.left = codedIS.readSInt32() + aleft;
break;
case OsmandOdb.MapTree.RIGHT_FIELD_NUMBER :
tree.right = codedIS.readSInt32() + aright;
break;
case OsmandOdb.MapTree.TOP_FIELD_NUMBER :
tree.top = codedIS.readSInt32() + atop;
break;
default:
// TODO skip unknown fields
return;
}
}
}
public RandomAccessFile getRaf() {
return raf;
}
public int readByte() throws IOException{
byte b = codedIS.readRawByte();
if(b < 0){
return b + 256;
} else {
return b;
}
}
public final int readInt() throws IOException {
int ch1 = readByte();
int ch2 = readByte();
int ch3 = readByte();
int ch4 = readByte();
return ((ch1 << 24) + (ch2 << 16) + (ch3 << 8) + (ch4 << 0));
}
public List<MapRoot> getMapIndexes() {
return mapIndexes;
}
public int getVersion() {
return version;
}
public static void main(String[] args) throws IOException {
RandomAccessFile raf = new RandomAccessFile(new File("e:\\Information\\OSM maps\\osmand\\Minsk.map.pbf"), "r");
BinaryMapIndexReader reader = new BinaryMapIndexReader(raf);
System.out.println(reader.getVersion());
for(MapRoot b : reader.getMapIndexes()) {
System.out.println(b.minZoom + " " + b.maxZoom + " " +b.trees.size() + " "
+ b.left + " " + b.right + " " + b.top + " " + b.bottom);
}
}
}

77
DataExtractionOSM/src/net/osmand/binary/BinaryMapIndexWriter.java
View file

@ -1,12 +1,14 @@
package net.osmand.binary;
import java.io.IOException;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Stack;
import net.osmand.Algoritms;
import net.osmand.data.index.IndexConstants;
import com.google.protobuf.CodedOutputStream;
import com.google.protobuf.WireFormat;
@ -45,21 +47,40 @@ public class BinaryMapIndexWriter {
private final static int MAP_ROOT_LEVEL_INIT = 3;
private final static int MAP_TREE = 4;
public BinaryMapIndexWriter(RandomAccessFile raf, CodedOutputStream codedOutputStream) throws IOException{
public BinaryMapIndexWriter(final RandomAccessFile raf) throws IOException{
this.raf = raf;
codedOutStream = codedOutputStream;
codedOutStream = CodedOutputStream.newInstance(new OutputStream() {
@Override
public void write(int b) throws IOException {
raf.write(b);
}
@Override
public void write(byte[] b) throws IOException {
raf.write(b);
}
@Override
public void write(byte[] b, int off, int len) throws IOException {
raf.write(b, off, len);
}
});
codedOutStream.writeInt32(OsmandOdb.OsmAndStructure.VERSION_FIELD_NUMBER, IndexConstants.BINARY_MAP_VERSION);
state.push(OSMAND_STRUCTURE_INIT);
}
private void preserveInt32Size() throws IOException {
codedOutStream.flush();
stackSizes.push(raf.getFilePointer());
codedOutStream.writeFixed32NoTag(0);
}
private void writeInt32Size() throws IOException{
codedOutStream.flush();
long filePointer = raf.getFilePointer();
Long old = stackSizes.pop();
int length = (int) (filePointer - old);
int length = (int) (filePointer - old - 4);
raf.seek(old);
raf.writeInt(length);
raf.seek(filePointer);
@ -79,7 +100,7 @@ public class BinaryMapIndexWriter {
writeInt32Size();
}
public void startWriteMapLevelIndex(int maxZoom, int minZoom, int leftX, int rightX, int topY, int bottomY) throws IOException{
public void startWriteMapLevelIndex(int minZoom, int maxZoom, int leftX, int rightX, int topY, int bottomY) throws IOException{
assert state.peek() == MAP_INDEX_INIT;
state.push(MAP_ROOT_LEVEL_INIT);
@ -104,6 +125,10 @@ public class BinaryMapIndexWriter {
}
public void startMapTreeElement(int leftX, int rightX, int topY, int bottomY) throws IOException{
startMapTreeElement(-1L, leftX, rightX, topY, bottomY);
}
public void startMapTreeElement(long baseId, int leftX, int rightX, int topY, int bottomY) throws IOException{
assert state.peek() == MAP_ROOT_LEVEL_INIT || state.peek() == MAP_TREE;
if(state.peek() == MAP_ROOT_LEVEL_INIT){
codedOutStream.writeTag(OsmandOdb.MapRootLevel.ROOT_FIELD_NUMBER, WireFormat.FieldType.MESSAGE.getWireType());
@ -120,7 +145,7 @@ public class BinaryMapIndexWriter {
codedOutStream.writeSInt32(OsmandOdb.MapTree.TOP_FIELD_NUMBER, topY - bounds.topY);
codedOutStream.writeSInt32(OsmandOdb.MapTree.BOTTOM_FIELD_NUMBER, bottomY - bounds.bottomY);
stackBounds.push(new Bounds(leftX, rightX, topY, bottomY));
stackBaseIds.push(-1L);
stackBaseIds.push(baseId);
stackStringTable.push(null);
}
@ -156,6 +181,11 @@ public class BinaryMapIndexWriter {
writeInt32Size();
}
public static int COORDINATES_SIZE = 0;
public static int ID_SIZE = 0;
public static int TYPES_SIZE = 0;
public static int MAP_DATA_SIZE = 0;
public void writeMapData(long id, byte[] nodes, byte[] types, String name, int highwayAttributes, byte[] restrictions) throws IOException{
assert state.peek() == MAP_TREE;
@ -168,19 +198,30 @@ public class BinaryMapIndexWriter {
// calculate size
int allSize = CodedOutputStream.computeTagSize(OsmandOdb.MapData.COORDINATES_FIELD_NUMBER);
int sizeCoordinates = 0;
int px = bounds.leftX / 2;
int py = bounds.topY / 2;
for(int i=0; i< nodes.length / 8; i++){
int x = Algoritms.parseIntFromBytes(nodes, i*8) - bounds.leftX;
int y = Algoritms.parseIntFromBytes(nodes, i*8 + 4) - bounds.topY;
sizeCoordinates += CodedOutputStream.computeInt32SizeNoTag(x);
sizeCoordinates += CodedOutputStream.computeInt32SizeNoTag(y);
int x = Algoritms.parseIntFromBytes(nodes, i*8);
int y = Algoritms.parseIntFromBytes(nodes, i*8 + 4);
sizeCoordinates += CodedOutputStream.computeSInt32SizeNoTag(x - px);
sizeCoordinates += CodedOutputStream.computeSInt32SizeNoTag(y - py);
px = x;
py = y;
}
allSize += sizeCoordinates;
// DEBUG
COORDINATES_SIZE += sizeCoordinates + CodedOutputStream.computeTagSize(OsmandOdb.MapData.COORDINATES_FIELD_NUMBER);
allSize += CodedOutputStream.computeTagSize(OsmandOdb.MapData.TYPES_FIELD_NUMBER);
allSize += CodedOutputStream.computeRawVarint32Size(types.length);
allSize += types.length;
// DEBUG
TYPES_SIZE += CodedOutputStream.computeTagSize(OsmandOdb.MapData.TYPES_FIELD_NUMBER) +
CodedOutputStream.computeRawVarint32Size(types.length) + types.length;
System.out.println(id + " " + CodedOutputStream.computeSInt64Size(OsmandOdb.MapData.ID_FIELD_NUMBER, id - stackBaseIds.peek()));
allSize += CodedOutputStream.computeSInt64Size(OsmandOdb.MapData.ID_FIELD_NUMBER, id - stackBaseIds.peek());
// DEBUG
ID_SIZE += CodedOutputStream.computeSInt64Size(OsmandOdb.MapData.ID_FIELD_NUMBER, id - stackBaseIds.peek());
int nameId = 0;
if(name != null){
@ -212,17 +253,25 @@ public class BinaryMapIndexWriter {
allSize += CodedOutputStream.computeInt32Size(OsmandOdb.MapData.HIGHWAYMETA_FIELD_NUMBER, highwayAttributes);
}
// DEBUG
MAP_DATA_SIZE += allSize;
// writing data
codedOutStream.writeTag(OsmandOdb.MapTree.LEAFS_FIELD_NUMBER, WireFormat.FieldType.MESSAGE.getWireType());
codedOutStream.writeRawVarint32(allSize);
codedOutStream.writeTag(OsmandOdb.MapData.COORDINATES_FIELD_NUMBER, WireFormat.FieldType.BYTES.getWireType());
codedOutStream.writeRawVarint32(sizeCoordinates);
px = bounds.leftX / 2;
py = bounds.topY / 2;
for (int i = 0; i < nodes.length / 8; i++) {
int x = Algoritms.parseIntFromBytes(nodes, i * 8) - bounds.leftX;
int y = Algoritms.parseIntFromBytes(nodes, i * 8 + 4) - bounds.topY;
codedOutStream.writeInt32NoTag(x);
codedOutStream.writeInt32NoTag(y);
int x = Algoritms.parseIntFromBytes(nodes, i*8);
int y = Algoritms.parseIntFromBytes(nodes, i*8 + 4);
codedOutStream.writeSInt32NoTag(x - px);
codedOutStream.writeSInt32NoTag(y - py);
px = x;
py = y;
}
codedOutStream.writeTag(OsmandOdb.MapData.TYPES_FIELD_NUMBER, WireFormat.FieldType.BYTES.getWireType());

1
DataExtractionOSM/src/net/osmand/data/index/DataIndexWriter.java
View file

@ -24,7 +24,6 @@ import net.osmand.data.City.CityType;
import net.osmand.data.index.IndexConstants.IndexBinaryMapRenderObject;
import net.osmand.data.index.IndexConstants.IndexBuildingTable;
import net.osmand.data.index.IndexConstants.IndexCityTable;
import net.osmand.data.index.IndexConstants.IndexMapRenderObject;
import net.osmand.data.index.IndexConstants.IndexPoiTable;
import net.osmand.data.index.IndexConstants.IndexStreetNodeTable;
import net.osmand.data.index.IndexConstants.IndexStreetTable;

2
DataExtractionOSM/src/net/osmand/data/index/batch.xml
View file

@ -11,7 +11,7 @@
<authorization_info cookieHSID=""
cookieSID=""
pagegen="" token=""
google_code_user="vics001" google_code_password="" />
google_code_user="" google_code_password="" />
<!-- There are 3 subprocess :
1. Download fresh osm files from servers to 'directory_for_osm_files' (override existings).
2. Generate index files from all files in 'directory_for_osm_files' and put all indexes into 'directory_for_index_files'

38
DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java
View file

@ -6,7 +6,6 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.sql.Connection;
import java.sql.DriverManager;
@ -29,7 +28,6 @@ import java.util.Map.Entry;
import net.osmand.Algoritms;
import net.osmand.IProgress;
import net.osmand.binary.BinaryMapIndexWriter;
import net.osmand.binary.OsmandOdb;
import net.osmand.data.Amenity;
import net.osmand.data.Building;
import net.osmand.data.City;
@ -78,8 +76,6 @@ import rtree.RTree;
import rtree.RTreeException;
import rtree.Rect;
import com.google.protobuf.CodedOutputStream;
/**
* http://wiki.openstreetmap.org/wiki/OSM_tags_for_routing#Is_inside.2Foutside
@ -117,7 +113,6 @@ public class IndexCreator {
private String poiFileName = null;
private String addressFileName = null;
private String mapFileName = null;
private String binaryMapFileName = null;
private Long lastModifiedDate = null;
@ -154,7 +149,7 @@ public class IndexCreator {
private RandomAccessFile mapRAFile;
private Connection mapConnection;
private PreparedStatement mapBinaryStat;
private static final int[] MAP_ZOOMS = new int[]{6, 9, 14, 22};
private static final int[] MAP_ZOOMS = new int[]{5, 9, 14, 22};
private RTree[] mapTree = null;
// MEMORY map : save it in memory while that is allowed
@ -1453,16 +1448,9 @@ public class IndexCreator {
assert IndexConstants.IndexBinaryMapRenderObject.values().length == 6;
PreparedStatement selectData = mapConnection.prepareStatement("SELECT * FROM " + IndexBinaryMapRenderObject.getTable() + " WHERE id = ?");
CodedOutputStream codedOutStream = CodedOutputStream.newInstance(new OutputStream() {
@Override
public void write(int b) throws IOException {
mapRAFile.write(b);
}
});
codedOutStream.writeInt32(OsmandOdb.OsmAndStructure.VERSION_FIELD_NUMBER, IndexConstants.BINARY_MAP_VERSION);
BinaryMapIndexWriter writer = new BinaryMapIndexWriter(mapRAFile, codedOutStream);
BinaryMapIndexWriter writer = new BinaryMapIndexWriter(mapRAFile);
writer.startWriteMapIndex();
for (int i = 0; i < MAP_ZOOMS.length - 1; i++) {
@ -1488,6 +1476,8 @@ public class IndexCreator {
public void writeBinaryMapTree(rtree.Node parent, RTree r, BinaryMapIndexWriter writer, PreparedStatement selectData) throws IOException, RTreeException, SQLException {
Element[] e = parent.getAllElements();
for (int i = 0; i < parent.getTotalElements(); i++) {
Rect re = e[i].getRect();
if(e[i].getElementType() == rtree.Node.LEAF_NODE){
@ -1502,12 +1492,14 @@ public class IndexCreator {
log.error("Something goes wrong with id = " + id);
}
} else {
long ptr = ((NonLeafElement) e[i]).getPtr();
writer.startMapTreeElement(re.getMinX(), re.getMaxX(), re.getMinY(), re.getMaxY());
rtree.Node ns = r.getReadNode(ptr);
writeBinaryMapTree(ns, r, writer, selectData);
writer.endWriteMapTreeElement();
}
long ptr = ((NonLeafElement) e[i]).getPtr();
rtree.Node ns = r.getReadNode(ptr);
writer.startMapTreeElement(re.getMinX(), re.getMaxX(), re.getMinY(), re.getMaxY());
writeBinaryMapTree(ns, r, writer, selectData);
writer.endWriteMapTreeElement();
}
}
}
@ -1906,7 +1898,7 @@ public class IndexCreator {
if (lastModifiedDate != null && mapFile.exists()) {
mapFile.setLastModified(lastModifiedDate);
}
File tempDBFile = new File(getTempMapDBFileName());
File tempDBFile = new File(workingDir, getTempMapDBFileName());
if(tempDBFile.exists()){
tempDBFile.delete();
}
@ -1968,6 +1960,10 @@ public class IndexCreator {
creator.setNodesDBFile(new File("e:/Information/OSM maps/osmand/minsk.tmp.odb"));
creator.generateIndexes(new File("e:/Information/OSM maps/belarus osm/minsk.osm"), new ConsoleProgressImplementation(3), null);
System.out.println("COORDINATES_SIZE " + BinaryMapIndexWriter.COORDINATES_SIZE);
System.out.println("TYPES_SIZE " + BinaryMapIndexWriter.TYPES_SIZE);
System.out.println("ID_SIZE " + BinaryMapIndexWriter.ID_SIZE);
System.out.println("MAP_DATA_SIZE " + BinaryMapIndexWriter.MAP_DATA_SIZE);
// creator.setNodesDBFile(new File("e:/Information/OSM maps/osmand/belarus_nodes.tmp.odb"));
// creator.generateIndexes(new File("e:/Information/OSM maps/belarus osm/belarus.osm.bz2"), new ConsoleProgressImplementation(3), null);