Android-builders-club/OsmAnd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

implement binary map inspector

Browse source 
git-svn-id: https://osmand.googlecode.com/svn/trunk@658 e29c36b1-1cfa-d876-8d93-3434fc2bb7b8
This commit is contained in:
Victor Shcherb 2010-11-12 11:53:56 +00:00
parent e4ca9a858e
commit b7a3a09b32
7 changed files with 257 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
DataExtractionOSM/src/net/osmand/ToDoConstants.java
View file

@ -7,15 +7,10 @@ package net.osmand;
*/
public class ToDoConstants {
// TODO max 101
// TODO max 104
// introduce bidforfix on site
// 1. Verify to use query
// !!! Fix phone, site information for POI INDEX
// Move poi index to 1 version and remove backward support (save int coordinates and save for name_en)
// Show in message and test for some indexes !!!
// TEST POI (phone, site, wiki), test show, update, delete
// For 0.5 release
// 101. 1) Introduce shell script to print information about indexes 2)Introduce activity to show active indexes
@ -23,11 +18,12 @@ public class ToDoConstants {
// 102. Refactoring rendering schema serializing to xml and prepare script to create fast java builder (support pure xml)
// Requires
// 86. Allow to add/edit custom tags to POI objects (Issue)
// 92. Replace poi index with standard map index and unify POI categories
// 69. Add phone and site information to POI (enable call to POI and open site)
// 92. Replace poi index with standard map index and unify POI categories (unify done +)
// 69. Add phone and site information to POI [+] (enable call to POI and open site)
// 97. For voice navigation consider current speed of vehicle. Especially when speed > 50 pronounce more than 200 m
// 98. Implement rendering of different app mode. For Car render streets name with large font.
// 96. Introduce settings for MPH, imperial units
// 103. Allow add poi with multiple tags (different types/subtypes for one object). Fix for constructing and updating on mobile.
// _19. colors for road trunk and motorway
@ -40,8 +36,7 @@ public class ToDoConstants {
// Not clear if it is really needed
// 66. Transport routing (show next stop, total distance, show stop get out, voice) (needed ?).
// 85. Enable on/off screen for bike navigation (?)
// 83. Add monitoring service to send locations to internet (?)
// 83. Add monitoring service to send location to internet immediately (?)
/////////////////////////// DONE //////////////////////////////
@ -50,7 +45,8 @@ public class ToDoConstants {
// 100. Show impoted gpx points (as favorites), sort the by distance
// 91. Invent binary format (minimize disk space, maximize speed)
// 94. Revise index to decrease their size (especially address) - replace to float lat/lon and remove for POI
// remove en_names from POI (possibly from address)
// remove en_names from POI (possibly from address)
// 85. Enable on/off screen for bike navigation (?)[not needed replace for continuous gps logging]
// DONE SWING

32
DataExtractionOSM/src/net/osmand/binary/BinaryIndexPart.java Normal file
View file

@ -0,0 +1,32 @@
package net.osmand.binary;
public class BinaryIndexPart {
String name;
int length;
int filePointer;
public int getLength() {
return length;
}
public void setLength(int length) {
this.length = length;
}
public int getFilePointer() {
return filePointer;
}
public void setFilePointer(int filePointer) {
this.filePointer = filePointer;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
}

108
DataExtractionOSM/src/net/osmand/binary/BinaryInspector.java Normal file
View file

@ -0,0 +1,108 @@
package net.osmand.binary;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.text.MessageFormat;
import java.util.Locale;
import net.osmand.binary.BinaryMapIndexReader.AddressRegion;
import net.osmand.binary.BinaryMapIndexReader.MapIndex;
import net.osmand.binary.BinaryMapIndexReader.MapRoot;
import net.osmand.binary.BinaryMapIndexReader.TransportIndex;
import net.osmand.osm.MapUtils;
public class BinaryInspector {
public static void main(String[] args) throws IOException {
args = new String[]{"E:\\Information\\OSM maps\\osmand\\Luxembourg.map.pbf"};
args = new String[]{"E:\\Information\\OSM maps\\osmand\\Minsk.map.pbf"};
// args = new String[]{"E:\\Information\\OSM maps\\osmand\\Belarus_4.map.pbf"};
// args = new String[]{"E:\\Information\\OSM maps\\osmand\\Netherlands.map.pbf"};
// args = new String[]{"E:\\Information\\OSM maps\\osm_map\\Netherlands\\Netherlands_trans.map.pbf"};
inspector(args);
}
public static void inspector(String[] args) throws IOException {
if(args == null || args.length == 0){
printUsage(null);
}
String f = args[0];
if(f.charAt(0) == '-'){
// command
} else {
File file = new File(f);
if(!file.exists()){
System.out.println("Binary OsmAnd index " + f + " was not found.");
return;
}
printFileInformation(file);
}
}
protected static String formatBounds(int left, int right, int top, int bottom){
double l = MapUtils.get31LongitudeX(left);
double r = MapUtils.get31LongitudeX(right);
double t = MapUtils.get31LatitudeY(top);
double b = MapUtils.get31LatitudeY(bottom);
MessageFormat format = new MessageFormat("(left top - right bottom) : {0}, {1} NE - {2}, {3} NE", Locale.US);
return format.format(new Object[]{l, t, r, b});
}
public static void printFileInformation(File file) throws IOException {
RandomAccessFile r = new RandomAccessFile(file.getAbsolutePath(), "r");
try {
BinaryMapIndexReader index = new BinaryMapIndexReader(r);
int i = 1;
System.out.println("Binary index " + file.getName() + " version = " + index.getVersion());
for(BinaryIndexPart p : index.getIndexes()){
String partname = "";
if(p instanceof MapIndex ){
partname = "Map";
} else if(p instanceof TransportIndex){
partname = "Transport";
} else if(p instanceof AddressRegion){
partname = "Address";
}
String name = p.getName() == null ? "" : p.getName();
System.out.println(MessageFormat.format("{0}. {1} data {3} - {2} bytes", i, partname, p.getLength(), name));
if(p instanceof TransportIndex){
TransportIndex ti = ((TransportIndex) p);
System.out.println("\t Bounds " + formatBounds(ti.getLeft(), ti.getRight(), ti.getTop(), ti.getBottom()));
} else if(p instanceof MapIndex){
MapIndex m = ((MapIndex) p);
for(MapRoot mi : m.getRoots()){
System.out.println(MessageFormat.format("\tMap level minZoom = {0}, maxZoom = {1}, size = {2} bytes \n\t\tBounds {3}",
mi.getMinZoom(), mi.getMaxZoom(), mi.getLength(),
formatBounds(mi.getLeft(), mi.getRight(), mi.getTop(), mi.getBottom())));
}
}
i++;
}
} catch (IOException e) {
System.err.println("File is not valid index : " + file.getAbsolutePath());
throw e;
}
}
public static void printUsage(String warning) {
if(warning != null){
System.out.println(warning);
}
System.out.println("Inspector is console utility for working with binary indexes of OsmAnd.");
System.out.println("It allows print info about file, extract parts and merge indexes.");
System.out.println("\nUsage : inspector [file]");
System.out.println("\tPrints information about [file] binary index of OsmAnd.");
}
}

137
DataExtractionOSM/src/net/osmand/binary/BinaryMapIndexReader.java
View file

@ -35,9 +35,11 @@ public class BinaryMapIndexReader {
private final RandomAccessFile raf;
private int version;
private List<MapRoot> mapIndexes = new ArrayList<MapRoot>();
private List<MapIndex> mapIndexes = new ArrayList<MapIndex>();
private List<AddressRegion> addressIndexes = new ArrayList<AddressRegion>();
private List<TransportIndex> transportIndexes = new ArrayList<TransportIndex>();
private List<BinaryIndexPart> indexes = new ArrayList<BinaryIndexPart>();
private CodedInputStreamRAF codedIS;
private final static Log log = LogUtil.getLog(BinaryMapIndexReader.class);
@ -48,6 +50,7 @@ public class BinaryMapIndexReader {
public BinaryMapIndexReader(final RandomAccessFile raf) throws IOException {
this.raf = raf;
codedIS = CodedInputStreamRAF.newInstance(raf, 1024);
codedIS.setSizeLimit(Integer.MAX_VALUE); // 2048 MB
init();
}
@ -66,31 +69,39 @@ public class BinaryMapIndexReader {
version = codedIS.readUInt32();
break;
case OsmandOdb.OsmAndStructure.MAPINDEX_FIELD_NUMBER:
int length = readInt();
int filePointer = codedIS.getTotalBytesRead();
int oldLimit = codedIS.pushLimit(length);
readMapIndex();
MapIndex mapIndex = new MapIndex();
mapIndex.length = readInt();
mapIndex.filePointer = codedIS.getTotalBytesRead();
int oldLimit = codedIS.pushLimit(mapIndex.length);
readMapIndex(mapIndex);
codedIS.popLimit(oldLimit);
codedIS.seek(filePointer + length);
codedIS.seek(mapIndex.filePointer + mapIndex.length);
mapIndexes.add(mapIndex);
indexes.add(mapIndex);
break;
case OsmandOdb.OsmAndStructure.ADDRESSINDEX_FIELD_NUMBER:
AddressRegion region = new AddressRegion();
region.length = readInt();
region.fileOffset = codedIS.getTotalBytesRead();
region.filePointer = codedIS.getTotalBytesRead();
oldLimit = codedIS.pushLimit(region.length);
readAddressIndex(region);
codedIS.popLimit(oldLimit);
codedIS.seek(region.fileOffset + region.length);
codedIS.seek(region.filePointer + region.length);
if(region.name != null){
addressIndexes.add(region);
indexes.add(region);
}
break;
case OsmandOdb.OsmAndStructure.TRANSPORTINDEX_FIELD_NUMBER:
TransportIndex ind = new TransportIndex();
ind.length = readInt();
ind.fileOffset = codedIS.getTotalBytesRead();
ind.filePointer = codedIS.getTotalBytesRead();
oldLimit = codedIS.pushLimit(ind.length);
readTransportIndex(ind);
codedIS.popLimit(oldLimit);
codedIS.seek(ind.fileOffset + ind.length);
codedIS.seek(ind.filePointer + ind.length);
transportIndexes.add(ind);
indexes.add(ind);
break;
case OsmandOdb.OsmAndStructure.VERSIONCONFIRM_FIELD_NUMBER :
int cversion = codedIS.readUInt32();
@ -103,6 +114,10 @@ public class BinaryMapIndexReader {
}
}
public List<BinaryIndexPart> getIndexes() {
return indexes;
}
private void skipUnknownField(int tag) throws IOException{
int wireType = WireFormat.getTagWireType(tag);
@ -124,6 +139,9 @@ public class BinaryMapIndexReader {
case OsmandOdb.OsmAndTransportIndex.ROUTES_FIELD_NUMBER :
skipUnknownField(t);
break;
case OsmandOdb.OsmAndTransportIndex.NAME_FIELD_NUMBER :
ind.setName(codedIS.readString());
break;
case OsmandOdb.OsmAndTransportIndex.STOPS_FIELD_NUMBER :
ind.stopsFileLength = readInt();
ind.stopsFileOffset = codedIS.getTotalBytesRead();
@ -374,7 +392,7 @@ public class BinaryMapIndexReader {
private TransportIndex getTransportIndex(int filePointer) {
TransportIndex ind = null;
for(TransportIndex i : transportIndexes){
if(i.fileOffset <= filePointer && (filePointer - i.fileOffset) < i.length){
if(i.filePointer <= filePointer && (filePointer - i.filePointer) < i.length){
ind = i;
break;
}
@ -613,9 +631,6 @@ public class BinaryMapIndexReader {
int tag = WireFormat.getTagFieldNumber(t);
switch (tag) {
case 0:
if(region.name != null){
addressIndexes.add(region);
}
return;
case OsmandOdb.OsmAndAddressIndex.NAME_FIELD_NUMBER :
region.name = codedIS.readString();
@ -648,19 +663,24 @@ public class BinaryMapIndexReader {
}
}
private void readMapIndex() throws IOException {
private void readMapIndex(MapIndex index) throws IOException {
while(true){
int t = codedIS.readTag();
int tag = WireFormat.getTagFieldNumber(t);
switch (tag) {
case 0:
return;
case OsmandOdb.OsmAndMapIndex.NAME_FIELD_NUMBER :
index.setName(codedIS.readString());
break;
case OsmandOdb.OsmAndMapIndex.LEVELS_FIELD_NUMBER :
int length = readInt();
int filePointer = codedIS.getTotalBytesRead();
int oldLimit = codedIS.pushLimit(length);
MapRoot mapRoot = readMapLevel();
mapIndexes.add(mapRoot);
mapRoot.length = length;
mapRoot.filePointer = filePointer;
index.getRoots().add(mapRoot);
codedIS.popLimit(oldLimit);
codedIS.seek(filePointer + length);
break;
@ -758,19 +778,21 @@ public class BinaryMapIndexReader {
req.numberOfAcceptedObjects = 0;
req.numberOfAcceptedSubtrees = 0;
req.numberOfReadSubtrees = 0;
for (MapRoot index : mapIndexes) {
if (index.minZoom <= req.zoom && index.maxZoom >= req.zoom) {
if(index.right < req.left || index.left > req.right || index.top > req.bottom || index.bottom < req.top){
continue;
}
for (MapTree tree : index.trees) {
if(tree.right < req.left || tree.left > req.right || tree.top > req.bottom || tree.bottom < req.top){
for (MapIndex mapIndex : mapIndexes) {
for (MapRoot index : mapIndex.getRoots()) {
if (index.minZoom <= req.zoom && index.maxZoom >= req.zoom) {
if (index.right < req.left || index.left > req.right || index.top > req.bottom || index.bottom < req.top) {
continue;
}
codedIS.seek(tree.filePointer);
int oldLimit = codedIS.pushLimit(tree.length);
searchMapTreeBounds(index.left, index.right, index.top, index.bottom, req);
codedIS.popLimit(oldLimit);
for (MapTree tree : index.trees) {
if (tree.right < req.left || tree.left > req.right || tree.top > req.bottom || tree.bottom < req.top) {
continue;
}
codedIS.seek(tree.filePointer);
int oldLimit = codedIS.pushLimit(tree.length);
searchMapTreeBounds(index.left, index.right, index.top, index.bottom, req);
codedIS.popLimit(oldLimit);
}
}
}
}
@ -1121,7 +1143,7 @@ public class BinaryMapIndexReader {
private void checkAddressIndex(int offset){
boolean ok = false;
for(AddressRegion r : addressIndexes){
if(offset >= r.fileOffset && offset <= (r.length + r.fileOffset)){
if(offset >= r.filePointer && offset <= (r.length + r.filePointer)){
ok = true;
break;
}
@ -1701,7 +1723,17 @@ public class BinaryMapIndexReader {
}
}
private static class MapRoot {
public static class MapIndex extends BinaryIndexPart {
List<MapRoot> roots = new ArrayList<MapRoot>();
public List<MapRoot> getRoots() {
return roots;
}
}
public static class MapRoot extends BinaryIndexPart {
int minZoom = 0;
int maxZoom = 0;
int left = 0;
@ -1709,6 +1741,27 @@ public class BinaryMapIndexReader {
int top = 0;
int bottom = 0;
public int getMinZoom() {
return minZoom;
}
public int getMaxZoom() {
return maxZoom;
}
public int getLeft() {
return left;
}
public int getRight() {
return right;
}
public int getTop() {
return top;
}
public int getBottom() {
return bottom;
}
List<MapTree> trees = new ArrayList<MapTree>();
}
@ -1728,9 +1781,7 @@ public class BinaryMapIndexReader {
}
private static class TransportIndex {
int fileOffset = 0;
int length = 0;
public static class TransportIndex extends BinaryIndexPart {
int left = 0;
int right = 0;
@ -1740,6 +1791,20 @@ public class BinaryMapIndexReader {
int stopsFileOffset = 0;
int stopsFileLength = 0;
public int getLeft() {
return left;
}
public int getRight() {
return right;
}
public int getTop() {
return top;
}
public int getBottom() {
return bottom;
}
IndexStringTable stringTable = null;
}
@ -1760,13 +1825,9 @@ public class BinaryMapIndexReader {
}
private static class AddressRegion {
String name;
public static class AddressRegion extends BinaryIndexPart {
String enName;
int fileOffset = 0;
int length = 0;
int postcodesOffset = -1;
int villagesOffset = -1;
int citiesOffset = -1;
@ -1854,4 +1915,4 @@ public class BinaryMapIndexReader {
System.out.println("Time " + (System.currentTimeMillis() - time));
}
}
}

8
DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java
View file

@ -2502,10 +2502,10 @@ public class IndexCreator {
creator.setIndexTransport(true);
creator.recreateOnlyBinaryFile = false;
creator.deleteDatabaseIndexes = false;
creator.deleteDatabaseIndexes = true;
// creator.setNodesDBFile(new File("e:/Information/OSM maps/osmand/minsk.tmp.odb"));
// creator.generateIndexes(new File("e:/Information/OSM maps/belarus osm/minsk.osm"), new ConsoleProgressImplementation(3), null);
creator.setNodesDBFile(new File("e:/Information/OSM maps/osmand/minsk.tmp.odb"));
creator.generateIndexes(new File("e:/Information/OSM maps/belarus osm/minsk.osm"), new ConsoleProgressImplementation(3), null);
// creator.setNodesDBFile(new File("e:/Information/OSM maps/osmand/belarus_nodes.tmp.odb"));
// creator.generateIndexes(new File("e:/Information/OSM maps/belarus osm/belarus.osm.bz2"), new ConsoleProgressImplementation(3), null);
@ -2525,7 +2525,7 @@ public class IndexCreator {
// creator.generateIndexes(new File("e:/Information/OSM maps/osm_map/forest_complex.osm"), new ConsoleProgressImplementation(25), null);
creator.generateIndexes(new File("e:/Information/OSM maps/osm_map/luxembourg.osm.pbf"), new ConsoleProgressImplementation(25), null);
// creator.generateIndexes(new File("e:/Information/OSM maps/osm_map/luxembourg.osm.pbf"), new ConsoleProgressImplementation(25), null);
System.out.println("WHOLE GENERATION TIME : " + (System.currentTimeMillis() - time));
System.out.println("COORDINATES_SIZE " + BinaryMapIndexWriter.COORDINATES_SIZE + " count " + BinaryMapIndexWriter.COORDINATES_COUNT);

5
OsmAnd/src/net/osmand/TransportIndexRepositoryBinary.java
View file

@ -227,6 +227,11 @@ public class TransportIndexRepositoryBinary implements TransportIndexRepository
@Override
public void close() {
try {
file.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}

1
OsmAnd/src/net/osmand/render/MapRenderRepositories.java
View file

@ -115,6 +115,7 @@ public class MapRenderRepositories {
try {
c.close();
} catch (IOException e) {
e.printStackTrace();
}
}