Android-builders-club/OsmAnd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

fix bugs implement binary index inspector tool

Browse source 
git-svn-id: https://osmand.googlecode.com/svn/trunk@660 e29c36b1-1cfa-d876-8d93-3434fc2bb7b8
This commit is contained in:
Victor Shcherb 2010-11-12 16:32:31 +00:00
parent b68a330e8f
commit 1d8b5a97d8
5 changed files with 203 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
DataExtractionOSM/build.xml
View file

@ -41,6 +41,9 @@
<include name="logging.properties"/>
<include name="OsmAndMapCreator.bat"/>
<include name="OsmAndMapCreator.sh"/>
<include name="inspector.bat"/>
<include name="inspector.sh"/>
<exclude name="lib/h2*"/>
</fileset>
</copy>
<zip destfile="build.zip">

1
DataExtractionOSM/inspector.bat Normal file
View file

@ -0,0 +1 @@
java.exe -Djava.util.logging.config.file=logging.properties -Xms64M -Xmx512M -cp "./OsmAndMapCreator.jar;./lib/*.jar" net.osmand.binary.BinaryInspector %1 %2 %3 %4 %5 %6 %7 %8 %9

7
DataExtractionOSM/inspector.sh Normal file
View file

@ -0,0 +1,7 @@
#!/bin/sh
# This file will start the Osm Extraction UI with custom memory settings for
# the JVM. With the below settings the heap size (Available memory for the application)
# will range from 64 megabyte up to 512 megabyte.
java -Djava.util.logging.config.file=logging.properties -Xms64M -Xmx512M -cp "./OsmAndMapCreator.jar:./lib/*.jar" net.osmand.binary.BinaryInspector %1 %2 %3 %4 %5 %6 %7 %8 %9

201
DataExtractionOSM/src/net/osmand/binary/BinaryInspector.java
View file

@ -1,10 +1,17 @@
package net.osmand.binary;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import net.osmand.binary.BinaryMapIndexReader.AddressRegion;
import net.osmand.binary.BinaryMapIndexReader.MapIndex;
@ -12,28 +19,72 @@ import net.osmand.binary.BinaryMapIndexReader.MapRoot;
import net.osmand.binary.BinaryMapIndexReader.TransportIndex;
import net.osmand.osm.MapUtils;
import com.google.protobuf.CodedOutputStream;
import com.google.protobuf.WireFormat;
public class BinaryInspector {
public static final int BUFFER_SIZE = 1 << 20;
public static void main(String[] args) throws IOException {
args = new String[]{"E:\\Information\\OSM maps\\osmand\\Luxembourg.map.pbf"};
// args = new String[]{"E:\\Information\\OSM maps\\osmand\\Minsk.map.pbf"};
// args = new String[]{"E:\\Information\\OSM maps\\osmand\\Belarus_4.map.pbf"};
// args = new String[]{"E:\\Information\\OSM maps\\osmand\\Netherlands.map.pbf"};
// args = new String[]{"E:\\Information\\OSM maps\\osm_map\\Netherlands\\Netherlands_trans.map.pbf"};
inspector(args);
// test cases show info
// inspector(new String[]{"E:\\Information\\OSM maps\\osmand\\Luxembourg.map.pbf"});
// inspector(new String[]{"E:\\Information\\OSM maps\\osmand\\Luxembourg-e.map.pbf"});
// inspector(new String[]{"E:\\Information\\OSM maps\\osmand\\Minsk.map.pbf"});
// inspector(new String[]{"E:\\Information\\OSM maps\\osmand\\Belarus_4.map.pbf"});
// inspector(new String[]{"E:\\Information\\OSM maps\\osmand\\Netherlands.map.pbf"});
//
// inspector(new String[]{"E:\\Information\\OSM maps\\osm_map\\Netherlands\\Netherlands_trans.map.pbf"});
// test case extract parts
// inspector(new String[]{"-c", "E:\\Information\\OSM maps\\osmand\\Netherlands-addr-trans.map.pbf",
// "E:\\Information\\OSM maps\\osmand\\Netherlands.map.pbf", "-1"});
// test case
// inspector(new String[]{"-c", "E:\\Information\\OSM maps\\osmand\\Netherlands-addr-trans.map.pbf",
// "E:\\Information\\OSM maps\\osmand\\Netherlands.map.pbf", "-1",
// "E:\\Information\\OSM maps\\osmand\\Belarus_4.map.pbf", "E:\\Information\\OSM maps\\osmand\\Minsk.map.pbf"});
// inspector(new String[]{"E:\\Information\\OSM maps\\osmand\\Netherlands-addr-trans.map.pbf"});
}
public static void inspector(String[] args) throws IOException {
if(args == null || args.length == 0){
printUsage(null);
return;
}
String f = args[0];
if(f.charAt(0) == '-'){
// command
if(f.equals("-c") || f.equals("-combine")) {
if(args.length < 4){
printUsage("Too few parameters to extract (require minimum 4)");
} else {
Map<File, String> parts = new LinkedHashMap<File, String>();
for (int i = 2; i < args.length; i++) {
File file = new File(args[i]);
if (!file.exists()) {
System.err.println("File to extract from doesn't exist " + args[i]);
return;
}
parts.put(file, null);
if(i < args.length - 1){
if(args[i+1].startsWith("-") || args[i+1].startsWith("+")){
parts.put(file, args[i+1]);
i++;
}
}
}
List<Integer> extracted = combineParts(new File(args[1]), parts);
if(extracted != null){
System.out.println("\n"+extracted.size()+" parts were successfully extracted to " + args[1]);
}
}
} else {
printUsage("Unknown command : "+ f);
}
} else {
File file = new File(f);
if(!file.exists()){
@ -43,8 +94,130 @@ public class BinaryInspector {
printFileInformation(file);
}
}
public static final void writeInt(CodedOutputStream ous, int v) throws IOException {
ous.writeRawByte((v >>> 24) & 0xFF);
ous.writeRawByte((v >>> 16) & 0xFF);
ous.writeRawByte((v >>> 8) & 0xFF);
ous.writeRawByte((v >>> 0) & 0xFF);
//written += 4;
}
@SuppressWarnings("unchecked")
public static List<Integer> combineParts(File fileToExtract, Map<File, String> partsToExtractFrom) throws IOException {
BinaryMapIndexReader[] indexes = new BinaryMapIndexReader[partsToExtractFrom.size()];
RandomAccessFile[] rafs = new RandomAccessFile[partsToExtractFrom.size()];
LinkedHashSet<Integer>[] partsSet = new LinkedHashSet[partsToExtractFrom.size()];
int c = 0;
Set<String> addressNames = new LinkedHashSet<String>();
int version = -1;
// Go through all files and validate conistency
for(File f : partsToExtractFrom.keySet()){
if(f.getAbsolutePath().equals(fileToExtract.getAbsolutePath())){
System.err.println("Error : Input file is equal to output file " + f.getAbsolutePath());
return null;
}
rafs[c] = new RandomAccessFile(f, "r");
indexes[c] = new BinaryMapIndexReader(rafs[c]);
partsSet[c] = new LinkedHashSet<Integer>();
if(version == -1){
version = indexes[c].getVersion();
} else {
if(indexes[c].getVersion() != version){
System.err.println("Error : Different input files has different input versions " + indexes[c].getVersion() + " != " + version);
return null;
}
}
LinkedHashSet<Integer> temp = new LinkedHashSet<Integer>();
String pattern = partsToExtractFrom.get(f);
boolean minus = true;
if(pattern != null){
minus = pattern.startsWith("-");
String[] split = pattern.substring(1).split(",");
for(String s : split){
temp.add(Integer.parseInt(s));
}
}
for (int i = 0; i < indexes[c].getIndexes().size(); i++) {
partsSet[c].add(i + 1);
}
if(minus){
partsSet[c].removeAll(temp);
} else {
partsSet[c].retainAll(temp);
}
c++;
}
// write files
FileOutputStream fout = new FileOutputStream(fileToExtract);
CodedOutputStream ous = CodedOutputStream.newInstance(fout, BUFFER_SIZE);
List<Integer> list = new ArrayList<Integer>();
byte[] BUFFER_TO_READ = new byte[BUFFER_SIZE];
ous.writeInt32(OsmandOdb.OsmAndStructure.VERSION_FIELD_NUMBER, version);
for (int k = 0; k < indexes.length; k++) {
LinkedHashSet<Integer> partSet = partsSet[k];
BinaryMapIndexReader index = indexes[k];
RandomAccessFile raf = rafs[k];
for (int i = 0; i < index.getIndexes().size(); i++) {
if (!partSet.contains(i + 1)) {
continue;
}
list.add(i + 1);
BinaryIndexPart part = index.getIndexes().get(i);
String map;
if (part instanceof AddressRegion) {
ous.writeTag(OsmandOdb.OsmAndStructure.ADDRESSINDEX_FIELD_NUMBER, WireFormat.WIRETYPE_FIXED32_LENGTH_DELIMITED);
map = "Address";
if (addressNames.contains(part.getName())) {
System.err.println("Error : going to merge 2 same addresses skip " + part.getName());
continue;
}
addressNames.add(part.getName());
} else if (part instanceof TransportIndex) {
ous.writeTag(OsmandOdb.OsmAndStructure.TRANSPORTINDEX_FIELD_NUMBER, WireFormat.WIRETYPE_FIXED32_LENGTH_DELIMITED);
map = "Transport";
} else if (part instanceof MapIndex) {
ous.writeTag(OsmandOdb.OsmAndStructure.MAPINDEX_FIELD_NUMBER, WireFormat.WIRETYPE_FIXED32_LENGTH_DELIMITED);
map = "Map";
} else {
throw new UnsupportedOperationException();
}
writeInt(ous, part.getLength());
raf.seek(part.getFilePointer());
int toRead = part.getLength();
while (toRead > 0) {
int read = raf.read(BUFFER_TO_READ);
if (read == -1) {
throw new IllegalArgumentException("Unexpected end of file");
}
if (toRead < read) {
read = toRead;
}
ous.writeRawBytes(BUFFER_TO_READ, 0, read);
toRead -= read;
}
System.out.println(MessageFormat.format("{2} part {0} is extracted {1} bytes", part.getName(), part.getLength(), map));
}
}
ous.writeInt32(OsmandOdb.OsmAndStructure.VERSIONCONFIRM_FIELD_NUMBER, version);
ous.flush();
fout.close();
return list;
}
protected static String formatBounds(int left, int right, int top, int bottom){
double l = MapUtils.get31LongitudeX(left);
double r = MapUtils.get31LongitudeX(right);
@ -102,8 +275,14 @@ public class BinaryInspector {
}
System.out.println("Inspector is console utility for working with binary indexes of OsmAnd.");
System.out.println("It allows print info about file, extract parts and merge indexes.");
System.out.println("\nUsage : inspector [file]");
System.out.println("\tPrints information about [file] binary index of OsmAnd.");
System.out.println("\nUsage for print info : inspector [file]");
System.out.println(" Prints information about [file] binary index of OsmAnd.");
System.out.println("\nUsage for combining indexes : inspector -c file_to_create (file_from_extract ((+|-)parts_to_extract)? )*");
System.out.println("\tCreate new file of extracted parts from input file. [parts_to_extract] could be parts to include or exclude.");
System.out.println(" Example : inspector -c output_file input_file +1,2,3\n\tExtracts 1, 2, 3 parts (could be find in print info)");
System.out.println(" Example : inspector -c output_file input_file -2,3\n\tExtracts all parts excluding 2, 3");
System.out.println(" Example : inspector -c output_file input_file1 input_file2 input_file3\n\tSimply combine 3 files");
System.out.println(" Example : inspector -c output_file input_file1 input_file2 -4\n\tCombine all parts of 1st file and all parts excluding 4th part of 2nd file");
}

4
DataExtractionOSM/src/rtree/CachedNodes.java
View file

@ -387,14 +387,14 @@ public class CachedNodes
if(fileName != null) {
Integer i = fileNamesMap.get(fileName);
if(i == null){
if(fileNamesMap.size() > 31){
if(fileNamesMap.size() > 1023){
throw new ArrayIndexOutOfBoundsException();
}
fileNamesMap.put(fileName, fileNamesMap.size());
i = fileNamesMap.get(fileName);
}
// System.out.println(idx + " " + fileName + " " + ((idx << 5)+ fileName.toLowerCase().hashCode() % 32));
return ((idx << 5)+ i);
return ((idx << 10)+ i);
} else{
System.out.println("CachedNodes.calKey: fileName null");
return 0;