diff --git a/DataExtractionOSM/build.xml b/DataExtractionOSM/build.xml index 66fef538dd..bfcf05f5e8 100644 --- a/DataExtractionOSM/build.xml +++ b/DataExtractionOSM/build.xml @@ -41,6 +41,9 @@ + + + diff --git a/DataExtractionOSM/inspector.bat b/DataExtractionOSM/inspector.bat new file mode 100644 index 0000000000..58c1f207ea --- /dev/null +++ b/DataExtractionOSM/inspector.bat @@ -0,0 +1 @@ +java.exe -Djava.util.logging.config.file=logging.properties -Xms64M -Xmx512M -cp "./OsmAndMapCreator.jar;./lib/*.jar" net.osmand.binary.BinaryInspector %1 %2 %3 %4 %5 %6 %7 %8 %9 \ No newline at end of file diff --git a/DataExtractionOSM/inspector.sh b/DataExtractionOSM/inspector.sh new file mode 100644 index 0000000000..d84fbcca0c --- /dev/null +++ b/DataExtractionOSM/inspector.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +# This file will start the Osm Extraction UI with custom memory settings for +# the JVM. With the below settings the heap size (Available memory for the application) +# will range from 64 megabyte up to 512 megabyte. + +java -Djava.util.logging.config.file=logging.properties -Xms64M -Xmx512M -cp "./OsmAndMapCreator.jar:./lib/*.jar" net.osmand.binary.BinaryInspector %1 %2 %3 %4 %5 %6 %7 %8 %9 diff --git a/DataExtractionOSM/src/net/osmand/binary/BinaryInspector.java b/DataExtractionOSM/src/net/osmand/binary/BinaryInspector.java index 21f34d38b9..1f3d6acece 100644 --- a/DataExtractionOSM/src/net/osmand/binary/BinaryInspector.java +++ b/DataExtractionOSM/src/net/osmand/binary/BinaryInspector.java @@ -1,10 +1,17 @@ package net.osmand.binary; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.io.RandomAccessFile; import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; import java.util.Locale; +import java.util.Map; +import java.util.Set; import net.osmand.binary.BinaryMapIndexReader.AddressRegion; import net.osmand.binary.BinaryMapIndexReader.MapIndex; @@ -12,28 +19,72 @@ import net.osmand.binary.BinaryMapIndexReader.MapRoot; import net.osmand.binary.BinaryMapIndexReader.TransportIndex; import net.osmand.osm.MapUtils; +import com.google.protobuf.CodedOutputStream; +import com.google.protobuf.WireFormat; + public class BinaryInspector { - + + public static final int BUFFER_SIZE = 1 << 20; public static void main(String[] args) throws IOException { - args = new String[]{"E:\\Information\\OSM maps\\osmand\\Luxembourg.map.pbf"}; -// args = new String[]{"E:\\Information\\OSM maps\\osmand\\Minsk.map.pbf"}; -// args = new String[]{"E:\\Information\\OSM maps\\osmand\\Belarus_4.map.pbf"}; -// args = new String[]{"E:\\Information\\OSM maps\\osmand\\Netherlands.map.pbf"}; -// args = new String[]{"E:\\Information\\OSM maps\\osm_map\\Netherlands\\Netherlands_trans.map.pbf"}; - - inspector(args); + // test cases show info +// inspector(new String[]{"E:\\Information\\OSM maps\\osmand\\Luxembourg.map.pbf"}); +// inspector(new String[]{"E:\\Information\\OSM maps\\osmand\\Luxembourg-e.map.pbf"}); +// inspector(new String[]{"E:\\Information\\OSM maps\\osmand\\Minsk.map.pbf"}); +// inspector(new String[]{"E:\\Information\\OSM maps\\osmand\\Belarus_4.map.pbf"}); +// inspector(new String[]{"E:\\Information\\OSM maps\\osmand\\Netherlands.map.pbf"}); +// +// inspector(new String[]{"E:\\Information\\OSM maps\\osm_map\\Netherlands\\Netherlands_trans.map.pbf"}); + + // test case extract parts +// inspector(new String[]{"-c", "E:\\Information\\OSM maps\\osmand\\Netherlands-addr-trans.map.pbf", +// "E:\\Information\\OSM maps\\osmand\\Netherlands.map.pbf", "-1"}); + + // test case +// inspector(new String[]{"-c", "E:\\Information\\OSM maps\\osmand\\Netherlands-addr-trans.map.pbf", +// "E:\\Information\\OSM maps\\osmand\\Netherlands.map.pbf", "-1", +// "E:\\Information\\OSM maps\\osmand\\Belarus_4.map.pbf", "E:\\Information\\OSM maps\\osmand\\Minsk.map.pbf"}); +// inspector(new String[]{"E:\\Information\\OSM maps\\osmand\\Netherlands-addr-trans.map.pbf"}); } + public static void inspector(String[] args) throws IOException { if(args == null || args.length == 0){ printUsage(null); + return; } String f = args[0]; if(f.charAt(0) == '-'){ // command + if(f.equals("-c") || f.equals("-combine")) { + if(args.length < 4){ + printUsage("Too few parameters to extract (require minimum 4)"); + } else { + Map parts = new LinkedHashMap(); + for (int i = 2; i < args.length; i++) { + File file = new File(args[i]); + if (!file.exists()) { + System.err.println("File to extract from doesn't exist " + args[i]); + return; + } + parts.put(file, null); + if(i < args.length - 1){ + if(args[i+1].startsWith("-") || args[i+1].startsWith("+")){ + parts.put(file, args[i+1]); + i++; + } + } + } + List extracted = combineParts(new File(args[1]), parts); + if(extracted != null){ + System.out.println("\n"+extracted.size()+" parts were successfully extracted to " + args[1]); + } + } + } else { + printUsage("Unknown command : "+ f); + } } else { File file = new File(f); if(!file.exists()){ @@ -43,8 +94,130 @@ public class BinaryInspector { printFileInformation(file); } } - + public static final void writeInt(CodedOutputStream ous, int v) throws IOException { + ous.writeRawByte((v >>> 24) & 0xFF); + ous.writeRawByte((v >>> 16) & 0xFF); + ous.writeRawByte((v >>> 8) & 0xFF); + ous.writeRawByte((v >>> 0) & 0xFF); + //written += 4; + } + @SuppressWarnings("unchecked") + public static List combineParts(File fileToExtract, Map partsToExtractFrom) throws IOException { + BinaryMapIndexReader[] indexes = new BinaryMapIndexReader[partsToExtractFrom.size()]; + RandomAccessFile[] rafs = new RandomAccessFile[partsToExtractFrom.size()]; + + LinkedHashSet[] partsSet = new LinkedHashSet[partsToExtractFrom.size()]; + int c = 0; + Set addressNames = new LinkedHashSet(); + + int version = -1; + // Go through all files and validate conistency + for(File f : partsToExtractFrom.keySet()){ + if(f.getAbsolutePath().equals(fileToExtract.getAbsolutePath())){ + System.err.println("Error : Input file is equal to output file " + f.getAbsolutePath()); + return null; + } + rafs[c] = new RandomAccessFile(f, "r"); + indexes[c] = new BinaryMapIndexReader(rafs[c]); + partsSet[c] = new LinkedHashSet(); + if(version == -1){ + version = indexes[c].getVersion(); + } else { + if(indexes[c].getVersion() != version){ + System.err.println("Error : Different input files has different input versions " + indexes[c].getVersion() + " != " + version); + return null; + } + } + + LinkedHashSet temp = new LinkedHashSet(); + String pattern = partsToExtractFrom.get(f); + boolean minus = true; + if(pattern != null){ + minus = pattern.startsWith("-"); + String[] split = pattern.substring(1).split(","); + for(String s : split){ + temp.add(Integer.parseInt(s)); + } + } + + for (int i = 0; i < indexes[c].getIndexes().size(); i++) { + partsSet[c].add(i + 1); + } + if(minus){ + partsSet[c].removeAll(temp); + } else { + partsSet[c].retainAll(temp); + } + + c++; + } + + // write files + FileOutputStream fout = new FileOutputStream(fileToExtract); + CodedOutputStream ous = CodedOutputStream.newInstance(fout, BUFFER_SIZE); + List list = new ArrayList(); + byte[] BUFFER_TO_READ = new byte[BUFFER_SIZE]; + + ous.writeInt32(OsmandOdb.OsmAndStructure.VERSION_FIELD_NUMBER, version); + + + for (int k = 0; k < indexes.length; k++) { + LinkedHashSet partSet = partsSet[k]; + BinaryMapIndexReader index = indexes[k]; + RandomAccessFile raf = rafs[k]; + for (int i = 0; i < index.getIndexes().size(); i++) { + if (!partSet.contains(i + 1)) { + continue; + } + list.add(i + 1); + + BinaryIndexPart part = index.getIndexes().get(i); + String map; + if (part instanceof AddressRegion) { + ous.writeTag(OsmandOdb.OsmAndStructure.ADDRESSINDEX_FIELD_NUMBER, WireFormat.WIRETYPE_FIXED32_LENGTH_DELIMITED); + map = "Address"; + if (addressNames.contains(part.getName())) { + System.err.println("Error : going to merge 2 same addresses skip " + part.getName()); + continue; + } + addressNames.add(part.getName()); + } else if (part instanceof TransportIndex) { + ous.writeTag(OsmandOdb.OsmAndStructure.TRANSPORTINDEX_FIELD_NUMBER, WireFormat.WIRETYPE_FIXED32_LENGTH_DELIMITED); + map = "Transport"; + } else if (part instanceof MapIndex) { + ous.writeTag(OsmandOdb.OsmAndStructure.MAPINDEX_FIELD_NUMBER, WireFormat.WIRETYPE_FIXED32_LENGTH_DELIMITED); + map = "Map"; + } else { + throw new UnsupportedOperationException(); + } + writeInt(ous, part.getLength()); + raf.seek(part.getFilePointer()); + int toRead = part.getLength(); + while (toRead > 0) { + int read = raf.read(BUFFER_TO_READ); + if (read == -1) { + throw new IllegalArgumentException("Unexpected end of file"); + } + if (toRead < read) { + read = toRead; + } + ous.writeRawBytes(BUFFER_TO_READ, 0, read); + toRead -= read; + } + System.out.println(MessageFormat.format("{2} part {0} is extracted {1} bytes", part.getName(), part.getLength(), map)); + } + } + + ous.writeInt32(OsmandOdb.OsmAndStructure.VERSIONCONFIRM_FIELD_NUMBER, version); + ous.flush(); + fout.close(); + + + return list; + } + + protected static String formatBounds(int left, int right, int top, int bottom){ double l = MapUtils.get31LongitudeX(left); double r = MapUtils.get31LongitudeX(right); @@ -102,8 +275,14 @@ public class BinaryInspector { } System.out.println("Inspector is console utility for working with binary indexes of OsmAnd."); System.out.println("It allows print info about file, extract parts and merge indexes."); - System.out.println("\nUsage : inspector [file]"); - System.out.println("\tPrints information about [file] binary index of OsmAnd."); + System.out.println("\nUsage for print info : inspector [file]"); + System.out.println(" Prints information about [file] binary index of OsmAnd."); + System.out.println("\nUsage for combining indexes : inspector -c file_to_create (file_from_extract ((+|-)parts_to_extract)? )*"); + System.out.println("\tCreate new file of extracted parts from input file. [parts_to_extract] could be parts to include or exclude."); + System.out.println(" Example : inspector -c output_file input_file +1,2,3\n\tExtracts 1, 2, 3 parts (could be find in print info)"); + System.out.println(" Example : inspector -c output_file input_file -2,3\n\tExtracts all parts excluding 2, 3"); + System.out.println(" Example : inspector -c output_file input_file1 input_file2 input_file3\n\tSimply combine 3 files"); + System.out.println(" Example : inspector -c output_file input_file1 input_file2 -4\n\tCombine all parts of 1st file and all parts excluding 4th part of 2nd file"); } diff --git a/DataExtractionOSM/src/rtree/CachedNodes.java b/DataExtractionOSM/src/rtree/CachedNodes.java index 515b6e67a5..7b847e04cc 100644 --- a/DataExtractionOSM/src/rtree/CachedNodes.java +++ b/DataExtractionOSM/src/rtree/CachedNodes.java @@ -387,14 +387,14 @@ public class CachedNodes if(fileName != null) { Integer i = fileNamesMap.get(fileName); if(i == null){ - if(fileNamesMap.size() > 31){ + if(fileNamesMap.size() > 1023){ throw new ArrayIndexOutOfBoundsException(); } fileNamesMap.put(fileName, fileNamesMap.size()); i = fileNamesMap.get(fileName); } // System.out.println(idx + " " + fileName + " " + ((idx << 5)+ fileName.toLowerCase().hashCode() % 32)); - return ((idx << 5)+ i); + return ((idx << 10)+ i); } else{ System.out.println("CachedNodes.calKey: fileName null"); return 0;