Write map coordinates to file

This commit is contained in:
Victor Shcherb 2012-02-19 02:21:43 +01:00
parent 4844bb98d6
commit c9b9466f2e
5 changed files with 7929 additions and 18249 deletions

View file

@ -0,0 +1,518 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (c) 2009, Rob Eden All Rights Reserved.
// Copyright (c) 2009, Jeff Randall All Rights Reserved.
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
///////////////////////////////////////////////////////////////////////////////
package gnu.trove.list;
//////////////////////////////////////////////////
// THIS IS A GENERATED CLASS. DO NOT HAND EDIT! //
//////////////////////////////////////////////////
import gnu.trove.function.*;
import gnu.trove.procedure.*;
import gnu.trove.TByteCollection;
import java.io.Serializable;
import java.util.Random;
/**
* Interface for Trove list implementations.
*/
public interface TByteList extends TByteCollection, Serializable {
/**
* Returns the value that is used to represent null. The default
* value is generally zero, but can be changed during construction
* of the collection.
*
* @return the value that represents null
*/
@Override
public byte getNoEntryValue();
/**
* Returns the number of values in the list.
*
* @return the number of values in the list.
*/
@Override
public int size();
/**
* Tests whether this list contains any values.
*
* @return true if the list is empty.
*/
@Override
public boolean isEmpty();
/**
* Adds <tt>val</tt> to the end of the list, growing as needed.
*
* @param val an <code>byte</code> value
* @return true if the list was modified by the add operation
*/
@Override
public boolean add(byte val);
/**
* Adds the values in the array <tt>vals</tt> to the end of the
* list, in order.
*
* @param vals an <code>byte[]</code> value
*/
public void add( byte[] vals );
/**
* Adds a subset of the values in the array <tt>vals</tt> to the
* end of the list, in order.
*
* @param vals an <code>byte[]</code> value
* @param offset the offset at which to start copying
* @param length the number of values to copy.
*/
public void add( byte[] vals, int offset, int length );
/**
* Inserts <tt>value</tt> into the list at <tt>offset</tt>. All
* values including and to the right of <tt>offset</tt> are shifted
* to the right.
*
* @param offset an <code>int</code> value
* @param value an <code>byte</code> value
*/
public void insert( int offset, byte value );
/**
* Inserts the array of <tt>values</tt> into the list at
* <tt>offset</tt>. All values including and to the right of
* <tt>offset</tt> are shifted to the right.
*
* @param offset an <code>int</code> value
* @param values an <code>byte[]</code> value
*/
public void insert( int offset, byte[] values );
/**
* Inserts a slice of the array of <tt>values</tt> into the list
* at <tt>offset</tt>. All values including and to the right of
* <tt>offset</tt> are shifted to the right.
*
* @param offset an <code>int</code> value
* @param values an <code>byte[]</code> value
* @param valOffset the offset in the values array at which to
* start copying.
* @param len the number of values to copy from the values array
*/
public void insert( int offset, byte[] values, int valOffset, int len );
/**
* Returns the value at the specified offset.
*
* @param offset an <code>int</code> value
* @return an <code>byte</code> value
*/
public byte get( int offset );
/**
* Sets the value at the specified offset.
*
* @param offset an <code>int</code> value
* @param val an <code>byte</code> value
*
* @return The value previously at the given index.
*/
public byte set( int offset, byte val );
/**
* Replace the values in the list starting at <tt>offset</tt> with
* the contents of the <tt>values</tt> array.
*
* @param offset the first offset to replace
* @param values the source of the new values
*/
public void set( int offset, byte[] values );
/**
* Replace the values in the list starting at <tt>offset</tt> with
* <tt>length</tt> values from the <tt>values</tt> array, starting
* at valOffset.
*
* @param offset the first offset to replace
* @param values the source of the new values
* @param valOffset the first value to copy from the values array
* @param length the number of values to copy
*/
public void set( int offset, byte[] values, int valOffset, int length );
/**
* Sets the value at the specified offset and returns the
* previously stored value.
*
* @param offset an <code>int</code> value
* @param val an <code>byte</code> value
* @return the value previously stored at offset.
*/
public byte replace( int offset, byte val );
/**
* Flushes the internal state of the list, resetting the capacity
* to the default.
*/
@Override
public void clear();
/**
* Removes <tt>value</tt> from the list.
*
* @param value an <code>byte</code> value
* @return true if the list was modified by the remove operation.
*/
@Override
public boolean remove( byte value );
/**
* Removes <tt>value</tt> at a given offset from the list.
*
* @param offset an <code>int</code> value that represents
* the offset to the element to be removed
* @return an <tt>byte</tt> that is the value removed.
*/
public byte removeAt( int offset );
/**
* Removes <tt>length</tt> values from the list, starting at
* <tt>offset</tt>
*
* @param offset an <code>int</code> value
* @param length an <code>int</code> value
*/
public void remove( int offset, int length );
/**
* Transform each value in the list using the specified function.
*
* @param function a <code>TByteFunction</code> value
*/
public void transformValues( TByteFunction function );
/**
* Reverse the order of the elements in the list.
*/
public void reverse();
/**
* Reverse the order of the elements in the range of the list.
*
* @param from the inclusive index at which to start reversing
* @param to the exclusive index at which to stop reversing
*/
public void reverse( int from, int to );
/**
* Shuffle the elements of the list using the specified random
* number generator.
*
* @param rand a <code>Random</code> value
*/
public void shuffle( Random rand );
/**
* Returns a sublist of this list.
*
* @param begin low endpoint (inclusive) of the subList.
* @param end high endpoint (exclusive) of the subList.
* @return sublist of this list from begin, inclusive to end, exclusive.
* @throws IndexOutOfBoundsException - endpoint out of range
* @throws IllegalArgumentException - endpoints out of order (end > begin)
*/
public TByteList subList( int begin, int end );
/**
* Copies the contents of the list into a native array.
*
* @return an <code>byte[]</code> value
*/
@Override
public byte[] toArray();
/**
* Copies a slice of the list into a native array.
*
* @param offset the offset at which to start copying
* @param len the number of values to copy.
* @return an <code>byte[]</code> value
*/
public byte[] toArray( int offset, int len );
/**
* Copies a slice of the list into a native array.
*
* <p>If the list fits in the specified array with room to spare (i.e.,
* the array has more elements than the list), the element in the array
* immediately following the end of the list is set to
* <tt>{@link #getNoEntryValue()}</tt>.
* (This is useful in determining the length of the list <i>only</i> if
* the caller knows that the list does not contain any "null" elements.)
*
* <p>NOTE: Trove does not allocate a new array if the array passed in is
* not large enough to hold all of the data elements. It will instead fill
* the array passed in.
*
* @param dest the array to copy into.
* @return the array passed in.
*/
@Override
public byte[] toArray( byte[] dest );
/**
* Copies a slice of the list into a native array.
*
* @param dest the array to copy into.
* @param offset the offset where the first value should be copied
* @param len the number of values to copy.
* @return the array passed in.
*/
public byte[] toArray( byte[] dest, int offset, int len );
/**
* Copies a slice of the list into a native array.
*
* @param dest the array to copy into.
* @param source_pos the offset of the first value to copy
* @param dest_pos the offset where the first value should be copied
* @param len the number of values to copy.
* @return the array passed in.
*/
public byte[] toArray( byte[] dest, int source_pos, int dest_pos, int len );
/**
* Applies the procedure to each value in the list in ascending
* (front to back) order.
*
* @param procedure a <code>TByteProcedure</code> value
* @return true if the procedure did not terminate prematurely.
*/
@Override
public boolean forEach( TByteProcedure procedure );
/**
* Applies the procedure to each value in the list in descending
* (back to front) order.
*
* @param procedure a <code>TByteProcedure</code> value
* @return true if the procedure did not terminate prematurely.
*/
public boolean forEachDescending( TByteProcedure procedure );
/**
* Sort the values in the list (ascending) using the Sun quicksort
* implementation.
*
* @see java.util.Arrays#sort
*/
public void sort();
/**
* Sort a slice of the list (ascending) using the Sun quicksort
* implementation.
*
* @param fromIndex the index at which to start sorting (inclusive)
* @param toIndex the index at which to stop sorting (exclusive)
* @see java.util.Arrays#sort
*/
public void sort( int fromIndex, int toIndex );
/**
* Fills every slot in the list with the specified value.
*
* @param val the value to use when filling
*/
public void fill( byte val );
/**
* Fills a range in the list with the specified value.
*
* @param fromIndex the offset at which to start filling (inclusive)
* @param toIndex the offset at which to stop filling (exclusive)
* @param val the value to use when filling
*/
public void fill( int fromIndex, int toIndex, byte val );
/**
* Performs a binary search for <tt>value</tt> in the entire list.
* Note that you <b>must</b> @{link #sort sort} the list before
* doing a search.
*
* @param value the value to search for
* @return the absolute offset in the list of the value, or its
* negative insertion point into the sorted list.
*/
public int binarySearch( byte value );
/**
* Performs a binary search for <tt>value</tt> in the specified
* range. Note that you <b>must</b> @{link #sort sort} the list
* or the range before doing a search.
*
* @param value the value to search for
* @param fromIndex the lower boundary of the range (inclusive)
* @param toIndex the upper boundary of the range (exclusive)
* @return the absolute offset in the list of the value, or its
* negative insertion point into the sorted list.
*/
public int binarySearch( byte value, int fromIndex, int toIndex );
/**
* Searches the list front to back for the index of
* <tt>value</tt>.
*
* @param value an <code>byte</code> value
* @return the first offset of the value, or -1 if it is not in
* the list.
* @see #binarySearch for faster searches on sorted lists
*/
public int indexOf( byte value );
/**
* Searches the list front to back for the index of
* <tt>value</tt>, starting at <tt>offset</tt>.
*
* @param offset the offset at which to start the linear search
* (inclusive)
* @param value an <code>byte</code> value
* @return the first offset of the value, or -1 if it is not in
* the list.
* @see #binarySearch for faster searches on sorted lists
*/
public int indexOf( int offset, byte value );
/**
* Searches the list back to front for the last index of
* <tt>value</tt>.
*
* @param value an <code>byte</code> value
* @return the last offset of the value, or -1 if it is not in
* the list.
* @see #binarySearch for faster searches on sorted lists
*/
public int lastIndexOf( byte value );
/**
* Searches the list back to front for the last index of
* <tt>value</tt>, starting at <tt>offset</tt>.
*
* @param offset the offset at which to start the linear search
* (exclusive)
* @param value an <code>byte</code> value
* @return the last offset of the value, or -1 if it is not in
* the list.
* @see #binarySearch for faster searches on sorted lists
*/
public int lastIndexOf( int offset, byte value );
/**
* Searches the list for <tt>value</tt>
*
* @param value an <code>byte</code> value
* @return true if value is in the list.
*/
@Override
public boolean contains( byte value );
/**
* Searches the list for values satisfying <tt>condition</tt> in
* the manner of the *nix <tt>grep</tt> utility.
*
* @param condition a condition to apply to each element in the list
* @return a list of values which match the condition.
*/
public TByteList grep( TByteProcedure condition );
/**
* Searches the list for values which do <b>not</b> satisfy
* <tt>condition</tt>. This is akin to *nix <code>grep -v</code>.
*
* @param condition a condition to apply to each element in the list
* @return a list of values which do not match the condition.
*/
public TByteList inverseGrep( TByteProcedure condition );
/**
* Finds the maximum value in the list.
*
* @return the largest value in the list.
* @exception IllegalStateException if the list is empty
*/
public byte max();
/**
* Finds the minimum value in the list.
*
* @return the smallest value in the list.
* @exception IllegalStateException if the list is empty
*/
public byte min();
}

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -1,12 +1,15 @@
package net.osmand.data.preparation;
import gnu.trove.TByteCollection;
import gnu.trove.list.TLongList;
import gnu.trove.list.array.TByteArrayList;
import gnu.trove.list.array.TIntArrayList;
import gnu.trove.list.array.TLongArrayList;
import java.io.IOException;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@ -15,6 +18,7 @@ import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.Stack;
@ -25,6 +29,7 @@ import net.osmand.Algoritms;
import net.osmand.binary.BinaryMapIndexReader;
import net.osmand.binary.OsmandOdb;
import net.osmand.binary.OsmandOdb.CityIndex;
import net.osmand.binary.OsmandOdb.MapData;
import net.osmand.binary.OsmandOdb.MapDataBlock;
import net.osmand.binary.OsmandOdb.OsmAndMapIndex.MapDataBox;
import net.osmand.binary.OsmandOdb.OsmAndMapIndex.MapEncodingRule;
@ -54,6 +59,7 @@ import net.osmand.osm.Node;
import net.osmand.osm.MapRenderingTypes.MapRulType;
import net.sf.junidecode.Junidecode;
import com.google.protobuf.ByteString;
import com.google.protobuf.CodedOutputStream;
import com.google.protobuf.MessageLite;
import com.google.protobuf.WireFormat;
@ -82,6 +88,7 @@ public class BinaryMapIndexWriter {
}
private Stack<Bounds> stackBounds = new Stack<Bounds>();
private Stack<Long> stackBaseIds = new Stack<Long>();
// internal constants to track state of index writing
private Stack<Integer> state = new Stack<Integer>();
@ -284,10 +291,6 @@ public class BinaryMapIndexWriter {
public static int STRING_TABLE_SIZE = 0;
protected static int codeCoordinateDifference(int x, int px){
// shift absolute coordinates first and get truncated
return (x >> SHIFT_COORDINATES) - (px >> SHIFT_COORDINATES);
}
@ -310,101 +313,118 @@ public class BinaryMapIndexWriter {
CodedOutputStream.computeRawVarint32Size(size) + size;
ref.writeReference(raf, getFilePointer());
codedOutStream.writeMessage(OsmAndMapIndex.BLOCKS_FIELD_NUMBER, builder.build());
MapDataBlock block = builder.build();
MAP_DATA_SIZE += block.getSerializedSize();
codedOutStream.writeMessage(OsmAndMapIndex.BLOCKS_FIELD_NUMBER, block);
}
/**
* Encode and write a varint. {@code value} is treated as unsigned, so it won't be sign-extended if negative.
*/
public void writeRawVarint32(TByteArrayList bf, int value) throws IOException {
while (true) {
if ((value & ~0x7F) == 0) {
writeRawByte(bf, value);
return;
} else {
writeRawByte(bf, (value & 0x7F) | 0x80);
value >>>= 7;
}
}
}
/** Write a single byte. */
public void writeRawByte(TByteArrayList bf, final int value) throws IOException {
bf.add((byte)value);
}
private TByteArrayList mapDataBuf = new TByteArrayList();
public void writeMapData(long diffId, byte[] nodes, byte[] types, Map<String,String> name, Map<String, Integer> stringTable,
public void writeMapData(long diffId, int pleft, int ptop, boolean area, byte[] coordinates,
byte[] innerPolygonTypes, byte[] types,
byte[] additionalTypes, Map<MapRulType, String> names, Map<String, Integer> stringTable,
MapDataBlock.Builder dataBlock) throws IOException{
Bounds bounds = stackBounds.peek();
MapData.Builder data = MapData.newBuilder();
// calculate size
int sizeCoordinates = 0;
int allSize = 0;
int px = bounds.leftX;
int py = bounds.topY;
for(int i=0; i< nodes.length / 8; i++){
int x = Algoritms.parseIntFromBytes(nodes, i * 8);
int y = Algoritms.parseIntFromBytes(nodes, i * 8 + 4);
sizeCoordinates += CodedOutputStream.computeSInt32SizeNoTag(codeCoordinateDifference(x, px));
sizeCoordinates += CodedOutputStream.computeSInt32SizeNoTag(codeCoordinateDifference(y, py));
px = x;
py = y;
COORDINATES_COUNT += 2;
mapDataBuf.clear();
int pcalcx = pleft;
int pcalcy = ptop;
for(int i=0; i< coordinates.length / 8; i++){
int x = Algoritms.parseIntFromBytes(coordinates, i * 8);
int y = Algoritms.parseIntFromBytes(coordinates, i * 8 + 4);
int tx = (x - pcalcx) >> SHIFT_COORDINATES;
int ty = (x - pcalcy) >> SHIFT_COORDINATES;
writeRawVarint32(mapDataBuf, tx);
writeRawVarint32(mapDataBuf, ty);
pcalcx = pcalcx + (tx << SHIFT_COORDINATES);
pcalcy = pcalcy + (ty << SHIFT_COORDINATES);
}
COORDINATES_SIZE += CodedOutputStream.computeRawVarint32Size(mapDataBuf.size()) +
CodedOutputStream.computeTagSize(MapData.COORDINATES_FIELD_NUMBER) + mapDataBuf.size();
if(area) {
data.setAreaCoordinates(ByteString.copyFrom(mapDataBuf.toArray()));
} else {
data.setCoordinates(ByteString.copyFrom(mapDataBuf.toArray()));
}
allSize += CodedOutputStream.computeRawVarint32Size(sizeCoordinates) +
CodedOutputStream.computeTagSize(OsmandOdb.MapData.COORDINATES_FIELD_NUMBER) + sizeCoordinates;
// DEBUG
COORDINATES_SIZE += allSize;
if(innerPolygonTypes.length > 0){
mapDataBuf.clear();
pcalcx = pleft;
pcalcy = ptop;
for(int i=0; i< innerPolygonTypes.length / 8; i++){
int x = Algoritms.parseIntFromBytes(coordinates, i * 8);
int y = Algoritms.parseIntFromBytes(coordinates, i * 8 + 4);
if(x == 0 && y == 0){
data.addPolygonInnerCoordinates(ByteString.copyFrom(mapDataBuf.toArray()));
mapDataBuf.clear();
pcalcx = pleft;
pcalcy = ptop;
} else {
int tx = (x - pcalcx) >> SHIFT_COORDINATES;
int ty = (x - pcalcy) >> SHIFT_COORDINATES;
allSize += CodedOutputStream.computeTagSize(OsmandOdb.MapData.TYPES_FIELD_NUMBER);
allSize += CodedOutputStream.computeRawVarint32Size(types.length);
allSize += types.length;
// DEBUG
TYPES_SIZE += CodedOutputStream.computeTagSize(OsmandOdb.MapData.TYPES_FIELD_NUMBER) +
CodedOutputStream.computeRawVarint32Size(types.length) + types.length;
allSize += CodedOutputStream.computeSInt64Size(OsmandOdb.MapData.ID_FIELD_NUMBER, diffId);
// DEBUG
ID_SIZE += CodedOutputStream.computeSInt64Size(OsmandOdb.MapData.ID_FIELD_NUMBER, diffId);
int nameId = 0;
if(name != null){
if(stringTable.containsKey(name)) {
nameId = stringTable.get(name);
} else {
nameId = stringTable.size();
stringTable.put(name, nameId);
writeRawVarint32(mapDataBuf, tx);
writeRawVarint32(mapDataBuf, ty);
pcalcx = pcalcx + (tx << SHIFT_COORDINATES);
pcalcy = pcalcy + (ty << SHIFT_COORDINATES);
}
}
allSize += CodedOutputStream.computeUInt32Size(OsmandOdb.MapData.S, nameId);
}
data.setTypes(ByteString.copyFrom(types));
TYPES_SIZE += CodedOutputStream.computeTagSize(OsmandOdb.MapData.TYPES_FIELD_NUMBER) +
CodedOutputStream.computeRawVarint32Size(types.length) + types.length;
if(additionalTypes.length > 0 ){
data.setAdditionalTypes(ByteString.copyFrom(additionalTypes));
TYPES_SIZE += CodedOutputStream.computeTagSize(OsmandOdb.MapData.ADDITIONALTYPES_FIELD_NUMBER) +
CodedOutputStream.computeRawVarint32Size(additionalTypes.length) + additionalTypes.length;
}
// DEBUG
MAP_DATA_SIZE += allSize;
// writing data
codedOutStream.writeTag(MapDataBlock.DATAOBJECTS_FIELD_NUMBER, WireFormat.FieldType.MESSAGE.getWireType());
codedOutStream.writeRawVarint32(allSize);
/// TODO !!!!
codedOutStream.writeTag(OsmandOdb.MapData.COORDINATES_FIELD_NUMBER, WireFormat.FieldType.BYTES.getWireType());
codedOutStream.writeRawVarint32(sizeCoordinates);
px = bounds.leftX;
py = bounds.topY;
for (int i = 0; i < nodes.length / 8; i++) {
int x = Algoritms.parseIntFromBytes(nodes, i * 8);
int y = Algoritms.parseIntFromBytes(nodes, i * 8 + 4);
codedOutStream.writeSInt32NoTag(codeCoordinateDifference(x, px));
codedOutStream.writeSInt32NoTag(codeCoordinateDifference(y, py));
px = x;
py = y;
mapDataBuf.clear();
for(Entry<MapRulType, String> s : names.entrySet()) {
writeRawVarint32(mapDataBuf, s.getKey().getTargetId());
Integer ls = stringTable.get(s.getValue());
if(ls == null){
ls = stringTable.size();
stringTable.put(s.getValue(), ls);
}
writeRawVarint32(mapDataBuf, ls);
}
STRING_TABLE_SIZE += mapDataBuf.size();
data.setStringNames(ByteString.copyFrom(mapDataBuf.toArray()));
codedOutStream.writeTag(OsmandOdb.MapData.TYPES_FIELD_NUMBER, WireFormat.FieldType.BYTES.getWireType());
codedOutStream.writeRawVarint32(types.length);
codedOutStream.writeRawBytes(types);
// TODO !!!
if(name != null){
codedOutStream.writeUInt32(OsmandOdb.MapData.STRINGID_FIELD_NUMBER, nameId);
}
codedOutStream.writeSInt64(OsmandOdb.MapData.ID_FIELD_NUMBER, diffId);
data.setId(diffId);
ID_SIZE += CodedOutputStream.computeSInt64Size(OsmandOdb.MapData.ID_FIELD_NUMBER, diffId);
}

View file

@ -107,6 +107,8 @@ message OsmAndMapIndex {
//shift from mapdatabox start to message MapDataBlock
optional fixed32 shiftToMapData = 5;
optional bool ocean = 6;
repeated MapDataBox boxes = 7;
}
@ -137,9 +139,9 @@ message MapDataBlock {
message MapData {
// array of delta x,y sint32 (CodedIinputStream) >> 5
// first x delta to Tree.left, y to delta Tree.top (next delta to previous)
optional bytes pointCoordinates = 1;
optional bytes lineCoordinates = 2;
optional bytes polygonOuterCoordinates = 3;
// point, line or outer polygon coordinates
optional bytes coordinates = 1;
optional bytes areaCoordinates = 2;
repeated bytes polygonInnerCoordinates = 4;
// array of raw var int types