Android-builders-club/OsmAnd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Read interpolation type

Browse source
This commit is contained in:
Victor Shcherb 2012-03-04 23:23:24 +01:00
parent 11ca506999
commit 61d80d9118
3 changed files with 105 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

72
DataExtractionOSM/src/net/osmand/data/preparation/BinaryMapIndexWriter.java
View file

@ -29,11 +29,15 @@ import org.apache.commons.logging.LogFactory;
import net.osmand.Algoritms;
import net.osmand.binary.BinaryMapIndexReader;
import net.osmand.binary.OsmandOdb;
import net.osmand.binary.OsmandOdb.AddressNameIndexDataAtom;
import net.osmand.binary.OsmandOdb.CityBlockIndex;
import net.osmand.binary.OsmandOdb.CityIndex;
import net.osmand.binary.OsmandOdb.MapData;
import net.osmand.binary.OsmandOdb.MapDataBlock;
import net.osmand.binary.OsmandOdb.OsmAndAddressIndex;
import net.osmand.binary.OsmandOdb.OsmAndAddressIndex.CitiesIndex;
import net.osmand.binary.OsmandOdb.OsmAndAddressNameIndexData.AddressNameIndexData;
import net.osmand.binary.OsmandOdb.OsmAndAddressNameIndexData;
import net.osmand.binary.OsmandOdb.OsmAndMapIndex.MapDataBox;
import net.osmand.binary.OsmandOdb.OsmAndMapIndex.MapEncodingRule;
import net.osmand.binary.OsmandOdb.OsmAndMapIndex.MapRootLevel;
@ -51,6 +55,7 @@ import net.osmand.binary.OsmandOdb.TransportRouteStop;
import net.osmand.binary.OsmandOdb.OsmAndCategoryTable.Builder;
import net.osmand.data.Building;
import net.osmand.data.City;
import net.osmand.data.City.CityType;
import net.osmand.data.IndexConstants;
import net.osmand.data.MapObject;
import net.osmand.data.PostCode;
@ -433,6 +438,54 @@ public class BinaryMapIndexWriter {
int len = writeInt32Size();
log.info("ADDRESS INDEX SIZE : " + len);
}
public void writeAddressNameIndex(Map<String, List<MapObject>> namesIndex) throws IOException {
checkPeekState(CITY_INDEX_INIT);
codedOutStream.writeTag(OsmAndAddressIndex.NAMEINDEX_FIELD_NUMBER, WireFormat.WIRETYPE_FIXED32_LENGTH_DELIMITED);
preserveInt32Size();
Map<String, BinaryFileReference> res = writeIndexedTable(OsmAndAddressNameIndexData.TABLE_FIELD_NUMBER, namesIndex.keySet());
for(Entry<String, List<MapObject>> entry : namesIndex.entrySet()) {
BinaryFileReference ref = res.get(entry.getKey());
codedOutStream.writeTag(OsmAndAddressNameIndexData.ATOM_FIELD_NUMBER, FieldType.MESSAGE.getWireType());
long pointer = getFilePointer();
if(ref != null) {
ref.writeReference(raf, getFilePointer());
}
AddressNameIndexData.Builder builder = AddressNameIndexData.newBuilder();
// collapse same name ?
for(MapObject o : entry.getValue()){
AddressNameIndexDataAtom.Builder atom = AddressNameIndexDataAtom.newBuilder();
atom.setName(o.getName());
if(checkEnNameToWrite(o)){
atom.setNameEn(o.getEnName());
}
int type = 1;
if (o instanceof City) {
CityType ct = ((City) o).getType();
if (ct != CityType.CITY && ct != CityType.TOWN) {
type = 3;
}
} else if(o instanceof PostCode) {
type = 2;
} else if(o instanceof Street) {
type = 4;
}
atom.setType(type);
atom.addShiftToIndex((int) (pointer - o.getFileOffset()));
if(o instanceof Street){
atom.addShiftToCityIndex((int) (pointer - ((Street) o).getCity().getFileOffset()));
}
builder.addAtom(atom.build());
}
codedOutStream.writeMessageNoTag(builder.build());
}
int len = writeInt32Size();
log.info("ADDRESS NAME INDEX SIZE : " + len);
}
private boolean checkEnNameToWrite(MapObject obj) {
if (obj.getEnName() == null || obj.getEnName().length() == 0) {
@ -476,6 +529,7 @@ public class BinaryMapIndexWriter {
ref.writeReference(raf, startMessage);
CityBlockIndex.Builder cityInd = OsmandOdb.CityBlockIndex.newBuilder();
cityInd.setShiftToCityIndex((int) (startMessage - startCityBlock));
long currentPointer = startMessage + 4;
int cx = MapUtils.get31TileNumberX(cityOrPostcode.getLocation().getLongitude());
int cy = MapUtils.get31TileNumberY(cityOrPostcode.getLocation().getLatitude());
@ -494,9 +548,16 @@ public class BinaryMapIndexWriter {
String postcodeFilter = cityOrPostcode instanceof PostCode ? cityOrPostcode.getName() : null;
for (Street s : streets) {
StreetIndex streetInd = createStreetAndBuildings(s, cx, cy, postcodeFilter, mapNodeToStreet, wayNodes);
currentPointer += CodedOutputStream.computeTagSize(CityBlockIndex.STREETS_FIELD_NUMBER);
if(currentPointer > Integer.MAX_VALUE) {
throw new IllegalArgumentException("File offset > 2 GB.");
}
s.setFileOffset((int) currentPointer);
currentPointer += CodedOutputStream.computeMessageSizeNoTag(streetInd);
cityInd.addStreets(streetInd);
}
codedOutStream.writeMessage(CitiesIndex.BLOCKS_FIELD_NUMBER, cityInd.build());
codedOutStream.writeMessageNoTag(cityInd.build());
}
public void startCityBlockIndex(int type) throws IOException {
@ -801,11 +862,12 @@ public class BinaryMapIndexWriter {
preserveInt32Size();
Map<PoiTileBox, List<BinaryFileReference>> fpToWriteSeeks = new LinkedHashMap<PoiTileBox, List<BinaryFileReference>>();
int previousSize = 0;
Map<String, BinaryFileReference> indexedTable = writeIndexedTable(OsmandOdb.OsmAndPoiNameIndex.TABLE_FIELD_NUMBER, namesIndex.keySet());
TODO;
for(Map.Entry<String, Set<PoiTileBox>> e : namesIndex.entrySet()) {
codedOutStream.writeTag(OsmandOdb.OsmAndPoiNameIndex.DATA_FIELD_NUMBER, WireFormat.WIRETYPE_FIXED32_LENGTH_DELIMITED);
BinaryFileReference nameTableRef = indexedTable.get(e.getKey());
nameTableRef.writeReference(raf, getFilePointer());
OsmAndPoiNameIndex.OsmAndPoiNameIndexData.Builder builder = OsmAndPoiNameIndex.OsmAndPoiNameIndexData.newBuilder();
List<PoiTileBox> tileBoxes = new ArrayList<PoiTileBox>(e.getValue());
for(PoiTileBox box : tileBoxes) {
@ -820,8 +882,8 @@ public class BinaryMapIndexWriter {
OsmAndPoiNameIndex.OsmAndPoiNameIndexData msg = builder.build();
codedOutStream.writeMessage(OsmandOdb.OsmAndPoiNameIndex.DATA_FIELD_NUMBER, msg);
long endPointer = codedOutStream.getWrittenBytes();
indexedTable.get(e.getKey()).writeReference(raf, (int) (endPointer - msg.getSerializedSize() - startPoiIndex.getStartPointer()));
long endPointer = getFilePointer();
// first message
int accumulateSize = 4;
for (int i = tileBoxes.size() - 1; i >= 0; i--) {

49
DataExtractionOSM/src/net/osmand/data/preparation/IndexAddressCreator.java
View file

@ -17,7 +17,6 @@ import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@ -31,6 +30,7 @@ import net.osmand.data.Building;
import net.osmand.data.City;
import net.osmand.data.City.CityType;
import net.osmand.data.DataTileManager;
import net.osmand.data.MapObject;
import net.osmand.data.PostCode;
import net.osmand.data.Street;
import net.osmand.data.WayBoundary;
@ -75,8 +75,8 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
private String[] normalizeSuffixes;
private boolean saveAddressWays;
private boolean DEBUG_FULL_NAMES = false; //true to see atached cityPart and boundaries to the street names
private final int ADDRESS_NAME_CHARACTERS_TO_INDEX = 4;
Connection mapConnection;
DBStreetDAO streetDAO;
@ -655,8 +655,6 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
public void iterateMainEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException {
// index not only buildings but also nodes that belongs to addr:interpolation ways
if (e.getTag(OSMTagKey.ADDR_HOUSE_NUMBER) != null && e.getTag(OSMTagKey.ADDR_STREET) != null) {
// TODO e.getTag(OSMTagKey.ADDR_CITY) could be used to find city however many cities could have same name!
// check that building is not registered already
boolean exist = streetDAO.findBuilding(e);
if (!exist) {
ctx.loadEntityData(e);
@ -770,18 +768,21 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
for(CityType t : cities.keySet()) {
if(t == CityType.CITY || t == CityType.TOWN){
cityTowns.addAll(cities.get(t));
} else if(t == CityType.SUBURB){
suburbs.addAll(cities.get(t));
} else {
villages.addAll(cities.get(t));
}
if(t == CityType.SUBURB){
suburbs.addAll(cities.get(t));
}
}
progress.startTask(Messages.getString("IndexCreator.SERIALIZING_ADRESS"), cityTowns.size() + villages.size() / 100 + 1); //$NON-NLS-1$
Map<String, List<MapObject>> namesIndex = new TreeMap<String, List<MapObject>>(Collator.getInstance());
Map<String, PostCode> postcodes = new TreeMap<String, PostCode>();
writeCityBlockIndex(writer, CITIES_TYPE, streetstat, waynodesStat, suburbs, cityTowns, postcodes, progress);
writeCityBlockIndex(writer, VILLAGES_TYPE, streetstat, waynodesStat, suburbs, villages, postcodes, progress);
writeCityBlockIndex(writer, CITIES_TYPE, streetstat, waynodesStat, suburbs, cityTowns, postcodes, namesIndex, progress);
writeCityBlockIndex(writer, VILLAGES_TYPE, streetstat, waynodesStat, null, villages, postcodes, namesIndex, progress);
// write postcodes
List<BinaryFileReference> refs = new ArrayList<BinaryFileReference>();
@ -793,6 +794,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
for (int i = 0; i < posts.size(); i++) {
PostCode postCode = posts.get(i);
BinaryFileReference ref = refs.get(i);
putNamedMapObject(namesIndex, postCode, ref.getStartPointer());
writer.writeCityIndex(postCode, new ArrayList<Street>(postCode.getStreets()), null, ref);
}
writer.endCityBlockIndex();
@ -800,7 +802,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
progress.finishTask();
TODO writeNameIndex;
writer.writeAddressNameIndex(namesIndex);
writer.endWriteAddressIndex();
writer.flush();
streetstat.close();
@ -809,10 +811,29 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
}
}
private void putNamedMapObject(Map<String, List<MapObject>> namesIndex, MapObject o, long fileOffset){
String name = o.getName();
if (name != null && name.length() > 0) {
if (name.length() > ADDRESS_NAME_CHARACTERS_TO_INDEX) {
name = name.substring(0, ADDRESS_NAME_CHARACTERS_TO_INDEX);
}
if (!namesIndex.containsKey(name)) {
namesIndex.put(name, new ArrayList<MapObject>());
}
namesIndex.get(name).add(o);
if (fileOffset > Integer.MAX_VALUE) {
throw new IllegalArgumentException("File offset > 2 GB.");
}
o.setFileOffset((int) fileOffset);
}
}
private void writeCityBlockIndex(BinaryMapIndexWriter writer, int type, PreparedStatement streetstat, PreparedStatement waynodesStat,
List<City> suburbs, List<City> cities, Map<String, PostCode> postcodes, IProgress progress) throws IOException, SQLException {
List<City> suburbs, List<City> cities, Map<String, PostCode> postcodes, Map<String, List<MapObject>> namesIndex, IProgress progress)
throws IOException, SQLException {
List<BinaryFileReference> refs = new ArrayList<BinaryFileReference>();
// 1. write cities
writer.startCityBlockIndex(type);
@ -822,6 +843,7 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
for(int i=0; i<cities.size(); i++) {
City city = cities.get(i);
BinaryFileReference ref = refs.get(i);
putNamedMapObject(namesIndex, city, ref.getStartPointer());
if(type == CITIES_TYPE) {
progress.progress(1);
} else {
@ -846,8 +868,9 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
long f = System.currentTimeMillis() - time;
writer.writeCityIndex(city, streets, streetNodes, ref);
int bCount = 0;
// register postcodes and name index
for (Street s : streets) {
bCount++;
putNamedMapObject(namesIndex, s, s.getFileOffset());
for (Building b : s.getBuildings()) {
bCount++;
if(city.getPostcode() != null && b.getPostcode() == null) {
@ -865,9 +888,9 @@ public class IndexAddressCreator extends AbstractIndexPartCreator{
}
if (f > 500) {
if(logMapDataWarn != null) {
logMapDataWarn.info("! " + city.getName() + " ! " + f + " " + bCount + " streets " + streets.size());
logMapDataWarn.info("! " + city.getName() + " ! " + f + " ms " + streets.size() + " streets " + bCount + " buildings");
} else {
log.info("! " + city.getName() + " ! " + f + " " + bCount + " streets " + streets.size());
log.info("! " + city.getName() + " ! " + f + " ms " + streets.size() + " streets " + bCount + " buildings");
}
}
}

3
DataExtractionOSM/src/osmand_odb.proto
View file

@ -195,6 +195,7 @@ message OsmAndAddressNameIndexData {
repeated AddressNameIndexData atom = 7;
message AddressNameIndexData {
// shift is measured from start (before length)
repeated AddressNameIndexDataAtom atom = 4;
}
}
@ -208,7 +209,7 @@ message AddressNameIndexDataAtom {
required uint32 type = 3;
// mixed
// shift from start AddressNameIndexDataAtom to ...Index
// shift from start AddressNameIndexData (!) to ...Index
repeated fixed32 shiftToIndex = 5;
// optional used in case of type=street
repeated fixed32 shiftToCityIndex = 6;