Speed up resource loading
This commit is contained in:
parent
a334193b14
commit
b112a8a6d3
13 changed files with 6257 additions and 70 deletions
|
@ -46,7 +46,7 @@ public class JUnitRouteTest {
|
||||||
int it = 0;
|
int it = 0;
|
||||||
for (File f : files) {
|
for (File f : files) {
|
||||||
RandomAccessFile raf = new RandomAccessFile(f, "r"); //$NON-NLS-1$ //$NON-NLS-2$
|
RandomAccessFile raf = new RandomAccessFile(f, "r"); //$NON-NLS-1$ //$NON-NLS-2$
|
||||||
rs[it++] = new BinaryMapIndexReader(raf, false);
|
rs[it++] = new BinaryMapIndexReader(raf);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -96,7 +96,7 @@ public class RouterTestsSuite {
|
||||||
int it = 0;
|
int it = 0;
|
||||||
for (File f : files) {
|
for (File f : files) {
|
||||||
RandomAccessFile raf = new RandomAccessFile(f, "r"); //$NON-NLS-1$ //$NON-NLS-2$
|
RandomAccessFile raf = new RandomAccessFile(f, "r"); //$NON-NLS-1$ //$NON-NLS-2$
|
||||||
rs[it++] = new BinaryMapIndexReader(raf, false);
|
rs[it++] = new BinaryMapIndexReader(raf);
|
||||||
}
|
}
|
||||||
|
|
||||||
boolean allSuccess = true;
|
boolean allSuccess = true;
|
||||||
|
|
|
@ -46,6 +46,18 @@ public class BinaryMapAddressReaderAdapter {
|
||||||
List<CitiesBlock> cities = new ArrayList<BinaryMapAddressReaderAdapter.CitiesBlock>();
|
List<CitiesBlock> cities = new ArrayList<BinaryMapAddressReaderAdapter.CitiesBlock>();
|
||||||
|
|
||||||
LatLon calculatedCenter = null;
|
LatLon calculatedCenter = null;
|
||||||
|
|
||||||
|
public String getEnName() {
|
||||||
|
return enName;
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<CitiesBlock> getCities() {
|
||||||
|
return cities;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getIndexNameOffset() {
|
||||||
|
return indexNameOffset;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static class CitiesBlock extends BinaryIndexPart {
|
public static class CitiesBlock extends BinaryIndexPart {
|
||||||
|
|
|
@ -58,16 +58,16 @@ public class BinaryMapIndexReader {
|
||||||
private final static Log log = LogUtil.getLog(BinaryMapIndexReader.class);
|
private final static Log log = LogUtil.getLog(BinaryMapIndexReader.class);
|
||||||
|
|
||||||
private final RandomAccessFile raf;
|
private final RandomAccessFile raf;
|
||||||
private int version;
|
/*private*/ int version;
|
||||||
private long dateCreated;
|
/*private */long dateCreated;
|
||||||
// keep them immutable inside
|
// keep them immutable inside
|
||||||
private boolean basemap = false;
|
/*private */ boolean basemap = false;
|
||||||
private List<MapIndex> mapIndexes = new ArrayList<MapIndex>();
|
/*private */List<MapIndex> mapIndexes = new ArrayList<MapIndex>();
|
||||||
private List<PoiRegion> poiIndexes = new ArrayList<PoiRegion>();
|
/*private */List<PoiRegion> poiIndexes = new ArrayList<PoiRegion>();
|
||||||
private List<AddressRegion> addressIndexes = new ArrayList<AddressRegion>();
|
/*private */List<AddressRegion> addressIndexes = new ArrayList<AddressRegion>();
|
||||||
private List<TransportIndex> transportIndexes = new ArrayList<TransportIndex>();
|
/*private */List<TransportIndex> transportIndexes = new ArrayList<TransportIndex>();
|
||||||
private List<RouteRegion> routingIndexes = new ArrayList<RouteRegion>();
|
/*private */List<RouteRegion> routingIndexes = new ArrayList<RouteRegion>();
|
||||||
private List<BinaryIndexPart> indexes = new ArrayList<BinaryIndexPart>();
|
/*private */List<BinaryIndexPart> indexes = new ArrayList<BinaryIndexPart>();
|
||||||
|
|
||||||
protected CodedInputStreamRAF codedIS;
|
protected CodedInputStreamRAF codedIS;
|
||||||
|
|
||||||
|
@ -80,7 +80,27 @@ public class BinaryMapIndexReader {
|
||||||
|
|
||||||
|
|
||||||
public BinaryMapIndexReader(final RandomAccessFile raf) throws IOException {
|
public BinaryMapIndexReader(final RandomAccessFile raf) throws IOException {
|
||||||
this(raf, false);
|
this.raf = raf;
|
||||||
|
codedIS = CodedInputStreamRAF.newInstance(raf, 1024 * 5);
|
||||||
|
codedIS.setSizeLimit(Integer.MAX_VALUE); // 2048 MB
|
||||||
|
transportAdapter = new BinaryMapTransportReaderAdapter(this);
|
||||||
|
addressAdapter = new BinaryMapAddressReaderAdapter(this);
|
||||||
|
poiAdapter = new BinaryMapPoiReaderAdapter(this);
|
||||||
|
routeAdapter = new BinaryMapRouteReaderAdapter(this);
|
||||||
|
init();
|
||||||
|
}
|
||||||
|
|
||||||
|
/*private */BinaryMapIndexReader(final RandomAccessFile raf, boolean init) throws IOException {
|
||||||
|
this.raf = raf;
|
||||||
|
codedIS = CodedInputStreamRAF.newInstance(raf, 1024 * 5);
|
||||||
|
codedIS.setSizeLimit(Integer.MAX_VALUE); // 2048 MB
|
||||||
|
transportAdapter = new BinaryMapTransportReaderAdapter(this);
|
||||||
|
addressAdapter = new BinaryMapAddressReaderAdapter(this);
|
||||||
|
poiAdapter = new BinaryMapPoiReaderAdapter(this);
|
||||||
|
routeAdapter = new BinaryMapRouteReaderAdapter(this);
|
||||||
|
if(init) {
|
||||||
|
init();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public BinaryMapIndexReader(final RandomAccessFile raf, BinaryMapIndexReader referenceToSameFile) throws IOException {
|
public BinaryMapIndexReader(final RandomAccessFile raf, BinaryMapIndexReader referenceToSameFile) throws IOException {
|
||||||
|
@ -101,23 +121,6 @@ public class BinaryMapIndexReader {
|
||||||
basemap = referenceToSameFile.basemap;
|
basemap = referenceToSameFile.basemap;
|
||||||
}
|
}
|
||||||
|
|
||||||
public BinaryMapIndexReader(final RandomAccessFile raf, boolean readOnlyMapData) throws IOException {
|
|
||||||
this.raf = raf;
|
|
||||||
codedIS = CodedInputStreamRAF.newInstance(raf, 1024 * 5);
|
|
||||||
codedIS.setSizeLimit(Integer.MAX_VALUE); // 2048 MB
|
|
||||||
if(!readOnlyMapData){
|
|
||||||
transportAdapter = new BinaryMapTransportReaderAdapter(this);
|
|
||||||
addressAdapter = new BinaryMapAddressReaderAdapter(this);
|
|
||||||
poiAdapter = new BinaryMapPoiReaderAdapter(this);
|
|
||||||
routeAdapter = new BinaryMapRouteReaderAdapter(this);
|
|
||||||
} else {
|
|
||||||
transportAdapter = null;
|
|
||||||
addressAdapter = null;
|
|
||||||
poiAdapter = null;
|
|
||||||
routeAdapter = null;
|
|
||||||
}
|
|
||||||
init();
|
|
||||||
}
|
|
||||||
|
|
||||||
public long getDateCreated() {
|
public long getDateCreated() {
|
||||||
return dateCreated;
|
return dateCreated;
|
||||||
|
@ -420,6 +423,10 @@ public class BinaryMapIndexReader {
|
||||||
return getTransportIndex(s.getFileOffset()) != null;
|
return getTransportIndex(s.getFileOffset()) != null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public List<TransportIndex> getTransportIndexes() {
|
||||||
|
return transportIndexes;
|
||||||
|
}
|
||||||
|
|
||||||
private TransportIndex getTransportIndex(int filePointer) {
|
private TransportIndex getTransportIndex(int filePointer) {
|
||||||
TransportIndex ind = null;
|
TransportIndex ind = null;
|
||||||
for(TransportIndex i : transportIndexes){
|
for(TransportIndex i : transportIndexes){
|
||||||
|
|
309
DataExtractionOSM/src/net/osmand/binary/CachedOsmandIndexes.java
Normal file
309
DataExtractionOSM/src/net/osmand/binary/CachedOsmandIndexes.java
Normal file
|
@ -0,0 +1,309 @@
|
||||||
|
package net.osmand.binary;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileInputStream;
|
||||||
|
import java.io.FileOutputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.RandomAccessFile;
|
||||||
|
|
||||||
|
import org.apache.commons.logging.Log;
|
||||||
|
|
||||||
|
import net.osmand.LogUtil;
|
||||||
|
import net.osmand.binary.BinaryMapAddressReaderAdapter.AddressRegion;
|
||||||
|
import net.osmand.binary.BinaryMapAddressReaderAdapter.CitiesBlock;
|
||||||
|
import net.osmand.binary.BinaryMapIndexReader.MapIndex;
|
||||||
|
import net.osmand.binary.BinaryMapIndexReader.MapRoot;
|
||||||
|
import net.osmand.binary.BinaryMapPoiReaderAdapter.PoiRegion;
|
||||||
|
import net.osmand.binary.BinaryMapRouteReaderAdapter.RouteRegion;
|
||||||
|
import net.osmand.binary.BinaryMapRouteReaderAdapter.RouteSubregion;
|
||||||
|
import net.osmand.binary.BinaryMapTransportReaderAdapter.IndexStringTable;
|
||||||
|
import net.osmand.binary.BinaryMapTransportReaderAdapter.TransportIndex;
|
||||||
|
import net.osmand.binary.OsmandIndex.AddressPart;
|
||||||
|
import net.osmand.binary.OsmandIndex.CityBlock;
|
||||||
|
import net.osmand.binary.OsmandIndex.FileIndex;
|
||||||
|
import net.osmand.binary.OsmandIndex.MapLevel;
|
||||||
|
import net.osmand.binary.OsmandIndex.MapPart;
|
||||||
|
import net.osmand.binary.OsmandIndex.OsmAndStoredIndex;
|
||||||
|
import net.osmand.binary.OsmandIndex.PoiPart;
|
||||||
|
import net.osmand.binary.OsmandIndex.RoutingPart;
|
||||||
|
import net.osmand.binary.OsmandIndex.RoutingSubregion;
|
||||||
|
import net.osmand.binary.OsmandIndex.TransportPart;
|
||||||
|
import net.osmand.osm.MapUtils;
|
||||||
|
|
||||||
|
public class CachedOsmandIndexes {
|
||||||
|
|
||||||
|
private OsmAndStoredIndex storedIndex;
|
||||||
|
private OsmAndStoredIndex.Builder storedIndexBuilder;
|
||||||
|
private Log log = LogUtil.getLog(CachedOsmandIndexes.class);
|
||||||
|
private boolean hasChanged = true;
|
||||||
|
|
||||||
|
public static final int VERSION = 1;
|
||||||
|
|
||||||
|
public void addToCache(BinaryMapIndexReader reader, File f) {
|
||||||
|
hasChanged = true;
|
||||||
|
if(storedIndexBuilder == null) {
|
||||||
|
storedIndexBuilder = OsmandIndex.OsmAndStoredIndex.newBuilder();
|
||||||
|
storedIndexBuilder.setVersion(VERSION);
|
||||||
|
storedIndexBuilder.setDateCreated(System.currentTimeMillis());
|
||||||
|
}
|
||||||
|
|
||||||
|
FileIndex.Builder fileIndex = OsmandIndex.FileIndex.newBuilder();
|
||||||
|
fileIndex.setDateModified(f.lastModified());
|
||||||
|
fileIndex.setSize(f.length());
|
||||||
|
fileIndex.setVersion(reader.getVersion());
|
||||||
|
fileIndex.setFileName(f.getName());
|
||||||
|
for(MapIndex index : reader.getMapIndexes()) {
|
||||||
|
MapPart.Builder map = OsmandIndex.MapPart.newBuilder();
|
||||||
|
map.setSize(index.getLength());
|
||||||
|
map.setOffset(index.getFilePointer());
|
||||||
|
if(index.getName() != null) {
|
||||||
|
map.setName(index.getName());
|
||||||
|
}
|
||||||
|
for(MapRoot mr : index.getRoots() ) {
|
||||||
|
MapLevel.Builder lev = OsmandIndex.MapLevel.newBuilder();
|
||||||
|
lev.setSize(mr.length);
|
||||||
|
lev.setOffset(mr.filePointer);
|
||||||
|
lev.setLeft(mr.left);
|
||||||
|
lev.setRight(mr.right);
|
||||||
|
lev.setTop(mr.top);
|
||||||
|
lev.setBottom(mr.bottom);
|
||||||
|
lev.setMinzoom(mr.minZoom);
|
||||||
|
lev.setMaxzoom(mr.maxZoom);
|
||||||
|
map.addLevels(lev);
|
||||||
|
}
|
||||||
|
fileIndex.addMapIndex(map);
|
||||||
|
}
|
||||||
|
|
||||||
|
for(AddressRegion index : reader.getAddressIndexes()) {
|
||||||
|
AddressPart.Builder addr = OsmandIndex.AddressPart.newBuilder();
|
||||||
|
addr.setSize(index.getLength());
|
||||||
|
addr.setOffset(index.getFilePointer());
|
||||||
|
if(index.getName() != null) {
|
||||||
|
addr.setName(index.getName());
|
||||||
|
}
|
||||||
|
if(index.getEnName() != null) {
|
||||||
|
addr.setNameEn(index.getEnName());
|
||||||
|
}
|
||||||
|
addr.setIndexNameOffset(index.getIndexNameOffset());
|
||||||
|
for(CitiesBlock mr : index.getCities() ) {
|
||||||
|
CityBlock.Builder cblock = OsmandIndex.CityBlock.newBuilder();
|
||||||
|
cblock.setSize(mr.length);
|
||||||
|
cblock.setOffset(mr.filePointer);
|
||||||
|
cblock.setType(mr.type);
|
||||||
|
addr.addCities(cblock);
|
||||||
|
}
|
||||||
|
fileIndex.addAddressIndex(addr);
|
||||||
|
}
|
||||||
|
|
||||||
|
for(PoiRegion index : reader.getPoiIndexes()) {
|
||||||
|
PoiPart.Builder poi = OsmandIndex.PoiPart.newBuilder();
|
||||||
|
poi.setSize(index.getLength());
|
||||||
|
poi.setOffset(index.getFilePointer());
|
||||||
|
if(index.getName() != null) {
|
||||||
|
poi.setName(index.getName());
|
||||||
|
}
|
||||||
|
poi.setLeft(MapUtils.get31TileNumberX(index.getLeftLongitude()));
|
||||||
|
poi.setRight(MapUtils.get31TileNumberX(index.getRightLongitude()));
|
||||||
|
poi.setTop(MapUtils.get31TileNumberY(index.getTopLatitude()));
|
||||||
|
poi.setBottom(MapUtils.get31TileNumberY(index.getBottomLatitude()));
|
||||||
|
fileIndex.addPoiIndex(poi.build());
|
||||||
|
}
|
||||||
|
|
||||||
|
for(TransportIndex index : reader.getTransportIndexes()) {
|
||||||
|
TransportPart.Builder transport = OsmandIndex.TransportPart.newBuilder();
|
||||||
|
transport.setSize(index.getLength());
|
||||||
|
transport.setOffset(index.getFilePointer());
|
||||||
|
if(index.getName() != null) {
|
||||||
|
transport.setName(index.getName());
|
||||||
|
}
|
||||||
|
transport.setLeft(index.getLeft());
|
||||||
|
transport.setRight(index.getRight());
|
||||||
|
transport.setTop(index.getTop());
|
||||||
|
transport.setBottom(index.getBottom());
|
||||||
|
transport.setStopsTableLength(index.stopsFileLength);
|
||||||
|
transport.setStopsTableOffset(index.stopsFileOffset);
|
||||||
|
transport.setStringTableLength(index.stringTable.length);
|
||||||
|
transport.setStringTableOffset(index.stringTable.fileOffset);
|
||||||
|
fileIndex.addTransportIndex(transport);
|
||||||
|
}
|
||||||
|
|
||||||
|
for(RouteRegion index : reader.getRoutingIndexes()) {
|
||||||
|
RoutingPart.Builder routing = OsmandIndex.RoutingPart.newBuilder();
|
||||||
|
routing.setSize(index.getLength());
|
||||||
|
routing.setOffset(index.getFilePointer());
|
||||||
|
if(index.getName() != null) {
|
||||||
|
routing.setName(index.getName());
|
||||||
|
}
|
||||||
|
for(RouteSubregion sub : index.getSubregions()) {
|
||||||
|
OsmandIndex.RoutingSubregion.Builder rpart = OsmandIndex.RoutingSubregion.newBuilder();
|
||||||
|
rpart.setSize(sub.length);
|
||||||
|
rpart.setOffset(sub.filePointer);
|
||||||
|
rpart.setLeft(sub.left);
|
||||||
|
rpart.setRight(sub.right);
|
||||||
|
rpart.setTop(sub.top);
|
||||||
|
rpart.setBottom(sub.bottom);
|
||||||
|
rpart.setShifToData(sub.shiftToData);
|
||||||
|
routing.addSubregions(rpart);
|
||||||
|
}
|
||||||
|
fileIndex.addRoutingIndex(routing);
|
||||||
|
}
|
||||||
|
|
||||||
|
storedIndexBuilder.addFileIndex(fileIndex);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public BinaryMapIndexReader getReader(File f) throws IOException {
|
||||||
|
RandomAccessFile mf = new RandomAccessFile(f, "r");
|
||||||
|
FileIndex found = null;
|
||||||
|
if (storedIndex != null) {
|
||||||
|
for (int i = 0; i < storedIndex.getFileIndexCount(); i++) {
|
||||||
|
FileIndex fi = storedIndex.getFileIndex(i);
|
||||||
|
if (f.length() == fi.getSize() && f.getName().equals(fi.getFileName())) {
|
||||||
|
// f.lastModified() == fi.getDateModified()
|
||||||
|
found = fi;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
BinaryMapIndexReader reader = null;
|
||||||
|
if (found == null) {
|
||||||
|
long val = System.currentTimeMillis();
|
||||||
|
reader = new BinaryMapIndexReader(mf);
|
||||||
|
addToCache(reader, f);
|
||||||
|
if (log.isDebugEnabled()) {
|
||||||
|
log.debug("Initializing db " + f.getAbsolutePath() + " " + (System.currentTimeMillis() - val ) + "ms"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
reader = initFileIndex(found, mf);
|
||||||
|
}
|
||||||
|
return reader;
|
||||||
|
}
|
||||||
|
|
||||||
|
private BinaryMapIndexReader initFileIndex(FileIndex found, RandomAccessFile mf) throws IOException {
|
||||||
|
BinaryMapIndexReader reader = new BinaryMapIndexReader(mf, false);
|
||||||
|
reader.version = found.getVersion();
|
||||||
|
reader.dateCreated =found.getDateModified();
|
||||||
|
|
||||||
|
for(MapPart index : found.getMapIndexList()) {
|
||||||
|
MapIndex mi = new MapIndex();
|
||||||
|
mi.length = (int) index.getSize();
|
||||||
|
mi.filePointer = (int) index.getOffset();
|
||||||
|
mi.name = index.getName();
|
||||||
|
|
||||||
|
for(MapLevel mr : index.getLevelsList()) {
|
||||||
|
MapRoot root = new MapRoot();
|
||||||
|
root.length = (int) mr.getSize();
|
||||||
|
root.filePointer = (int) mr.getOffset();
|
||||||
|
root.left = mr.getLeft();
|
||||||
|
root.right = mr.getRight();
|
||||||
|
root.top = mr.getTop();
|
||||||
|
root.bottom = mr.getBottom();
|
||||||
|
root.minZoom = mr.getMinzoom();
|
||||||
|
root.maxZoom = mr.getMaxzoom();
|
||||||
|
mi.roots.add(root);
|
||||||
|
}
|
||||||
|
reader.mapIndexes.add(mi);
|
||||||
|
reader.indexes.add(mi);
|
||||||
|
reader.basemap = reader.basemap || mi.isBaseMap();
|
||||||
|
}
|
||||||
|
|
||||||
|
for(AddressPart index : found.getAddressIndexList()) {
|
||||||
|
AddressRegion mi = new AddressRegion();
|
||||||
|
mi.length = (int) index.getSize();
|
||||||
|
mi.filePointer = (int) index.getOffset();
|
||||||
|
mi.name = index.getName();
|
||||||
|
mi.enName = index.getNameEn();
|
||||||
|
mi.indexNameOffset = index.getIndexNameOffset();
|
||||||
|
for(CityBlock mr : index.getCitiesList() ) {
|
||||||
|
CitiesBlock cblock = new CitiesBlock();
|
||||||
|
cblock.length = (int) mr.getSize();
|
||||||
|
cblock.filePointer = (int) mr.getOffset();
|
||||||
|
cblock.type = mr.getType();
|
||||||
|
mi.cities.add(cblock);
|
||||||
|
}
|
||||||
|
reader.addressIndexes.add(mi);
|
||||||
|
reader.indexes.add(mi);
|
||||||
|
}
|
||||||
|
|
||||||
|
for(PoiPart index : found.getPoiIndexList()) {
|
||||||
|
PoiRegion mi = new PoiRegion();
|
||||||
|
mi.length = (int) index.getSize();
|
||||||
|
mi.filePointer = (int) index.getOffset();
|
||||||
|
mi.name = index.getName();
|
||||||
|
mi.leftLongitude = MapUtils.get31LongitudeX(index.getLeft());
|
||||||
|
mi.rightLongitude = MapUtils.get31LongitudeX(index.getRight());
|
||||||
|
mi.topLatitude =MapUtils.get31LatitudeY(index.getTop());
|
||||||
|
mi.bottomLatitude = MapUtils.get31LatitudeY(index.getBottom());
|
||||||
|
reader.poiIndexes.add(mi);
|
||||||
|
reader.indexes.add(mi);
|
||||||
|
}
|
||||||
|
|
||||||
|
for(TransportPart index : found.getTransportIndexList()) {
|
||||||
|
TransportIndex mi = new TransportIndex();
|
||||||
|
mi.length = (int) index.getSize();
|
||||||
|
mi.filePointer = (int) index.getOffset();
|
||||||
|
mi.name = index.getName();
|
||||||
|
mi.left = index.getLeft();
|
||||||
|
mi.right =index.getRight();
|
||||||
|
mi.top = index.getTop();
|
||||||
|
mi.bottom = index.getBottom();
|
||||||
|
mi.stopsFileLength = index.getStopsTableLength();
|
||||||
|
mi.stopsFileOffset = index.getStopsTableOffset();
|
||||||
|
mi.stringTable = new IndexStringTable();
|
||||||
|
mi.stringTable.fileOffset = index.getStringTableOffset();
|
||||||
|
mi.stringTable.length = index.getStringTableLength();
|
||||||
|
reader.transportIndexes.add(mi);
|
||||||
|
reader.indexes.add(mi);
|
||||||
|
}
|
||||||
|
|
||||||
|
for(RoutingPart index : found.getRoutingIndexList()) {
|
||||||
|
RouteRegion mi = new RouteRegion();
|
||||||
|
mi.length = (int) index.getSize();
|
||||||
|
mi.filePointer = (int) index.getOffset();
|
||||||
|
mi.name = index.getName();
|
||||||
|
|
||||||
|
for(RoutingSubregion mr : index.getSubregionsList()) {
|
||||||
|
RouteSubregion sub = new RouteSubregion(mi);
|
||||||
|
sub.length = (int) mr.getSize();
|
||||||
|
sub.filePointer = (int) mr.getOffset();
|
||||||
|
sub.left = mr.getLeft();
|
||||||
|
sub.right = mr.getRight();
|
||||||
|
sub.top = mr.getTop();
|
||||||
|
sub.bottom = mr.getBottom();
|
||||||
|
sub.shiftToData = mr.getShifToData();
|
||||||
|
mi.subregions.add(sub);
|
||||||
|
}
|
||||||
|
reader.routingIndexes.add(mi);
|
||||||
|
reader.indexes.add(mi);
|
||||||
|
}
|
||||||
|
|
||||||
|
return reader;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void readFromFile(File f, int version) throws IOException {
|
||||||
|
long time = System.currentTimeMillis();
|
||||||
|
FileInputStream is = new FileInputStream(f);
|
||||||
|
try {
|
||||||
|
storedIndex = OsmandIndex.OsmAndStoredIndex.newBuilder().mergeFrom(is).build();
|
||||||
|
hasChanged = false;
|
||||||
|
if(storedIndex.getVersion() != version){
|
||||||
|
storedIndex = null;
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
is.close();
|
||||||
|
}
|
||||||
|
log.info("Initialize cache " + (System.currentTimeMillis() - time));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void writeToFile(File f) throws IOException {
|
||||||
|
if (hasChanged) {
|
||||||
|
FileOutputStream outputStream = new FileOutputStream(f);
|
||||||
|
try {
|
||||||
|
storedIndexBuilder.build().writeTo(outputStream);
|
||||||
|
} finally {
|
||||||
|
outputStream.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
5749
DataExtractionOSM/src/net/osmand/binary/OsmandIndex.java
Normal file
5749
DataExtractionOSM/src/net/osmand/binary/OsmandIndex.java
Normal file
File diff suppressed because it is too large
Load diff
|
@ -110,7 +110,7 @@ public class MapClusterLayer implements MapPanelLayer {
|
||||||
for (File f : new File(DataExtractionSettings.getSettings().getBinaryFilesDir()).listFiles()) {
|
for (File f : new File(DataExtractionSettings.getSettings().getBinaryFilesDir()).listFiles()) {
|
||||||
if (f.getName().endsWith(".obf")) {
|
if (f.getName().endsWith(".obf")) {
|
||||||
RandomAccessFile raf = new RandomAccessFile(f, "r"); //$NON-NLS-1$ //$NON-NLS-2$
|
RandomAccessFile raf = new RandomAccessFile(f, "r"); //$NON-NLS-1$ //$NON-NLS-2$
|
||||||
rs.add(new BinaryMapIndexReader(raf, false));
|
rs.add(new BinaryMapIndexReader(raf));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
BinaryRoutePlanner router = new BinaryRoutePlanner(NativeSwingRendering.getDefaultFromSettings(),
|
BinaryRoutePlanner router = new BinaryRoutePlanner(NativeSwingRendering.getDefaultFromSettings(),
|
||||||
|
|
|
@ -594,7 +594,7 @@ public class MapRouterLayer implements MapPanelLayer {
|
||||||
int it = 0;
|
int it = 0;
|
||||||
for (File f : files) {
|
for (File f : files) {
|
||||||
RandomAccessFile raf = new RandomAccessFile(f, "r"); //$NON-NLS-1$ //$NON-NLS-2$
|
RandomAccessFile raf = new RandomAccessFile(f, "r"); //$NON-NLS-1$ //$NON-NLS-2$
|
||||||
rs[it++] = new BinaryMapIndexReader(raf, false);
|
rs[it++] = new BinaryMapIndexReader(raf);
|
||||||
}
|
}
|
||||||
String m = DataExtractionSettings.getSettings().getRouteMode();
|
String m = DataExtractionSettings.getSettings().getRouteMode();
|
||||||
String[] props = m.split("\\,");
|
String[] props = m.split("\\,");
|
||||||
|
|
105
DataExtractionOSM/src/osmand_index.proto
Normal file
105
DataExtractionOSM/src/osmand_index.proto
Normal file
|
@ -0,0 +1,105 @@
|
||||||
|
// Highly coupled with protobuf 2.3.0
|
||||||
|
option java_package = "net.osmand.binary";
|
||||||
|
//protoc --java_out=DataExtractionOSM/src DataExtractionOSM/src/osmand_index.proto
|
||||||
|
|
||||||
|
// C++ # cd OsmAnd
|
||||||
|
// # protoc --proto_path=../DataExtractionOSM/src --cpp_out=jni/osmand/proto ../DataExtractionOSM/src/osmand_odb.proto
|
||||||
|
|
||||||
|
message OsmAndStoredIndex {
|
||||||
|
required uint32 version = 1;
|
||||||
|
|
||||||
|
// System.currentTimeMillis()
|
||||||
|
required int64 dateCreated = 18;
|
||||||
|
|
||||||
|
repeated FileIndex fileIndex = 7;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
message FileIndex {
|
||||||
|
required int64 size = 1;
|
||||||
|
required int64 dateModified = 2;
|
||||||
|
required string fileName = 3;
|
||||||
|
required int32 version = 4;
|
||||||
|
repeated AddressPart addressIndex = 8;
|
||||||
|
repeated TransportPart transportIndex = 9;
|
||||||
|
repeated PoiPart poiIndex = 10;
|
||||||
|
repeated MapPart mapIndex = 11;
|
||||||
|
repeated RoutingPart routingIndex = 12;
|
||||||
|
}
|
||||||
|
|
||||||
|
message AddressPart {
|
||||||
|
required int64 size = 1;
|
||||||
|
required int64 offset = 2;
|
||||||
|
optional string name = 3;
|
||||||
|
optional string nameEn = 4;
|
||||||
|
optional int32 indexNameOffset = 5;
|
||||||
|
repeated CityBlock cities = 8;
|
||||||
|
}
|
||||||
|
message CityBlock {
|
||||||
|
required int64 size = 1;
|
||||||
|
required int64 offset = 2;
|
||||||
|
required int32 type = 3;
|
||||||
|
}
|
||||||
|
|
||||||
|
message PoiPart {
|
||||||
|
required int64 size = 1;
|
||||||
|
required int64 offset = 2;
|
||||||
|
optional string name = 3;
|
||||||
|
required int32 left = 4;
|
||||||
|
required int32 right = 5;
|
||||||
|
required int32 top = 6;
|
||||||
|
required int32 bottom = 7;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
message MapLevel {
|
||||||
|
required int64 size = 1;
|
||||||
|
required int64 offset = 2;
|
||||||
|
required int32 left = 4;
|
||||||
|
required int32 right = 5;
|
||||||
|
required int32 top = 6;
|
||||||
|
required int32 bottom = 7;
|
||||||
|
optional int32 minzoom = 8;
|
||||||
|
optional int32 maxzoom = 9;
|
||||||
|
}
|
||||||
|
|
||||||
|
message MapPart {
|
||||||
|
required int64 size = 1;
|
||||||
|
required int64 offset = 2;
|
||||||
|
optional string name = 3;
|
||||||
|
|
||||||
|
repeated MapLevel levels = 5;
|
||||||
|
}
|
||||||
|
|
||||||
|
message RoutingSubregion {
|
||||||
|
required int64 size = 1;
|
||||||
|
required int64 offset = 2;
|
||||||
|
|
||||||
|
required int32 left = 4;
|
||||||
|
required int32 right = 5;
|
||||||
|
required int32 top = 6;
|
||||||
|
required int32 bottom = 7;
|
||||||
|
required uint32 shifToData = 8;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
message RoutingPart {
|
||||||
|
required int64 size = 1;
|
||||||
|
required int64 offset = 2;
|
||||||
|
optional string name = 3;
|
||||||
|
|
||||||
|
repeated RoutingSubregion subregions =5;
|
||||||
|
}
|
||||||
|
message TransportPart {
|
||||||
|
required int64 size = 1;
|
||||||
|
required int64 offset = 2;
|
||||||
|
optional string name = 3;
|
||||||
|
optional int32 left = 4;
|
||||||
|
optional int32 right = 5;
|
||||||
|
optional int32 top = 6;
|
||||||
|
optional int32 bottom = 7;
|
||||||
|
optional uint32 stringTableOffset = 8;
|
||||||
|
optional uint32 stringTableLength = 9;
|
||||||
|
optional uint32 stopsTableOffset = 10;
|
||||||
|
optional uint32 stopsTableLength = 11;
|
||||||
|
}
|
|
@ -89,6 +89,11 @@
|
||||||
<arguments>
|
<arguments>
|
||||||
</arguments>
|
</arguments>
|
||||||
</buildCommand>
|
</buildCommand>
|
||||||
|
<buildCommand>
|
||||||
|
<name>org.eclipse.xtext.ui.shared.xtextBuilder</name>
|
||||||
|
<arguments>
|
||||||
|
</arguments>
|
||||||
|
</buildCommand>
|
||||||
</buildSpec>
|
</buildSpec>
|
||||||
<natures>
|
<natures>
|
||||||
<nature>com.android.ide.eclipse.adt.AndroidNature</nature>
|
<nature>com.android.ide.eclipse.adt.AndroidNature</nature>
|
||||||
|
@ -97,6 +102,7 @@
|
||||||
<nature>org.eclipse.cdt.core.ccnature</nature>
|
<nature>org.eclipse.cdt.core.ccnature</nature>
|
||||||
<nature>org.eclipse.cdt.managedbuilder.core.managedBuildNature</nature>
|
<nature>org.eclipse.cdt.managedbuilder.core.managedBuildNature</nature>
|
||||||
<nature>org.eclipse.cdt.managedbuilder.core.ScannerConfigNature</nature>
|
<nature>org.eclipse.cdt.managedbuilder.core.ScannerConfigNature</nature>
|
||||||
|
<nature>org.eclipse.xtext.ui.shared.xtextNature</nature>
|
||||||
</natures>
|
</natures>
|
||||||
<linkedResources>
|
<linkedResources>
|
||||||
<link>
|
<link>
|
||||||
|
|
|
@ -24,6 +24,7 @@ import net.osmand.LogUtil;
|
||||||
import net.osmand.ResultMatcher;
|
import net.osmand.ResultMatcher;
|
||||||
import net.osmand.Version;
|
import net.osmand.Version;
|
||||||
import net.osmand.binary.BinaryMapIndexReader;
|
import net.osmand.binary.BinaryMapIndexReader;
|
||||||
|
import net.osmand.binary.CachedOsmandIndexes;
|
||||||
import net.osmand.data.Amenity;
|
import net.osmand.data.Amenity;
|
||||||
import net.osmand.data.AmenityType;
|
import net.osmand.data.AmenityType;
|
||||||
import net.osmand.data.IndexConstants;
|
import net.osmand.data.IndexConstants;
|
||||||
|
@ -69,6 +70,7 @@ public class ResourceManager {
|
||||||
public static final String VOICE_PATH = APP_DIR + IndexConstants.VOICE_INDEX_DIR;
|
public static final String VOICE_PATH = APP_DIR + IndexConstants.VOICE_INDEX_DIR;
|
||||||
public static final String GPX_PATH = APP_DIR + "tracks";
|
public static final String GPX_PATH = APP_DIR + "tracks";
|
||||||
public static final String MAPS_PATH = APP_DIR;
|
public static final String MAPS_PATH = APP_DIR;
|
||||||
|
public static final String INDEXES_CACHE = APP_DIR + "ind.cache";
|
||||||
public static final String BACKUP_PATH = APP_DIR + "backup/";
|
public static final String BACKUP_PATH = APP_DIR + "backup/";
|
||||||
public static final String TILES_PATH = APP_DIR+"tiles/"; //$NON-NLS-1$
|
public static final String TILES_PATH = APP_DIR+"tiles/"; //$NON-NLS-1$
|
||||||
public static final String TEMP_SOURCE_TO_LOAD = "temp"; //$NON-NLS-1$
|
public static final String TEMP_SOURCE_TO_LOAD = "temp"; //$NON-NLS-1$
|
||||||
|
@ -526,12 +528,34 @@ public class ResourceManager {
|
||||||
file.mkdirs();
|
file.mkdirs();
|
||||||
List<String> warnings = new ArrayList<String>();
|
List<String> warnings = new ArrayList<String>();
|
||||||
renderer.clearAllResources();
|
renderer.clearAllResources();
|
||||||
|
CachedOsmandIndexes cachedOsmandIndexes = new CachedOsmandIndexes();
|
||||||
|
File indCache = context.getSettings().extendOsmandPath(INDEXES_CACHE);
|
||||||
|
if(indCache.exists()) {
|
||||||
|
try {
|
||||||
|
cachedOsmandIndexes.readFromFile(indCache, CachedOsmandIndexes.VERSION);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.error(e.getMessage(), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
if (file.exists() && file.canRead()) {
|
if (file.exists() && file.canRead()) {
|
||||||
|
long val = System.currentTimeMillis();
|
||||||
for (File f : file.listFiles()) {
|
for (File f : file.listFiles()) {
|
||||||
if (f.getName().endsWith(IndexConstants.BINARY_MAP_INDEX_EXT)) {
|
if (f.getName().endsWith(IndexConstants.BINARY_MAP_INDEX_EXT)) {
|
||||||
progress.startTask(context.getString(R.string.indexing_map) + " " + f.getName(), -1); //$NON-NLS-1$
|
progress.startTask(context.getString(R.string.indexing_map) + " " + f.getName(), -1); //$NON-NLS-1$
|
||||||
try {
|
try {
|
||||||
BinaryMapIndexReader index = renderer.initializeNewResource(progress, f);
|
BinaryMapIndexReader index = null;
|
||||||
|
try {
|
||||||
|
index = cachedOsmandIndexes.getReader(f);
|
||||||
|
|
||||||
|
if (index.getVersion() != IndexConstants.BINARY_MAP_VERSION) {
|
||||||
|
index = null;
|
||||||
|
}
|
||||||
|
if (index != null) {
|
||||||
|
renderer.initializeNewResource(progress, f, index);
|
||||||
|
}
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error(String.format("File %s could not be read", f.getName()), e);
|
||||||
|
}
|
||||||
if (index == null || (Version.isFreeVersion(context) && f.getName().contains("_wiki"))) {
|
if (index == null || (Version.isFreeVersion(context) && f.getName().contains("_wiki"))) {
|
||||||
warnings.add(MessageFormat.format(context.getString(R.string.version_index_is_not_supported), f.getName())); //$NON-NLS-1$
|
warnings.add(MessageFormat.format(context.getString(R.string.version_index_is_not_supported), f.getName())); //$NON-NLS-1$
|
||||||
} else {
|
} else {
|
||||||
|
@ -584,6 +608,14 @@ public class ResourceManager {
|
||||||
warnings.add(MessageFormat.format(context.getString(R.string.old_map_index_is_not_supported), f.getName())); //$NON-NLS-1$
|
warnings.add(MessageFormat.format(context.getString(R.string.old_map_index_is_not_supported), f.getName())); //$NON-NLS-1$
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
log.debug("All map files initialized " + (System.currentTimeMillis() - val) + " ms");
|
||||||
|
}
|
||||||
|
if(!indCache.exists() || indCache.canWrite()){
|
||||||
|
try {
|
||||||
|
cachedOsmandIndexes.writeToFile(indCache);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.error("Index file could not be written", e);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return warnings;
|
return warnings;
|
||||||
}
|
}
|
||||||
|
|
|
@ -355,7 +355,7 @@ public class LocalIndexHelper {
|
||||||
private void updateObfFileInformation(LocalIndexInfo info, File mapFile) {
|
private void updateObfFileInformation(LocalIndexInfo info, File mapFile) {
|
||||||
try {
|
try {
|
||||||
RandomAccessFile mf = new RandomAccessFile(mapFile, "r");
|
RandomAccessFile mf = new RandomAccessFile(mapFile, "r");
|
||||||
BinaryMapIndexReader reader = new BinaryMapIndexReader(mf, false);
|
BinaryMapIndexReader reader = new BinaryMapIndexReader(mf);
|
||||||
|
|
||||||
info.setNotSupported(reader.getVersion() != IndexConstants.BINARY_MAP_VERSION);
|
info.setNotSupported(reader.getVersion() != IndexConstants.BINARY_MAP_VERSION);
|
||||||
List<BinaryIndexPart> indexes = reader.getIndexes();
|
List<BinaryIndexPart> indexes = reader.getIndexes();
|
||||||
|
|
|
@ -104,58 +104,25 @@ public class MapRenderRepositories {
|
||||||
return context;
|
return context;
|
||||||
}
|
}
|
||||||
|
|
||||||
public BinaryMapIndexReader initializeNewResource(final IProgress progress, File file) {
|
public void initializeNewResource(final IProgress progress, File file, BinaryMapIndexReader reader) {
|
||||||
long start = System.currentTimeMillis();
|
long start = System.currentTimeMillis();
|
||||||
if (files.containsKey(file.getAbsolutePath())) {
|
if (files.containsKey(file.getAbsolutePath())) {
|
||||||
closeConnection(files.get(file.getAbsolutePath()), file.getAbsolutePath());
|
closeConnection(files.get(file.getAbsolutePath()), file.getAbsolutePath());
|
||||||
|
|
||||||
}
|
}
|
||||||
RandomAccessFile raf = null;
|
|
||||||
BinaryMapIndexReader reader = null;
|
|
||||||
try {
|
|
||||||
raf = new RandomAccessFile(file, "r"); //$NON-NLS-1$
|
|
||||||
reader = new BinaryMapIndexReader(raf);
|
|
||||||
if (reader.getVersion() != IndexConstants.BINARY_MAP_VERSION) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
files.put(file.getAbsolutePath(), reader);
|
files.put(file.getAbsolutePath(), reader);
|
||||||
|
|
||||||
} catch (IOException e) {
|
|
||||||
log.error("No connection or unsupported version", e); //$NON-NLS-1$
|
|
||||||
if (raf != null) {
|
|
||||||
try {
|
|
||||||
raf.close();
|
|
||||||
} catch (IOException e1) {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
} catch (OutOfMemoryError oome) {
|
|
||||||
if (raf != null) {
|
|
||||||
try {
|
|
||||||
raf.close();
|
|
||||||
} catch (IOException e1) {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
throw oome;
|
|
||||||
}
|
|
||||||
long val = System.currentTimeMillis();
|
|
||||||
if (log.isDebugEnabled()) {
|
|
||||||
log.debug("Initializing db " + file.getAbsolutePath() + " " + (val - start) + "ms"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
|
|
||||||
}
|
|
||||||
NativeOsmandLibrary nativeLib = prefs.NATIVE_RENDERING.get() ? NativeOsmandLibrary.getLoadedLibrary() : null;
|
NativeOsmandLibrary nativeLib = prefs.NATIVE_RENDERING.get() ? NativeOsmandLibrary.getLoadedLibrary() : null;
|
||||||
if (nativeLib != null) {
|
if (nativeLib != null) {
|
||||||
start = val;
|
|
||||||
if (!nativeLib.initMapFile(file.getAbsolutePath())) {
|
if (!nativeLib.initMapFile(file.getAbsolutePath())) {
|
||||||
log.debug("Initializing native db " + file.getAbsolutePath() + " failed!"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
|
log.debug("Initializing native db " + file.getAbsolutePath() + " failed!"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
|
||||||
} else {
|
} else {
|
||||||
nativeFiles.add(file.getAbsolutePath());
|
nativeFiles.add(file.getAbsolutePath());
|
||||||
val = System.currentTimeMillis();
|
long val = System.currentTimeMillis();
|
||||||
if (log.isDebugEnabled()) {
|
if (log.isDebugEnabled()) {
|
||||||
log.debug("Initializing native db " + file.getAbsolutePath() + " " + (val - start) + "ms"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
|
log.debug("Initializing native db " + file.getAbsolutePath() + " " + (val - start) + "ms"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return reader;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public RotatedTileBox getBitmapLocation() {
|
public RotatedTileBox getBitmapLocation() {
|
||||||
|
|
Loading…
Reference in a new issue