Android-builders-club/OsmAnd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Speedup route calculation and optimize memory

Browse source
This commit is contained in:
Victor Shcherb 2012-09-28 18:54:57 +02:00
parent f8673d5d98
commit ccb1bcdfdb
4 changed files with 108 additions and 173 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
DataExtractionOSM/src/net/osmand/router/BinaryRoutePlanner.java
View file

@ -181,7 +181,7 @@ public class BinaryRoutePlanner {
} }
if (RoutingContext.SHOW_GC_SIZE) { if (RoutingContext.SHOW_GC_SIZE) {
int sz = ctx.global.size; int sz = ctx.global.size;
System.out.println("Subregion size " + ctx.subregionTiles.size() + " " + " tiles " + ctx.tiles.size()); log.warn("Subregion size " + ctx.subregionTiles.size() + " " + " tiles " + ctx.indexedSubregions.size());
ctx.runGCUsedMemory(); ctx.runGCUsedMemory();
long h1 = ctx.runGCUsedMemory(); long h1 = ctx.runGCUsedMemory();
ctx.unloadAllData(); ctx.unloadAllData();

2
DataExtractionOSM/src/net/osmand/router/RoutingConfiguration.java
View file

@ -24,7 +24,7 @@ public class RoutingConfiguration {
// 1.1 tile load parameters (should not affect routing) // 1.1 tile load parameters (should not affect routing)
public int ZOOM_TO_LOAD_TILES = 13; // 12?, 14? public int ZOOM_TO_LOAD_TILES = 16;
public int memoryLimitation; public int memoryLimitation;
// 1.2 Dynamic road prioritizing (heuristic) // 1.2 Dynamic road prioritizing (heuristic)

274
DataExtractionOSM/src/net/osmand/router/RoutingContext.java
View file

@ -1,10 +1,12 @@
package net.osmand.router; package net.osmand.router;
import gnu.trove.iterator.TIntObjectIterator; import gnu.trove.iterator.TIntObjectIterator;
import gnu.trove.iterator.TLongIterator;
import gnu.trove.map.TLongObjectMap; import gnu.trove.map.TLongObjectMap;
import gnu.trove.map.hash.TIntObjectHashMap; import gnu.trove.map.hash.TIntObjectHashMap;
import gnu.trove.map.hash.TLongObjectHashMap; import gnu.trove.map.hash.TLongObjectHashMap;
import gnu.trove.set.TLongSet; import gnu.trove.set.TLongSet;
import gnu.trove.set.hash.TIntHashSet;
import gnu.trove.set.hash.TLongHashSet; import gnu.trove.set.hash.TLongHashSet;
import java.io.IOException; import java.io.IOException;
@ -51,9 +53,10 @@ public class RoutingContext {
public List<RouteSegmentResult> previouslyCalculatedRoute; public List<RouteSegmentResult> previouslyCalculatedRoute;
// 2. Routing memory cache (big objects) // 2. Routing memory cache (big objects)
TIntObjectHashMap<RoutingTile> tiles = new TIntObjectHashMap<RoutingContext.RoutingTile>(); TLongObjectHashMap<List<RoutingSubregionTile>> indexedSubregions = new TLongObjectHashMap<List<RoutingSubregionTile>>();
// Map<Integer, RoutingTile> tiles = new LinkedHashMap<Integer, RoutingContext.RoutingTile>(); TLongObjectHashMap<List<RouteDataObject>> tileRoutes = new TLongObjectHashMap<List<RouteDataObject>>();
// need to be array list
// need to be array list and keep sorted Another option to use hashmap but it is more memory expensive
List<RoutingSubregionTile> subregionTiles = new ArrayList<RoutingSubregionTile>(); List<RoutingSubregionTile> subregionTiles = new ArrayList<RoutingSubregionTile>();
// 3. Warm object caches // 3. Warm object caches
@ -87,8 +90,6 @@ public class RoutingContext {
public TileStatistics global = new TileStatistics(); public TileStatistics global = new TileStatistics();
public static final boolean SHOW_GC_SIZE = false; public static final boolean SHOW_GC_SIZE = false;
public RoutingContext(RoutingContext cp) { public RoutingContext(RoutingContext cp) {
this.config = cp.config; this.config = cp.config;
this.map.putAll(cp.map); this.map.putAll(cp.map);
@ -218,7 +219,13 @@ public class RoutingContext {
if(!getRouter().acceptLine(o)){ if(!getRouter().acceptLine(o)){
return; return;
} }
getRoutingTile(x31, y31, false).registerRouteDataObject(o); long tileId = getRoutingTile(x31, y31, false);
List<RouteDataObject> routes = tileRoutes.get(tileId);
if(routes == null){
routes = new ArrayList<RouteDataObject>();
tileRoutes.put(tileId, routes);
}
routes.add(o);
} }
public void unloadAllData() { public void unloadAllData() {
@ -236,11 +243,13 @@ public class RoutingContext {
} }
} }
subregionTiles.clear(); subregionTiles.clear();
tiles.clear(); tileRoutes.clear();
indexedSubregions.clear();
} }
private int searchSubregionTile(RouteSubregion subregion){ private int searchSubregionTile(RouteSubregion subregion){
RoutingSubregionTile key = new RoutingSubregionTile(subregion); RoutingSubregionTile key = new RoutingSubregionTile(subregion);
long now = System.nanoTime();
int ind = Collections.binarySearch(subregionTiles, key, new Comparator<RoutingSubregionTile>() { int ind = Collections.binarySearch(subregionTiles, key, new Comparator<RoutingSubregionTile>() {
@Override @Override
public int compare(RoutingSubregionTile o1, RoutingSubregionTile o2) { public int compare(RoutingSubregionTile o1, RoutingSubregionTile o2) {
@ -261,14 +270,38 @@ public class RoutingContext {
} }
} }
} }
timeToLoadHeaders += (System.nanoTime() - now);
return ind; return ind;
} }
public RouteSegment loadRouteSegment(int x31, int y31) { public RouteSegment loadRouteSegment(int x31, int y31) {
final RoutingTile tile = getRoutingTile(x31, y31, true); long tileId = getRoutingTile(x31, y31, true);
return tile.getSegment(x31, y31, this); TLongObjectHashMap<RouteDataObject> excludeDuplications = new TLongObjectHashMap<RouteDataObject>();
RouteSegment original = null;
if (tileRoutes.containsKey(tileId)) {
List<RouteDataObject> routes = tileRoutes.get(tileId);
if (routes != null) {
for (RouteDataObject ro : routes) {
for (int i = 0; i < ro.pointsX.length; i++) {
if (ro.getPoint31XTile(i) == x31 && ro.getPoint31YTile(i) == y31) {
excludeDuplications.put(ro.id, ro);
RouteSegment segment = new RouteSegment(ro, i);
segment.next = original;
original = segment;
}
}
}
}
}
List<RoutingSubregionTile> subregions = indexedSubregions.get(tileId);
if (subregions != null) {
for (RoutingSubregionTile rs : subregions) {
original = rs.loadRouteSegment(x31, y31, this, excludeDuplications, original);
}
}
return original;
} }
private void loadSubregionTile(final RoutingSubregionTile ts) { private void loadSubregionTile(final RoutingSubregionTile ts) {
@ -331,13 +364,14 @@ public class RoutingContext {
global.size += ts.tileStatistics.size; global.size += ts.tileStatistics.size;
} }
private void loadTileHeaders(RoutingTile tile) { private List<RoutingSubregionTile> loadTileHeaders(final int x31, final int y31) {
final int zoomToLoad = 31 - tile.getZoom(); final int zoomToLoad = 31 - config.ZOOM_TO_LOAD_TILES;
final int tileX = tile.getTileX(); int tileX = x31 >> zoomToLoad;
final int tileY = tile.getTileY(); int tileY = y31 >> zoomToLoad;
SearchRequest<RouteDataObject> request = BinaryMapIndexReader.buildSearchRouteRequest(tileX << zoomToLoad, SearchRequest<RouteDataObject> request = BinaryMapIndexReader.buildSearchRouteRequest(tileX << zoomToLoad,
(tileX + 1) << zoomToLoad, tileY << zoomToLoad, (tileY + 1) << zoomToLoad, null); (tileX + 1) << zoomToLoad, tileY << zoomToLoad, (tileY + 1) << zoomToLoad, null);
List<RoutingSubregionTile> collection = null;
for (Entry<BinaryMapIndexReader, List<RouteSubregion>> r : map.entrySet()) { for (Entry<BinaryMapIndexReader, List<RouteSubregion>> r : map.entrySet()) {
if(nativeLib == null) { if(nativeLib == null) {
try { try {
@ -354,7 +388,10 @@ public class RoutingContext {
} else { } else {
found = subregionTiles.get(ind); found = subregionTiles.get(ind);
} }
tile.searchSubregionAndAdd(sr, found); if(collection == null) {
collection = new ArrayList<RoutingContext.RoutingSubregionTile>(4);
}
collection.add(found);
} }
timeToLoadHeaders += (System.nanoTime() - now); timeToLoadHeaders += (System.nanoTime() - now);
} }
@ -366,43 +403,31 @@ public class RoutingContext {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
} }
tile.setHeadersLoaded(); return collection;
} }
public void loadTileData(int x31, int y31, int zoomAround, final List<RouteDataObject> toFillIn) { public void loadTileData(int x31, int y31, int zoomAround, final List<RouteDataObject> toFillIn) {
int coordinatesShift = (1 << (31 - zoomAround)); int coordinatesShift = (1 << (31 - zoomAround));
// put in map to avoid duplicate map loading // put in map to avoid duplicate map loading
TIntObjectHashMap<RoutingTile> ts = new TIntObjectHashMap<RoutingContext.RoutingTile>(); // TIntObjectHashMap<RoutingTile> ts = new TIntObjectHashMap<RoutingContext.RoutingTile>();
TLongHashSet ts = new TLongHashSet();
long now = System.nanoTime(); long now = System.nanoTime();
RoutingTile rt = getRoutingTile(x31 - coordinatesShift, y31 - coordinatesShift, true); ts.add(getRoutingTile(x31 - coordinatesShift, y31 - coordinatesShift, true));
ts.put(rt.getId(), rt); ts.add(getRoutingTile(x31 + coordinatesShift, y31 - coordinatesShift, true));
rt = getRoutingTile(x31 + coordinatesShift, y31 - coordinatesShift, true); ts.add(getRoutingTile(x31 - coordinatesShift, y31 + coordinatesShift, true));
ts.put(rt.getId(), rt); ts.add(getRoutingTile(x31 + coordinatesShift, y31 + coordinatesShift, true));
rt = getRoutingTile(x31 - coordinatesShift, y31 + coordinatesShift, true); TLongIterator it = ts.iterator();
ts.put(rt.getId(), rt); while(it.hasNext()){
rt = getRoutingTile(x31 + coordinatesShift, y31 + coordinatesShift, true); getAllObjects(it.next(), toFillIn);
ts.put(rt.getId(), rt);
Iterator<RoutingTile> it = ts.valueCollection().iterator();
while (it.hasNext()) {
RoutingTile tl = it.next();
tl.getAllObjects(toFillIn, this);
} }
timeToFindInitialSegments += (System.nanoTime() - now); timeToFindInitialSegments += (System.nanoTime() - now);
} }
private RoutingTile getRoutingTile(int x31, int y31, boolean load){ private long getRoutingTile(int x31, int y31, boolean load){
// long now = System.nanoTime(); // long now = System.nanoTime();
int xloc = x31 >> (31 - config.ZOOM_TO_LOAD_TILES); long xloc = x31 >> (31 - config.ZOOM_TO_LOAD_TILES);
int yloc = y31 >> (31 - config.ZOOM_TO_LOAD_TILES); long yloc = y31 >> (31 - config.ZOOM_TO_LOAD_TILES);
int l = (xloc << config.ZOOM_TO_LOAD_TILES) + yloc; long tileId = (xloc << config.ZOOM_TO_LOAD_TILES) + yloc;
RoutingTile tile = tiles.get(l);
if(tile == null) {
tile = new RoutingTile(xloc, yloc, config.ZOOM_TO_LOAD_TILES);
tiles.put(l, tile);
}
if (load) { if (load) {
if (getCurrentEstimatedSize() > 0.95 * config.memoryLimitation) { if (getCurrentEstimatedSize() > 0.95 * config.memoryLimitation) {
int sz1 = getCurrentEstimatedSize(); int sz1 = getCurrentEstimatedSize();
@ -422,25 +447,30 @@ public class RoutingContext {
log.warn("Used after " + h2 / mb + " of " + Runtime.getRuntime().totalMemory() / mb + " max " log.warn("Used after " + h2 / mb + " of " + Runtime.getRuntime().totalMemory() / mb + " max "
+ Runtime.getRuntime().maxMemory() / mb); + Runtime.getRuntime().maxMemory() / mb);
} else { } else {
// float mb = (1 << 20); // float mb = (1 << 20);
// int sz2 = getCurrentEstimatedSize(); // int sz2 = getCurrentEstimatedSize();
// log.warn("Unload tiles : occupied before " + sz1 / mb + " Mb - now " + sz2 / mb + "MB "); // log.warn("Unload tiles : occupied before " + sz1 / mb + " Mb - now " + sz2 / mb + "MB ");
// log.warn("Memory free " + Runtime.getRuntime().freeMemory() / mb + " of " + Runtime.getRuntime().totalMemory() / mb // log.warn("Memory free " + Runtime.getRuntime().freeMemory() / mb + " of " + Runtime.getRuntime().totalMemory() / mb
// + " max " + Runtime.getRuntime().maxMemory() / mb); // + " max " + Runtime.getRuntime().maxMemory() / mb);
} }
} }
if (!tile.isHeadersLoaded()) { if (!indexedSubregions.containsKey(tileId)) {
loadTileHeaders(tile); List<RoutingSubregionTile> collection = loadTileHeaders(x31, y31);
indexedSubregions.put(tileId, collection);
} }
for (RoutingSubregionTile ts : tile.subregions) { List<RoutingSubregionTile> subregions = indexedSubregions.get(tileId);
if (!ts.isLoaded()) { if (subregions != null) {
loadSubregionTile(ts); for (RoutingSubregionTile ts : subregions) {
if (!ts.isLoaded()) {
loadSubregionTile(ts);
}
} }
} }
} }
// timeToLoad += (System.nanoTime() - now); // timeToLoad += (System.nanoTime() - now);
return tile; return tileId;
} }
public boolean checkIfMemoryLimitCritical() { public boolean checkIfMemoryLimitCritical() {
@ -482,7 +512,6 @@ public class RoutingContext {
unloadedTiles ++; unloadedTiles ++;
global.size -= unload.tileStatistics.size; global.size -= unload.tileStatistics.size;
// tile could be cleaned from routing tiles and deleted from whole list // tile could be cleaned from routing tiles and deleted from whole list
// List<RoutingTile> ts = getRoutingTiles(tile.subregion.left, tile.subregion.top, tile.subregion.right, tile.subregion.bottom);
} }
for(RoutingSubregionTile t : subregionTiles) { for(RoutingSubregionTile t : subregionTiles) {
@ -490,6 +519,29 @@ public class RoutingContext {
} }
} }
private void getAllObjects(long tileId, final List<RouteDataObject> toFillIn) {
TLongObjectHashMap<RouteDataObject> excludeDuplications = new TLongObjectHashMap<RouteDataObject>();
if (tileRoutes.containsKey(tileId)) {
List<RouteDataObject> routes = tileRoutes.get(tileId);
if (routes != null) {
for (RouteDataObject ro : routes) {
if (!excludeDuplications.contains(ro.id)) {
excludeDuplications.put(ro.id, ro);
toFillIn.add(ro);
}
}
}
}
List<RoutingSubregionTile> subregions = indexedSubregions.get(tileId);
if (subregions != null) {
for (RoutingSubregionTile rs : subregions) {
rs.loadAllObjects(toFillIn, this, excludeDuplications);
}
}
}
protected static long runGCUsedMemory() { protected static long runGCUsedMemory() {
Runtime runtime = Runtime.getRuntime(); Runtime runtime = Runtime.getRuntime();
long usedMem1 = runtime.totalMemory() - runtime.freeMemory(); long usedMem1 = runtime.totalMemory() - runtime.freeMemory();
@ -661,122 +713,6 @@ public class RoutingContext {
} }
} }
public static class RoutingTile {
private int tileX;
private int tileY;
private int zoom;
private int isLoaded = 0;
private List<RouteDataObject> routes = null;
private List<RoutingSubregionTile> subregions = new ArrayList<RoutingContext.RoutingSubregionTile>(4);
public RoutingTile(int tileX, int tileY, int zoom) {
this.tileX = tileX;
this.tileY = tileY;
this.zoom = zoom;
}
public boolean isHeadersLoaded(){
return isLoaded > 0;
}
public void setHeadersLoaded(){
isLoaded = 1;
}
public boolean isEmpty(){
return (routes == null || routes.isEmpty()) && subregions.isEmpty();
}
public RoutingSubregionTile searchSubregionAndAdd(RouteSubregion s, RoutingSubregionTile rt) {
for(int i=0; i<subregions.size(); i++) {
RoutingSubregionTile ts = subregions.get(i);
if(ts.subregion == s){
if(rt != null) {
subregions.set(i, rt);
return rt;
}
return ts;
}
}
if(rt != null) {
subregions.add(rt);
return rt;
}
return null;
}
public void getAllObjects(final List<RouteDataObject> toFillIn, RoutingContext ctx) {
TLongObjectHashMap<RouteDataObject> excludeDuplications = new TLongObjectHashMap<RouteDataObject>();
if (routes != null) {
for (RouteDataObject ro : routes) {
if (!excludeDuplications.contains(ro.id)) {
excludeDuplications.put(ro.id, ro);
toFillIn.add(ro);
}
}
}
for (RoutingSubregionTile rs : subregions) {
rs.loadAllObjects(toFillIn, ctx, excludeDuplications);
}
}
public RouteSegment getSegment(int x31, int y31, RoutingContext ctx) {
TLongObjectHashMap<RouteDataObject> excludeDuplications = new TLongObjectHashMap<RouteDataObject>();
RouteSegment original = null;
if (routes != null) {
for (RouteDataObject ro : routes) {
for (int i = 0; i < ro.pointsX.length; i++) {
if (ro.getPoint31XTile(i) == x31 && ro.getPoint31YTile(i) == y31) {
excludeDuplications.put(ro.id, ro);
RouteSegment segment = new RouteSegment(ro, i);
segment.next = original;
original = segment;
}
}
}
}
for (RoutingSubregionTile rs : subregions) {
original = rs.loadRouteSegment(x31, y31, ctx, excludeDuplications, original);
}
return original;
}
public void registerRouteDataObject(RouteDataObject r) {
if(routes == null){
routes = new ArrayList<RouteDataObject>();
}
routes.add(r);
}
public int getId(){
return (tileX << zoom) + tileY;
}
public int getZoom() {
return zoom;
}
public int getTileX() {
return tileX;
}
public int getTileY() {
return tileY;
}
public boolean checkContains(int x31, int y31) {
return tileX == (x31 >> (31 - zoom)) && tileY == (y31 >> (31 - zoom));
}
@Override
public String toString() {
return "Tile " + tileX + "/" + tileY ;
}
}
protected static class TileStatistics { protected static class TileStatistics {
public int size = 0; public int size = 0;
public int allRoutes = 0; public int allRoutes = 0;

3
DataExtractionOSM/src/net/osmand/router/routing.xml
View file

@ -5,8 +5,7 @@
<attribute name="heuristicCoefficient" value="1.0" /> <attribute name="heuristicCoefficient" value="1.0" />
<!-- 1.1 tile load parameters (should not affect routing) --> <!-- 1.1 tile load parameters (should not affect routing) -->
<!-- by default 13 --> <!-- by default 16 -->
<!-- OPTIMAL -->
<attribute name="zoomToLoadTiles" value="16" /> <attribute name="zoomToLoadTiles" value="16" />
<!-- by default it is 15 --> <!-- by default it is 15 -->
<attribute name="memoryLimitInMB" value="50" /> <attribute name="memoryLimitInMB" value="50" />