diff --git a/DataExtractionOSM/src/net/osmand/data/DataTileManager.java b/DataExtractionOSM/src/net/osmand/data/DataTileManager.java index a6ceae6a6d..120eca56e4 100644 --- a/DataExtractionOSM/src/net/osmand/data/DataTileManager.java +++ b/DataExtractionOSM/src/net/osmand/data/DataTileManager.java @@ -1,26 +1,22 @@ package net.osmand.data; +import gnu.trove.map.hash.TLongObjectHashMap; + import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; import net.osmand.osm.MapUtils; - /** * * @param - object to store in that manager */ public class DataTileManager { - private int zoom = 15; + private final int zoom; - /** - * map for objects stores as 'xTile_yTile' -> List - */ - private Map> objects = new HashMap>(); + private TLongObjectHashMap> objects = new TLongObjectHashMap>(); public DataTileManager(){ zoom = 15; @@ -33,30 +29,18 @@ public class DataTileManager { public int getZoom() { return zoom; } - - public void setZoom(int zoom) { - // it is required to reindex all stored objects - if(!isEmpty()){ - throw new UnsupportedOperationException(); - } - this.zoom = zoom; - } + public boolean isEmpty(){ - for(String s : objects.keySet()){ - if(!objects.get(s).isEmpty()){ - return false; - } - } - return true; + return getObjectsCount() == 0; } + @SuppressWarnings("rawtypes") public int getObjectsCount(){ int x = 0; - for(String s : objects.keySet()){ - x += objects.get(s).size(); + for(List s : objects.valueCollection()){ + x += s.size(); } - return x; } @@ -66,10 +50,11 @@ public class DataTileManager { } } + @SuppressWarnings({ "rawtypes", "unchecked" }) public List getAllObjects(){ List l = new ArrayList(); - for(String s : objects.keySet()){ - l.addAll(objects.get(s)); + for(List s : objects.valueCollection()){ + l.addAll(s); } return l; } @@ -146,27 +131,29 @@ public class DataTileManager { return result; } - private String evTile(int tileX, int tileY){ - return tileX +"_"+tileY; //$NON-NLS-1$ + private long evTile(int tileX, int tileY){ + long tx = tileX; + long ty = tileY; + return ((tx) << zoom) + ty; } - public String evaluateTile(double latitude, double longitude){ + public long evaluateTile(double latitude, double longitude){ int tileX = (int) MapUtils.getTileNumberX(zoom, longitude); int tileY = (int) MapUtils.getTileNumberY(zoom, latitude); return evTile(tileX, tileY); } public void unregisterObject(double latitude, double longitude, T object){ - String tile = evaluateTile(latitude, longitude); + long tile = evaluateTile(latitude, longitude); if(objects.containsKey(tile)){ objects.get(tile).remove(object); } } - public String registerObject(double latitude, double longitude, T object){ - String tile = evaluateTile(latitude, longitude); + public long registerObject(double latitude, double longitude, T object){ + long tile = evaluateTile(latitude, longitude); if(!objects.containsKey(tile)){ objects.put(tile, new ArrayList()); } diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/DBDialect.java b/DataExtractionOSM/src/net/osmand/data/preparation/DBDialect.java index e15103af87..1dc8f43279 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/DBDialect.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/DBDialect.java @@ -20,7 +20,8 @@ public enum DBDialect { DERBY, H2, NOSQL, - SQLITE; + SQLITE, + SQLITE_IN_MEMORY; public void deleteTableIfExists(String table, Statement stat) throws SQLException { if(this == DERBY){ @@ -87,14 +88,15 @@ public enum DBDialect { throw new SQLException(status.ToString()); } return dbAccessor; - } else if (DBDialect.SQLITE == this) { + } else if (DBDialect.SQLITE == this || DBDialect.SQLITE_IN_MEMORY == this) { try { Class.forName("org.sqlite.JDBC"); } catch (ClassNotFoundException e) { log.error("Illegal configuration", e); throw new IllegalStateException(e); } - Connection connection = DriverManager.getConnection("jdbc:sqlite:" + fileName); + Connection connection = DriverManager.getConnection("jdbc:sqlite:" + (DBDialect.SQLITE_IN_MEMORY == this? ":memory:": + fileName)); Statement statement = connection.createStatement(); statement.executeUpdate("PRAGMA synchronous = 0"); //no journaling, saves some I/O access, but database can go corrupt diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java b/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java index 72dd2634a2..82e6eec4ec 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java @@ -692,7 +692,7 @@ public class IndexCreator { deleteDatabaseIndexes); indexMapCreator.commitAndCloseFiles(getRTreeMapIndexNonPackFileName(), getRTreeMapIndexPackFileName(), deleteDatabaseIndexes); - indexRouteCreator.commitAndCloseFiles(getRTreeRouteIndexPackFileName(), getRTreeRouteIndexPackFileName(), + indexRouteCreator.commitAndCloseFiles(getRTreeRouteIndexNonPackFileName(), getRTreeRouteIndexPackFileName(), deleteDatabaseIndexes); if (mapConnection != null) { diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/IndexRouteCreator.java b/DataExtractionOSM/src/net/osmand/data/preparation/IndexRouteCreator.java index f221469e3b..cf4b4aad40 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/IndexRouteCreator.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/IndexRouteCreator.java @@ -238,17 +238,17 @@ public class IndexRouteCreator extends AbstractIndexPartCreator { if (routeTree != null) { RandomAccessFile file = routeTree.getFileHdr().getFile(); file.close(); - if (rTreeMapIndexNonPackFileName != null) { - File f = new File(rTreeMapIndexNonPackFileName); - if (f.exists() && deleteDatabaseIndexes) { - f.delete(); - } + } + if (rTreeMapIndexNonPackFileName != null) { + File f = new File(rTreeMapIndexNonPackFileName); + if (f.exists() && deleteDatabaseIndexes) { + f.delete(); } - if (rTreeMapIndexPackFileName != null) { - File f = new File(rTreeMapIndexPackFileName); - if (f.exists() && deleteDatabaseIndexes) { - f.delete(); - } + } + if (rTreeMapIndexPackFileName != null) { + File f = new File(rTreeMapIndexPackFileName); + if (f.exists() && deleteDatabaseIndexes) { + f.delete(); } } closeAllPreparedStatements(); diff --git a/DataExtractionOSM/src/net/osmand/osm/util/MinskTransReader.java b/DataExtractionOSM/src/net/osmand/osm/util/MinskTransReader.java index cf2aee8739..d12714b583 100644 --- a/DataExtractionOSM/src/net/osmand/osm/util/MinskTransReader.java +++ b/DataExtractionOSM/src/net/osmand/osm/util/MinskTransReader.java @@ -224,8 +224,7 @@ public class MinskTransReader { OsmBaseStorage storage = new OsmBaseStorage(); final Map definedRoutes = new HashMap(); - final DataTileManager busStops = new DataTileManager(); - busStops.setZoom(17); + final DataTileManager busStops = new DataTileManager(17); storage.getFilters().add(new IOsmStorageFilter(){ @Override diff --git a/DataExtractionOSM/src/net/osmand/swing/MapClusterLayer.java b/DataExtractionOSM/src/net/osmand/swing/MapClusterLayer.java index c4a409e0da..b1cf11f904 100644 --- a/DataExtractionOSM/src/net/osmand/swing/MapClusterLayer.java +++ b/DataExtractionOSM/src/net/osmand/swing/MapClusterLayer.java @@ -87,8 +87,7 @@ public class MapClusterLayer implements MapPanelLayer { try { List ways = clustering(latitude, longitude); if (!ANIMATE_CLUSTERING) { - DataTileManager points = new DataTileManager(); - points.setZoom(11); + DataTileManager points = new DataTileManager(11); for (RouteSegment s : ways) { Way w = new Way(-1); for (int i = 0; i < s.getRoad().getPointsLength(); i++) { @@ -132,8 +131,7 @@ public class MapClusterLayer implements MapPanelLayer { log.info("ROAD TO START " + highway + " " + //road.getName() + " " + road.id); } - final DataTileManager points = new DataTileManager(); - points.setZoom(11); + final DataTileManager points = new DataTileManager(11); map.setPoints(points); ctx.setVisitor(new RouteSegmentVisitor() { diff --git a/DataExtractionOSM/src/net/osmand/swing/MapRouterLayer.java b/DataExtractionOSM/src/net/osmand/swing/MapRouterLayer.java index da25b0a0e3..6aabcd8aeb 100644 --- a/DataExtractionOSM/src/net/osmand/swing/MapRouterLayer.java +++ b/DataExtractionOSM/src/net/osmand/swing/MapRouterLayer.java @@ -176,8 +176,7 @@ public class MapRouterLayer implements MapPanelLayer { public void run() { List ways = selfRoute(startRoute, endRoute, intermediates, null); if (ways != null) { - DataTileManager points = new DataTileManager(); - points.setZoom(11); + DataTileManager points = new DataTileManager(11); for (Way w : ways) { LatLon n = w.getLatLon(); points.registerObject(n.getLatitude(), n.getLongitude(), w); @@ -205,8 +204,7 @@ public class MapRouterLayer implements MapPanelLayer { public void run() { List ways = selfRoute(startRoute, endRoute, intermediates, previousRoute); if (ways != null) { - DataTileManager points = new DataTileManager(); - points.setZoom(11); + DataTileManager points = new DataTileManager(11); for (Way w : ways) { LatLon n = w.getLatLon(); points.registerObject(n.getLatitude(), n.getLongitude(), w); @@ -229,8 +227,7 @@ public class MapRouterLayer implements MapPanelLayer { @Override public void run() { List ways = route_YOURS(startRoute, endRoute); - DataTileManager points = new DataTileManager(); - points.setZoom(11); + DataTileManager points = new DataTileManager(11); for(Way w : ways){ LatLon n = w.getLatLon(); points.registerObject(n.getLatitude(), n.getLongitude(), w); @@ -250,8 +247,7 @@ public class MapRouterLayer implements MapPanelLayer { @Override public void run() { List ways = route_CloudMate(startRoute, endRoute); - DataTileManager points = new DataTileManager(); - points.setZoom(11); + DataTileManager points = new DataTileManager(11); for (Way w : ways) { LatLon n = w.getLatLon(); points.registerObject(n.getLatitude(), n.getLongitude(), w); @@ -644,8 +640,7 @@ public class MapRouterLayer implements MapPanelLayer { } } - final DataTileManager points = new DataTileManager(); - points.setZoom(11); + final DataTileManager points = new DataTileManager(11); map.setPoints(points); ctx.setVisitor(new RouteSegmentVisitor() { diff --git a/build-scripts/indexes-batch-generate-inmem.xml b/build-scripts/indexes-batch-generate-inmem.xml new file mode 100644 index 0000000000..4e67275ec6 --- /dev/null +++ b/build-scripts/indexes-batch-generate-inmem.xml @@ -0,0 +1,24 @@ + + + + + + + + + + + diff --git a/build-scripts/indexes-generate-inmem.sh b/build-scripts/indexes-generate-inmem.sh new file mode 100755 index 0000000000..f2f3d5e41f --- /dev/null +++ b/build-scripts/indexes-generate-inmem.sh @@ -0,0 +1,14 @@ +# remove backup and create new backup +# we should not rm, just do incremental updates for now! rm -rf backup + +# remove all previous files +mkdir ~/indexes +mkdir ~/indexes/uploaded + +rm -rf .work +mkdir .work +mkdir .work/osm +if [ -z $INDEXES_FILE ]; then INDEXES_FILE="build-scripts/regions/indexes.xml"; echo "$INDEXES_FILE"; fi + +echo 'Running java net.osmand.data.index.IndexBatchCreator with $INDEXES_FILE' +java -XX:+UseParallelGC -Xmx8096M -Xmn512M -Djava.util.logging.config.file=build-scripts/batch-logging.properties -cp "DataExtractionOSM/OsmAndMapCreator.jar:DataExtractionOSM/lib/*.jar" net.osmand.data.index.IndexBatchCreator build-scripts/indexes-batch-generate-inmem.xml "$INDEXES_FILE"