diff --git a/DataExtractionOSM/.classpath b/DataExtractionOSM/.classpath index 8d48231432..1695f5f3b1 100644 --- a/DataExtractionOSM/.classpath +++ b/DataExtractionOSM/.classpath @@ -3,8 +3,7 @@ - - + diff --git a/DataExtractionOSM/lib/lucene-core-3.0.1.jar b/DataExtractionOSM/lib/lucene-core-3.0.1.jar deleted file mode 100644 index 411524695e..0000000000 Binary files a/DataExtractionOSM/lib/lucene-core-3.0.1.jar and /dev/null differ diff --git a/DataExtractionOSM/src/com/osmand/DefaultLauncherConstants.java b/DataExtractionOSM/src/com/osmand/DefaultLauncherConstants.java deleted file mode 100644 index 1ff15aca48..0000000000 --- a/DataExtractionOSM/src/com/osmand/DefaultLauncherConstants.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.osmand; - - -/** - * This is temp class where all path & machine specific properties are written - */ -public abstract class DefaultLauncherConstants { - - - - // Application constants - public static String APP_NAME = "OsmAnd"; - public static String APP_VERSION = "0.1"; - - - // Download manager tile settings - public static int TILE_DOWNLOAD_THREADS = 4; - public static int TILE_DOWNLOAD_SECONDS_TO_WORK = 25; - public static final int TILE_DOWNLOAD_MAX_ERRORS = -1; - -} diff --git a/DataExtractionOSM/src/com/osmand/ToDoConstants.java b/DataExtractionOSM/src/com/osmand/ToDoConstants.java index 14a2a01e56..f81db5b6a1 100644 --- a/DataExtractionOSM/src/com/osmand/ToDoConstants.java +++ b/DataExtractionOSM/src/com/osmand/ToDoConstants.java @@ -14,7 +14,6 @@ public class ToDoConstants { // TODO ANDROID // 1. POI search near to map location (show categories & type). First cut. (implement incremental search) -// 0. Minimize memory used for index & improve time for reading index // 3. Revise osmand UI. Preparing new icons. // 2. Showing compass on the map : use device compass if exists(?) // 5. Search for city/streets/buildings @@ -33,6 +32,9 @@ public class ToDoConstants { // 23. Implement moving point from center to bottom (for rotating map). // It is not very useful to see what was before. +// 24. Implement ResourceManager on Low memory (clear previous all addresses cities, remove all amenities cache) +// Use async loading tile thread, to preload amenities also. + // FIXME Bugs Android : // 0. FIX TODO for partial loading rotated map // 1. When firstly run osmand navigation (from notification bar) show map & go to menu shows desktop. @@ -43,8 +45,7 @@ public class ToDoConstants { // TODO SWING: // 1. Download tiles without using dir tiles // 2. Configure file log & see log from file - // 3. Reinvent index mechanism (save in zip file with tile indexes, save city/town addresses separately, read partially !) - // 4. Invent different file extensions for poi.index, address.index,... + // 5. Implement supress warning for duplicate id // DONE ANDROID : @@ -52,10 +53,12 @@ public class ToDoConstants { // 10. Specify auto-rotating map (bearing of your direction) // 22. Investigate 3D tile view (how it is done in osmand). Looking not very good, because of // angle of perspective (best perspective angle = 60) use -// android.graphics.Camera.rotateX(6), getMatrix(m), canvas.concat(m) (find example in internet) +// android.graphics.Camera.rotateX(60), getMatrix(m), canvas.concat(m) (find example in internet) // Problems : to calculate how to drag point on map, to calculate how many tiles are needed, is location visible .... +// 0. Minimize memory used for index & improve time for reading index // DONE SWING - + // 3. Reinvent index mechanism (save in zip file with tile indexes, save city/town addresses separately, read partially !) + // 4. Invent different file extensions for poi.index, address.index,... } diff --git a/DataExtractionOSM/src/com/osmand/data/Amenity.java b/DataExtractionOSM/src/com/osmand/data/Amenity.java index 290f55ad55..c3c35b6409 100644 --- a/DataExtractionOSM/src/com/osmand/data/Amenity.java +++ b/DataExtractionOSM/src/com/osmand/data/Amenity.java @@ -7,7 +7,7 @@ import com.osmand.Algoritms; import com.osmand.osm.Entity; import com.osmand.osm.OSMSettings.OSMTagKey; -public class Amenity extends MapObject { +public class Amenity extends MapObject { // http://wiki.openstreetmap.org/wiki/Amenity public enum AmenityType { SUSTENANCE, // restaurant, cafe ... @@ -89,9 +89,10 @@ public class Amenity extends MapObject { private AmenityType type; public Amenity(Entity entity){ - this.entity = entity; + super(entity); this.type = getType(entity); this.subType = getSubType(entity); + } public Amenity(){ @@ -168,6 +169,8 @@ public class Amenity extends MapObject { } - + public void doDataPreparation() { + + } } diff --git a/DataExtractionOSM/src/com/osmand/data/Building.java b/DataExtractionOSM/src/com/osmand/data/Building.java index e267cd6456..2a9e630fd1 100644 --- a/DataExtractionOSM/src/com/osmand/data/Building.java +++ b/DataExtractionOSM/src/com/osmand/data/Building.java @@ -2,7 +2,7 @@ package com.osmand.data; import com.osmand.osm.Entity; -public class Building extends MapObject { +public class Building extends MapObject { public Building(Entity e){ super(e); diff --git a/DataExtractionOSM/src/com/osmand/data/City.java b/DataExtractionOSM/src/com/osmand/data/City.java index bdf602680f..63156b06db 100644 --- a/DataExtractionOSM/src/com/osmand/data/City.java +++ b/DataExtractionOSM/src/com/osmand/data/City.java @@ -9,7 +9,7 @@ import com.osmand.osm.Entity; import com.osmand.osm.Node; import com.osmand.osm.OSMSettings.OSMTagKey; -public class City extends MapObject { +public class City extends MapObject { public enum CityType { // that's tricky way to play with that numbers (to avoid including suburbs in city & vice verse) @@ -67,6 +67,10 @@ public class City extends MapObject { return streets.remove(name.toLowerCase()); } + public void removeAllStreets(){ + streets.clear(); + } + public Street registerStreet(Street street){ String name = street.getName().toLowerCase(); if(!Algoritms.isEmpty(name)){ diff --git a/DataExtractionOSM/src/com/osmand/data/MapObject.java b/DataExtractionOSM/src/com/osmand/data/MapObject.java index 5bf0166783..81e2a4e75b 100644 --- a/DataExtractionOSM/src/com/osmand/data/MapObject.java +++ b/DataExtractionOSM/src/com/osmand/data/MapObject.java @@ -5,26 +5,27 @@ import com.osmand.osm.LatLon; import com.osmand.osm.MapUtils; import com.osmand.osm.OSMSettings.OSMTagKey; -public abstract class MapObject implements Comparable> { +public abstract class MapObject implements Comparable { protected String name = null; protected LatLon location = null; protected Long id = null; - // could be null - protected T entity = null; public MapObject(){} - public MapObject(T e){ - entity = e; + public MapObject(Entity e){ + setEntity(e); } - public T getEntity(){ - return entity; - } - public void setEntity(T e){ - entity = e; + public void setEntity(Entity e){ + this.id = e.getId(); + if(this.name == null){ + this.name = e.getTag(OSMTagKey.NAME); + } + if(this.location == null){ + this.location = MapUtils.getCenter(e); + } } public void setId(Long id) { @@ -35,10 +36,6 @@ public abstract class MapObject implements Comparable implements Comparable implements Comparable implements Comparable o) { + public int compareTo(MapObject o) { return getName().compareTo(o.getName()); } + + public void doDataPreparation() { + + } } diff --git a/DataExtractionOSM/src/com/osmand/data/Region.java b/DataExtractionOSM/src/com/osmand/data/Region.java index 7e881ac1c6..cb7d0389c2 100644 --- a/DataExtractionOSM/src/com/osmand/data/Region.java +++ b/DataExtractionOSM/src/com/osmand/data/Region.java @@ -10,24 +10,14 @@ import java.util.Map; import com.osmand.Algoritms; import com.osmand.data.City.CityType; -import com.osmand.osm.Entity; import com.osmand.osm.LatLon; import com.osmand.osm.MapUtils; import com.osmand.osm.Node; import com.osmand.osm.io.OsmBaseStorage; -public class Region extends MapObject { - +public class Region extends MapObject { private DataTileManager amenities = new DataTileManager(); private OsmBaseStorage storage; - - private static class CityComparator implements Comparator{ - @Override - public int compare(City o1, City o2) { - return o1.getName().compareTo(o2.getName()); - } - } - private DataTileManager cityManager = new DataTileManager(); private Map> cities = new HashMap>(); { @@ -37,6 +27,13 @@ public class Region extends MapObject { } } + private static class CityComparator implements Comparator{ + @Override + public int compare(City o1, City o2) { + return o1.getName().compareTo(o2.getName()); + } + } + public Region(){ name = "Region"; } @@ -98,7 +95,7 @@ public class Region extends MapObject { City closest = null; double relDist = Double.POSITIVE_INFINITY; for (City c : cityManager.getClosestObjects(point.getLatitude(), point.getLongitude())) { - double rel = MapUtils.getDistance(c.getEntity(), point) / c.getType().getRadius(); + double rel = MapUtils.getDistance(c.getLocation(), point) / c.getType().getRadius(); if (rel < relDist) { closest = c; relDist = rel; diff --git a/DataExtractionOSM/src/com/osmand/data/Street.java b/DataExtractionOSM/src/com/osmand/data/Street.java index ce64375a13..dbab879afd 100644 --- a/DataExtractionOSM/src/com/osmand/data/Street.java +++ b/DataExtractionOSM/src/com/osmand/data/Street.java @@ -13,7 +13,7 @@ import com.osmand.osm.Node; import com.osmand.osm.Way; import com.osmand.osm.OSMSettings.OSMTagKey; -public class Street extends MapObject { +public class Street extends MapObject { private List buildings = new ArrayList(); private List wayNodes = new ArrayList(); @@ -42,18 +42,7 @@ public class Street extends MapObject { return buildings; } - public LatLon getLocation(){ - if(entity == null){ - calculateCenter(); - } - return entity == null ? null : entity.getLatLon(); - } - protected void calculateCenter(){ - if(wayNodes.size() == 1){ - entity = wayNodes.get(0); - return; - } List nodes = new ArrayList(); for(Way w : wayNodes){ nodes.addAll(w.getNodes()); @@ -65,8 +54,8 @@ public class Street extends MapObject { if (n != null) { double nd = MapUtils.getDistance(n, c); if (nd < dist) { - entity = n; dist = nd; + location = n.getLatLon(); } } } @@ -74,13 +63,16 @@ public class Street extends MapObject { @Override public void setName(String name) { - if(name.equals(getName())){ + if (name.equals(getName())) { return; } - Street unregisterStreet = city.unregisterStreet(getName()); - assert unregisterStreet == this; - super.setName(name); - city.registerStreet(this); + if (city.getStreet(getName()) == this) { + city.unregisterStreet(getName()); + super.setName(name); + city.registerStreet(this); + } else { + super.setName(name); + } } @@ -89,17 +81,28 @@ public class Street extends MapObject { } public void doDataPreparation() { - calculateCenter(); Collections.sort(buildings, new Comparator(){ - @Override public int compare(Building o1, Building o2) { int i1 = Algoritms.extractFirstIntegerNumber(o1.getName()); int i2 = Algoritms.extractFirstIntegerNumber(o2.getName()); return i1 - i2; } - }); + calculateCenter(); + if(location == null){ + List nodes = new ArrayList(); + for(Building b : buildings){ + nodes.add(b.getLocation()); + } + location = MapUtils.getWeightCenter(nodes); + } + if (wayNodes.size() > 0) { + this.id = wayNodes.get(0).getId(); + } else { + this.id = buildings.get(0).getId(); + } + } } diff --git a/DataExtractionOSM/src/com/osmand/data/index/DataIndexReader.java b/DataExtractionOSM/src/com/osmand/data/index/DataIndexReader.java new file mode 100644 index 0000000000..0d5dad635e --- /dev/null +++ b/DataExtractionOSM/src/com/osmand/data/index/DataIndexReader.java @@ -0,0 +1,109 @@ +package com.osmand.data.index; + +import java.io.File; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; + +import com.osmand.LogUtil; +import com.osmand.data.Building; +import com.osmand.data.City; +import com.osmand.data.Street; +import com.osmand.data.City.CityType; +import com.osmand.data.index.IndexConstants.IndexBuildingTable; +import com.osmand.data.index.IndexConstants.IndexCityTable; +import com.osmand.data.index.IndexConstants.IndexStreetTable; + +public class DataIndexReader { + private static final Log log = LogUtil.getLog(DataIndexReader.class); + + public Connection getConnection(File file) throws SQLException{ + try { + Class.forName("org.sqlite.JDBC"); + } catch (ClassNotFoundException e) { + log.error("Illegal configuration", e); + throw new IllegalStateException(e); + } + return DriverManager.getConnection("jdbc:sqlite:"+file.getAbsolutePath()); + } + + + public List readCities(Connection c) throws SQLException{ + List cities = new ArrayList(); + Statement stat = c.createStatement(); + ResultSet set = stat.executeQuery(IndexConstants.generateSelectSQL(IndexCityTable.values())); + while(set.next()){ + City city = new City(CityType.valueFromString(set.getString(IndexCityTable.CITY_TYPE.ordinal() + 1))); + city.setName(set.getString(IndexCityTable.NAME.ordinal() + 1)); + city.setLocation(set.getDouble(IndexCityTable.LATITUDE.ordinal() + 1), + set.getDouble(IndexCityTable.LONGITUDE.ordinal() + 1)); + city.setId(set.getLong(IndexCityTable.ID.ordinal() + 1)); + cities.add(city); + + } + set.close(); + stat.close(); + return cities; + } + + public List readStreets(Connection c, City city) throws SQLException{ + List streets = new ArrayList(); + Statement stat = c.createStatement(); + ResultSet set = stat.executeQuery(IndexConstants.generateSelectSQL(IndexStreetTable.values(), + IndexStreetTable.CITY.toString() +" = " + city.getId())); + while(set.next()){ + Street street = new Street(city); + street.setName(set.getString(IndexStreetTable.NAME.ordinal() + 1)); + street.setLocation(set.getDouble(IndexStreetTable.LATITUDE.ordinal() + 1), + set.getDouble(IndexStreetTable.LONGITUDE.ordinal() + 1)); + street.setId(set.getLong(IndexStreetTable.ID.ordinal() + 1)); + streets.add(street); + } + set.close(); + stat.close(); + return streets; + } + + public List readBuildings(Connection c, Street street) throws SQLException{ + List buildings = new ArrayList(); + Statement stat = c.createStatement(); + ResultSet set = stat.executeQuery(IndexConstants.generateSelectSQL(IndexBuildingTable.values(), + IndexBuildingTable.STREET.toString() +" = " + street.getId())); + while(set.next()){ + Building building = new Building(); + building.setName(set.getString(IndexBuildingTable.NAME.ordinal() + 1)); + building.setLocation(set.getDouble(IndexBuildingTable.LATITUDE.ordinal() + 1), + set.getDouble(IndexBuildingTable.LONGITUDE.ordinal() + 1)); + building.setId(set.getLong(IndexBuildingTable.ID.ordinal() + 1)); + buildings.add(building); + } + set.close(); + stat.close(); + return buildings; + } + + public void testIndex(File f) throws SQLException { + Connection c = getConnection(f); + try { + for (City city : readCities(c)) { + System.out.println("CITY " + city.getName()); + for (Street s : readStreets(c, city)) { + System.out.println("\tSTREET " + s.getName()); + for (Building b : readBuildings(c, s)) { + System.out.println("\t\tBULDING " + b.getName()); + } + } + + } + } finally { + c.close(); + } + } + +} diff --git a/DataExtractionOSM/src/com/osmand/data/index/DataIndexWriter.java b/DataExtractionOSM/src/com/osmand/data/index/DataIndexWriter.java new file mode 100644 index 0000000000..e60dfdbcf9 --- /dev/null +++ b/DataExtractionOSM/src/com/osmand/data/index/DataIndexWriter.java @@ -0,0 +1,225 @@ +package com.osmand.data.index; + +import java.io.File; +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.logging.Log; + +import com.osmand.LogUtil; +import com.osmand.data.Amenity; +import com.osmand.data.Building; +import com.osmand.data.City; +import com.osmand.data.Region; +import com.osmand.data.Street; +import com.osmand.data.Amenity.AmenityType; +import com.osmand.data.City.CityType; +import com.osmand.data.index.IndexConstants.IndexBuildingTable; +import com.osmand.data.index.IndexConstants.IndexCityTable; +import com.osmand.data.index.IndexConstants.IndexPoiTable; +import com.osmand.data.index.IndexConstants.IndexStreetNodeTable; +import com.osmand.data.index.IndexConstants.IndexStreetTable; +import com.osmand.osm.Node; +import com.osmand.osm.Way; + + +public class DataIndexWriter { + + private final File workingDir; + private final Region region; + private static final Log log = LogUtil.getLog(DataIndexWriter.class); + + private static final int BATCH_SIZE = 1000; + + public DataIndexWriter(File workingDir, Region region){ + this.workingDir = workingDir; + this.region = region; + } + + protected File checkFile(String name) throws IOException { + String fileName = name; + File f = new File(workingDir, fileName); + f.mkdirs(); + // remove existing file + if (f.exists()) { + log.warn("Remove existing index : " + f.getAbsolutePath()); + f.delete(); + } + return f; + } + + + public DataIndexWriter writePOI() throws IOException, SQLException { + File file = checkFile(IndexConstants.POI_INDEX_DIR+region.getName()+IndexConstants.POI_INDEX_EXT); + long now = System.currentTimeMillis(); + try { + Class.forName("org.sqlite.JDBC"); + } catch (ClassNotFoundException e) { + log.error("Illegal configuration", e); + throw new IllegalStateException(e); + } + Connection conn = DriverManager.getConnection("jdbc:sqlite:"+file.getAbsolutePath()); + try { + Statement stat = conn.createStatement(); + assert IndexPoiTable.values().length == 6; + stat.executeUpdate(IndexConstants.generateCreateSQL(IndexPoiTable.values())); + stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexPoiTable.values())); + stat.close(); + + PreparedStatement prep = conn.prepareStatement( + IndexConstants.generatePrepareStatementToInsert(IndexPoiTable.getTable(), 6)); + conn.setAutoCommit(false); + int currentCount = 0; + for (Amenity a : region.getAmenityManager().getAllObjects()) { + prep.setLong(IndexPoiTable.ID.ordinal() + 1, a.getId()); + prep.setDouble(IndexPoiTable.LATITUDE.ordinal() + 1, a.getLocation().getLatitude()); + prep.setDouble(IndexPoiTable.LONGITUDE.ordinal() + 1, a.getLocation().getLongitude()); + prep.setString(IndexPoiTable.NAME.ordinal() + 1, a.getName()); + prep.setString(IndexPoiTable.TYPE.ordinal() + 1, AmenityType.valueToString(a.getType())); + prep.setString(IndexPoiTable.SUBTYPE.ordinal() + 1, a.getSubType()); + prep.addBatch(); + currentCount++; + if(currentCount >= BATCH_SIZE){ + prep.executeBatch(); + currentCount = 0; + } + } + if(currentCount > 0){ + prep.executeBatch(); + } + prep.close(); + conn.setAutoCommit(true); + } finally { + conn.close(); + log.info(String.format("Indexing poi done in %s ms.", System.currentTimeMillis() - now)); + } + return this; + } + + public DataIndexWriter writeAddress() throws IOException, SQLException{ + File file = checkFile(IndexConstants.ADDRESS_INDEX_DIR+region.getName()+IndexConstants.ADDRESS_INDEX_EXT); + long now = System.currentTimeMillis(); + try { + Class.forName("org.sqlite.JDBC"); + } catch (ClassNotFoundException e) { + log.error("Illegal configuration", e); + throw new IllegalStateException(e); + } + Connection conn = DriverManager.getConnection("jdbc:sqlite:"+file.getAbsolutePath()); + try { + Statement stat = conn.createStatement(); + assert IndexCityTable.values().length == 5; + assert IndexBuildingTable.values().length == 5; + assert IndexStreetNodeTable.values().length == 5; + assert IndexStreetTable.values().length == 5; + + stat.executeUpdate(IndexConstants.generateCreateSQL(IndexCityTable.values())); + stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexCityTable.values())); + stat.executeUpdate(IndexConstants.generateCreateSQL(IndexBuildingTable.values())); + stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexBuildingTable.values())); + stat.executeUpdate(IndexConstants.generateCreateSQL(IndexStreetNodeTable.values())); + stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexStreetNodeTable.values())); + stat.executeUpdate(IndexConstants.generateCreateSQL(IndexStreetTable.values())); + stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexStreetTable.values())); + stat.close(); + + PreparedStatement prepCity = conn.prepareStatement( + IndexConstants.generatePrepareStatementToInsert(IndexCityTable.getTable(), 5)); + PreparedStatement prepStreet = conn.prepareStatement( + IndexConstants.generatePrepareStatementToInsert(IndexStreetTable.getTable(), 5)); + PreparedStatement prepBuilding = conn.prepareStatement( + IndexConstants.generatePrepareStatementToInsert(IndexBuildingTable.getTable(), 5)); + PreparedStatement prepStreetNode = conn.prepareStatement( + IndexConstants.generatePrepareStatementToInsert(IndexStreetNodeTable.getTable(), 5)); + Map count = new HashMap(); + count.put(prepStreet, 0); + count.put(prepCity, 0); + count.put(prepStreetNode, 0); + count.put(prepBuilding, 0); + conn.setAutoCommit(false); + + for(CityType t : CityType.values()){ + for(City city : region.getCitiesByType(t)) { + if(city.getId() == null || city.getLocation() == null){ + continue; + } + prepCity.setLong(IndexCityTable.ID.ordinal() + 1, city.getId()); + prepCity.setDouble(IndexCityTable.LATITUDE.ordinal() + 1, city.getLocation().getLatitude()); + prepCity.setDouble(IndexCityTable.LONGITUDE.ordinal() + 1, city.getLocation().getLongitude()); + prepCity.setString(IndexCityTable.NAME.ordinal() + 1, city.getName()); + prepCity.setString(IndexCityTable.CITY_TYPE.ordinal() + 1, CityType.valueToString(city.getType())); + addBatch(count, prepCity); + + for(Street street : city.getStreets()){ + if(street.getId() == null || street.getLocation() == null){ + continue; + } + prepStreet.setLong(IndexStreetTable.ID.ordinal() + 1, street.getId()); + prepStreet.setDouble(IndexStreetTable.LATITUDE.ordinal() + 1, street.getLocation().getLatitude()); + prepStreet.setDouble(IndexStreetTable.LONGITUDE.ordinal() + 1, street.getLocation().getLongitude()); + prepStreet.setString(IndexStreetTable.NAME.ordinal() + 1, street.getName()); + prepStreet.setLong(IndexStreetTable.CITY.ordinal() + 1, city.getId()); + addBatch(count, prepStreet); + for(Way way : street.getWayNodes()){ + for(Node n : way.getNodes()){ + if(n == null){ + continue; + } + prepStreetNode.setLong(IndexStreetNodeTable.ID.ordinal() + 1, n.getId()); + prepStreetNode.setDouble(IndexStreetNodeTable.LATITUDE.ordinal() + 1, n.getLatitude()); + prepStreetNode.setDouble(IndexStreetNodeTable.LONGITUDE.ordinal() + 1, n.getLongitude()); + prepStreetNode.setLong(IndexStreetNodeTable.WAY.ordinal() + 1, way.getId()); + prepStreetNode.setLong(IndexStreetNodeTable.STREET.ordinal() + 1, street.getId()); + addBatch(count, prepStreetNode); + } + + } + + for(Building building : street.getBuildings()){ + if(building.getId() == null || building.getLocation() == null){ + continue; + } + prepBuilding.setLong(IndexBuildingTable.ID.ordinal() + 1, building.getId()); + prepBuilding.setDouble(IndexBuildingTable.LATITUDE.ordinal() + 1, building.getLocation().getLatitude()); + prepBuilding.setDouble(IndexBuildingTable.LONGITUDE.ordinal() + 1, building.getLocation().getLongitude()); + prepBuilding.setString(IndexBuildingTable.NAME.ordinal() + 1, building.getName()); + prepBuilding.setLong(IndexBuildingTable.STREET.ordinal() + 1, street.getId()); + addBatch(count, prepBuilding); + } + } + + } + } + + for(PreparedStatement p : count.keySet()){ + if(count.get(p) > 0){ + p.executeBatch(); + } + p.close(); + } + conn.setAutoCommit(true); + } finally { + conn.close(); + log.info(String.format("Indexing address done in %s ms.", System.currentTimeMillis() - now)); + } + return this; + } + + private void addBatch(Map count, PreparedStatement p) throws SQLException{ + p.addBatch(); + if(count.get(p) >= BATCH_SIZE){ + p.executeBatch(); + count.put(p, 0); + } else { + count.put(p, count.get(p) + 1); + } + } + + +} diff --git a/DataExtractionOSM/src/com/osmand/data/index/IndexConstants.java b/DataExtractionOSM/src/com/osmand/data/index/IndexConstants.java new file mode 100644 index 0000000000..683c1e6fad --- /dev/null +++ b/DataExtractionOSM/src/com/osmand/data/index/IndexConstants.java @@ -0,0 +1,280 @@ +package com.osmand.data.index; + +public class IndexConstants { + + public static final String POI_INDEX_DIR = "POI/"; + public static final String ADDRESS_INDEX_DIR = "Address/"; + + public static final String POI_INDEX_EXT = ".poi.odb"; + public static final String ADDRESS_INDEX_EXT = ".addr.odb"; + + public interface IndexColumn { + public boolean isIndex(); + + public String getType(); + + public String getTableName(); + } + + public static String[] generateColumnNames(IndexColumn[] columns) { + String[] columnNames = new String[columns.length]; + for (int i = 0; i < columnNames.length; i++) { + columnNames[i] = columns[i].toString(); + } + return columnNames; + } + + public static String generateCreateSQL(IndexColumn[] columns){ + StringBuilder b = new StringBuilder(); + b.append("create table ").append(columns[0].getTableName()).append(" ("); + boolean first = true; + for(IndexColumn c : columns){ + if(first) { + first = false; + } else { + b.append(", "); + } + b.append(c.toString()); + if(c.getType() != null){ + b.append(" ").append(c.getType()); + } + } + b.append(" ); "); + return b.toString(); + } + + public static String generateSelectSQL(IndexColumn[] select){ + return generateSelectSQL(select, null); + } + + public static String generateSelectSQL(IndexColumn[] select, String where){ + StringBuilder b = new StringBuilder(); + b.append("select "); + boolean first = true; + for(IndexColumn c : select){ + if(first) { + first = false; + } else { + b.append(", "); + } + b.append(c.toString()); + } + b.append(" FROM ").append(select[0].getTableName()); + if(where != null){ + b.append(" WHERE " ).append(where); + } + b.append(" ; "); + return b.toString(); + } + + public static String generatePrepareStatementToInsert(String tableName, int numColumns){ + StringBuilder b = new StringBuilder(); + b.append("insert into ").append(tableName).append(" values ("); + for(int i=0; i< numColumns; i++){ + if(i > 0){ + b.append(", "); + } + b.append("?"); + } + b.append(");"); + return b.toString(); + } + + public static String generateCreateIndexSQL(IndexColumn[] columns){ + StringBuilder b = new StringBuilder(); + String tableName = columns[0].getTableName(); + b.append("create index ").append(tableName).append("_index ON ").append(tableName).append(" ("); + boolean first = true; + for(IndexColumn c : columns){ + if(!c.isIndex()){ + continue; + } + if(first) { + first = false; + } else { + b.append(", "); + } + b.append(c.toString()); + } + b.append(" ); "); + if(first){ + return null; + } + return b.toString(); + } + + + public enum IndexPoiTable implements IndexColumn { + ID("long"), LATITUDE("double", true), LONGITUDE("double", true), NAME, TYPE, SUBTYPE; + boolean index = false; + String type = null; + private IndexPoiTable(){} + private IndexPoiTable(String type){ + this.type = type; + } + private IndexPoiTable(String type, boolean index){ this(type); this.index = index;} + + public static String getTable(){ + return "poi"; + } + + public String getTableName(){ + return getTable(); + } + + @Override + public String getType() { + return type; + } + + @Override + public boolean isIndex() { + return index; + } + } + + + public enum IndexCityTable implements IndexColumn { + ID("long"), LATITUDE("double", true), LONGITUDE("double", true), NAME, CITY_TYPE; + boolean index = false; + String type = null; + + private IndexCityTable() { + } + + private IndexCityTable(String type) { + this.type = type; + } + + private IndexCityTable(String type, boolean index) { + this(type); + this.index = index; + } + + public static String getTable() { + return "city"; + } + + public String getTableName() { + return getTable(); + } + + @Override + public String getType() { + return type; + } + + @Override + public boolean isIndex() { + return index; + } + } + + public enum IndexStreetTable implements IndexColumn { + ID("long"), LATITUDE("double", true), LONGITUDE("double", true), NAME, CITY("long", true); + boolean index = false; + String type = null; + + private IndexStreetTable() { + } + + private IndexStreetTable(String type) { + this.type = type; + } + + private IndexStreetTable(String type, boolean index) { + this(type); + this.index = index; + } + + public static String getTable() { + return "street"; + } + + public String getTableName() { + return getTable(); + } + + @Override + public String getType() { + return type; + } + + @Override + public boolean isIndex() { + return index; + } + } + + public enum IndexStreetNodeTable implements IndexColumn { + ID("long"), LATITUDE("double"), LONGITUDE("double"), STREET("long", true), WAY("long", true); + boolean index = false; + String type = null; + + private IndexStreetNodeTable() { + } + + private IndexStreetNodeTable(String type) { + this.type = type; + } + + private IndexStreetNodeTable(String type, boolean index) { + this(type); + this.index = index; + } + + public static String getTable() { + return "street_node"; + } + + public String getTableName() { + return getTable(); + } + + @Override + public String getType() { + return type; + } + + @Override + public boolean isIndex() { + return index; + } + } + + public enum IndexBuildingTable implements IndexColumn { + ID("long"), LATITUDE("double"), LONGITUDE("double"), NAME, STREET("long", true); + boolean index = false; + String type = null; + + private IndexBuildingTable() { + } + + private IndexBuildingTable(String type) { + this.type = type; + } + + private IndexBuildingTable(String type, boolean index) { + this(type); + this.index = index; + } + + public static String getTable() { + return "building"; + } + + public String getTableName() { + return getTable(); + } + + @Override + public String getType() { + return type; + } + + @Override + public boolean isIndex() { + return index; + } + } +} diff --git a/DataExtractionOSM/src/com/osmand/data/preparation/DataExtraction.java b/DataExtractionOSM/src/com/osmand/data/preparation/DataExtraction.java index da780907fc..308f7c7c32 100644 --- a/DataExtractionOSM/src/com/osmand/data/preparation/DataExtraction.java +++ b/DataExtractionOSM/src/com/osmand/data/preparation/DataExtraction.java @@ -2,20 +2,21 @@ package com.osmand.data.preparation; import java.io.File; import java.io.FileInputStream; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; import java.util.ArrayList; -import java.util.List; - -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.stream.XMLStreamException; +import java.util.LinkedHashMap; +import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.tools.bzip2.CBZip2InputStream; -import org.apache.tools.bzip2.CBZip2OutputStream; import org.xml.sax.SAXException; import com.osmand.Algoritms; @@ -26,7 +27,6 @@ import com.osmand.data.DataTileManager; import com.osmand.data.Region; import com.osmand.data.Street; import com.osmand.data.City.CityType; -import com.osmand.impl.ConsoleProgressImplementation; import com.osmand.osm.Entity; import com.osmand.osm.LatLon; import com.osmand.osm.MapUtils; @@ -36,13 +36,9 @@ import com.osmand.osm.Way; import com.osmand.osm.OSMSettings.OSMTagKey; import com.osmand.osm.io.IOsmStorageFilter; import com.osmand.osm.io.OsmBaseStorage; -import com.osmand.osm.io.OsmStorageWriter; import com.osmand.swing.DataExtractionSettings; -// TO implement -// 1. Full structured search for town/street/building. - /** * http://wiki.openstreetmap.org/wiki/OSM_tags_for_routing#Is_inside.2Foutside * http://wiki.openstreetmap.org/wiki/Relations/Proposed/Postal_Addresses @@ -76,89 +72,163 @@ import com.osmand.swing.DataExtractionSettings; */ public class DataExtraction { private static final Log log = LogFactory.getLog(DataExtraction.class); - -// public static void main(String[] args) throws ParserConfigurationException, SAXException, IOException, XMLStreamException { -// new DataExtraction().testReadingOsmFile(); -// } - - // External files - public static String pathToTestDataDir = "E:\\Information\\OSM maps\\"; - public static String pathToOsmFile = pathToTestDataDir + "minsk.osm"; - public static String pathToOsmBz2File = pathToTestDataDir + "belarus_2010_04_01.osm.bz2"; - public static String pathToWorkingDir = pathToTestDataDir +"osmand\\"; - public static String pathToDirWithTiles = pathToWorkingDir +"tiles"; - public static String writeTestOsmFile = "C:\\1_tmp.osm"; // could be null - wo writing - private static boolean parseSmallFile = true; - private static boolean parseOSM = true; + public static final int BATCH_SIZE = 5000; + public static final String NODES_DB = "nodes.db"; + private final boolean loadAllObjects; - private final boolean normalizeStreets; - private final boolean indexAddress; - private final boolean indexPOI; + private File workingDir = null; - - - /////////////////////////////////////////// - // Test method for local purposes - public void testReadingOsmFile() throws ParserConfigurationException, SAXException, IOException, XMLStreamException { - String f; - if(parseSmallFile){ - f = pathToOsmFile; - } else { - f = pathToOsmBz2File; - } - long st = System.currentTimeMillis(); - - Region country; - if(parseOSM){ - country = readCountry(f, new ConsoleProgressImplementation(), null); - } else { - country = new Region(); - country.setStorage(new OsmBaseStorage()); - } - - - - List interestedObjects = new ArrayList(); - // add interested objects - if (writeTestOsmFile != null) { - OsmStorageWriter writer = new OsmStorageWriter(); - OutputStream output = new FileOutputStream(writeTestOsmFile); - if (writeTestOsmFile.endsWith(".bz2")) { - output.write('B'); - output.write('Z'); - output = new CBZip2OutputStream(output); - } - - writer.saveStorage(output, country.getStorage(), interestedObjects, false); - output.close(); - } - - System.out.println(); - System.out.println("USED Memory " + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1e6); - System.out.println("TIME : " + (System.currentTimeMillis() - st)); - } - - public DataExtraction(){ - this.indexPOI = true; - this.indexAddress = true; - this.loadAllObjects = false; - this.normalizeStreets = true; - } - - public DataExtraction(boolean indexAddress, boolean indexPOI, boolean normalizeStreets, boolean loadAllObjects){ + public DataExtraction(boolean indexAddress, boolean indexPOI, boolean normalizeStreets, boolean loadAllObjects, File workingDir){ this.indexAddress = indexAddress; this.indexPOI = indexPOI; this.normalizeStreets = normalizeStreets; this.loadAllObjects = loadAllObjects; + this.workingDir = workingDir; } - public Region readCountry(String path, IProgress progress, IOsmStorageFilter addFilter) throws IOException, SAXException{ + protected class DataExtractionOsmFilter implements IOsmStorageFilter { + final ArrayList places; + final ArrayList buildings; + final ArrayList amenities; + final ArrayList ways; + + int currentCount = 0; + private Connection conn; + private PreparedStatement prep; + + public DataExtractionOsmFilter(ArrayList amenities, ArrayList buildings, ArrayList places, + ArrayList ways) { + this.amenities = amenities; + this.buildings = buildings; + this.places = places; + this.ways = ways; + } + + public void initDatabase() throws SQLException { + try { + Class.forName("org.sqlite.JDBC"); + } catch (ClassNotFoundException e) { + log.error("Illegal configuration", e); + throw new IllegalStateException(e); + } + + File file = new File(workingDir, NODES_DB); + // to save space + if(file.exists()){ + file.delete(); + } + // creating nodes db to fast access for all nodes + conn = DriverManager.getConnection("jdbc:sqlite:" + file.getAbsolutePath()); + + // prepare tables + Statement stat = conn.createStatement(); + stat.executeUpdate("drop table if exists node;"); + stat.executeUpdate("create table node (id long, latitude double, longitude double);"); + stat.executeUpdate("create index IdIndex ON node (id);"); + stat.close(); + + prep = conn.prepareStatement("insert into node values (?, ?, ?);"); + conn.setAutoCommit(false); + } + + public void correlateData(OsmBaseStorage storage, IProgress progress) throws SQLException { + if (currentCount > 0) { + prep.executeBatch(); + } + prep.close(); + conn.setAutoCommit(true); + final PreparedStatement pselect = conn.prepareStatement("select * from node where id = ?"); + Map map = new LinkedHashMap(); + progress.startTask("Correlating data...", storage.getRegisteredEntities().size()); + for (Entity e : storage.getRegisteredEntities().values()) { + progress.progress(1); + if (e instanceof Way) { + map.clear(); + for (Long i : ((Way) e).getNodeIds()) { + pselect.setLong(1, i); + if (pselect.execute()) { + ResultSet rs = pselect.getResultSet(); + if (rs.next()) { + map.put(i, new Node(rs.getDouble(2), rs.getDouble(3), rs.getLong(1))); + } + rs.close(); + } + } + e.initializeLinks(map); + } + } + } + + public void close() { + if (conn != null) { + try { + conn.close(); + } catch (SQLException e) { + } + } + } + + @Override + public boolean acceptEntityToLoad(OsmBaseStorage storage, Entity e) { + boolean processed = false; + if (indexAddress) { + if ("yes".equals(e.getTag(OSMTagKey.BUILDING))) { + if (e.getTag(OSMTagKey.ADDR_HOUSE_NUMBER) != null && e.getTag(OSMTagKey.ADDR_STREET) != null) { + buildings.add(e); + processed = true; + } + } + } + if (indexPOI && Amenity.isAmenity(e)) { + amenities.add(e); + processed = true; + } + if (e instanceof Node && e.getTag(OSMTagKey.PLACE) != null) { + places.add((Node) e); + processed = true; + } + if (indexAddress) { + // suppose that streets are way for car + if (e instanceof Way && OSMSettings.wayForCar(e.getTag(OSMTagKey.HIGHWAY)) && e.getTag(OSMTagKey.NAME) != null) { + ways.add((Way) e); + processed = true; + } + } + // put all nodes into temporary db to get only required nodes after loading all data + try { + if (e instanceof Node && indexAddress) { + currentCount++; + prep.setLong(1, e.getId()); + prep.setDouble(2, ((Node) e).getLatitude()); + prep.setDouble(3, ((Node) e).getLongitude()); + prep.addBatch(); + if (currentCount >= BATCH_SIZE) { + prep.executeBatch(); + currentCount = 0; + } + } + } catch (SQLException ex) { + log.error("Could not save node", ex); + } + return processed || loadAllObjects; + } + + } + + + public Region readCountry(String path, IProgress progress, IOsmStorageFilter addFilter) throws IOException, SAXException, SQLException{ + // data to load & index + final ArrayList places = new ArrayList(); + final ArrayList buildings = new ArrayList(); + final ArrayList amenities = new ArrayList(); + final ArrayList ways = new ArrayList(); + File f = new File(path); InputStream stream = new FileInputStream(f); InputStream streamFile = stream; @@ -174,51 +244,33 @@ public class DataExtraction { if(progress != null){ progress.startTask("Loading file " + path, -1); } - - // preloaded data - final ArrayList places = new ArrayList(); - final ArrayList buildings = new ArrayList(); - final ArrayList amenities = new ArrayList(); - final ArrayList ways = new ArrayList(); - - IOsmStorageFilter filter = new IOsmStorageFilter(){ - @Override - public boolean acceptEntityToLoad(OsmBaseStorage storage, Entity e) { - if (indexAddress) { - if ("yes".equals(e.getTag(OSMTagKey.BUILDING))) { - if (e.getTag(OSMTagKey.ADDR_HOUSE_NUMBER) != null && e.getTag(OSMTagKey.ADDR_STREET) != null) { - buildings.add(e); - return true; - } - } - } - if (indexPOI && Amenity.isAmenity(e)) { - amenities.add(new Amenity(e)); - return true; - } - if (e instanceof Node && e.getTag(OSMTagKey.PLACE) != null) { - places.add((Node) e); - return true; - } - if (indexAddress) { - if (e instanceof Way && OSMSettings.wayForCar(e.getTag(OSMTagKey.HIGHWAY))) { - ways.add((Way) e); - return true; - } - } - return (e instanceof Node && indexAddress) || loadAllObjects; - } - }; - - OsmBaseStorage storage = new OsmBaseStorage(); - if (addFilter != null) { + OsmBaseStorage storage = new OsmBaseStorage(); + if (addFilter != null) { storage.getFilters().add(addFilter); } - storage.getFilters().add(filter); - storage.parseOSM(stream, progress, streamFile); - if (log.isDebugEnabled()) { - log.debug("File parsed : " + (System.currentTimeMillis() - st)); + DataExtractionOsmFilter filter = new DataExtractionOsmFilter(amenities, buildings, places, ways); + storage.getFilters().add(filter); + // 0. Loading osm file + try { + // 0.1 init database to store temporary data + filter.initDatabase(); + + // 0.2 parsing osm itself + storage.parseOSM(stream, progress, streamFile); + if (log.isInfoEnabled()) { + log.info("File parsed : " + (System.currentTimeMillis() - st)); + } + progress.finishTask(); + + // 0.3 Correlating data (linking way & node) + filter.correlateData(storage, progress); + + } finally { + if (log.isInfoEnabled()) { + log.info("File indexed : " + (System.currentTimeMillis() - st)); + } + filter.close(); } // 1. Initialize region @@ -228,9 +280,9 @@ public class DataExtraction { country.setStorage(storage); // 2. Reading amenities - if(indexPOI){ - readingAmenities(amenities, country); - } + if (indexPOI) { + readingAmenities(amenities, country); + } // 3. Reading cities readingCities(places, country); @@ -247,7 +299,7 @@ public class DataExtraction { // 6. normalizing streets normalizingStreets(progress, country); } - + // 7. Call data preparation to sort cities, calculate center location, assign id to objects country.doDataPreparation(); return country; } @@ -311,9 +363,9 @@ public class DataExtraction { } - private void readingAmenities(final ArrayList amenities, Region country) { - for(Amenity a: amenities){ - country.registerAmenity(a); + private void readingAmenities(final ArrayList amenities, Region country) { + for(Entity a: amenities){ + country.registerAmenity(new Amenity(a)); } } diff --git a/DataExtractionOSM/src/com/osmand/data/preparation/DataIndexBuilder.java b/DataExtractionOSM/src/com/osmand/data/preparation/DataIndexBuilder.java deleted file mode 100644 index b690ec96ab..0000000000 --- a/DataExtractionOSM/src/com/osmand/data/preparation/DataIndexBuilder.java +++ /dev/null @@ -1,94 +0,0 @@ -package com.osmand.data.preparation; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; - -import javax.xml.stream.XMLStreamException; - -import com.osmand.data.Amenity; -import com.osmand.data.Region; -import com.osmand.osm.io.OsmStorageWriter; - - -public class DataIndexBuilder { - - private final File workingDir; - private final Region region; - private boolean zipped = true; - - public DataIndexBuilder(File workingDir, Region region){ - this.workingDir = workingDir; - this.region = region; - } - - public void setZipped(boolean zipped) { - this.zipped = zipped; - } - - public boolean isZipped() { - return zipped; - } - - protected OutputStream checkFile(String name) throws IOException { - String fileName = name; - if (zipped) { - // change name - name = new File(name).getName(); - fileName += ".zip"; - } - File f = new File(workingDir, fileName); - f.mkdirs(); - // remove existing file - if (f.exists()) { - f.delete(); - } - OutputStream output = new FileOutputStream(f); - if(zipped){ - ZipOutputStream zipStream = new ZipOutputStream(output); - zipStream.setLevel(5); - zipStream.putNextEntry(new ZipEntry(name)); - output = zipStream; - } - return output; - } - - - public DataIndexBuilder buildPOI() throws XMLStreamException, IOException, SQLException { - - List list = region.getAmenityManager().getAllObjects(); - List interestedObjects = new ArrayList(list.size()); - for(Amenity a : list) { - interestedObjects.add(a.getEntity().getId()); - } - OutputStream output = checkFile("POI/"+region.getName()+".osmand"); - try { - OsmStorageWriter writer = new OsmStorageWriter(); - writer.savePOIIndex(output, region); - } finally { - output.close(); - } - OsmStorageWriter writer = new OsmStorageWriter(); - writer.saveLuceneIndex(new File(workingDir, "lucene"), region); - writer.saveSQLLitePOIIndex(new File(workingDir, "POI/"+region.getName()+".db"), region); - - return this; - } - - public DataIndexBuilder buildAddress() throws XMLStreamException, IOException{ - OutputStream output = checkFile("Address/"+region.getName()+".osmand"); - try { - OsmStorageWriter writer = new OsmStorageWriter(); - writer.saveAddressIndex(output, region); - } finally { - output.close(); - } - return this; - } -} diff --git a/DataExtractionOSM/src/com/osmand/data/preparation/MapTileDownloader.java b/DataExtractionOSM/src/com/osmand/data/preparation/MapTileDownloader.java index 181b2de568..67b3129044 100644 --- a/DataExtractionOSM/src/com/osmand/data/preparation/MapTileDownloader.java +++ b/DataExtractionOSM/src/com/osmand/data/preparation/MapTileDownloader.java @@ -19,10 +19,18 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.logging.Log; import com.osmand.Algoritms; -import com.osmand.DefaultLauncherConstants; import com.osmand.LogUtil; public class MapTileDownloader { + // Application constants + public static String APP_NAME = "OsmAnd"; + public static String APP_VERSION = "0.1"; + + + // Download manager tile settings + public static int TILE_DOWNLOAD_THREADS = 4; + public static int TILE_DOWNLOAD_SECONDS_TO_WORK = 25; + public static final int TILE_DOWNLOAD_MAX_ERRORS = -1; private static MapTileDownloader downloader = null; private static Log log = LogUtil.getLog(MapTileDownloader.class); @@ -40,7 +48,7 @@ public class MapTileDownloader { public static MapTileDownloader getInstance(){ if(downloader == null){ - downloader = new MapTileDownloader(DefaultLauncherConstants.TILE_DOWNLOAD_THREADS); + downloader = new MapTileDownloader(TILE_DOWNLOAD_THREADS); } return downloader; } @@ -89,7 +97,7 @@ public class MapTileDownloader { public MapTileDownloader(int numberOfThreads){ - threadPoolExecutor = new ThreadPoolExecutor(numberOfThreads, numberOfThreads, DefaultLauncherConstants.TILE_DOWNLOAD_SECONDS_TO_WORK, + threadPoolExecutor = new ThreadPoolExecutor(numberOfThreads, numberOfThreads, TILE_DOWNLOAD_SECONDS_TO_WORK, TimeUnit.SECONDS, new LinkedBlockingQueue()); // 1.6 method but very useful to kill non-running threads // threadPoolExecutor.allowCoreThreadTimeOut(true); @@ -129,8 +137,8 @@ public class MapTileDownloader { } public void requestToDownload(DownloadRequest request){ - if(DefaultLauncherConstants.TILE_DOWNLOAD_MAX_ERRORS > 0 && - currentErrors > DefaultLauncherConstants.TILE_DOWNLOAD_MAX_ERRORS){ + if(TILE_DOWNLOAD_MAX_ERRORS > 0 && + currentErrors > TILE_DOWNLOAD_MAX_ERRORS){ return; } if(request.url == null){ @@ -163,8 +171,7 @@ public class MapTileDownloader { request.fileToSave.getParentFile().mkdirs(); URL url = new URL(request.url); URLConnection connection = url.openConnection(); - connection.setRequestProperty("User-Agent", DefaultLauncherConstants.APP_NAME + "/" - + DefaultLauncherConstants.APP_VERSION); + connection.setRequestProperty("User-Agent", APP_NAME + "/" + APP_VERSION); BufferedInputStream inputStream = new BufferedInputStream(connection.getInputStream(), 8 * 1024); FileOutputStream stream = null; try { diff --git a/DataExtractionOSM/src/com/osmand/IMapLocationListener.java b/DataExtractionOSM/src/com/osmand/map/IMapLocationListener.java similarity index 81% rename from DataExtractionOSM/src/com/osmand/IMapLocationListener.java rename to DataExtractionOSM/src/com/osmand/map/IMapLocationListener.java index 0499c6090f..2c635a6368 100644 --- a/DataExtractionOSM/src/com/osmand/IMapLocationListener.java +++ b/DataExtractionOSM/src/com/osmand/map/IMapLocationListener.java @@ -1,4 +1,4 @@ -package com.osmand; +package com.osmand.map; public interface IMapLocationListener { void locationChanged(double newLatitude, double newLongitude, Object source); diff --git a/DataExtractionOSM/src/com/osmand/osm/MapUtils.java b/DataExtractionOSM/src/com/osmand/osm/MapUtils.java index 3c6d584a24..9f302fbc92 100644 --- a/DataExtractionOSM/src/com/osmand/osm/MapUtils.java +++ b/DataExtractionOSM/src/com/osmand/osm/MapUtils.java @@ -53,7 +53,7 @@ public class MapUtils { * Gets distance in meters */ public static double getDistance(LatLon l1, LatLon l2){ - return getDistance(l1, l2); + return getDistance(l1.getLatitude(), l1.getLongitude(), l2.getLatitude(), l2.getLongitude()); } public static LatLon getCenter(Entity e){ @@ -184,15 +184,14 @@ public class MapUtils { } } - public static void sortListOfMapObject(List> list, final double lat, final double lon){ - Collections.sort(list, new Comparator>() { + public static void sortListOfMapObject(List list, final double lat, final double lon){ + Collections.sort(list, new Comparator() { @Override - public int compare(MapObject o1, MapObject o2) { + public int compare(MapObject o1, MapObject o2) { return Double.compare(MapUtils.getDistance(o1.getLocation(), lat, lon), MapUtils.getDistance(o2.getLocation(), lat, lon)); } }); } - } diff --git a/DataExtractionOSM/src/com/osmand/osm/io/OsmBaseStorage.java b/DataExtractionOSM/src/com/osmand/osm/io/OsmBaseStorage.java index e5693e3c87..bfdbeaec3e 100644 --- a/DataExtractionOSM/src/com/osmand/osm/io/OsmBaseStorage.java +++ b/DataExtractionOSM/src/com/osmand/osm/io/OsmBaseStorage.java @@ -193,7 +193,7 @@ public class OsmBaseStorage extends DefaultHandler { if(acceptEntityToLoad(currentParsedEntity)){ Entity oldEntity = entities.put(currentParsedEntity.getId(), currentParsedEntity); if(oldEntity!= null){ - throw new UnsupportedOperationException("Entity with id=" + oldEntity.getId() +" is duplicated in osm map"); + // throw new UnsupportedOperationException("Entity with id=" + oldEntity.getId() +" is duplicated in osm map"); } } else { // System.gc(); diff --git a/DataExtractionOSM/src/com/osmand/osm/io/OsmIndexStorage.java b/DataExtractionOSM/src/com/osmand/osm/io/OsmIndexStorage.java deleted file mode 100644 index c8dab70205..0000000000 --- a/DataExtractionOSM/src/com/osmand/osm/io/OsmIndexStorage.java +++ /dev/null @@ -1,107 +0,0 @@ -package com.osmand.osm.io; - -import org.xml.sax.Attributes; -import org.xml.sax.SAXException; - -import com.osmand.data.Amenity; -import com.osmand.data.Building; -import com.osmand.data.City; -import com.osmand.data.MapObject; -import com.osmand.data.Region; -import com.osmand.data.Street; -import com.osmand.data.Amenity.AmenityType; -import com.osmand.data.City.CityType; -import com.osmand.osm.Entity; - -public class OsmIndexStorage extends OsmBaseStorage { - protected static final String ELEM_OSMAND = "osmand"; - protected static final String ELEM_INDEX = "index"; - protected static final String ELEM_CITY = "city"; - protected static final String ELEM_STREET = "street"; - protected static final String ELEM_BUILDING = "building"; - protected static final String ELEM_AMENITY = "amenity"; - - protected static final String ATTR_CITYTYPE = "citytype"; - protected static final String ATTR_TYPE = "type"; - protected static final String ATTR_SUBTYPE = "subtype"; - protected static final String ATTR_NAME = "name"; - - public static final String OSMAND_VERSION = "0.1"; - - protected Region region; - - - public OsmIndexStorage(Region region){ - this.region = region; - } - - public Region getRegion() { - return region; - } - - protected City currentParsedCity = null; - protected Street currentParsedStreet = null; - - @Override - protected void initRootElement(String uri, String localName, String name, Attributes attributes) throws OsmVersionNotSupported { - if(ELEM_OSM.equals(name)){ - if(!supportedVersions.contains(attributes.getValue(ATTR_VERSION))){ - throw new OsmVersionNotSupported(); - } - } else if(ELEM_OSMAND.equals(name)){ - if(!OSMAND_VERSION.equals(attributes.getValue(ATTR_VERSION))){ - throw new OsmVersionNotSupported(); - } - } else { - throw new OsmVersionNotSupported(); - } - parseStarted = true; - } - - public void parseMapObject(MapObject c, Attributes attributes){ - double lat = parseDouble(attributes, ATTR_LAT, 0); - double lon = parseDouble(attributes, ATTR_LON, 0); - long id = parseId(attributes, ATTR_ID, -1); - c.setId(id); - if(lat != 0 || lon != 0){ - c.setLocation(lat, lon); - } - c.setName(attributes.getValue(ATTR_NAME)); - } - - @Override - public void startElement(String uri, String localName, String name, Attributes attributes) throws SAXException { - name = saxParser.isNamespaceAware() ? localName : name; - if(!parseStarted){ - initRootElement(uri, localName, name, attributes); - } else if(ELEM_INDEX.equals(name)){ - } else if(ELEM_CITY.equals(name)){ - CityType t = CityType.valueFromString(attributes.getValue(ATTR_CITYTYPE)); - City c = new City(t); - parseMapObject(c, attributes); - region.registerCity(c); - currentParsedCity = c; - } else if(ELEM_STREET.equals(name)){ - assert currentParsedCity != null; - Street street = new Street(currentParsedCity); - parseMapObject(street, attributes); - currentParsedCity.registerStreet(street); - currentParsedStreet = street; - } else if(ELEM_BUILDING.equals(name)){ - assert currentParsedStreet != null; - Building building = new Building(); - parseMapObject(building, attributes); - currentParsedStreet.registerBuilding(building); - } else if(ELEM_AMENITY.equals(name)){ - Amenity a = new Amenity(); - a.setType(AmenityType.fromString(attributes.getValue(ATTR_TYPE))); - a.setSubType(attributes.getValue(ATTR_SUBTYPE)); - parseMapObject(a, attributes); - region.registerAmenity(a); - - } else { - super.startElement(uri, localName, name, attributes); - } - - } -} diff --git a/DataExtractionOSM/src/com/osmand/osm/io/OsmLuceneRepository.java b/DataExtractionOSM/src/com/osmand/osm/io/OsmLuceneRepository.java deleted file mode 100644 index 2944662587..0000000000 --- a/DataExtractionOSM/src/com/osmand/osm/io/OsmLuceneRepository.java +++ /dev/null @@ -1,168 +0,0 @@ -package com.osmand.osm.io; - -import java.io.File; -import java.io.IOException; -import java.text.DecimalFormat; -import java.text.DecimalFormatSymbols; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Locale; - -import org.apache.commons.logging.Log; -import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.queryParser.ParseException; -import org.apache.lucene.queryParser.QueryParser; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.TopScoreDocCollector; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.util.Version; - -import com.osmand.IProgress; -import com.osmand.LogUtil; -import com.osmand.data.Amenity; -import com.osmand.data.Amenity.AmenityType; - -public class OsmLuceneRepository { - private static final Log log = LogUtil.getLog(OsmLuceneRepository.class); - private static final int MAX_POI_HITS = 1000; - private static DecimalFormat fmtLatitude = new DecimalFormat("000.000000", new DecimalFormatSymbols(Locale.US)); - private static DecimalFormat negFmtLatitude = new DecimalFormat("00.000000", new DecimalFormatSymbols(Locale.US)); - private static DecimalFormat fmtLongitude = new DecimalFormat("0000.000000", new DecimalFormatSymbols(Locale.US)); - private static DecimalFormat negFmtLongitude = new DecimalFormat("000.000000", new DecimalFormatSymbols(Locale.US)); - - - public static String formatLatitude(double latitude){ - if(latitude <0 ){ - return negFmtLatitude.format(latitude); - } else { - return fmtLatitude.format(latitude); - } - } - - public static String formatLongitude(double longitude){ - if(longitude <0 ){ - return negFmtLongitude.format(longitude); - } else { - return fmtLongitude.format(longitude); - } - } - - private List internalSearch(double topLatitude, double leftLongitude, double bottomLatitude, double rightLongitude){ - queryFormat.setLength(0); - queryFormat.append("latitude:[").append(formatLatitude(bottomLatitude)).append(" TO ").append(formatLatitude(topLatitude)).append( - "]").append(" AND longitude:[").append(formatLongitude(leftLongitude)).append(" TO ").append( - formatLongitude(rightLongitude)).append("]"); - - TopScoreDocCollector collector = TopScoreDocCollector.create(MAX_POI_HITS, true); - try { - Query q = new QueryParser(Version.LUCENE_30, "id", new StandardAnalyzer(Version.LUCENE_30)).parse(queryFormat.toString()); - long now = System.currentTimeMillis(); - amenityIndexSearcher.search(q, collector); - ScoreDoc[] hits = collector.topDocs().scoreDocs; - - List result = new ArrayList(hits.length); - for (int i = 0; i < hits.length; i++) { - result.add(convertAmenity(amenityIndexSearcher.doc(hits[i].doc))); - } - if (log.isDebugEnabled()) { - log.debug(String.format("Search for %s done in %s ms found %s.", q, System.currentTimeMillis() - now, hits.length)); - } - return result; - } catch (IOException e) { - log.error("Failed to search.", e); - throw new RuntimeException(e); - } catch (ParseException e) { - log.error("Invalid query.", e); - return new ArrayList(); - } - } - - private List cachedAmenities = null; - private double cTopLatitude; - private double cBottomLatitude; - private double cLeftLongitude; - private double cRightLongitude; - - private final StringBuilder queryFormat = new StringBuilder(); - private IndexSearcher amenityIndexSearcher; - - private boolean isLoading = false; - - protected synchronized void loadAmenitiesInAnotherThread(final double topLatitude, final double leftLongitude, final double bottomLatitude, final double rightLongitude){ - isLoading = true; - new Thread(new Runnable(){ - - @Override - public void run() { - try { - cachedAmenities = internalSearch(topLatitude, leftLongitude, bottomLatitude, rightLongitude); - cTopLatitude = topLatitude; - cLeftLongitude = leftLongitude; - cBottomLatitude = bottomLatitude ; - cRightLongitude = rightLongitude; - } finally { - synchronized (this) { - isLoading = false; - } - } - } - }, "Searching in index...").start(); - } - public synchronized List searchAmenities(double topLatitude, double leftLongitude, double bottomLatitude, double rightLongitude) { - if(amenityIndexSearcher == null){ - return Collections.emptyList(); - } - // TODO take into account that right could be -53 & left = 175 (normalized coordinates - if (cTopLatitude >= topLatitude && cLeftLongitude <= leftLongitude && cRightLongitude >= rightLongitude - && cBottomLatitude <= bottomLatitude) { - return cachedAmenities; - } - if(!isLoading){ - double h = (topLatitude - bottomLatitude); - double w = (rightLongitude - leftLongitude); - topLatitude += h; - leftLongitude -= w; - bottomLatitude -= h; - rightLongitude += w; - loadAmenitiesInAnotherThread(topLatitude, leftLongitude, bottomLatitude, rightLongitude); - } - return Collections.emptyList(); - } - - public void indexing(final IProgress progress, File dir) { - long start = System.currentTimeMillis(); - progress.startTask("Indexing lucene", -1); - try { - amenityIndexSearcher = new IndexSearcher(FSDirectory.open(dir)); - } catch (Exception t) { - log.error("Failed to initialize searcher.", t); - throw new RuntimeException(t); - } - if (log.isDebugEnabled()) { - log.debug("Finished index lucene " + dir.getAbsolutePath() + " " + (System.currentTimeMillis() - start) + "ms"); - } - } - - protected Amenity convertAmenity(Document document) { - try { - Amenity am = new Amenity(); - am.setName(document.get("name")); - am.setId(Long.parseLong(document.get("id"))); - am.setSubType(document.get("subtype")); - am.setType(AmenityType.fromString(document.get("type"))); - double longitude = fmtLongitude.parse(document.get("longitude")).doubleValue(); - double latitude = fmtLatitude.parse(document.get("latitude")).doubleValue(); - am.setLocation(latitude, longitude); - return am; - } catch (java.text.ParseException e) { - return null; - } - } - - - -} diff --git a/DataExtractionOSM/src/com/osmand/osm/io/OsmStorageWriter.java b/DataExtractionOSM/src/com/osmand/osm/io/OsmStorageWriter.java index c3d6c1e52a..a32c3094b8 100644 --- a/DataExtractionOSM/src/com/osmand/osm/io/OsmStorageWriter.java +++ b/DataExtractionOSM/src/com/osmand/osm/io/OsmStorageWriter.java @@ -16,24 +16,9 @@ import static com.osmand.osm.io.OsmBaseStorage.ELEM_OSM; import static com.osmand.osm.io.OsmBaseStorage.ELEM_RELATION; import static com.osmand.osm.io.OsmBaseStorage.ELEM_TAG; import static com.osmand.osm.io.OsmBaseStorage.ELEM_WAY; -import static com.osmand.osm.io.OsmIndexStorage.ATTR_CITYTYPE; -import static com.osmand.osm.io.OsmIndexStorage.ATTR_NAME; -import static com.osmand.osm.io.OsmIndexStorage.ATTR_SUBTYPE; -import static com.osmand.osm.io.OsmIndexStorage.ELEM_AMENITY; -import static com.osmand.osm.io.OsmIndexStorage.ELEM_BUILDING; -import static com.osmand.osm.io.OsmIndexStorage.ELEM_CITY; -import static com.osmand.osm.io.OsmIndexStorage.ELEM_OSMAND; -import static com.osmand.osm.io.OsmIndexStorage.ELEM_STREET; -import static com.osmand.osm.io.OsmIndexStorage.OSMAND_VERSION; -import java.io.File; import java.io.IOException; import java.io.OutputStream; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.sql.Statement; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -44,31 +29,9 @@ import java.util.Map.Entry; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamWriter; -import org.apache.commons.logging.Log; -import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.Field.Index; -import org.apache.lucene.document.Field.Store; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriter.MaxFieldLength; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.store.LockObtainFailedException; -import org.apache.lucene.util.Version; - import com.osmand.Algoritms; -import com.osmand.LogUtil; -import com.osmand.data.Amenity; -import com.osmand.data.Building; -import com.osmand.data.City; import com.osmand.data.MapObject; -import com.osmand.data.Region; -import com.osmand.data.Street; -import com.osmand.data.Amenity.AmenityType; -import com.osmand.data.City.CityType; import com.osmand.osm.Entity; -import com.osmand.osm.LatLon; import com.osmand.osm.Node; import com.osmand.osm.Relation; import com.osmand.osm.Way; @@ -79,10 +42,6 @@ public class OsmStorageWriter { private final String INDENT = " "; private final String INDENT2 = INDENT + INDENT; - private final String INDENT3 = INDENT + INDENT + INDENT; - private static final Log log = LogUtil.getLog(OsmStorageWriter.class); - - private static final Version VERSION = Version.LUCENE_30; public OsmStorageWriter(){ @@ -182,169 +141,14 @@ public class OsmStorageWriter { return "node"; } - public void saveLuceneIndex(File dir, Region region) throws CorruptIndexException, LockObtainFailedException, IOException{ - long now = System.currentTimeMillis(); - IndexWriter writer = null; - try { - // Make a lucene writer and create new Lucene index with arg3 = true - writer = new IndexWriter(FSDirectory.open(dir), new StandardAnalyzer(VERSION), true, MaxFieldLength.LIMITED); - for (Amenity a : region.getAmenityManager().getAllObjects()) { - index(a, writer); - } - writer.optimize(); - } finally { - try { - writer.close(); - } catch (Exception t) { - log.error("Failed to close index.", t); - throw new RuntimeException(t); - } - log.info(String.format("Indexing done in %s ms.", System.currentTimeMillis() - now)); - } - - } - // TODO externalize strings - protected void index(Amenity amenity, IndexWriter writer) throws CorruptIndexException, IOException { - Document document = new Document(); - document.add(new Field("id",""+amenity.getEntity().getId(),Store.YES, Index.NOT_ANALYZED)); - LatLon latLon = amenity.getEntity().getLatLon(); - document.add(new Field("name",amenity.getName(),Store.YES, Index.NOT_ANALYZED)); - document.add(new Field("longitude",OsmLuceneRepository.formatLongitude(latLon.getLongitude()),Store.YES, Index.ANALYZED)); - document.add(new Field("latitude",OsmLuceneRepository.formatLatitude(latLon.getLatitude()),Store.YES, Index.ANALYZED)); - document.add(new Field("type",amenity.getType().name(),Store.YES, Index.ANALYZED)); - document.add(new Field("subtype",amenity.getSubType(),Store.YES, Index.ANALYZED)); - //for (Entry entry:amenity.getNode().getTags().entrySet()) { - // document.add(new Field(entry.getKey(),entry.getValue(),Store.YES, Index.NOT_ANALYZED)); - //} - writer.addDocument(document); - } - - - public void saveSQLLitePOIIndex(File file, Region region) throws SQLException{ - long now = System.currentTimeMillis(); - try { - Class.forName("org.sqlite.JDBC"); - } catch (ClassNotFoundException e) { - log.error("Illegal configuration", e); - throw new IllegalStateException(e); - } - Connection conn = DriverManager.getConnection("jdbc:sqlite:"+file.getAbsolutePath()); - - final int batchSize = 500; - try { - Statement stat = conn.createStatement(); - stat.executeUpdate("create table poi (id long, latitude double, longitude double, name, type, subtype);"); - stat.executeUpdate("create index LatLonIndex ON poi (latitude, longitude);"); - PreparedStatement prep = conn.prepareStatement( - "insert into poi values (?, ?, ?, ?, ? ,? );"); - conn.setAutoCommit(false); - int currentCount = 0; - for (Amenity a : region.getAmenityManager().getAllObjects()) { - prep.setLong(1, a.getId()); - prep.setDouble(2, a.getLocation().getLatitude()); - prep.setDouble(3, a.getLocation().getLongitude()); - prep.setString(4, a.getName()); - prep.setString(5, AmenityType.valueToString(a.getType())); - prep.setString(6, a.getSubType()); - prep.addBatch(); - currentCount++; - if(currentCount >= batchSize){ - prep.executeBatch(); - currentCount = 0; - } - - } - if(currentCount > 0){ - prep.executeBatch(); - } - conn.setAutoCommit(true); - } finally { - conn.close(); - log.info(String.format("Indexing sqllite done in %s ms.", System.currentTimeMillis() - now)); - } - - } - - - public void savePOIIndex(OutputStream output, Region region) throws XMLStreamException, IOException { - PropertyManager propertyManager = new PropertyManager(PropertyManager.CONTEXT_WRITER); - XMLStreamWriter streamWriter = new XMLStreamWriterImpl(output, propertyManager); - - writeStartElement(streamWriter, ELEM_OSMAND, ""); - streamWriter.writeAttribute(ATTR_VERSION, OSMAND_VERSION); - List amenities = region.getAmenityManager().getAllObjects(); - for(Amenity n : amenities){ - if (couldBeWrited(n)) { - writeStartElement(streamWriter, ELEM_AMENITY, INDENT); - writeAttributesMapObject(streamWriter, n); - streamWriter.writeAttribute(ATTR_TYPE, AmenityType.valueToString(n.getType())); - streamWriter.writeAttribute(ATTR_SUBTYPE, n.getSubType()); - writeEndElement(streamWriter, INDENT); - } - } - writeEndElement(streamWriter, ""); // osmand - streamWriter.writeEndDocument(); - streamWriter.flush(); - } - - private void writeCity(XMLStreamWriter streamWriter, City c) throws XMLStreamException{ - writeStartElement(streamWriter, ELEM_CITY, INDENT); - writeAttributesMapObject(streamWriter, c); - streamWriter.writeAttribute(ATTR_CITYTYPE, CityType.valueToString(c.getType())); - for(Street s : c.getStreets()){ - if (couldBeWrited(s)) { - writeStartElement(streamWriter, ELEM_STREET, INDENT2); - writeAttributesMapObject(streamWriter, s); - for(Building b : s.getBuildings()) { - if (couldBeWrited(b)) { - writeStartElement(streamWriter, ELEM_BUILDING, INDENT3); - writeAttributesMapObject(streamWriter, b); - writeEndElement(streamWriter, INDENT3); - } - } - writeEndElement(streamWriter, INDENT2); - } - } - writeEndElement(streamWriter, INDENT); - } - - public void saveAddressIndex(OutputStream output, Region region) throws XMLStreamException, IOException { - PropertyManager propertyManager = new PropertyManager(PropertyManager.CONTEXT_WRITER); - XMLStreamWriter streamWriter = new XMLStreamWriterImpl(output, propertyManager); - - writeStartElement(streamWriter, ELEM_OSMAND, ""); - streamWriter.writeAttribute(ATTR_VERSION, OSMAND_VERSION); - for(CityType t : CityType.values()){ - Collection cities = region.getCitiesByType(t); - if(cities != null){ - for(City c : cities){ - if (couldBeWrited(c)) { - writeCity(streamWriter, c); - } - } - } - } - writeEndElement(streamWriter, ""); // osmand - streamWriter.writeEndDocument(); - streamWriter.flush(); - } - - public boolean couldBeWrited(MapObject e){ + + public boolean couldBeWrited(MapObject e){ if(!Algoritms.isEmpty(e.getName()) && e.getLocation() != null){ return true; } return false; } - - public void writeAttributesMapObject(XMLStreamWriter streamWriter, MapObject e) throws XMLStreamException{ - LatLon location = e.getLocation(); - streamWriter.writeAttribute(ATTR_LAT, location.getLatitude()+""); - streamWriter.writeAttribute(ATTR_LON, location.getLongitude()+""); - streamWriter.writeAttribute(ATTR_NAME, e.getName()); - streamWriter.writeAttribute(ATTR_ID, e.getId()+""); - } - private void writeStartElement(XMLStreamWriter writer, String name, String indent) throws XMLStreamException{ diff --git a/DataExtractionOSM/src/com/osmand/swing/MapPanel.java b/DataExtractionOSM/src/com/osmand/swing/MapPanel.java index 52b6e945a5..cb9eb9cacc 100644 --- a/DataExtractionOSM/src/com/osmand/swing/MapPanel.java +++ b/DataExtractionOSM/src/com/osmand/swing/MapPanel.java @@ -48,12 +48,12 @@ import javax.swing.UIManager; import org.apache.commons.logging.Log; -import com.osmand.IMapLocationListener; import com.osmand.LogUtil; import com.osmand.data.DataTileManager; import com.osmand.data.preparation.MapTileDownloader; import com.osmand.data.preparation.MapTileDownloader.DownloadRequest; import com.osmand.data.preparation.MapTileDownloader.IMapDownloaderCallback; +import com.osmand.map.IMapLocationListener; import com.osmand.map.ITileSource; import com.osmand.map.TileSourceManager; import com.osmand.map.TileSourceManager.TileSourceTemplate; diff --git a/DataExtractionOSM/src/com/osmand/swing/OsmExtractionUI.java b/DataExtractionOSM/src/com/osmand/swing/OsmExtractionUI.java index 14d7fbb101..443da889d6 100644 --- a/DataExtractionOSM/src/com/osmand/swing/OsmExtractionUI.java +++ b/DataExtractionOSM/src/com/osmand/swing/OsmExtractionUI.java @@ -64,7 +64,6 @@ import org.xml.sax.SAXException; import com.osmand.Algoritms; import com.osmand.ExceptionHandler; -import com.osmand.IMapLocationListener; import com.osmand.data.Amenity; import com.osmand.data.Building; import com.osmand.data.City; @@ -74,12 +73,14 @@ import com.osmand.data.Region; import com.osmand.data.Street; import com.osmand.data.Amenity.AmenityType; import com.osmand.data.City.CityType; +import com.osmand.data.index.DataIndexReader; +import com.osmand.data.index.DataIndexWriter; +import com.osmand.data.index.IndexConstants; import com.osmand.data.preparation.DataExtraction; -import com.osmand.data.preparation.DataIndexBuilder; +import com.osmand.map.IMapLocationListener; import com.osmand.osm.Entity; import com.osmand.osm.LatLon; import com.osmand.osm.MapUtils; -import com.osmand.osm.Node; import com.osmand.osm.Way; import com.osmand.osm.io.IOsmStorageFilter; import com.osmand.osm.io.OsmBoundsFilter; @@ -232,15 +233,14 @@ public class OsmExtractionUI implements IMapLocationListener { treePlaces.setEditable(true); treePlaces.setCellEditor(new RegionCellEditor(treePlaces, (DefaultTreeCellRenderer) treePlaces.getCellRenderer())); treePlaces.addTreeSelectionListener(new TreeSelectionListener() { - @SuppressWarnings("unchecked") @Override public void valueChanged(TreeSelectionEvent e) { if (e.getPath() != null) { if (e.getPath().getLastPathComponent() instanceof DataExtractionTreeNode) { Object o = ((DataExtractionTreeNode) e.getPath().getLastPathComponent()).getModelObject(); - if (o instanceof MapObject) { - MapObject c = (MapObject) o; + if (o instanceof MapObject) { + MapObject c = (MapObject) o; LatLon location = c.getLocation(); if(location != null){ if(o instanceof Street){ @@ -279,8 +279,8 @@ public class OsmExtractionUI implements IMapLocationListener { if(((DataExtractionTreeNode) node).getModelObject() instanceof Region){ Region r = (Region) ((DataExtractionTreeNode) node).getModelObject(); r.setName(node.getUserObject().toString()); - } else if(((DataExtractionTreeNode) node).getModelObject() instanceof MapObject){ - MapObject r = (MapObject) ((DataExtractionTreeNode) node).getModelObject(); + } else if(((DataExtractionTreeNode) node).getModelObject() instanceof MapObject){ + MapObject r = (MapObject) ((DataExtractionTreeNode) node).getModelObject(); r.setName(node.getUserObject().toString()); } } @@ -355,28 +355,29 @@ public class OsmExtractionUI implements IMapLocationListener { @Override public void run() { dlg.startTask("Generating indices...", -1); - DataIndexBuilder builder = new DataIndexBuilder(DataExtractionSettings.getSettings().getDefaultWorkingDir(), region); + DataIndexWriter builder = new DataIndexWriter(DataExtractionSettings.getSettings().getDefaultWorkingDir(), region); StringBuilder msg = new StringBuilder(); try { msg.append("Indices for ").append(region.getName()); - if(buildPoiIndex.isEnabled()){ + if(buildPoiIndex.isSelected()){ dlg.startTask("Generating POI index...", -1); - builder.buildPOI(); + builder.writePOI(); msg.append(", POI index ").append("successfully created"); } - if(buildAddressIndex.isEnabled()){ + if(buildAddressIndex.isSelected()){ dlg.startTask("Generating address index...", -1); - builder.buildAddress(); + builder.writeAddress(); msg.append(", Address index ").append("successfully created"); } + new DataIndexReader().testIndex(new File( + DataExtractionSettings.getSettings().getDefaultWorkingDir(), + IndexConstants.ADDRESS_INDEX_DIR+region.getName()+IndexConstants.ADDRESS_INDEX_EXT)); msg.append("."); JOptionPane pane = new JOptionPane(msg); JDialog dialog = pane.createDialog(frame, "Generation data"); dialog.setVisible(true); } catch (SQLException e1) { throw new IllegalArgumentException(e1); - } catch (XMLStreamException e1) { - throw new IllegalArgumentException(e1); } catch (IOException e1) { throw new IllegalArgumentException(e1); } @@ -441,10 +442,10 @@ public class OsmExtractionUI implements IMapLocationListener { @Override public void valueChanged(ListSelectionEvent e) { if(searchList.getSelectedValue() != null){ - Node node = ((City)searchList.getSelectedValue()).getEntity(); + LatLon node = ((City)searchList.getSelectedValue()).getLocation(); String text = "Lat : " + node.getLatitude() + " Lon " + node.getLongitude(); if(selectedCity != null){ - text += " distance " + MapUtils.getDistance(selectedCity.getEntity(), node); + text += " distance " + MapUtils.getDistance(node, node); } mapPanel.setLatLon(node.getLatitude(), node.getLongitude()); } @@ -627,7 +628,8 @@ public class OsmExtractionUI implements IMapLocationListener { Region res; try { DataExtraction dataExtraction = new DataExtraction(buildAddressIndex.isSelected(), buildPoiIndex.isSelected(), - normalizingStreets.isSelected(), loadingAllData.isSelected()); + normalizingStreets.isSelected(), loadingAllData.isSelected(), + DataExtractionSettings.getSettings().getDefaultWorkingDir()); if(!buildAddressIndex.isSelected()){ buildAddressIndex.setEnabled(false); } @@ -639,6 +641,8 @@ public class OsmExtractionUI implements IMapLocationListener { throw new IllegalArgumentException(e); } catch (SAXException e) { throw new IllegalStateException(e); + } catch (SQLException e) { + throw new IllegalStateException(e); } dlg.setResult(res); } @@ -783,7 +787,7 @@ public class OsmExtractionUI implements IMapLocationListener { Object node = tree.getLastSelectedPathComponent(); if (node instanceof DataExtractionTreeNode) { DataExtractionTreeNode treeNode = (DataExtractionTreeNode) node; - if (treeNode.getModelObject() instanceof Region || treeNode.getModelObject() instanceof MapObject) { + if (treeNode.getModelObject() instanceof Region || treeNode.getModelObject() instanceof MapObject) { return true; } } diff --git a/OsmAnd/.classpath b/OsmAnd/.classpath index eec77f1a36..605aecf7de 100644 --- a/OsmAnd/.classpath +++ b/OsmAnd/.classpath @@ -5,6 +5,5 @@ - diff --git a/OsmAnd/lib/lucene-core-3.0.1.jar b/OsmAnd/lib/lucene-core-3.0.1.jar deleted file mode 100644 index 411524695e..0000000000 Binary files a/OsmAnd/lib/lucene-core-3.0.1.jar and /dev/null differ diff --git a/OsmAnd/src/com/osmand/AmenityIndexRepository.java b/OsmAnd/src/com/osmand/AmenityIndexRepository.java new file mode 100644 index 0000000000..5ee1bc59d2 --- /dev/null +++ b/OsmAnd/src/com/osmand/AmenityIndexRepository.java @@ -0,0 +1,137 @@ +package com.osmand; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; + +import android.database.Cursor; +import android.database.sqlite.SQLiteDatabase; + +import com.osmand.data.Amenity; +import com.osmand.data.Amenity.AmenityType; +import com.osmand.data.index.IndexConstants; +import com.osmand.data.index.IndexConstants.IndexPoiTable; +import com.osmand.osm.LatLon; + +public class AmenityIndexRepository { + private static final Log log = LogUtil.getLog(AmenityIndexRepository.class); + public final static int LIMIT_AMENITIES = 500; + + + private SQLiteDatabase db; + private double dataTopLatitude; + private double dataBottomLatitude; + private double dataLeftLongitude; + private double dataRightLongitude; + + private String name; + + // cache amenities + private List cachedAmenities = new ArrayList(); + private double cTopLatitude; + private double cBottomLatitude; + private double cLeftLongitude; + private double cRightLongitude; + + + + private final String[] columns = IndexConstants.generateColumnNames(IndexPoiTable.values()); + public List searchAmenities(double topLatitude, double leftLongitude, double bottomLatitude, double rightLongitude, int limit, List amenities){ + long now = System.currentTimeMillis(); + Cursor query = db.query(IndexPoiTable.getTable(), columns, "? < latitude AND latitude < ? AND ? < longitude AND longitude < ?", + new String[]{Double.toString(bottomLatitude), + Double.toString(topLatitude), Double.toString(leftLongitude), Double.toString(rightLongitude)}, null, null, null); + if(query.moveToFirst()){ + do { + Amenity am = new Amenity(); + am.setId(query.getLong(IndexPoiTable.ID.ordinal())); + am.setLocation(query.getDouble(IndexPoiTable.LATITUDE.ordinal()), + query.getDouble(IndexPoiTable.LONGITUDE.ordinal())); + am.setName(query.getString(IndexPoiTable.NAME.ordinal() )); + am.setType(AmenityType.fromString(query.getString(IndexPoiTable.TYPE.ordinal()))); + am.setSubType(query.getString(IndexPoiTable.SUBTYPE.ordinal())); + amenities.add(am); + if(limit != -1 && amenities.size() >= limit){ + break; + } + } while(query.moveToNext()); + } + query.deactivate(); + + if (log.isDebugEnabled()) { + log.debug(String.format("Search for %s done in %s ms found %s.", + topLatitude + " " + leftLongitude, System.currentTimeMillis() - now, amenities.size())); + } + return amenities; + } + + + public void evaluateCachedAmenities(double topLatitude, double leftLongitude, double bottomLatitude, double rightLongitude, List toFill){ + cachedAmenities.clear(); + cTopLatitude = topLatitude + (topLatitude -bottomLatitude); + cBottomLatitude = bottomLatitude - (topLatitude -bottomLatitude); + cLeftLongitude = leftLongitude - (rightLongitude - leftLongitude); + cRightLongitude = rightLongitude + (rightLongitude - leftLongitude); + searchAmenities(topLatitude, leftLongitude, bottomLatitude, rightLongitude, -1, cachedAmenities); + checkCachedAmenities(topLatitude, leftLongitude, bottomLatitude, rightLongitude, toFill); + } + + public boolean checkCachedAmenities(double topLatitude, double leftLongitude, double bottomLatitude, double rightLongitude, List toFill){ + if (db == null) { + return true; + } + if (cTopLatitude >= topLatitude && cLeftLongitude <= leftLongitude && cRightLongitude >= rightLongitude + && cBottomLatitude <= bottomLatitude) { + for(Amenity a : cachedAmenities){ + LatLon location = a.getLocation(); + if (location.getLatitude() <= topLatitude && location.getLongitude() >= leftLongitude && location.getLongitude() <= rightLongitude + && location.getLatitude() >= bottomLatitude) { + toFill.add(a); + } + } + return true; + } + return false; + } + + public void initialize(final IProgress progress, File file) { + long start = System.currentTimeMillis(); + if(db != null){ + // close previous db + db.close(); + } + db = SQLiteDatabase.openOrCreateDatabase(file, null); + name = file.getName().substring(0, file.getName().indexOf('.')); + Cursor query = db.query(IndexPoiTable.getTable(), new String[]{"MAX(latitude)", "MAX(longitude)", "MIN(latitude)", "MIN(longitude)"}, null, null,null, null, null); + if(query.moveToFirst()){ + dataTopLatitude = query.getDouble(0); + dataRightLongitude = query.getDouble(1); + dataBottomLatitude = query.getDouble(2); + dataLeftLongitude = query.getDouble(3); + } + if (log.isDebugEnabled()) { + log.debug("Initializing db " + file.getAbsolutePath() + " " + (System.currentTimeMillis() - start) + "ms"); + } + + } + + public String getName() { + return name; + } + + + public boolean checkContains(double topLatitude, double leftLongitude, double bottomLatitude, double rightLongitude){ + if(rightLongitude < dataLeftLongitude || leftLongitude > dataRightLongitude){ + return false; + } + if(topLatitude < dataBottomLatitude || bottomLatitude > dataTopLatitude){ + return false; + } + return true; + } + + + +} diff --git a/OsmAnd/src/com/osmand/OsmSQLLiteRepository.java b/OsmAnd/src/com/osmand/OsmSQLLiteRepository.java deleted file mode 100644 index 9f1a5c9efa..0000000000 --- a/OsmAnd/src/com/osmand/OsmSQLLiteRepository.java +++ /dev/null @@ -1,122 +0,0 @@ -package com.osmand; - -import java.io.File; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import org.apache.commons.logging.Log; - -import android.database.Cursor; -import android.database.sqlite.SQLiteDatabase; - -import com.osmand.data.Amenity; -import com.osmand.data.Amenity.AmenityType; - -public class OsmSQLLiteRepository { - private static final Log log = LogUtil.getLog(OsmSQLLiteRepository.class); - - private SQLiteDatabase db; - - private List cachedAmenities = null; - private double cTopLatitude; - private double cBottomLatitude; - private double cLeftLongitude; - private double cRightLongitude; - - private boolean isLoading = false; - - protected synchronized void loadAmenitiesInAnotherThread(final double topLatitude, final double leftLongitude, final double bottomLatitude, final double rightLongitude){ - isLoading = true; - new Thread(new Runnable(){ - - @Override - public void run() { - try { - cachedAmenities = internalSearch(topLatitude, leftLongitude, bottomLatitude, rightLongitude); - cTopLatitude = topLatitude; - cLeftLongitude = leftLongitude; - cBottomLatitude = bottomLatitude ; - cRightLongitude = rightLongitude; - } finally { - synchronized (this) { - isLoading = false; - } - } - } - }, "Searching in index...").start(); - } - - - private List internalSearch(double topLatitude, double leftLongitude, double bottomLatitude, double rightLongitude){ - long now = System.currentTimeMillis(); - Cursor query = db.query("poi", null, "? < latitude AND latitude < ? AND ? < longitude AND longitude < ?", - new String[]{Double.toString(bottomLatitude), - Double.toString(topLatitude), Double.toString(leftLongitude), Double.toString(rightLongitude)}, null, null, null); - List amenities = new ArrayList(); - int idIndex = query.getColumnIndex("id"); - int latitudeIndex = query.getColumnIndex("latitude"); - int longitudeIndex = query.getColumnIndex("longitude"); - int nameIndex = query.getColumnIndex("name"); - int typeIndex = query.getColumnIndex("type"); - int subtypeIndex = query.getColumnIndex("subtype"); - if(query.moveToFirst()){ - do { - Amenity am = new Amenity(); - if(idIndex != -1){ - am.setId(query.getLong(idIndex)); - } - if(latitudeIndex != -1 && longitudeIndex != -1){ - am.setLocation(query.getDouble(latitudeIndex), query.getDouble(longitudeIndex)); - } - if(nameIndex != -1){ - am.setName(query.getString(nameIndex)); - } - if(typeIndex != -1){ - am.setType(AmenityType.fromString(query.getString(typeIndex))); - } - if(subtypeIndex != -1){ - am.setSubType(query.getString(subtypeIndex)); - } - amenities.add(am); - } while(query.moveToNext()); - } - query.deactivate(); - - if (log.isDebugEnabled()) { - log.debug(String.format("Search for %s done in %s ms found %s.", - topLatitude + " " + leftLongitude, System.currentTimeMillis() - now, amenities.size())); - } - return amenities; - } - public synchronized List searchAmenities(double topLatitude, double leftLongitude, double bottomLatitude, double rightLongitude) { - if (db == null) { - return Collections.emptyList(); - } - if (cTopLatitude >= topLatitude && cLeftLongitude <= leftLongitude && cRightLongitude >= rightLongitude - && cBottomLatitude <= bottomLatitude) { - return cachedAmenities; - } - if(!isLoading){ - double h = (topLatitude - bottomLatitude); - double w = (rightLongitude - leftLongitude); - topLatitude += h; - leftLongitude -= w; - bottomLatitude -= h; - rightLongitude += w; - loadAmenitiesInAnotherThread(topLatitude, leftLongitude, bottomLatitude, rightLongitude); - } - return Collections.emptyList(); - } - - public void initialize(final IProgress progress, File file) { - long start = System.currentTimeMillis(); - db = SQLiteDatabase.openOrCreateDatabase(file, null); - if (log.isDebugEnabled()) { - log.debug("Initializing db " + file.getAbsolutePath() + " " + (System.currentTimeMillis() - start) + "ms"); - } - } - - - -} diff --git a/OsmAnd/src/com/osmand/ResourceManager.java b/OsmAnd/src/com/osmand/ResourceManager.java index aaf90992b9..de60a0012a 100644 --- a/OsmAnd/src/com/osmand/ResourceManager.java +++ b/OsmAnd/src/com/osmand/ResourceManager.java @@ -1,35 +1,27 @@ package com.osmand; import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; import java.util.ArrayList; -import java.util.Enumeration; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Stack; import java.util.TreeMap; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; import org.apache.commons.logging.Log; -import org.xml.sax.SAXException; import android.graphics.Bitmap; import android.graphics.BitmapFactory; import android.os.Environment; import com.osmand.data.Amenity; -import com.osmand.data.DataTileManager; import com.osmand.data.Region; +import com.osmand.data.index.IndexConstants; import com.osmand.data.preparation.MapTileDownloader; import com.osmand.data.preparation.MapTileDownloader.DownloadRequest; import com.osmand.data.preparation.MapTileDownloader.IMapDownloaderCallback; import com.osmand.map.ITileSource; -import com.osmand.osm.LatLon; -import com.osmand.osm.io.OsmIndexStorage; -import com.osmand.osm.io.OsmLuceneRepository; +import com.osmand.osm.MapUtils; /** * Resource manager is responsible to work with all resources @@ -41,10 +33,9 @@ import com.osmand.osm.io.OsmLuceneRepository; */ public class ResourceManager { - private static final String POI_PATH = "osmand/poi/"; - private static final String ADDRESS_PATH = "osmand/address/"; + private static final String POI_PATH = "osmand/" + IndexConstants.POI_INDEX_DIR; + private static final String ADDRESS_PATH = "osmand/" + IndexConstants.ADDRESS_INDEX_DIR; private static final String TILES_PATH = "osmand/tiles/"; - private static final String LUCENE_PATH = "osmand/lucene/"; private static final Log log = LogUtil.getLog(ResourceManager.class); @@ -60,21 +51,21 @@ public class ResourceManager { // it is not good investigated but no more than 64 (satellite images) protected final int maxImgCacheSize = 64; - private DataTileManager poiIndex = null; - - private Map addressMap = new TreeMap(); - protected Map cacheOfImages = new LinkedHashMap(); protected File dirWithTiles ; private MapTileDownloader downloader = MapTileDownloader.getInstance(); + + // Indexes + private Map addressMap = new TreeMap(); + + protected List amenityRepositories = new ArrayList(); + public AsyncLoadingThread asyncLoadingTiles = new AsyncLoadingThread(); - protected OsmLuceneRepository amenityIndexSearcher = new OsmLuceneRepository(); - protected OsmSQLLiteRepository amenityRepository = new OsmSQLLiteRepository(); public ResourceManager() { @@ -87,7 +78,9 @@ public class ResourceManager { } - /// Working with tiles /// + ////////////////////////////////////////////// Working with tiles //////////////////////////////////////////////// + + public Bitmap getTileImageForMapAsync(ITileSource map, int x, int y, int zoom, boolean loadFromInternetIfNeeded) { return getTileImageForMap(map, x, y, zoom, loadFromInternetIfNeeded, false); } @@ -120,6 +113,121 @@ public class ResourceManager { + private Bitmap getRequestedImageTile(TileLoadDownloadRequest req){ + if(req.fileToLoad == null || req.dirWithTiles == null){ + return null; + } + File en = new File(req.dirWithTiles, req.fileToLoad); + if (cacheOfImages.size() > maxImgCacheSize) { + onLowMemory(); + } + + if (!downloader.isFileCurrentlyDownloaded(en) && req.dirWithTiles.canRead()) { + if (en.exists()) { + long time = System.currentTimeMillis(); + cacheOfImages.put(req.fileToLoad, BitmapFactory.decodeFile(en.getAbsolutePath())); + if (log.isDebugEnabled()) { + log.debug("Loaded file : " + req.fileToLoad + " " + -(time - System.currentTimeMillis()) + " ms"); + } + } + + if(cacheOfImages.get(req.fileToLoad) == null && req.url != null){ + // TODO we could check that network is available (context is required) +// ConnectivityManager mgr = (ConnectivityManager) getContext().getSystemService(Context.CONNECTIVITY_SERVICE); +// NetworkInfo info = mgr.getActiveNetworkInfo(); +// if (info != null && info.isConnected()) { +// downloader.requestToDownload(req); +// } + downloader.requestToDownload(req); + } + } + return cacheOfImages.get(req.fileToLoad); + } + + ////////////////////////////////////////////// Working with indexes //////////////////////////////////////////////// + // POI INDEX // + public void indexingPoi(final IProgress progress) { + File file = new File(Environment.getExternalStorageDirectory(), POI_PATH); + amenityRepositories.clear(); + if (file.exists() && file.canRead()) { + for (File f : file.listFiles()) { + if (f.getName().endsWith(IndexConstants.POI_INDEX_EXT)) { + AmenityIndexRepository repository = new AmenityIndexRepository(); + progress.startTask("Indexing poi " + f.getName(), -1); + repository.initialize(progress, f); + amenityRepositories.add(repository); + } + } + } + } + + + public void indexingAddresses(final IProgress progress){ + File file = new File(Environment.getExternalStorageDirectory(), ADDRESS_PATH); + addressMap.clear(); + if (file.exists() && file.canRead()) { + for (File f : file.listFiles()) { + if (f.getName().endsWith(IndexConstants.ADDRESS_INDEX_EXT)) { + // TODO fill in address index + + } + } + } + } + + ////////////////////////////////////////////// Working with amenities //////////////////////////////////////////////// + public List searchAmenities(double latitude, double longitude, int zoom, int limit) { + double tileNumberX = Math.floor(MapUtils.getTileNumberX(zoom, longitude)); + double tileNumberY = Math.floor(MapUtils.getTileNumberY(zoom, latitude)); + double topLatitude = MapUtils.getLatitudeFromTile(zoom, tileNumberY); + double bottomLatitude = MapUtils.getLatitudeFromTile(zoom, tileNumberY + 1); + double leftLongitude = MapUtils.getLongitudeFromTile(zoom, tileNumberX); + double rightLongitude = MapUtils.getLongitudeFromTile(zoom, tileNumberX + 1); + List amenities = new ArrayList(); + for(AmenityIndexRepository index : amenityRepositories){ + if(index.checkContains(topLatitude, leftLongitude, bottomLatitude, rightLongitude)){ + if(!index.checkCachedAmenities(topLatitude, leftLongitude, bottomLatitude, rightLongitude, amenities)){ + index.searchAmenities(topLatitude, leftLongitude, bottomLatitude, rightLongitude, limit, amenities); + } + } + } + + return amenities; + } + + public void searchAmenitiesAsync(double topLatitude, double leftLongitude, double bottomLatitude, double rightLongitude, List toFill){ + for(AmenityIndexRepository index : amenityRepositories){ + if(index.checkContains(topLatitude, leftLongitude, bottomLatitude, rightLongitude)){ + if(!index.checkCachedAmenities(topLatitude, leftLongitude, bottomLatitude, rightLongitude, toFill)){ + // TODO add request to another thread + index.evaluateCachedAmenities(topLatitude, leftLongitude, bottomLatitude, rightLongitude, toFill); + } + } + } + } + + + + /// On low memory method /// + public void onLowMemory() { + log.info("On low memory : cleaning tiles - size = " + cacheOfImages.size()); + ArrayList list = new ArrayList(cacheOfImages.keySet()); + // remove first images (as we think they are older) + for (int i = 0; i < list.size()/2; i ++) { + Bitmap bmp = cacheOfImages.remove(list.get(i)); + if(bmp != null){ + bmp.recycle(); + } + } + // TODO clear amenity indexes & addresses indexes + System.gc(); + } + + + + + + private static class TileLoadDownloadRequest extends DownloadRequest { public final String fileToLoad; @@ -131,8 +239,9 @@ public class ResourceManager { this.dirWithTiles = dirWithTiles; this.fileToLoad = fileToLoad; } - } + + public class AsyncLoadingThread extends Thread { Stack requests = new Stack(); @@ -170,182 +279,4 @@ public class ResourceManager { requests.push(req); } }; - - private Bitmap getRequestedImageTile(TileLoadDownloadRequest req){ - if(req.fileToLoad == null || req.dirWithTiles == null){ - return null; - } - File en = new File(req.dirWithTiles, req.fileToLoad); - if (cacheOfImages.size() > maxImgCacheSize) { - onLowMemory(); - } - - if (!downloader.isFileCurrentlyDownloaded(en) && req.dirWithTiles.canRead()) { - if (en.exists()) { - long time = System.currentTimeMillis(); - cacheOfImages.put(req.fileToLoad, BitmapFactory.decodeFile(en.getAbsolutePath())); - if (log.isDebugEnabled()) { - log.debug("Loaded file : " + req.fileToLoad + " " + -(time - System.currentTimeMillis()) + " ms"); - } - } - - if(cacheOfImages.get(req.fileToLoad) == null && req.url != null){ - // TODO we could check that network is available (context is required) -// ConnectivityManager mgr = (ConnectivityManager) getContext().getSystemService(Context.CONNECTIVITY_SERVICE); -// NetworkInfo info = mgr.getActiveNetworkInfo(); -// if (info != null && info.isConnected()) { -// downloader.requestToDownload(req); -// } - downloader.requestToDownload(req); - } - } - return cacheOfImages.get(req.fileToLoad); - } - - private interface IndexVisitor { - /** - * returns if entry was visited succesfully - */ - public boolean visitEntry(String entryName, InputStream stream) throws IOException, SAXException; - } - - - public void indexingFiles(String pathToIndex, String ext, IProgress progress, String objectToIndex, IndexVisitor visitor) { - File file = new File(Environment.getExternalStorageDirectory(), pathToIndex); - if (file.exists() && file.canRead()) { - for (File f : file.listFiles()) { - InputStream stream = null; - ZipFile zipFile = null; - Enumeration entries = null; - try { - if (f.getName().endsWith(".zip")) { - zipFile = new ZipFile(f); - entries = zipFile.entries(); - } else { - stream = new FileInputStream(f); - } - } catch (IOException e) { - log.error("Can't read file " + f.getAbsolutePath(), e); - continue; - } - String entryName = f.getName(); - do { - try { - if (entries != null && entries.hasMoreElements()) { - ZipEntry entry = entries.nextElement(); - entryName = entry.getName(); - stream = zipFile.getInputStream(entry); - } - - if (entryName != null && entryName.endsWith(ext)) { - long start = System.currentTimeMillis(); - if (log.isDebugEnabled()) { - log.debug("Starting index " + objectToIndex + " " + f.getAbsolutePath()); - } - - if (progress != null) { - progress.startTask("Indexing " + objectToIndex + " " + f.getName(), stream.available()); - } - visitor.visitEntry(f.getName(), stream); - if (log.isDebugEnabled()) { - log.debug("Finished index " + objectToIndex + " " + f.getAbsolutePath() + " " - + (System.currentTimeMillis() - start) + "ms"); - } - } - } catch (IOException e) { - log.error("Can't read file " + f.getAbsolutePath(), e); - } catch (SAXException e) { - log.error("Can't read file " + f.getAbsolutePath(), e); - } finally { - Algoritms.closeStream(stream); - } - } while (zipFile != null && entries.hasMoreElements()); - } - } - } - - // POI INDEX // - public void indexingPoi(final IProgress progress) { - if (poiIndex == null) { - poiIndex = new DataTileManager(); - indexingFiles(POI_PATH, ".osmand", progress, "POI", new IndexVisitor() { - @Override - public boolean visitEntry(String entryName, InputStream stream) throws IOException, SAXException { - OsmIndexStorage storage = new OsmIndexStorage(new Region()); - storage.parseOSM(stream, progress); - Region region = ((OsmIndexStorage) storage).getRegion(); - for (Amenity a : region.getAmenityManager().getAllObjects()) { - LatLon location = a.getLocation(); - poiIndex.registerObject(location.getLatitude(), location.getLongitude(), a); - } - return true; - } - - }); - - File file = new File(Environment.getExternalStorageDirectory(), POI_PATH); - if (file.exists() && file.canRead()) { - for (File f : file.listFiles()) { - if(f.getName().endsWith(".db")){ - progress.startTask("Indexing poi " + f.getName(), -1); - amenityRepository.initialize(progress, f); - } - } - } - } - } - - public void indexingLucene(final IProgress progress){ - // read index - File file = new File(Environment.getExternalStorageDirectory(), LUCENE_PATH); - if (file.exists() && file.canRead()) { - amenityIndexSearcher.indexing(progress, file); - } - } - - public void indexingAddresses(final IProgress progress){ - indexingFiles(ADDRESS_PATH, ".osmand", progress, "address", new IndexVisitor() { - @Override - public boolean visitEntry(String entryName, InputStream stream) throws IOException, SAXException { - String name = entryName.substring(0, entryName.indexOf('.')); - Region region = new Region(); - region.setName(name); - addressMap.put(name, region); - OsmIndexStorage storage = new OsmIndexStorage(region); - storage.parseOSM(stream, progress); - return true; - } - }); - } - - public DataTileManager getPoiIndex() { - if(poiIndex == null){ - indexingPoi(null); - } - return poiIndex; - } - - public OsmSQLLiteRepository getAmenityRepository() { - return amenityRepository; - } - - - public OsmLuceneRepository getAmenityIndexSearcher(){ - return amenityIndexSearcher; - } - - - /// On low memory method /// - public void onLowMemory() { - log.info("On low memory : cleaning tiles - size = " + cacheOfImages.size()); - ArrayList list = new ArrayList(cacheOfImages.keySet()); - // remove first images (as we think they are older) - for (int i = 0; i < list.size()/2; i ++) { - Bitmap bmp = cacheOfImages.remove(list.get(i)); - if(bmp != null){ - bmp.recycle(); - } - } - System.gc(); - } } diff --git a/OsmAnd/src/com/osmand/activities/MainMenuActivity.java b/OsmAnd/src/com/osmand/activities/MainMenuActivity.java index a78efb4385..1c360dd462 100644 --- a/OsmAnd/src/com/osmand/activities/MainMenuActivity.java +++ b/OsmAnd/src/com/osmand/activities/MainMenuActivity.java @@ -93,7 +93,6 @@ public class MainMenuActivity extends Activity { try { ResourceManager.getResourceManager().indexingPoi(impl); ResourceManager.getResourceManager().indexingAddresses(impl); - ResourceManager.getResourceManager().indexingLucene(impl); } finally { dlg.dismiss(); } diff --git a/OsmAnd/src/com/osmand/activities/MapActivity.java b/OsmAnd/src/com/osmand/activities/MapActivity.java index dc5efc955c..dc428b12b4 100644 --- a/OsmAnd/src/com/osmand/activities/MapActivity.java +++ b/OsmAnd/src/com/osmand/activities/MapActivity.java @@ -22,11 +22,11 @@ import android.widget.ImageButton; import android.widget.Toast; import android.widget.ZoomControls; -import com.osmand.IMapLocationListener; import com.osmand.OsmandSettings; import com.osmand.R; import com.osmand.ResourceManager; import com.osmand.data.preparation.MapTileDownloader; +import com.osmand.map.IMapLocationListener; import com.osmand.osm.LatLon; import com.osmand.views.OsmandMapTileView; import com.osmand.views.POIMapLayer; @@ -64,7 +64,6 @@ public class MapActivity extends Activity implements LocationListener, IMapLocat MapTileDownloader.getInstance().addDownloaderCallback(mapView); mapView.setMapLocationListener(this); poiMapLayer = new POIMapLayer(); - poiMapLayer.setNodeManager(ResourceManager.getResourceManager().getPoiIndex()); mapView.addLayer(poiMapLayer); locationLayer = new PointLocationLayer(); mapView.addLayer(locationLayer); diff --git a/OsmAnd/src/com/osmand/activities/SearchActivity.java b/OsmAnd/src/com/osmand/activities/SearchActivity.java index 74fd084bbe..2751eba32f 100644 --- a/OsmAnd/src/com/osmand/activities/SearchActivity.java +++ b/OsmAnd/src/com/osmand/activities/SearchActivity.java @@ -24,7 +24,6 @@ import com.osmand.OsmandSettings; import com.osmand.R; import com.osmand.ResourceManager; import com.osmand.data.Amenity; -import com.osmand.data.DataTileManager; import com.osmand.data.Amenity.AmenityType; import com.osmand.osm.LatLon; import com.osmand.osm.MapUtils; @@ -67,11 +66,11 @@ public class SearchActivity extends ListActivity { } private void createAmenityFilter() { - DataTileManager poiIndex = ResourceManager.getResourceManager().getPoiIndex(); + ResourceManager resourceManager = ResourceManager.getResourceManager(); filter = new TreeMap>(); LatLon lastKnownMapLocation = OsmandSettings.getLastKnownMapLocation(this); - List closestAmenities = poiIndex.getClosestObjects(lastKnownMapLocation.getLatitude(), - lastKnownMapLocation.getLongitude(), 0, 5); + List closestAmenities = resourceManager.searchAmenities(lastKnownMapLocation.getLatitude(), + lastKnownMapLocation.getLongitude(), 13, 500); MapUtils.sortListOfMapObject(closestAmenities, lastKnownMapLocation.getLatitude(), lastKnownMapLocation.getLongitude()); for (Amenity n : closestAmenities) { AmenityType type = n.getType(); diff --git a/OsmAnd/src/com/osmand/views/OsmandMapTileView.java b/OsmAnd/src/com/osmand/views/OsmandMapTileView.java index e7fb3d7919..61d9bfd8c6 100644 --- a/OsmAnd/src/com/osmand/views/OsmandMapTileView.java +++ b/OsmAnd/src/com/osmand/views/OsmandMapTileView.java @@ -22,13 +22,13 @@ import android.view.SurfaceHolder; import android.view.SurfaceView; import android.view.SurfaceHolder.Callback; -import com.osmand.IMapLocationListener; import com.osmand.LogUtil; import com.osmand.OsmandSettings; import com.osmand.ResourceManager; import com.osmand.data.preparation.MapTileDownloader; import com.osmand.data.preparation.MapTileDownloader.DownloadRequest; import com.osmand.data.preparation.MapTileDownloader.IMapDownloaderCallback; +import com.osmand.map.IMapLocationListener; import com.osmand.map.ITileSource; import com.osmand.osm.MapUtils; import com.osmand.views.AnimateDraggingMapThread.AnimateDraggingCallback; diff --git a/OsmAnd/src/com/osmand/views/POIMapLayer.java b/OsmAnd/src/com/osmand/views/POIMapLayer.java index 8ec70ebd15..bc641724d6 100644 --- a/OsmAnd/src/com/osmand/views/POIMapLayer.java +++ b/OsmAnd/src/com/osmand/views/POIMapLayer.java @@ -1,5 +1,6 @@ package com.osmand.views; +import java.util.ArrayList; import java.util.List; import android.graphics.Canvas; @@ -8,21 +9,18 @@ import android.graphics.Paint; import android.view.MotionEvent; import android.widget.Toast; -import com.osmand.OsmSQLLiteRepository; import com.osmand.ResourceManager; import com.osmand.data.Amenity; -import com.osmand.data.DataTileManager; import com.osmand.osm.MapUtils; -import com.osmand.osm.io.OsmLuceneRepository; public class POIMapLayer implements OsmandMapLayer { private static final int radiusClick = 2; // for 15 level zoom - private DataTileManager nodeManager = null; - private Paint pointUI; private Paint pointAltUI; private OsmandMapTileView view; - private List objects; + private List objects = new ArrayList(); + + private ResourceManager resourceManager; @@ -57,29 +55,15 @@ public class POIMapLayer implements OsmandMapLayer { } - public void setNodeManager(DataTileManager nodeManager) { - this.nodeManager = nodeManager; - if(view != null){ - view.prepareImage(); - } - } - - public DataTileManager getNodeManager() { - return nodeManager; - } - @Override public void initLayer(OsmandMapTileView view) { this.view = view; - pointUI = new Paint(); - pointUI.setColor(Color.BLUE); - pointUI.setAlpha(150); - pointUI.setAntiAlias(true); - + pointAltUI = new Paint(); pointAltUI.setColor(Color.GREEN); pointAltUI.setAlpha(150); pointAltUI.setAntiAlias(true); + resourceManager = ResourceManager.getResourceManager(); } public int getRadiusPoi(int zoom){ @@ -92,7 +76,6 @@ public class POIMapLayer implements OsmandMapLayer { @Override public void onDraw(Canvas canvas) { - if (view.getZoom() >= 15) { double tileNumberX = MapUtils.getTileNumberX(view.getZoom(), view.getLongitude()); double tileNumberY = MapUtils.getTileNumberY(view.getZoom(), view.getLatitude()); @@ -103,42 +86,18 @@ public class POIMapLayer implements OsmandMapLayer { double topLatitude = MapUtils.getLatitudeFromTile(view.getZoom(), yTileUp); double leftLongitude = MapUtils.getLongitudeFromTile(view.getZoom(), xTileLeft); double bottomLatitude = MapUtils.getLatitudeFromTile(view.getZoom(), yTileDown); - double rightLongitude= MapUtils.getLongitudeFromTile(view.getZoom(), xTileRight); - - OsmLuceneRepository searcher = ResourceManager.getResourceManager().getAmenityIndexSearcher(); - if (searcher != null) { - objects = searcher.searchAmenities(topLatitude, leftLongitude, bottomLatitude, rightLongitude); - for (Amenity o : objects) { - double tileX = MapUtils.getTileNumberX(view.getZoom(), o.getLocation().getLongitude()); - int x = (int) ((tileX - xTileLeft) * getTileSize()); - double tileY = MapUtils.getTileNumberY(view.getZoom(), o.getLocation().getLatitude()); - int y = (int) ((tileY - yTileUp) * getTileSize()); - canvas.drawCircle(x, y, getRadiusPoi(view.getZoom()), pointAltUI); - } - } - - OsmSQLLiteRepository sqlLite = ResourceManager.getResourceManager().getAmenityRepository(); - if (sqlLite != null) { - objects = sqlLite.searchAmenities(topLatitude, leftLongitude, bottomLatitude, rightLongitude); - for (Amenity o : objects) { - double tileX = MapUtils.getTileNumberX(view.getZoom(), o.getLocation().getLongitude()); - int x = (int) ((tileX - xTileLeft) * getTileSize()); - double tileY = MapUtils.getTileNumberY(view.getZoom(), o.getLocation().getLatitude()); - int y = (int) ((tileY - yTileUp) * getTileSize()); - canvas.drawCircle(x, y, getRadiusPoi(view.getZoom()), pointAltUI); - } + double rightLongitude = MapUtils.getLongitudeFromTile(view.getZoom(), xTileRight); + + objects.clear(); + resourceManager.searchAmenitiesAsync(topLatitude, leftLongitude, bottomLatitude, rightLongitude, objects); + for (Amenity o : objects) { + double tileX = MapUtils.getTileNumberX(view.getZoom(), o.getLocation().getLongitude()); + int x = (int) ((tileX - xTileLeft) * getTileSize()); + double tileY = MapUtils.getTileNumberY(view.getZoom(), o.getLocation().getLatitude()); + int y = (int) ((tileY - yTileUp) * getTileSize()); + canvas.drawCircle(x, y, getRadiusPoi(view.getZoom()), pointAltUI); } - if (nodeManager != null) { - List objects = nodeManager.getObjects(topLatitude, leftLongitude, bottomLatitude, rightLongitude); - for (Amenity o : objects) { - double tileX = MapUtils.getTileNumberX(view.getZoom(), o.getLocation().getLongitude()); - int x = (int) ((tileX - xTileLeft) * getTileSize()); - double tileY = MapUtils.getTileNumberY(view.getZoom(), o.getLocation().getLatitude()); - int y = (int) ((tileY - yTileUp) * getTileSize()); - canvas.drawCircle(x, y, getRadiusPoi(view.getZoom()), pointAltUI); - } - } - + } }