Android-builders-club/OsmAnd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

reinvent index mechanism

Browse source 
git-svn-id: https://osmand.googlecode.com/svn/trunk@84 e29c36b1-1cfa-d876-8d93-3434fc2bb7b8
This commit is contained in:
Victor Shcherb 2010-05-27 11:02:02 +00:00
parent 30641e9ce6
commit 4b7408da3f
34 changed files with 1205 additions and 1212 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
DataExtractionOSM/.classpath
View file

@ -3,8 +3,7 @@
<classpathentry kind="src" path="src"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="lib" path="lib/bzip2-20090327.jar"/>
<classpathentry kind="lib" path="lib/commons-logging-1.1.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-core-3.0.1.jar" sourcepath="C:/docs/lucene/lucene-src.zip"/>
<classpathentry kind="lib" path="lib/sqlitejdbc-v056.jar"/>
<classpathentry kind="lib" path="lib/commons-logging-1.1.1.jar"/>
<classpathentry kind="output" path="bin"/>
</classpath>

BIN
DataExtractionOSM/lib/lucene-core-3.0.1.jar
View file

Binary file not shown.

21
DataExtractionOSM/src/com/osmand/DefaultLauncherConstants.java
View file

@ -1,21 +0,0 @@
package com.osmand;
/**
* This is temp class where all path & machine specific properties are written
*/
public abstract class DefaultLauncherConstants {
// Application constants
public static String APP_NAME = "OsmAnd";
public static String APP_VERSION = "0.1";
// Download manager tile settings
public static int TILE_DOWNLOAD_THREADS = 4;
public static int TILE_DOWNLOAD_SECONDS_TO_WORK = 25;
public static final int TILE_DOWNLOAD_MAX_ERRORS = -1;
}

13
DataExtractionOSM/src/com/osmand/ToDoConstants.java
View file

@ -14,7 +14,6 @@ public class ToDoConstants {
// TODO ANDROID
// 1. POI search near to map location (show categories & type). First cut. (implement incremental search)
// 0. Minimize memory used for index & improve time for reading index
// 3. Revise osmand UI. Preparing new icons.
// 2. Showing compass on the map : use device compass if exists(?)
// 5. Search for city/streets/buildings
@ -33,6 +32,9 @@ public class ToDoConstants {
// 23. Implement moving point from center to bottom (for rotating map).
// It is not very useful to see what was before.
// 24. Implement ResourceManager on Low memory (clear previous all addresses cities, remove all amenities cache)
// Use async loading tile thread, to preload amenities also.
// FIXME Bugs Android :
// 0. FIX TODO for partial loading rotated map
// 1. When firstly run osmand navigation (from notification bar) show map & go to menu shows desktop.
@ -43,8 +45,7 @@ public class ToDoConstants {
// TODO SWING:
// 1. Download tiles without using dir tiles
// 2. Configure file log & see log from file
// 3. Reinvent index mechanism (save in zip file with tile indexes, save city/town addresses separately, read partially !)
// 4. Invent different file extensions for poi.index, address.index,...
// 5. Implement supress warning for duplicate id
// DONE ANDROID :
@ -52,10 +53,12 @@ public class ToDoConstants {
// 10. Specify auto-rotating map (bearing of your direction)
// 22. Investigate 3D tile view (how it is done in osmand). Looking not very good, because of
// angle of perspective (best perspective angle = 60) use
// android.graphics.Camera.rotateX(6), getMatrix(m), canvas.concat(m) (find example in internet)
// android.graphics.Camera.rotateX(60), getMatrix(m), canvas.concat(m) (find example in internet)
// Problems : to calculate how to drag point on map, to calculate how many tiles are needed, is location visible ....
// 0. Minimize memory used for index & improve time for reading index
// DONE SWING
// 3. Reinvent index mechanism (save in zip file with tile indexes, save city/town addresses separately, read partially !)
// 4. Invent different file extensions for poi.index, address.index,...
}

9
DataExtractionOSM/src/com/osmand/data/Amenity.java
View file

@ -7,7 +7,7 @@ import com.osmand.Algoritms;
import com.osmand.osm.Entity;
import com.osmand.osm.OSMSettings.OSMTagKey;
public class Amenity extends MapObject<Entity> {
public class Amenity extends MapObject {
// http://wiki.openstreetmap.org/wiki/Amenity
public enum AmenityType {
SUSTENANCE, // restaurant, cafe ...
@ -89,9 +89,10 @@ public class Amenity extends MapObject<Entity> {
private AmenityType type;
public Amenity(Entity entity){
this.entity = entity;
super(entity);
this.type = getType(entity);
this.subType = getSubType(entity);
}
public Amenity(){
@ -168,6 +169,8 @@ public class Amenity extends MapObject<Entity> {
}
public void doDataPreparation() {
}
}

2
DataExtractionOSM/src/com/osmand/data/Building.java
View file

@ -2,7 +2,7 @@ package com.osmand.data;
import com.osmand.osm.Entity;
public class Building extends MapObject<Entity> {
public class Building extends MapObject {
public Building(Entity e){
super(e);

6
DataExtractionOSM/src/com/osmand/data/City.java
View file

@ -9,7 +9,7 @@ import com.osmand.osm.Entity;
import com.osmand.osm.Node;
import com.osmand.osm.OSMSettings.OSMTagKey;
public class City extends MapObject<Node> {
public class City extends MapObject {
public enum CityType {
// that's tricky way to play with that numbers (to avoid including suburbs in city & vice verse)
@ -67,6 +67,10 @@ public class City extends MapObject<Node> {
return streets.remove(name.toLowerCase());
}
public void removeAllStreets(){
streets.clear();
}
public Street registerStreet(Street street){
String name = street.getName().toLowerCase();
if(!Algoritms.isEmpty(name)){

45
DataExtractionOSM/src/com/osmand/data/MapObject.java
View file

@ -5,26 +5,27 @@ import com.osmand.osm.LatLon;
import com.osmand.osm.MapUtils;
import com.osmand.osm.OSMSettings.OSMTagKey;
public abstract class MapObject<T extends Entity> implements Comparable<MapObject<T>> {
public abstract class MapObject implements Comparable<MapObject> {
protected String name = null;
protected LatLon location = null;
protected Long id = null;
// could be null
protected T entity = null;
public MapObject(){}
public MapObject(T e){
entity = e;
public MapObject(Entity e){
setEntity(e);
}
public T getEntity(){
return entity;
}
public void setEntity(T e){
entity = e;
public void setEntity(Entity e){
this.id = e.getId();
if(this.name == null){
this.name = e.getTag(OSMTagKey.NAME);
}
if(this.location == null){
this.location = MapUtils.getCenter(e);
}
}
public void setId(Long id) {
@ -35,10 +36,6 @@ public abstract class MapObject<T extends Entity> implements Comparable<MapObjec
if(id != null){
return id;
}
T e = getEntity();
if(e != null){
return e.getId();
}
return null;
}
@ -46,13 +43,8 @@ public abstract class MapObject<T extends Entity> implements Comparable<MapObjec
if (this.name != null) {
return this.name;
}
Entity e = getEntity();
if (e != null) {
String name = getEntity().getTag(OSMTagKey.NAME);
if (name != null) {
return name;
}
return e.getId() + "";
if (id != null) {
return id + "";
} else {
return "";
}
@ -63,10 +55,7 @@ public abstract class MapObject<T extends Entity> implements Comparable<MapObjec
}
public LatLon getLocation(){
if(location != null){
return location;
}
return MapUtils.getCenter(getEntity());
return location;
}
public void setLocation(double latitude, double longitude){
@ -74,8 +63,12 @@ public abstract class MapObject<T extends Entity> implements Comparable<MapObjec
}
@Override
public int compareTo(MapObject<T> o) {
public int compareTo(MapObject o) {
return getName().compareTo(o.getName());
}
public void doDataPreparation() {
}
}

21
DataExtractionOSM/src/com/osmand/data/Region.java
View file

@ -10,24 +10,14 @@ import java.util.Map;
import com.osmand.Algoritms;
import com.osmand.data.City.CityType;
import com.osmand.osm.Entity;
import com.osmand.osm.LatLon;
import com.osmand.osm.MapUtils;
import com.osmand.osm.Node;
import com.osmand.osm.io.OsmBaseStorage;
public class Region extends MapObject<Entity> {
public class Region extends MapObject {
private DataTileManager<Amenity> amenities = new DataTileManager<Amenity>();
private OsmBaseStorage storage;
private static class CityComparator implements Comparator<City>{
@Override
public int compare(City o1, City o2) {
return o1.getName().compareTo(o2.getName());
}
}
private DataTileManager<City> cityManager = new DataTileManager<City>();
private Map<CityType, List<City>> cities = new HashMap<CityType, List<City>>();
{
@ -37,6 +27,13 @@ public class Region extends MapObject<Entity> {
}
}
private static class CityComparator implements Comparator<City>{
@Override
public int compare(City o1, City o2) {
return o1.getName().compareTo(o2.getName());
}
}
public Region(){
name = "Region";
}
@ -98,7 +95,7 @@ public class Region extends MapObject<Entity> {
City closest = null;
double relDist = Double.POSITIVE_INFINITY;
for (City c : cityManager.getClosestObjects(point.getLatitude(), point.getLongitude())) {
double rel = MapUtils.getDistance(c.getEntity(), point) / c.getType().getRadius();
double rel = MapUtils.getDistance(c.getLocation(), point) / c.getType().getRadius();
if (rel < relDist) {
closest = c;
relDist = rel;

45
DataExtractionOSM/src/com/osmand/data/Street.java
View file

@ -13,7 +13,7 @@ import com.osmand.osm.Node;
import com.osmand.osm.Way;
import com.osmand.osm.OSMSettings.OSMTagKey;
public class Street extends MapObject<Entity> {
public class Street extends MapObject {
private List<Building> buildings = new ArrayList<Building>();
private List<Way> wayNodes = new ArrayList<Way>();
@ -42,18 +42,7 @@ public class Street extends MapObject<Entity> {
return buildings;
}
public LatLon getLocation(){
if(entity == null){
calculateCenter();
}
return entity == null ? null : entity.getLatLon();
}
protected void calculateCenter(){
if(wayNodes.size() == 1){
entity = wayNodes.get(0);
return;
}
List<Node> nodes = new ArrayList<Node>();
for(Way w : wayNodes){
nodes.addAll(w.getNodes());
@ -65,8 +54,8 @@ public class Street extends MapObject<Entity> {
if (n != null) {
double nd = MapUtils.getDistance(n, c);
if (nd < dist) {
entity = n;
dist = nd;
location = n.getLatLon();
}
}
}
@ -74,13 +63,16 @@ public class Street extends MapObject<Entity> {
@Override
public void setName(String name) {
if(name.equals(getName())){
if (name.equals(getName())) {
return;
}
Street unregisterStreet = city.unregisterStreet(getName());
assert unregisterStreet == this;
super.setName(name);
city.registerStreet(this);
if (city.getStreet(getName()) == this) {
city.unregisterStreet(getName());
super.setName(name);
city.registerStreet(this);
} else {
super.setName(name);
}
}
@ -89,17 +81,28 @@ public class Street extends MapObject<Entity> {
}
public void doDataPreparation() {
calculateCenter();
Collections.sort(buildings, new Comparator<Building>(){
@Override
public int compare(Building o1, Building o2) {
int i1 = Algoritms.extractFirstIntegerNumber(o1.getName());
int i2 = Algoritms.extractFirstIntegerNumber(o2.getName());
return i1 - i2;
}
});
calculateCenter();
if(location == null){
List<LatLon> nodes = new ArrayList<LatLon>();
for(Building b : buildings){
nodes.add(b.getLocation());
}
location = MapUtils.getWeightCenter(nodes);
}
if (wayNodes.size() > 0) {
this.id = wayNodes.get(0).getId();
} else {
this.id = buildings.get(0).getId();
}
}
}

109
DataExtractionOSM/src/com/osmand/data/index/DataIndexReader.java Normal file
View file

@ -0,0 +1,109 @@
package com.osmand.data.index;
import java.io.File;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.logging.Log;
import com.osmand.LogUtil;
import com.osmand.data.Building;
import com.osmand.data.City;
import com.osmand.data.Street;
import com.osmand.data.City.CityType;
import com.osmand.data.index.IndexConstants.IndexBuildingTable;
import com.osmand.data.index.IndexConstants.IndexCityTable;
import com.osmand.data.index.IndexConstants.IndexStreetTable;
public class DataIndexReader {
private static final Log log = LogUtil.getLog(DataIndexReader.class);
public Connection getConnection(File file) throws SQLException{
try {
Class.forName("org.sqlite.JDBC");
} catch (ClassNotFoundException e) {
log.error("Illegal configuration", e);
throw new IllegalStateException(e);
}
return DriverManager.getConnection("jdbc:sqlite:"+file.getAbsolutePath());
}
public List<City> readCities(Connection c) throws SQLException{
List<City> cities = new ArrayList<City>();
Statement stat = c.createStatement();
ResultSet set = stat.executeQuery(IndexConstants.generateSelectSQL(IndexCityTable.values()));
while(set.next()){
City city = new City(CityType.valueFromString(set.getString(IndexCityTable.CITY_TYPE.ordinal() + 1)));
city.setName(set.getString(IndexCityTable.NAME.ordinal() + 1));
city.setLocation(set.getDouble(IndexCityTable.LATITUDE.ordinal() + 1),
set.getDouble(IndexCityTable.LONGITUDE.ordinal() + 1));
city.setId(set.getLong(IndexCityTable.ID.ordinal() + 1));
cities.add(city);
}
set.close();
stat.close();
return cities;
}
public List<Street> readStreets(Connection c, City city) throws SQLException{
List<Street> streets = new ArrayList<Street>();
Statement stat = c.createStatement();
ResultSet set = stat.executeQuery(IndexConstants.generateSelectSQL(IndexStreetTable.values(),
IndexStreetTable.CITY.toString() +" = " + city.getId()));
while(set.next()){
Street street = new Street(city);
street.setName(set.getString(IndexStreetTable.NAME.ordinal() + 1));
street.setLocation(set.getDouble(IndexStreetTable.LATITUDE.ordinal() + 1),
set.getDouble(IndexStreetTable.LONGITUDE.ordinal() + 1));
street.setId(set.getLong(IndexStreetTable.ID.ordinal() + 1));
streets.add(street);
}
set.close();
stat.close();
return streets;
}
public List<Building> readBuildings(Connection c, Street street) throws SQLException{
List<Building> buildings = new ArrayList<Building>();
Statement stat = c.createStatement();
ResultSet set = stat.executeQuery(IndexConstants.generateSelectSQL(IndexBuildingTable.values(),
IndexBuildingTable.STREET.toString() +" = " + street.getId()));
while(set.next()){
Building building = new Building();
building.setName(set.getString(IndexBuildingTable.NAME.ordinal() + 1));
building.setLocation(set.getDouble(IndexBuildingTable.LATITUDE.ordinal() + 1),
set.getDouble(IndexBuildingTable.LONGITUDE.ordinal() + 1));
building.setId(set.getLong(IndexBuildingTable.ID.ordinal() + 1));
buildings.add(building);
}
set.close();
stat.close();
return buildings;
}
public void testIndex(File f) throws SQLException {
Connection c = getConnection(f);
try {
for (City city : readCities(c)) {
System.out.println("CITY " + city.getName());
for (Street s : readStreets(c, city)) {
System.out.println("\tSTREET " + s.getName());
for (Building b : readBuildings(c, s)) {
System.out.println("\t\tBULDING " + b.getName());
}
}
}
} finally {
c.close();
}
}
}

225
DataExtractionOSM/src/com/osmand/data/index/DataIndexWriter.java Normal file
View file

@ -0,0 +1,225 @@
package com.osmand.data.index;
import java.io.File;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.logging.Log;
import com.osmand.LogUtil;
import com.osmand.data.Amenity;
import com.osmand.data.Building;
import com.osmand.data.City;
import com.osmand.data.Region;
import com.osmand.data.Street;
import com.osmand.data.Amenity.AmenityType;
import com.osmand.data.City.CityType;
import com.osmand.data.index.IndexConstants.IndexBuildingTable;
import com.osmand.data.index.IndexConstants.IndexCityTable;
import com.osmand.data.index.IndexConstants.IndexPoiTable;
import com.osmand.data.index.IndexConstants.IndexStreetNodeTable;
import com.osmand.data.index.IndexConstants.IndexStreetTable;
import com.osmand.osm.Node;
import com.osmand.osm.Way;
public class DataIndexWriter {
private final File workingDir;
private final Region region;
private static final Log log = LogUtil.getLog(DataIndexWriter.class);
private static final int BATCH_SIZE = 1000;
public DataIndexWriter(File workingDir, Region region){
this.workingDir = workingDir;
this.region = region;
}
protected File checkFile(String name) throws IOException {
String fileName = name;
File f = new File(workingDir, fileName);
f.mkdirs();
// remove existing file
if (f.exists()) {
log.warn("Remove existing index : " + f.getAbsolutePath());
f.delete();
}
return f;
}
public DataIndexWriter writePOI() throws IOException, SQLException {
File file = checkFile(IndexConstants.POI_INDEX_DIR+region.getName()+IndexConstants.POI_INDEX_EXT);
long now = System.currentTimeMillis();
try {
Class.forName("org.sqlite.JDBC");
} catch (ClassNotFoundException e) {
log.error("Illegal configuration", e);
throw new IllegalStateException(e);
}
Connection conn = DriverManager.getConnection("jdbc:sqlite:"+file.getAbsolutePath());
try {
Statement stat = conn.createStatement();
assert IndexPoiTable.values().length == 6;
stat.executeUpdate(IndexConstants.generateCreateSQL(IndexPoiTable.values()));
stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexPoiTable.values()));
stat.close();
PreparedStatement prep = conn.prepareStatement(
IndexConstants.generatePrepareStatementToInsert(IndexPoiTable.getTable(), 6));
conn.setAutoCommit(false);
int currentCount = 0;
for (Amenity a : region.getAmenityManager().getAllObjects()) {
prep.setLong(IndexPoiTable.ID.ordinal() + 1, a.getId());
prep.setDouble(IndexPoiTable.LATITUDE.ordinal() + 1, a.getLocation().getLatitude());
prep.setDouble(IndexPoiTable.LONGITUDE.ordinal() + 1, a.getLocation().getLongitude());
prep.setString(IndexPoiTable.NAME.ordinal() + 1, a.getName());
prep.setString(IndexPoiTable.TYPE.ordinal() + 1, AmenityType.valueToString(a.getType()));
prep.setString(IndexPoiTable.SUBTYPE.ordinal() + 1, a.getSubType());
prep.addBatch();
currentCount++;
if(currentCount >= BATCH_SIZE){
prep.executeBatch();
currentCount = 0;
}
}
if(currentCount > 0){
prep.executeBatch();
}
prep.close();
conn.setAutoCommit(true);
} finally {
conn.close();
log.info(String.format("Indexing poi done in %s ms.", System.currentTimeMillis() - now));
}
return this;
}
public DataIndexWriter writeAddress() throws IOException, SQLException{
File file = checkFile(IndexConstants.ADDRESS_INDEX_DIR+region.getName()+IndexConstants.ADDRESS_INDEX_EXT);
long now = System.currentTimeMillis();
try {
Class.forName("org.sqlite.JDBC");
} catch (ClassNotFoundException e) {
log.error("Illegal configuration", e);
throw new IllegalStateException(e);
}
Connection conn = DriverManager.getConnection("jdbc:sqlite:"+file.getAbsolutePath());
try {
Statement stat = conn.createStatement();
assert IndexCityTable.values().length == 5;
assert IndexBuildingTable.values().length == 5;
assert IndexStreetNodeTable.values().length == 5;
assert IndexStreetTable.values().length == 5;
stat.executeUpdate(IndexConstants.generateCreateSQL(IndexCityTable.values()));
stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexCityTable.values()));
stat.executeUpdate(IndexConstants.generateCreateSQL(IndexBuildingTable.values()));
stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexBuildingTable.values()));
stat.executeUpdate(IndexConstants.generateCreateSQL(IndexStreetNodeTable.values()));
stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexStreetNodeTable.values()));
stat.executeUpdate(IndexConstants.generateCreateSQL(IndexStreetTable.values()));
stat.executeUpdate(IndexConstants.generateCreateIndexSQL(IndexStreetTable.values()));
stat.close();
PreparedStatement prepCity = conn.prepareStatement(
IndexConstants.generatePrepareStatementToInsert(IndexCityTable.getTable(), 5));
PreparedStatement prepStreet = conn.prepareStatement(
IndexConstants.generatePrepareStatementToInsert(IndexStreetTable.getTable(), 5));
PreparedStatement prepBuilding = conn.prepareStatement(
IndexConstants.generatePrepareStatementToInsert(IndexBuildingTable.getTable(), 5));
PreparedStatement prepStreetNode = conn.prepareStatement(
IndexConstants.generatePrepareStatementToInsert(IndexStreetNodeTable.getTable(), 5));
Map<PreparedStatement, Integer> count = new HashMap<PreparedStatement, Integer>();
count.put(prepStreet, 0);
count.put(prepCity, 0);
count.put(prepStreetNode, 0);
count.put(prepBuilding, 0);
conn.setAutoCommit(false);
for(CityType t : CityType.values()){
for(City city : region.getCitiesByType(t)) {
if(city.getId() == null || city.getLocation() == null){
continue;
}
prepCity.setLong(IndexCityTable.ID.ordinal() + 1, city.getId());
prepCity.setDouble(IndexCityTable.LATITUDE.ordinal() + 1, city.getLocation().getLatitude());
prepCity.setDouble(IndexCityTable.LONGITUDE.ordinal() + 1, city.getLocation().getLongitude());
prepCity.setString(IndexCityTable.NAME.ordinal() + 1, city.getName());
prepCity.setString(IndexCityTable.CITY_TYPE.ordinal() + 1, CityType.valueToString(city.getType()));
addBatch(count, prepCity);
for(Street street : city.getStreets()){
if(street.getId() == null || street.getLocation() == null){
continue;
}
prepStreet.setLong(IndexStreetTable.ID.ordinal() + 1, street.getId());
prepStreet.setDouble(IndexStreetTable.LATITUDE.ordinal() + 1, street.getLocation().getLatitude());
prepStreet.setDouble(IndexStreetTable.LONGITUDE.ordinal() + 1, street.getLocation().getLongitude());
prepStreet.setString(IndexStreetTable.NAME.ordinal() + 1, street.getName());
prepStreet.setLong(IndexStreetTable.CITY.ordinal() + 1, city.getId());
addBatch(count, prepStreet);
for(Way way : street.getWayNodes()){
for(Node n : way.getNodes()){
if(n == null){
continue;
}
prepStreetNode.setLong(IndexStreetNodeTable.ID.ordinal() + 1, n.getId());
prepStreetNode.setDouble(IndexStreetNodeTable.LATITUDE.ordinal() + 1, n.getLatitude());
prepStreetNode.setDouble(IndexStreetNodeTable.LONGITUDE.ordinal() + 1, n.getLongitude());
prepStreetNode.setLong(IndexStreetNodeTable.WAY.ordinal() + 1, way.getId());
prepStreetNode.setLong(IndexStreetNodeTable.STREET.ordinal() + 1, street.getId());
addBatch(count, prepStreetNode);
}
}
for(Building building : street.getBuildings()){
if(building.getId() == null || building.getLocation() == null){
continue;
}
prepBuilding.setLong(IndexBuildingTable.ID.ordinal() + 1, building.getId());
prepBuilding.setDouble(IndexBuildingTable.LATITUDE.ordinal() + 1, building.getLocation().getLatitude());
prepBuilding.setDouble(IndexBuildingTable.LONGITUDE.ordinal() + 1, building.getLocation().getLongitude());
prepBuilding.setString(IndexBuildingTable.NAME.ordinal() + 1, building.getName());
prepBuilding.setLong(IndexBuildingTable.STREET.ordinal() + 1, street.getId());
addBatch(count, prepBuilding);
}
}
}
}
for(PreparedStatement p : count.keySet()){
if(count.get(p) > 0){
p.executeBatch();
}
p.close();
}
conn.setAutoCommit(true);
} finally {
conn.close();
log.info(String.format("Indexing address done in %s ms.", System.currentTimeMillis() - now));
}
return this;
}
private void addBatch(Map<PreparedStatement, Integer> count, PreparedStatement p) throws SQLException{
p.addBatch();
if(count.get(p) >= BATCH_SIZE){
p.executeBatch();
count.put(p, 0);
} else {
count.put(p, count.get(p) + 1);
}
}
}

280
DataExtractionOSM/src/com/osmand/data/index/IndexConstants.java Normal file
View file

@ -0,0 +1,280 @@
package com.osmand.data.index;
public class IndexConstants {
public static final String POI_INDEX_DIR = "POI/";
public static final String ADDRESS_INDEX_DIR = "Address/";
public static final String POI_INDEX_EXT = ".poi.odb";
public static final String ADDRESS_INDEX_EXT = ".addr.odb";
public interface IndexColumn {
public boolean isIndex();
public String getType();
public String getTableName();
}
public static String[] generateColumnNames(IndexColumn[] columns) {
String[] columnNames = new String[columns.length];
for (int i = 0; i < columnNames.length; i++) {
columnNames[i] = columns[i].toString();
}
return columnNames;
}
public static String generateCreateSQL(IndexColumn[] columns){
StringBuilder b = new StringBuilder();
b.append("create table ").append(columns[0].getTableName()).append(" (");
boolean first = true;
for(IndexColumn c : columns){
if(first) {
first = false;
} else {
b.append(", ");
}
b.append(c.toString());
if(c.getType() != null){
b.append(" ").append(c.getType());
}
}
b.append(" ); ");
return b.toString();
}
public static String generateSelectSQL(IndexColumn[] select){
return generateSelectSQL(select, null);
}
public static String generateSelectSQL(IndexColumn[] select, String where){
StringBuilder b = new StringBuilder();
b.append("select ");
boolean first = true;
for(IndexColumn c : select){
if(first) {
first = false;
} else {
b.append(", ");
}
b.append(c.toString());
}
b.append(" FROM ").append(select[0].getTableName());
if(where != null){
b.append(" WHERE " ).append(where);
}
b.append(" ; ");
return b.toString();
}
public static String generatePrepareStatementToInsert(String tableName, int numColumns){
StringBuilder b = new StringBuilder();
b.append("insert into ").append(tableName).append(" values (");
for(int i=0; i< numColumns; i++){
if(i > 0){
b.append(", ");
}
b.append("?");
}
b.append(");");
return b.toString();
}
public static String generateCreateIndexSQL(IndexColumn[] columns){
StringBuilder b = new StringBuilder();
String tableName = columns[0].getTableName();
b.append("create index ").append(tableName).append("_index ON ").append(tableName).append(" (");
boolean first = true;
for(IndexColumn c : columns){
if(!c.isIndex()){
continue;
}
if(first) {
first = false;
} else {
b.append(", ");
}
b.append(c.toString());
}
b.append(" ); ");
if(first){
return null;
}
return b.toString();
}
public enum IndexPoiTable implements IndexColumn {
ID("long"), LATITUDE("double", true), LONGITUDE("double", true), NAME, TYPE, SUBTYPE;
boolean index = false;
String type = null;
private IndexPoiTable(){}
private IndexPoiTable(String type){
this.type = type;
}
private IndexPoiTable(String type, boolean index){ this(type); this.index = index;}
public static String getTable(){
return "poi";
}
public String getTableName(){
return getTable();
}
@Override
public String getType() {
return type;
}
@Override
public boolean isIndex() {
return index;
}
}
public enum IndexCityTable implements IndexColumn {
ID("long"), LATITUDE("double", true), LONGITUDE("double", true), NAME, CITY_TYPE;
boolean index = false;
String type = null;
private IndexCityTable() {
}
private IndexCityTable(String type) {
this.type = type;
}
private IndexCityTable(String type, boolean index) {
this(type);
this.index = index;
}
public static String getTable() {
return "city";
}
public String getTableName() {
return getTable();
}
@Override
public String getType() {
return type;
}
@Override
public boolean isIndex() {
return index;
}
}
public enum IndexStreetTable implements IndexColumn {
ID("long"), LATITUDE("double", true), LONGITUDE("double", true), NAME, CITY("long", true);
boolean index = false;
String type = null;
private IndexStreetTable() {
}
private IndexStreetTable(String type) {
this.type = type;
}
private IndexStreetTable(String type, boolean index) {
this(type);
this.index = index;
}
public static String getTable() {
return "street";
}
public String getTableName() {
return getTable();
}
@Override
public String getType() {
return type;
}
@Override
public boolean isIndex() {
return index;
}
}
public enum IndexStreetNodeTable implements IndexColumn {
ID("long"), LATITUDE("double"), LONGITUDE("double"), STREET("long", true), WAY("long", true);
boolean index = false;
String type = null;
private IndexStreetNodeTable() {
}
private IndexStreetNodeTable(String type) {
this.type = type;
}
private IndexStreetNodeTable(String type, boolean index) {
this(type);
this.index = index;
}
public static String getTable() {
return "street_node";
}
public String getTableName() {
return getTable();
}
@Override
public String getType() {
return type;
}
@Override
public boolean isIndex() {
return index;
}
}
public enum IndexBuildingTable implements IndexColumn {
ID("long"), LATITUDE("double"), LONGITUDE("double"), NAME, STREET("long", true);
boolean index = false;
String type = null;
private IndexBuildingTable() {
}
private IndexBuildingTable(String type) {
this.type = type;
}
private IndexBuildingTable(String type, boolean index) {
this(type);
this.index = index;
}
public static String getTable() {
return "building";
}
public String getTableName() {
return getTable();
}
@Override
public String getType() {
return type;
}
@Override
public boolean isIndex() {
return index;
}
}
}

312
DataExtractionOSM/src/com/osmand/data/preparation/DataExtraction.java
View file

@ -2,20 +2,21 @@ package com.osmand.data.preparation;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.stream.XMLStreamException;
import java.util.LinkedHashMap;
import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.tools.bzip2.CBZip2InputStream;
import org.apache.tools.bzip2.CBZip2OutputStream;
import org.xml.sax.SAXException;
import com.osmand.Algoritms;
@ -26,7 +27,6 @@ import com.osmand.data.DataTileManager;
import com.osmand.data.Region;
import com.osmand.data.Street;
import com.osmand.data.City.CityType;
import com.osmand.impl.ConsoleProgressImplementation;
import com.osmand.osm.Entity;
import com.osmand.osm.LatLon;
import com.osmand.osm.MapUtils;
@ -36,13 +36,9 @@ import com.osmand.osm.Way;
import com.osmand.osm.OSMSettings.OSMTagKey;
import com.osmand.osm.io.IOsmStorageFilter;
import com.osmand.osm.io.OsmBaseStorage;
import com.osmand.osm.io.OsmStorageWriter;
import com.osmand.swing.DataExtractionSettings;
// TO implement
// 1. Full structured search for town/street/building.
/**
* http://wiki.openstreetmap.org/wiki/OSM_tags_for_routing#Is_inside.2Foutside
* http://wiki.openstreetmap.org/wiki/Relations/Proposed/Postal_Addresses
@ -76,89 +72,163 @@ import com.osmand.swing.DataExtractionSettings;
*/
public class DataExtraction {
private static final Log log = LogFactory.getLog(DataExtraction.class);
// public static void main(String[] args) throws ParserConfigurationException, SAXException, IOException, XMLStreamException {
// new DataExtraction().testReadingOsmFile();
// }
// External files
public static String pathToTestDataDir = "E:\\Information\\OSM maps\\";
public static String pathToOsmFile = pathToTestDataDir + "minsk.osm";
public static String pathToOsmBz2File = pathToTestDataDir + "belarus_2010_04_01.osm.bz2";
public static String pathToWorkingDir = pathToTestDataDir +"osmand\\";
public static String pathToDirWithTiles = pathToWorkingDir +"tiles";
public static String writeTestOsmFile = "C:\\1_tmp.osm"; // could be null - wo writing
private static boolean parseSmallFile = true;
private static boolean parseOSM = true;
public static final int BATCH_SIZE = 5000;
public static final String NODES_DB = "nodes.db";
private final boolean loadAllObjects;
private final boolean normalizeStreets;
private final boolean indexAddress;
private final boolean indexPOI;
private File workingDir = null;
///////////////////////////////////////////
// Test method for local purposes
public void testReadingOsmFile() throws ParserConfigurationException, SAXException, IOException, XMLStreamException {
String f;
if(parseSmallFile){
f = pathToOsmFile;
} else {
f = pathToOsmBz2File;
}
long st = System.currentTimeMillis();
Region country;
if(parseOSM){
country = readCountry(f, new ConsoleProgressImplementation(), null);
} else {
country = new Region();
country.setStorage(new OsmBaseStorage());
}
List<Long> interestedObjects = new ArrayList<Long>();
// add interested objects
if (writeTestOsmFile != null) {
OsmStorageWriter writer = new OsmStorageWriter();
OutputStream output = new FileOutputStream(writeTestOsmFile);
if (writeTestOsmFile.endsWith(".bz2")) {
output.write('B');
output.write('Z');
output = new CBZip2OutputStream(output);
}
writer.saveStorage(output, country.getStorage(), interestedObjects, false);
output.close();
}
System.out.println();
System.out.println("USED Memory " + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1e6);
System.out.println("TIME : " + (System.currentTimeMillis() - st));
}
public DataExtraction(){
this.indexPOI = true;
this.indexAddress = true;
this.loadAllObjects = false;
this.normalizeStreets = true;
}
public DataExtraction(boolean indexAddress, boolean indexPOI, boolean normalizeStreets, boolean loadAllObjects){
public DataExtraction(boolean indexAddress, boolean indexPOI, boolean normalizeStreets, boolean loadAllObjects, File workingDir){
this.indexAddress = indexAddress;
this.indexPOI = indexPOI;
this.normalizeStreets = normalizeStreets;
this.loadAllObjects = loadAllObjects;
this.workingDir = workingDir;
}
public Region readCountry(String path, IProgress progress, IOsmStorageFilter addFilter) throws IOException, SAXException{
protected class DataExtractionOsmFilter implements IOsmStorageFilter {
final ArrayList<Node> places;
final ArrayList<Entity> buildings;
final ArrayList<Entity> amenities;
final ArrayList<Way> ways;
int currentCount = 0;
private Connection conn;
private PreparedStatement prep;
public DataExtractionOsmFilter(ArrayList<Entity> amenities, ArrayList<Entity> buildings, ArrayList<Node> places,
ArrayList<Way> ways) {
this.amenities = amenities;
this.buildings = buildings;
this.places = places;
this.ways = ways;
}
public void initDatabase() throws SQLException {
try {
Class.forName("org.sqlite.JDBC");
} catch (ClassNotFoundException e) {
log.error("Illegal configuration", e);
throw new IllegalStateException(e);
}
File file = new File(workingDir, NODES_DB);
// to save space
if(file.exists()){
file.delete();
}
// creating nodes db to fast access for all nodes
conn = DriverManager.getConnection("jdbc:sqlite:" + file.getAbsolutePath());
// prepare tables
Statement stat = conn.createStatement();
stat.executeUpdate("drop table if exists node;");
stat.executeUpdate("create table node (id long, latitude double, longitude double);");
stat.executeUpdate("create index IdIndex ON node (id);");
stat.close();
prep = conn.prepareStatement("insert into node values (?, ?, ?);");
conn.setAutoCommit(false);
}
public void correlateData(OsmBaseStorage storage, IProgress progress) throws SQLException {
if (currentCount > 0) {
prep.executeBatch();
}
prep.close();
conn.setAutoCommit(true);
final PreparedStatement pselect = conn.prepareStatement("select * from node where id = ?");
Map<Long, Entity> map = new LinkedHashMap<Long, Entity>();
progress.startTask("Correlating data...", storage.getRegisteredEntities().size());
for (Entity e : storage.getRegisteredEntities().values()) {
progress.progress(1);
if (e instanceof Way) {
map.clear();
for (Long i : ((Way) e).getNodeIds()) {
pselect.setLong(1, i);
if (pselect.execute()) {
ResultSet rs = pselect.getResultSet();
if (rs.next()) {
map.put(i, new Node(rs.getDouble(2), rs.getDouble(3), rs.getLong(1)));
}
rs.close();
}
}
e.initializeLinks(map);
}
}
}
public void close() {
if (conn != null) {
try {
conn.close();
} catch (SQLException e) {
}
}
}
@Override
public boolean acceptEntityToLoad(OsmBaseStorage storage, Entity e) {
boolean processed = false;
if (indexAddress) {
if ("yes".equals(e.getTag(OSMTagKey.BUILDING))) {
if (e.getTag(OSMTagKey.ADDR_HOUSE_NUMBER) != null && e.getTag(OSMTagKey.ADDR_STREET) != null) {
buildings.add(e);
processed = true;
}
}
}
if (indexPOI && Amenity.isAmenity(e)) {
amenities.add(e);
processed = true;
}
if (e instanceof Node && e.getTag(OSMTagKey.PLACE) != null) {
places.add((Node) e);
processed = true;
}
if (indexAddress) {
// suppose that streets are way for car
if (e instanceof Way && OSMSettings.wayForCar(e.getTag(OSMTagKey.HIGHWAY)) && e.getTag(OSMTagKey.NAME) != null) {
ways.add((Way) e);
processed = true;
}
}
// put all nodes into temporary db to get only required nodes after loading all data
try {
if (e instanceof Node && indexAddress) {
currentCount++;
prep.setLong(1, e.getId());
prep.setDouble(2, ((Node) e).getLatitude());
prep.setDouble(3, ((Node) e).getLongitude());
prep.addBatch();
if (currentCount >= BATCH_SIZE) {
prep.executeBatch();
currentCount = 0;
}
}
} catch (SQLException ex) {
log.error("Could not save node", ex);
}
return processed || loadAllObjects;
}
}
public Region readCountry(String path, IProgress progress, IOsmStorageFilter addFilter) throws IOException, SAXException, SQLException{
// data to load & index
final ArrayList<Node> places = new ArrayList<Node>();
final ArrayList<Entity> buildings = new ArrayList<Entity>();
final ArrayList<Entity> amenities = new ArrayList<Entity>();
final ArrayList<Way> ways = new ArrayList<Way>();
File f = new File(path);
InputStream stream = new FileInputStream(f);
InputStream streamFile = stream;
@ -174,51 +244,33 @@ public class DataExtraction {
if(progress != null){
progress.startTask("Loading file " + path, -1);
}
// preloaded data
final ArrayList<Node> places = new ArrayList<Node>();
final ArrayList<Entity> buildings = new ArrayList<Entity>();
final ArrayList<Amenity> amenities = new ArrayList<Amenity>();
final ArrayList<Way> ways = new ArrayList<Way>();
IOsmStorageFilter filter = new IOsmStorageFilter(){
@Override
public boolean acceptEntityToLoad(OsmBaseStorage storage, Entity e) {
if (indexAddress) {
if ("yes".equals(e.getTag(OSMTagKey.BUILDING))) {
if (e.getTag(OSMTagKey.ADDR_HOUSE_NUMBER) != null && e.getTag(OSMTagKey.ADDR_STREET) != null) {
buildings.add(e);
return true;
}
}
}
if (indexPOI && Amenity.isAmenity(e)) {
amenities.add(new Amenity(e));
return true;
}
if (e instanceof Node && e.getTag(OSMTagKey.PLACE) != null) {
places.add((Node) e);
return true;
}
if (indexAddress) {
if (e instanceof Way && OSMSettings.wayForCar(e.getTag(OSMTagKey.HIGHWAY))) {
ways.add((Way) e);
return true;
}
}
return (e instanceof Node && indexAddress) || loadAllObjects;
}
};
OsmBaseStorage storage = new OsmBaseStorage();
if (addFilter != null) {
OsmBaseStorage storage = new OsmBaseStorage();
if (addFilter != null) {
storage.getFilters().add(addFilter);
}
storage.getFilters().add(filter);
storage.parseOSM(stream, progress, streamFile);
if (log.isDebugEnabled()) {
log.debug("File parsed : " + (System.currentTimeMillis() - st));
DataExtractionOsmFilter filter = new DataExtractionOsmFilter(amenities, buildings, places, ways);
storage.getFilters().add(filter);
// 0. Loading osm file
try {
// 0.1 init database to store temporary data
filter.initDatabase();
// 0.2 parsing osm itself
storage.parseOSM(stream, progress, streamFile);
if (log.isInfoEnabled()) {
log.info("File parsed : " + (System.currentTimeMillis() - st));
}
progress.finishTask();
// 0.3 Correlating data (linking way & node)
filter.correlateData(storage, progress);
} finally {
if (log.isInfoEnabled()) {
log.info("File indexed : " + (System.currentTimeMillis() - st));
}
filter.close();
}
// 1. Initialize region
@ -228,9 +280,9 @@ public class DataExtraction {
country.setStorage(storage);
// 2. Reading amenities
if(indexPOI){
readingAmenities(amenities, country);
}
if (indexPOI) {
readingAmenities(amenities, country);
}
// 3. Reading cities
readingCities(places, country);
@ -247,7 +299,7 @@ public class DataExtraction {
// 6. normalizing streets
normalizingStreets(progress, country);
}
// 7. Call data preparation to sort cities, calculate center location, assign id to objects
country.doDataPreparation();
return country;
}
@ -311,9 +363,9 @@ public class DataExtraction {
}
private void readingAmenities(final ArrayList<Amenity> amenities, Region country) {
for(Amenity a: amenities){
country.registerAmenity(a);
private void readingAmenities(final ArrayList<Entity> amenities, Region country) {
for(Entity a: amenities){
country.registerAmenity(new Amenity(a));
}
}

94
DataExtractionOSM/src/com/osmand/data/preparation/DataIndexBuilder.java
View file

@ -1,94 +0,0 @@
package com.osmand.data.preparation;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import javax.xml.stream.XMLStreamException;
import com.osmand.data.Amenity;
import com.osmand.data.Region;
import com.osmand.osm.io.OsmStorageWriter;
public class DataIndexBuilder {
private final File workingDir;
private final Region region;
private boolean zipped = true;
public DataIndexBuilder(File workingDir, Region region){
this.workingDir = workingDir;
this.region = region;
}
public void setZipped(boolean zipped) {
this.zipped = zipped;
}
public boolean isZipped() {
return zipped;
}
protected OutputStream checkFile(String name) throws IOException {
String fileName = name;
if (zipped) {
// change name
name = new File(name).getName();
fileName += ".zip";
}
File f = new File(workingDir, fileName);
f.mkdirs();
// remove existing file
if (f.exists()) {
f.delete();
}
OutputStream output = new FileOutputStream(f);
if(zipped){
ZipOutputStream zipStream = new ZipOutputStream(output);
zipStream.setLevel(5);
zipStream.putNextEntry(new ZipEntry(name));
output = zipStream;
}
return output;
}
public DataIndexBuilder buildPOI() throws XMLStreamException, IOException, SQLException {
List<Amenity> list = region.getAmenityManager().getAllObjects();
List<Long> interestedObjects = new ArrayList<Long>(list.size());
for(Amenity a : list) {
interestedObjects.add(a.getEntity().getId());
}
OutputStream output = checkFile("POI/"+region.getName()+".osmand");
try {
OsmStorageWriter writer = new OsmStorageWriter();
writer.savePOIIndex(output, region);
} finally {
output.close();
}
OsmStorageWriter writer = new OsmStorageWriter();
writer.saveLuceneIndex(new File(workingDir, "lucene"), region);
writer.saveSQLLitePOIIndex(new File(workingDir, "POI/"+region.getName()+".db"), region);
return this;
}
public DataIndexBuilder buildAddress() throws XMLStreamException, IOException{
OutputStream output = checkFile("Address/"+region.getName()+".osmand");
try {
OsmStorageWriter writer = new OsmStorageWriter();
writer.saveAddressIndex(output, region);
} finally {
output.close();
}
return this;
}
}

21
DataExtractionOSM/src/com/osmand/data/preparation/MapTileDownloader.java
View file

@ -19,10 +19,18 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import com.osmand.Algoritms;
import com.osmand.DefaultLauncherConstants;
import com.osmand.LogUtil;
public class MapTileDownloader {
// Application constants
public static String APP_NAME = "OsmAnd";
public static String APP_VERSION = "0.1";
// Download manager tile settings
public static int TILE_DOWNLOAD_THREADS = 4;
public static int TILE_DOWNLOAD_SECONDS_TO_WORK = 25;
public static final int TILE_DOWNLOAD_MAX_ERRORS = -1;
private static MapTileDownloader downloader = null;
private static Log log = LogUtil.getLog(MapTileDownloader.class);
@ -40,7 +48,7 @@ public class MapTileDownloader {
public static MapTileDownloader getInstance(){
if(downloader == null){
downloader = new MapTileDownloader(DefaultLauncherConstants.TILE_DOWNLOAD_THREADS);
downloader = new MapTileDownloader(TILE_DOWNLOAD_THREADS);
}
return downloader;
}
@ -89,7 +97,7 @@ public class MapTileDownloader {
public MapTileDownloader(int numberOfThreads){
threadPoolExecutor = new ThreadPoolExecutor(numberOfThreads, numberOfThreads, DefaultLauncherConstants.TILE_DOWNLOAD_SECONDS_TO_WORK,
threadPoolExecutor = new ThreadPoolExecutor(numberOfThreads, numberOfThreads, TILE_DOWNLOAD_SECONDS_TO_WORK,
TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
// 1.6 method but very useful to kill non-running threads
// threadPoolExecutor.allowCoreThreadTimeOut(true);
@ -129,8 +137,8 @@ public class MapTileDownloader {
}
public void requestToDownload(DownloadRequest request){
if(DefaultLauncherConstants.TILE_DOWNLOAD_MAX_ERRORS > 0 &&
currentErrors > DefaultLauncherConstants.TILE_DOWNLOAD_MAX_ERRORS){
if(TILE_DOWNLOAD_MAX_ERRORS > 0 &&
currentErrors > TILE_DOWNLOAD_MAX_ERRORS){
return;
}
if(request.url == null){
@ -163,8 +171,7 @@ public class MapTileDownloader {
request.fileToSave.getParentFile().mkdirs();
URL url = new URL(request.url);
URLConnection connection = url.openConnection();
connection.setRequestProperty("User-Agent", DefaultLauncherConstants.APP_NAME + "/"
+ DefaultLauncherConstants.APP_VERSION);
connection.setRequestProperty("User-Agent", APP_NAME + "/" + APP_VERSION);
BufferedInputStream inputStream = new BufferedInputStream(connection.getInputStream(), 8 * 1024);
FileOutputStream stream = null;
try {

2
DataExtractionOSM/src/com/osmand/IMapLocationListener.java → DataExtractionOSM/src/com/osmand/map/IMapLocationListener.java
View file

@ -1,4 +1,4 @@
package com.osmand;
package com.osmand.map;
public interface IMapLocationListener {
void locationChanged(double newLatitude, double newLongitude, Object source);

9
DataExtractionOSM/src/com/osmand/osm/MapUtils.java
View file

@ -53,7 +53,7 @@ public class MapUtils {
* Gets distance in meters
*/
public static double getDistance(LatLon l1, LatLon l2){
return getDistance(l1, l2);
return getDistance(l1.getLatitude(), l1.getLongitude(), l2.getLatitude(), l2.getLongitude());
}
public static LatLon getCenter(Entity e){
@ -184,15 +184,14 @@ public class MapUtils {
}
}
public static void sortListOfMapObject(List<? extends MapObject<?>> list, final double lat, final double lon){
Collections.sort(list, new Comparator<MapObject<?>>() {
public static void sortListOfMapObject(List<? extends MapObject> list, final double lat, final double lon){
Collections.sort(list, new Comparator<MapObject>() {
@Override
public int compare(MapObject<?> o1, MapObject<?> o2) {
public int compare(MapObject o1, MapObject o2) {
return Double.compare(MapUtils.getDistance(o1.getLocation(), lat, lon), MapUtils.getDistance(o2.getLocation(),
lat, lon));
}
});
}
}

2
DataExtractionOSM/src/com/osmand/osm/io/OsmBaseStorage.java
View file

@ -193,7 +193,7 @@ public class OsmBaseStorage extends DefaultHandler {
if(acceptEntityToLoad(currentParsedEntity)){
Entity oldEntity = entities.put(currentParsedEntity.getId(), currentParsedEntity);
if(oldEntity!= null){
throw new UnsupportedOperationException("Entity with id=" + oldEntity.getId() +" is duplicated in osm map");
// throw new UnsupportedOperationException("Entity with id=" + oldEntity.getId() +" is duplicated in osm map");
}
} else {
// System.gc();

107
DataExtractionOSM/src/com/osmand/osm/io/OsmIndexStorage.java
View file

@ -1,107 +0,0 @@
package com.osmand.osm.io;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import com.osmand.data.Amenity;
import com.osmand.data.Building;
import com.osmand.data.City;
import com.osmand.data.MapObject;
import com.osmand.data.Region;
import com.osmand.data.Street;
import com.osmand.data.Amenity.AmenityType;
import com.osmand.data.City.CityType;
import com.osmand.osm.Entity;
public class OsmIndexStorage extends OsmBaseStorage {
protected static final String ELEM_OSMAND = "osmand";
protected static final String ELEM_INDEX = "index";
protected static final String ELEM_CITY = "city";
protected static final String ELEM_STREET = "street";
protected static final String ELEM_BUILDING = "building";
protected static final String ELEM_AMENITY = "amenity";
protected static final String ATTR_CITYTYPE = "citytype";
protected static final String ATTR_TYPE = "type";
protected static final String ATTR_SUBTYPE = "subtype";
protected static final String ATTR_NAME = "name";
public static final String OSMAND_VERSION = "0.1";
protected Region region;
public OsmIndexStorage(Region region){
this.region = region;
}
public Region getRegion() {
return region;
}
protected City currentParsedCity = null;
protected Street currentParsedStreet = null;
@Override
protected void initRootElement(String uri, String localName, String name, Attributes attributes) throws OsmVersionNotSupported {
if(ELEM_OSM.equals(name)){
if(!supportedVersions.contains(attributes.getValue(ATTR_VERSION))){
throw new OsmVersionNotSupported();
}
} else if(ELEM_OSMAND.equals(name)){
if(!OSMAND_VERSION.equals(attributes.getValue(ATTR_VERSION))){
throw new OsmVersionNotSupported();
}
} else {
throw new OsmVersionNotSupported();
}
parseStarted = true;
}
public void parseMapObject(MapObject<? extends Entity> c, Attributes attributes){
double lat = parseDouble(attributes, ATTR_LAT, 0);
double lon = parseDouble(attributes, ATTR_LON, 0);
long id = parseId(attributes, ATTR_ID, -1);
c.setId(id);
if(lat != 0 || lon != 0){
c.setLocation(lat, lon);
}
c.setName(attributes.getValue(ATTR_NAME));
}
@Override
public void startElement(String uri, String localName, String name, Attributes attributes) throws SAXException {
name = saxParser.isNamespaceAware() ? localName : name;
if(!parseStarted){
initRootElement(uri, localName, name, attributes);
} else if(ELEM_INDEX.equals(name)){
} else if(ELEM_CITY.equals(name)){
CityType t = CityType.valueFromString(attributes.getValue(ATTR_CITYTYPE));
City c = new City(t);
parseMapObject(c, attributes);
region.registerCity(c);
currentParsedCity = c;
} else if(ELEM_STREET.equals(name)){
assert currentParsedCity != null;
Street street = new Street(currentParsedCity);
parseMapObject(street, attributes);
currentParsedCity.registerStreet(street);
currentParsedStreet = street;
} else if(ELEM_BUILDING.equals(name)){
assert currentParsedStreet != null;
Building building = new Building();
parseMapObject(building, attributes);
currentParsedStreet.registerBuilding(building);
} else if(ELEM_AMENITY.equals(name)){
Amenity a = new Amenity();
a.setType(AmenityType.fromString(attributes.getValue(ATTR_TYPE)));
a.setSubType(attributes.getValue(ATTR_SUBTYPE));
parseMapObject(a, attributes);
region.registerAmenity(a);
} else {
super.startElement(uri, localName, name, attributes);
}
}
}

168
DataExtractionOSM/src/com/osmand/osm/io/OsmLuceneRepository.java
View file

@ -1,168 +0,0 @@
package com.osmand.osm.io;
import java.io.File;
import java.io.IOException;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import org.apache.commons.logging.Log;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
import com.osmand.IProgress;
import com.osmand.LogUtil;
import com.osmand.data.Amenity;
import com.osmand.data.Amenity.AmenityType;
public class OsmLuceneRepository {
private static final Log log = LogUtil.getLog(OsmLuceneRepository.class);
private static final int MAX_POI_HITS = 1000;
private static DecimalFormat fmtLatitude = new DecimalFormat("000.000000", new DecimalFormatSymbols(Locale.US));
private static DecimalFormat negFmtLatitude = new DecimalFormat("00.000000", new DecimalFormatSymbols(Locale.US));
private static DecimalFormat fmtLongitude = new DecimalFormat("0000.000000", new DecimalFormatSymbols(Locale.US));
private static DecimalFormat negFmtLongitude = new DecimalFormat("000.000000", new DecimalFormatSymbols(Locale.US));
public static String formatLatitude(double latitude){
if(latitude <0 ){
return negFmtLatitude.format(latitude);
} else {
return fmtLatitude.format(latitude);
}
}
public static String formatLongitude(double longitude){
if(longitude <0 ){
return negFmtLongitude.format(longitude);
} else {
return fmtLongitude.format(longitude);
}
}
private List<Amenity> internalSearch(double topLatitude, double leftLongitude, double bottomLatitude, double rightLongitude){
queryFormat.setLength(0);
queryFormat.append("latitude:[").append(formatLatitude(bottomLatitude)).append(" TO ").append(formatLatitude(topLatitude)).append(
"]").append(" AND longitude:[").append(formatLongitude(leftLongitude)).append(" TO ").append(
formatLongitude(rightLongitude)).append("]");
TopScoreDocCollector collector = TopScoreDocCollector.create(MAX_POI_HITS, true);
try {
Query q = new QueryParser(Version.LUCENE_30, "id", new StandardAnalyzer(Version.LUCENE_30)).parse(queryFormat.toString());
long now = System.currentTimeMillis();
amenityIndexSearcher.search(q, collector);
ScoreDoc[] hits = collector.topDocs().scoreDocs;
List<Amenity> result = new ArrayList<Amenity>(hits.length);
for (int i = 0; i < hits.length; i++) {
result.add(convertAmenity(amenityIndexSearcher.doc(hits[i].doc)));
}
if (log.isDebugEnabled()) {
log.debug(String.format("Search for %s done in %s ms found %s.", q, System.currentTimeMillis() - now, hits.length));
}
return result;
} catch (IOException e) {
log.error("Failed to search.", e);
throw new RuntimeException(e);
} catch (ParseException e) {
log.error("Invalid query.", e);
return new ArrayList<Amenity>();
}
}
private List<Amenity> cachedAmenities = null;
private double cTopLatitude;
private double cBottomLatitude;
private double cLeftLongitude;
private double cRightLongitude;
private final StringBuilder queryFormat = new StringBuilder();
private IndexSearcher amenityIndexSearcher;
private boolean isLoading = false;
protected synchronized void loadAmenitiesInAnotherThread(final double topLatitude, final double leftLongitude, final double bottomLatitude, final double rightLongitude){
isLoading = true;
new Thread(new Runnable(){
@Override
public void run() {
try {
cachedAmenities = internalSearch(topLatitude, leftLongitude, bottomLatitude, rightLongitude);
cTopLatitude = topLatitude;
cLeftLongitude = leftLongitude;
cBottomLatitude = bottomLatitude ;
cRightLongitude = rightLongitude;
} finally {
synchronized (this) {
isLoading = false;
}
}
}
}, "Searching in index...").start();
}
public synchronized List<Amenity> searchAmenities(double topLatitude, double leftLongitude, double bottomLatitude, double rightLongitude) {
if(amenityIndexSearcher == null){
return Collections.emptyList();
}
// TODO take into account that right could be -53 & left = 175 (normalized coordinates
if (cTopLatitude >= topLatitude && cLeftLongitude <= leftLongitude && cRightLongitude >= rightLongitude
&& cBottomLatitude <= bottomLatitude) {
return cachedAmenities;
}
if(!isLoading){
double h = (topLatitude - bottomLatitude);
double w = (rightLongitude - leftLongitude);
topLatitude += h;
leftLongitude -= w;
bottomLatitude -= h;
rightLongitude += w;
loadAmenitiesInAnotherThread(topLatitude, leftLongitude, bottomLatitude, rightLongitude);
}
return Collections.emptyList();
}
public void indexing(final IProgress progress, File dir) {
long start = System.currentTimeMillis();
progress.startTask("Indexing lucene", -1);
try {
amenityIndexSearcher = new IndexSearcher(FSDirectory.open(dir));
} catch (Exception t) {
log.error("Failed to initialize searcher.", t);
throw new RuntimeException(t);
}
if (log.isDebugEnabled()) {
log.debug("Finished index lucene " + dir.getAbsolutePath() + " " + (System.currentTimeMillis() - start) + "ms");
}
}
protected Amenity convertAmenity(Document document) {
try {
Amenity am = new Amenity();
am.setName(document.get("name"));
am.setId(Long.parseLong(document.get("id")));
am.setSubType(document.get("subtype"));
am.setType(AmenityType.fromString(document.get("type")));
double longitude = fmtLongitude.parse(document.get("longitude")).doubleValue();
double latitude = fmtLatitude.parse(document.get("latitude")).doubleValue();
am.setLocation(latitude, longitude);
return am;
} catch (java.text.ParseException e) {
return null;
}
}
}

200
DataExtractionOSM/src/com/osmand/osm/io/OsmStorageWriter.java
View file

@ -16,24 +16,9 @@ import static com.osmand.osm.io.OsmBaseStorage.ELEM_OSM;
import static com.osmand.osm.io.OsmBaseStorage.ELEM_RELATION;
import static com.osmand.osm.io.OsmBaseStorage.ELEM_TAG;
import static com.osmand.osm.io.OsmBaseStorage.ELEM_WAY;
import static com.osmand.osm.io.OsmIndexStorage.ATTR_CITYTYPE;
import static com.osmand.osm.io.OsmIndexStorage.ATTR_NAME;
import static com.osmand.osm.io.OsmIndexStorage.ATTR_SUBTYPE;
import static com.osmand.osm.io.OsmIndexStorage.ELEM_AMENITY;
import static com.osmand.osm.io.OsmIndexStorage.ELEM_BUILDING;
import static com.osmand.osm.io.OsmIndexStorage.ELEM_CITY;
import static com.osmand.osm.io.OsmIndexStorage.ELEM_OSMAND;
import static com.osmand.osm.io.OsmIndexStorage.ELEM_STREET;
import static com.osmand.osm.io.OsmIndexStorage.OSMAND_VERSION;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@ -44,31 +29,9 @@ import java.util.Map.Entry;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamWriter;
import org.apache.commons.logging.Log;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriter.MaxFieldLength;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.Version;
import com.osmand.Algoritms;
import com.osmand.LogUtil;
import com.osmand.data.Amenity;
import com.osmand.data.Building;
import com.osmand.data.City;
import com.osmand.data.MapObject;
import com.osmand.data.Region;
import com.osmand.data.Street;
import com.osmand.data.Amenity.AmenityType;
import com.osmand.data.City.CityType;
import com.osmand.osm.Entity;
import com.osmand.osm.LatLon;
import com.osmand.osm.Node;
import com.osmand.osm.Relation;
import com.osmand.osm.Way;
@ -79,10 +42,6 @@ public class OsmStorageWriter {
private final String INDENT = " ";
private final String INDENT2 = INDENT + INDENT;
private final String INDENT3 = INDENT + INDENT + INDENT;
private static final Log log = LogUtil.getLog(OsmStorageWriter.class);
private static final Version VERSION = Version.LUCENE_30;
public OsmStorageWriter(){
@ -182,169 +141,14 @@ public class OsmStorageWriter {
return "node";
}
public void saveLuceneIndex(File dir, Region region) throws CorruptIndexException, LockObtainFailedException, IOException{
long now = System.currentTimeMillis();
IndexWriter writer = null;
try {
// Make a lucene writer and create new Lucene index with arg3 = true
writer = new IndexWriter(FSDirectory.open(dir), new StandardAnalyzer(VERSION), true, MaxFieldLength.LIMITED);
for (Amenity a : region.getAmenityManager().getAllObjects()) {
index(a, writer);
}
writer.optimize();
} finally {
try {
writer.close();
} catch (Exception t) {
log.error("Failed to close index.", t);
throw new RuntimeException(t);
}
log.info(String.format("Indexing done in %s ms.", System.currentTimeMillis() - now));
}
}
// TODO externalize strings
protected void index(Amenity amenity, IndexWriter writer) throws CorruptIndexException, IOException {
Document document = new Document();
document.add(new Field("id",""+amenity.getEntity().getId(),Store.YES, Index.NOT_ANALYZED));
LatLon latLon = amenity.getEntity().getLatLon();
document.add(new Field("name",amenity.getName(),Store.YES, Index.NOT_ANALYZED));
document.add(new Field("longitude",OsmLuceneRepository.formatLongitude(latLon.getLongitude()),Store.YES, Index.ANALYZED));
document.add(new Field("latitude",OsmLuceneRepository.formatLatitude(latLon.getLatitude()),Store.YES, Index.ANALYZED));
document.add(new Field("type",amenity.getType().name(),Store.YES, Index.ANALYZED));
document.add(new Field("subtype",amenity.getSubType(),Store.YES, Index.ANALYZED));
//for (Entry<String, String> entry:amenity.getNode().getTags().entrySet()) {
// document.add(new Field(entry.getKey(),entry.getValue(),Store.YES, Index.NOT_ANALYZED));
//}
writer.addDocument(document);
}
public void saveSQLLitePOIIndex(File file, Region region) throws SQLException{
long now = System.currentTimeMillis();
try {
Class.forName("org.sqlite.JDBC");
} catch (ClassNotFoundException e) {
log.error("Illegal configuration", e);
throw new IllegalStateException(e);
}
Connection conn = DriverManager.getConnection("jdbc:sqlite:"+file.getAbsolutePath());
final int batchSize = 500;
try {
Statement stat = conn.createStatement();
stat.executeUpdate("create table poi (id long, latitude double, longitude double, name, type, subtype);");
stat.executeUpdate("create index LatLonIndex ON poi (latitude, longitude);");
PreparedStatement prep = conn.prepareStatement(
"insert into poi values (?, ?, ?, ?, ? ,? );");
conn.setAutoCommit(false);
int currentCount = 0;
for (Amenity a : region.getAmenityManager().getAllObjects()) {
prep.setLong(1, a.getId());
prep.setDouble(2, a.getLocation().getLatitude());
prep.setDouble(3, a.getLocation().getLongitude());
prep.setString(4, a.getName());
prep.setString(5, AmenityType.valueToString(a.getType()));
prep.setString(6, a.getSubType());
prep.addBatch();
currentCount++;
if(currentCount >= batchSize){
prep.executeBatch();
currentCount = 0;
}
}
if(currentCount > 0){
prep.executeBatch();
}
conn.setAutoCommit(true);
} finally {
conn.close();
log.info(String.format("Indexing sqllite done in %s ms.", System.currentTimeMillis() - now));
}
}
public void savePOIIndex(OutputStream output, Region region) throws XMLStreamException, IOException {
PropertyManager propertyManager = new PropertyManager(PropertyManager.CONTEXT_WRITER);
XMLStreamWriter streamWriter = new XMLStreamWriterImpl(output, propertyManager);
writeStartElement(streamWriter, ELEM_OSMAND, "");
streamWriter.writeAttribute(ATTR_VERSION, OSMAND_VERSION);
List<Amenity> amenities = region.getAmenityManager().getAllObjects();
for(Amenity n : amenities){
if (couldBeWrited(n)) {
writeStartElement(streamWriter, ELEM_AMENITY, INDENT);
writeAttributesMapObject(streamWriter, n);
streamWriter.writeAttribute(ATTR_TYPE, AmenityType.valueToString(n.getType()));
streamWriter.writeAttribute(ATTR_SUBTYPE, n.getSubType());
writeEndElement(streamWriter, INDENT);
}
}
writeEndElement(streamWriter, ""); // osmand
streamWriter.writeEndDocument();
streamWriter.flush();
}
private void writeCity(XMLStreamWriter streamWriter, City c) throws XMLStreamException{
writeStartElement(streamWriter, ELEM_CITY, INDENT);
writeAttributesMapObject(streamWriter, c);
streamWriter.writeAttribute(ATTR_CITYTYPE, CityType.valueToString(c.getType()));
for(Street s : c.getStreets()){
if (couldBeWrited(s)) {
writeStartElement(streamWriter, ELEM_STREET, INDENT2);
writeAttributesMapObject(streamWriter, s);
for(Building b : s.getBuildings()) {
if (couldBeWrited(b)) {
writeStartElement(streamWriter, ELEM_BUILDING, INDENT3);
writeAttributesMapObject(streamWriter, b);
writeEndElement(streamWriter, INDENT3);
}
}
writeEndElement(streamWriter, INDENT2);
}
}
writeEndElement(streamWriter, INDENT);
}
public void saveAddressIndex(OutputStream output, Region region) throws XMLStreamException, IOException {
PropertyManager propertyManager = new PropertyManager(PropertyManager.CONTEXT_WRITER);
XMLStreamWriter streamWriter = new XMLStreamWriterImpl(output, propertyManager);
writeStartElement(streamWriter, ELEM_OSMAND, "");
streamWriter.writeAttribute(ATTR_VERSION, OSMAND_VERSION);
for(CityType t : CityType.values()){
Collection<City> cities = region.getCitiesByType(t);
if(cities != null){
for(City c : cities){
if (couldBeWrited(c)) {
writeCity(streamWriter, c);
}
}
}
}
writeEndElement(streamWriter, ""); // osmand
streamWriter.writeEndDocument();
streamWriter.flush();
}
public boolean couldBeWrited(MapObject<? extends Entity> e){
public boolean couldBeWrited(MapObject e){
if(!Algoritms.isEmpty(e.getName()) && e.getLocation() != null){
return true;
}
return false;
}
public void writeAttributesMapObject(XMLStreamWriter streamWriter, MapObject<? extends Entity> e) throws XMLStreamException{
LatLon location = e.getLocation();
streamWriter.writeAttribute(ATTR_LAT, location.getLatitude()+"");
streamWriter.writeAttribute(ATTR_LON, location.getLongitude()+"");
streamWriter.writeAttribute(ATTR_NAME, e.getName());
streamWriter.writeAttribute(ATTR_ID, e.getId()+"");
}
private void writeStartElement(XMLStreamWriter writer, String name, String indent) throws XMLStreamException{

2
DataExtractionOSM/src/com/osmand/swing/MapPanel.java
View file

@ -48,12 +48,12 @@ import javax.swing.UIManager;
import org.apache.commons.logging.Log;
import com.osmand.IMapLocationListener;
import com.osmand.LogUtil;
import com.osmand.data.DataTileManager;
import com.osmand.data.preparation.MapTileDownloader;
import com.osmand.data.preparation.MapTileDownloader.DownloadRequest;
import com.osmand.data.preparation.MapTileDownloader.IMapDownloaderCallback;
import com.osmand.map.IMapLocationListener;
import com.osmand.map.ITileSource;
import com.osmand.map.TileSourceManager;
import com.osmand.map.TileSourceManager.TileSourceTemplate;

42
DataExtractionOSM/src/com/osmand/swing/OsmExtractionUI.java
View file

@ -64,7 +64,6 @@ import org.xml.sax.SAXException;
import com.osmand.Algoritms;
import com.osmand.ExceptionHandler;
import com.osmand.IMapLocationListener;
import com.osmand.data.Amenity;
import com.osmand.data.Building;
import com.osmand.data.City;
@ -74,12 +73,14 @@ import com.osmand.data.Region;
import com.osmand.data.Street;
import com.osmand.data.Amenity.AmenityType;
import com.osmand.data.City.CityType;
import com.osmand.data.index.DataIndexReader;
import com.osmand.data.index.DataIndexWriter;
import com.osmand.data.index.IndexConstants;
import com.osmand.data.preparation.DataExtraction;
import com.osmand.data.preparation.DataIndexBuilder;
import com.osmand.map.IMapLocationListener;
import com.osmand.osm.Entity;
import com.osmand.osm.LatLon;
import com.osmand.osm.MapUtils;
import com.osmand.osm.Node;
import com.osmand.osm.Way;
import com.osmand.osm.io.IOsmStorageFilter;
import com.osmand.osm.io.OsmBoundsFilter;
@ -232,15 +233,14 @@ public class OsmExtractionUI implements IMapLocationListener {
treePlaces.setEditable(true);
treePlaces.setCellEditor(new RegionCellEditor(treePlaces, (DefaultTreeCellRenderer) treePlaces.getCellRenderer()));
treePlaces.addTreeSelectionListener(new TreeSelectionListener() {
@SuppressWarnings("unchecked")
@Override
public void valueChanged(TreeSelectionEvent e) {
if (e.getPath() != null) {
if (e.getPath().getLastPathComponent() instanceof DataExtractionTreeNode) {
Object o = ((DataExtractionTreeNode) e.getPath().getLastPathComponent()).getModelObject();
if (o instanceof MapObject<?>) {
MapObject<Entity> c = (MapObject<Entity>) o;
if (o instanceof MapObject) {
MapObject c = (MapObject) o;
LatLon location = c.getLocation();
if(location != null){
if(o instanceof Street){
@ -279,8 +279,8 @@ public class OsmExtractionUI implements IMapLocationListener {
if(((DataExtractionTreeNode) node).getModelObject() instanceof Region){
Region r = (Region) ((DataExtractionTreeNode) node).getModelObject();
r.setName(node.getUserObject().toString());
} else if(((DataExtractionTreeNode) node).getModelObject() instanceof MapObject<?>){
MapObject<?> r = (MapObject<?>) ((DataExtractionTreeNode) node).getModelObject();
} else if(((DataExtractionTreeNode) node).getModelObject() instanceof MapObject){
MapObject r = (MapObject) ((DataExtractionTreeNode) node).getModelObject();
r.setName(node.getUserObject().toString());
}
}
@ -355,28 +355,29 @@ public class OsmExtractionUI implements IMapLocationListener {
@Override
public void run() {
dlg.startTask("Generating indices...", -1);
DataIndexBuilder builder = new DataIndexBuilder(DataExtractionSettings.getSettings().getDefaultWorkingDir(), region);
DataIndexWriter builder = new DataIndexWriter(DataExtractionSettings.getSettings().getDefaultWorkingDir(), region);
StringBuilder msg = new StringBuilder();
try {
msg.append("Indices for ").append(region.getName());
if(buildPoiIndex.isEnabled()){
if(buildPoiIndex.isSelected()){
dlg.startTask("Generating POI index...", -1);
builder.buildPOI();
builder.writePOI();
msg.append(", POI index ").append("successfully created");
}
if(buildAddressIndex.isEnabled()){
if(buildAddressIndex.isSelected()){
dlg.startTask("Generating address index...", -1);
builder.buildAddress();
builder.writeAddress();
msg.append(", Address index ").append("successfully created");
}
new DataIndexReader().testIndex(new File(
DataExtractionSettings.getSettings().getDefaultWorkingDir(),
IndexConstants.ADDRESS_INDEX_DIR+region.getName()+IndexConstants.ADDRESS_INDEX_EXT));
msg.append(".");
JOptionPane pane = new JOptionPane(msg);
JDialog dialog = pane.createDialog(frame, "Generation data");
dialog.setVisible(true);
} catch (SQLException e1) {
throw new IllegalArgumentException(e1);
} catch (XMLStreamException e1) {
throw new IllegalArgumentException(e1);
} catch (IOException e1) {
throw new IllegalArgumentException(e1);
}
@ -441,10 +442,10 @@ public class OsmExtractionUI implements IMapLocationListener {
@Override
public void valueChanged(ListSelectionEvent e) {
if(searchList.getSelectedValue() != null){
Node node = ((City)searchList.getSelectedValue()).getEntity();
LatLon node = ((City)searchList.getSelectedValue()).getLocation();
String text = "Lat : " + node.getLatitude() + " Lon " + node.getLongitude();
if(selectedCity != null){
text += " distance " + MapUtils.getDistance(selectedCity.getEntity(), node);
text += " distance " + MapUtils.getDistance(node, node);
}
mapPanel.setLatLon(node.getLatitude(), node.getLongitude());
}
@ -627,7 +628,8 @@ public class OsmExtractionUI implements IMapLocationListener {
Region res;
try {
DataExtraction dataExtraction = new DataExtraction(buildAddressIndex.isSelected(), buildPoiIndex.isSelected(),
normalizingStreets.isSelected(), loadingAllData.isSelected());
normalizingStreets.isSelected(), loadingAllData.isSelected(),
DataExtractionSettings.getSettings().getDefaultWorkingDir());
if(!buildAddressIndex.isSelected()){
buildAddressIndex.setEnabled(false);
}
@ -639,6 +641,8 @@ public class OsmExtractionUI implements IMapLocationListener {
throw new IllegalArgumentException(e);
} catch (SAXException e) {
throw new IllegalStateException(e);
} catch (SQLException e) {
throw new IllegalStateException(e);
}
dlg.setResult(res);
}
@ -783,7 +787,7 @@ public class OsmExtractionUI implements IMapLocationListener {
Object node = tree.getLastSelectedPathComponent();
if (node instanceof DataExtractionTreeNode) {
DataExtractionTreeNode treeNode = (DataExtractionTreeNode) node;
if (treeNode.getModelObject() instanceof Region || treeNode.getModelObject() instanceof MapObject<?>) {
if (treeNode.getModelObject() instanceof Region || treeNode.getModelObject() instanceof MapObject) {
return true;
}
}

1
OsmAnd/.classpath
View file

@ -5,6 +5,5 @@
<classpathentry kind="src" path="gen"/>
<classpathentry kind="con" path="com.android.ide.eclipse.adt.ANDROID_FRAMEWORK"/>
<classpathentry kind="lib" path="lib/bzip2-20090327.jar"/>
<classpathentry kind="lib" path="lib/lucene-core-3.0.1.jar" sourcepath="C:/docs/lucene/lucene-src.zip"/>
<classpathentry kind="output" path="bin"/>
</classpath>

BIN
OsmAnd/lib/lucene-core-3.0.1.jar
View file

Binary file not shown.

137
OsmAnd/src/com/osmand/AmenityIndexRepository.java Normal file
View file

@ -0,0 +1,137 @@
package com.osmand;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.logging.Log;
import android.database.Cursor;
import android.database.sqlite.SQLiteDatabase;
import com.osmand.data.Amenity;
import com.osmand.data.Amenity.AmenityType;
import com.osmand.data.index.IndexConstants;
import com.osmand.data.index.IndexConstants.IndexPoiTable;
import com.osmand.osm.LatLon;
public class AmenityIndexRepository {
private static final Log log = LogUtil.getLog(AmenityIndexRepository.class);
public final static int LIMIT_AMENITIES = 500;
private SQLiteDatabase db;
private double dataTopLatitude;
private double dataBottomLatitude;
private double dataLeftLongitude;
private double dataRightLongitude;
private String name;
// cache amenities
private List<Amenity> cachedAmenities = new ArrayList<Amenity>();
private double cTopLatitude;
private double cBottomLatitude;
private double cLeftLongitude;
private double cRightLongitude;
private final String[] columns = IndexConstants.generateColumnNames(IndexPoiTable.values());
public List<Amenity> searchAmenities(double topLatitude, double leftLongitude, double bottomLatitude, double rightLongitude, int limit, List<Amenity> amenities){
long now = System.currentTimeMillis();
Cursor query = db.query(IndexPoiTable.getTable(), columns, "? < latitude AND latitude < ? AND ? < longitude AND longitude < ?",
new String[]{Double.toString(bottomLatitude),
Double.toString(topLatitude), Double.toString(leftLongitude), Double.toString(rightLongitude)}, null, null, null);
if(query.moveToFirst()){
do {
Amenity am = new Amenity();
am.setId(query.getLong(IndexPoiTable.ID.ordinal()));
am.setLocation(query.getDouble(IndexPoiTable.LATITUDE.ordinal()),
query.getDouble(IndexPoiTable.LONGITUDE.ordinal()));
am.setName(query.getString(IndexPoiTable.NAME.ordinal() ));
am.setType(AmenityType.fromString(query.getString(IndexPoiTable.TYPE.ordinal())));
am.setSubType(query.getString(IndexPoiTable.SUBTYPE.ordinal()));
amenities.add(am);
if(limit != -1 && amenities.size() >= limit){
break;
}
} while(query.moveToNext());
}
query.deactivate();
if (log.isDebugEnabled()) {
log.debug(String.format("Search for %s done in %s ms found %s.",
topLatitude + " " + leftLongitude, System.currentTimeMillis() - now, amenities.size()));
}
return amenities;
}
public void evaluateCachedAmenities(double topLatitude, double leftLongitude, double bottomLatitude, double rightLongitude, List<Amenity> toFill){
cachedAmenities.clear();
cTopLatitude = topLatitude + (topLatitude -bottomLatitude);
cBottomLatitude = bottomLatitude - (topLatitude -bottomLatitude);
cLeftLongitude = leftLongitude - (rightLongitude - leftLongitude);
cRightLongitude = rightLongitude + (rightLongitude - leftLongitude);
searchAmenities(topLatitude, leftLongitude, bottomLatitude, rightLongitude, -1, cachedAmenities);
checkCachedAmenities(topLatitude, leftLongitude, bottomLatitude, rightLongitude, toFill);
}
public boolean checkCachedAmenities(double topLatitude, double leftLongitude, double bottomLatitude, double rightLongitude, List<Amenity> toFill){
if (db == null) {
return true;
}
if (cTopLatitude >= topLatitude && cLeftLongitude <= leftLongitude && cRightLongitude >= rightLongitude
&& cBottomLatitude <= bottomLatitude) {
for(Amenity a : cachedAmenities){
LatLon location = a.getLocation();
if (location.getLatitude() <= topLatitude && location.getLongitude() >= leftLongitude && location.getLongitude() <= rightLongitude
&& location.getLatitude() >= bottomLatitude) {
toFill.add(a);
}
}
return true;
}
return false;
}
public void initialize(final IProgress progress, File file) {
long start = System.currentTimeMillis();
if(db != null){
// close previous db
db.close();
}
db = SQLiteDatabase.openOrCreateDatabase(file, null);
name = file.getName().substring(0, file.getName().indexOf('.'));
Cursor query = db.query(IndexPoiTable.getTable(), new String[]{"MAX(latitude)", "MAX(longitude)", "MIN(latitude)", "MIN(longitude)"}, null, null,null, null, null);
if(query.moveToFirst()){
dataTopLatitude = query.getDouble(0);
dataRightLongitude = query.getDouble(1);
dataBottomLatitude = query.getDouble(2);
dataLeftLongitude = query.getDouble(3);
}
if (log.isDebugEnabled()) {
log.debug("Initializing db " + file.getAbsolutePath() + " " + (System.currentTimeMillis() - start) + "ms");
}
}
public String getName() {
return name;
}
public boolean checkContains(double topLatitude, double leftLongitude, double bottomLatitude, double rightLongitude){
if(rightLongitude < dataLeftLongitude || leftLongitude > dataRightLongitude){
return false;
}
if(topLatitude < dataBottomLatitude || bottomLatitude > dataTopLatitude){
return false;
}
return true;
}
}

122
OsmAnd/src/com/osmand/OsmSQLLiteRepository.java
View file

@ -1,122 +0,0 @@
package com.osmand;
import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.apache.commons.logging.Log;
import android.database.Cursor;
import android.database.sqlite.SQLiteDatabase;
import com.osmand.data.Amenity;
import com.osmand.data.Amenity.AmenityType;
public class OsmSQLLiteRepository {
private static final Log log = LogUtil.getLog(OsmSQLLiteRepository.class);
private SQLiteDatabase db;
private List<Amenity> cachedAmenities = null;
private double cTopLatitude;
private double cBottomLatitude;
private double cLeftLongitude;
private double cRightLongitude;
private boolean isLoading = false;
protected synchronized void loadAmenitiesInAnotherThread(final double topLatitude, final double leftLongitude, final double bottomLatitude, final double rightLongitude){
isLoading = true;
new Thread(new Runnable(){
@Override
public void run() {
try {
cachedAmenities = internalSearch(topLatitude, leftLongitude, bottomLatitude, rightLongitude);
cTopLatitude = topLatitude;
cLeftLongitude = leftLongitude;
cBottomLatitude = bottomLatitude ;
cRightLongitude = rightLongitude;
} finally {
synchronized (this) {
isLoading = false;
}
}
}
}, "Searching in index...").start();
}
private List<Amenity> internalSearch(double topLatitude, double leftLongitude, double bottomLatitude, double rightLongitude){
long now = System.currentTimeMillis();
Cursor query = db.query("poi", null, "? < latitude AND latitude < ? AND ? < longitude AND longitude < ?",
new String[]{Double.toString(bottomLatitude),
Double.toString(topLatitude), Double.toString(leftLongitude), Double.toString(rightLongitude)}, null, null, null);
List<Amenity> amenities = new ArrayList<Amenity>();
int idIndex = query.getColumnIndex("id");
int latitudeIndex = query.getColumnIndex("latitude");
int longitudeIndex = query.getColumnIndex("longitude");
int nameIndex = query.getColumnIndex("name");
int typeIndex = query.getColumnIndex("type");
int subtypeIndex = query.getColumnIndex("subtype");
if(query.moveToFirst()){
do {
Amenity am = new Amenity();
if(idIndex != -1){
am.setId(query.getLong(idIndex));
}
if(latitudeIndex != -1 && longitudeIndex != -1){
am.setLocation(query.getDouble(latitudeIndex), query.getDouble(longitudeIndex));
}
if(nameIndex != -1){
am.setName(query.getString(nameIndex));
}
if(typeIndex != -1){
am.setType(AmenityType.fromString(query.getString(typeIndex)));
}
if(subtypeIndex != -1){
am.setSubType(query.getString(subtypeIndex));
}
amenities.add(am);
} while(query.moveToNext());
}
query.deactivate();
if (log.isDebugEnabled()) {
log.debug(String.format("Search for %s done in %s ms found %s.",
topLatitude + " " + leftLongitude, System.currentTimeMillis() - now, amenities.size()));
}
return amenities;
}
public synchronized List<Amenity> searchAmenities(double topLatitude, double leftLongitude, double bottomLatitude, double rightLongitude) {
if (db == null) {
return Collections.emptyList();
}
if (cTopLatitude >= topLatitude && cLeftLongitude <= leftLongitude && cRightLongitude >= rightLongitude
&& cBottomLatitude <= bottomLatitude) {
return cachedAmenities;
}
if(!isLoading){
double h = (topLatitude - bottomLatitude);
double w = (rightLongitude - leftLongitude);
topLatitude += h;
leftLongitude -= w;
bottomLatitude -= h;
rightLongitude += w;
loadAmenitiesInAnotherThread(topLatitude, leftLongitude, bottomLatitude, rightLongitude);
}
return Collections.emptyList();
}
public void initialize(final IProgress progress, File file) {
long start = System.currentTimeMillis();
db = SQLiteDatabase.openOrCreateDatabase(file, null);
if (log.isDebugEnabled()) {
log.debug("Initializing db " + file.getAbsolutePath() + " " + (System.currentTimeMillis() - start) + "ms");
}
}
}

331
OsmAnd/src/com/osmand/ResourceManager.java
View file

@ -1,35 +1,27 @@
package com.osmand;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Stack;
import java.util.TreeMap;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import org.apache.commons.logging.Log;
import org.xml.sax.SAXException;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.os.Environment;
import com.osmand.data.Amenity;
import com.osmand.data.DataTileManager;
import com.osmand.data.Region;
import com.osmand.data.index.IndexConstants;
import com.osmand.data.preparation.MapTileDownloader;
import com.osmand.data.preparation.MapTileDownloader.DownloadRequest;
import com.osmand.data.preparation.MapTileDownloader.IMapDownloaderCallback;
import com.osmand.map.ITileSource;
import com.osmand.osm.LatLon;
import com.osmand.osm.io.OsmIndexStorage;
import com.osmand.osm.io.OsmLuceneRepository;
import com.osmand.osm.MapUtils;
/**
* Resource manager is responsible to work with all resources
@ -41,10 +33,9 @@ import com.osmand.osm.io.OsmLuceneRepository;
*/
public class ResourceManager {
private static final String POI_PATH = "osmand/poi/";
private static final String ADDRESS_PATH = "osmand/address/";
private static final String POI_PATH = "osmand/" + IndexConstants.POI_INDEX_DIR;
private static final String ADDRESS_PATH = "osmand/" + IndexConstants.ADDRESS_INDEX_DIR;
private static final String TILES_PATH = "osmand/tiles/";
private static final String LUCENE_PATH = "osmand/lucene/";
private static final Log log = LogUtil.getLog(ResourceManager.class);
@ -60,21 +51,21 @@ public class ResourceManager {
// it is not good investigated but no more than 64 (satellite images)
protected final int maxImgCacheSize = 64;
private DataTileManager<Amenity> poiIndex = null;
private Map<String, Region> addressMap = new TreeMap<String, Region>();
protected Map<String, Bitmap> cacheOfImages = new LinkedHashMap<String, Bitmap>();
protected File dirWithTiles ;
private MapTileDownloader downloader = MapTileDownloader.getInstance();
// Indexes
private Map<String, Region> addressMap = new TreeMap<String, Region>();
protected List<AmenityIndexRepository> amenityRepositories = new ArrayList<AmenityIndexRepository>();
public AsyncLoadingThread asyncLoadingTiles = new AsyncLoadingThread();
protected OsmLuceneRepository amenityIndexSearcher = new OsmLuceneRepository();
protected OsmSQLLiteRepository amenityRepository = new OsmSQLLiteRepository();
public ResourceManager() {
@ -87,7 +78,9 @@ public class ResourceManager {
}
/// Working with tiles ///
////////////////////////////////////////////// Working with tiles ////////////////////////////////////////////////
public Bitmap getTileImageForMapAsync(ITileSource map, int x, int y, int zoom, boolean loadFromInternetIfNeeded) {
return getTileImageForMap(map, x, y, zoom, loadFromInternetIfNeeded, false);
}
@ -120,6 +113,121 @@ public class ResourceManager {
private Bitmap getRequestedImageTile(TileLoadDownloadRequest req){
if(req.fileToLoad == null || req.dirWithTiles == null){
return null;
}
File en = new File(req.dirWithTiles, req.fileToLoad);
if (cacheOfImages.size() > maxImgCacheSize) {
onLowMemory();
}
if (!downloader.isFileCurrentlyDownloaded(en) && req.dirWithTiles.canRead()) {
if (en.exists()) {
long time = System.currentTimeMillis();
cacheOfImages.put(req.fileToLoad, BitmapFactory.decodeFile(en.getAbsolutePath()));
if (log.isDebugEnabled()) {
log.debug("Loaded file : " + req.fileToLoad + " " + -(time - System.currentTimeMillis()) + " ms");
}
}
if(cacheOfImages.get(req.fileToLoad) == null && req.url != null){
// TODO we could check that network is available (context is required)
// ConnectivityManager mgr = (ConnectivityManager) getContext().getSystemService(Context.CONNECTIVITY_SERVICE);
// NetworkInfo info = mgr.getActiveNetworkInfo();
// if (info != null && info.isConnected()) {
// downloader.requestToDownload(req);
// }
downloader.requestToDownload(req);
}
}
return cacheOfImages.get(req.fileToLoad);
}
////////////////////////////////////////////// Working with indexes ////////////////////////////////////////////////
// POI INDEX //
public void indexingPoi(final IProgress progress) {
File file = new File(Environment.getExternalStorageDirectory(), POI_PATH);
amenityRepositories.clear();
if (file.exists() && file.canRead()) {
for (File f : file.listFiles()) {
if (f.getName().endsWith(IndexConstants.POI_INDEX_EXT)) {
AmenityIndexRepository repository = new AmenityIndexRepository();
progress.startTask("Indexing poi " + f.getName(), -1);
repository.initialize(progress, f);
amenityRepositories.add(repository);
}
}
}
}
public void indexingAddresses(final IProgress progress){
File file = new File(Environment.getExternalStorageDirectory(), ADDRESS_PATH);
addressMap.clear();
if (file.exists() && file.canRead()) {
for (File f : file.listFiles()) {
if (f.getName().endsWith(IndexConstants.ADDRESS_INDEX_EXT)) {
// TODO fill in address index
}
}
}
}
////////////////////////////////////////////// Working with amenities ////////////////////////////////////////////////
public List<Amenity> searchAmenities(double latitude, double longitude, int zoom, int limit) {
double tileNumberX = Math.floor(MapUtils.getTileNumberX(zoom, longitude));
double tileNumberY = Math.floor(MapUtils.getTileNumberY(zoom, latitude));
double topLatitude = MapUtils.getLatitudeFromTile(zoom, tileNumberY);
double bottomLatitude = MapUtils.getLatitudeFromTile(zoom, tileNumberY + 1);
double leftLongitude = MapUtils.getLongitudeFromTile(zoom, tileNumberX);
double rightLongitude = MapUtils.getLongitudeFromTile(zoom, tileNumberX + 1);
List<Amenity> amenities = new ArrayList<Amenity>();
for(AmenityIndexRepository index : amenityRepositories){
if(index.checkContains(topLatitude, leftLongitude, bottomLatitude, rightLongitude)){
if(!index.checkCachedAmenities(topLatitude, leftLongitude, bottomLatitude, rightLongitude, amenities)){
index.searchAmenities(topLatitude, leftLongitude, bottomLatitude, rightLongitude, limit, amenities);
}
}
}
return amenities;
}
public void searchAmenitiesAsync(double topLatitude, double leftLongitude, double bottomLatitude, double rightLongitude, List<Amenity> toFill){
for(AmenityIndexRepository index : amenityRepositories){
if(index.checkContains(topLatitude, leftLongitude, bottomLatitude, rightLongitude)){
if(!index.checkCachedAmenities(topLatitude, leftLongitude, bottomLatitude, rightLongitude, toFill)){
// TODO add request to another thread
index.evaluateCachedAmenities(topLatitude, leftLongitude, bottomLatitude, rightLongitude, toFill);
}
}
}
}
/// On low memory method ///
public void onLowMemory() {
log.info("On low memory : cleaning tiles - size = " + cacheOfImages.size());
ArrayList<String> list = new ArrayList<String>(cacheOfImages.keySet());
// remove first images (as we think they are older)
for (int i = 0; i < list.size()/2; i ++) {
Bitmap bmp = cacheOfImages.remove(list.get(i));
if(bmp != null){
bmp.recycle();
}
}
// TODO clear amenity indexes & addresses indexes
System.gc();
}
private static class TileLoadDownloadRequest extends DownloadRequest {
public final String fileToLoad;
@ -131,8 +239,9 @@ public class ResourceManager {
this.dirWithTiles = dirWithTiles;
this.fileToLoad = fileToLoad;
}
}
public class AsyncLoadingThread extends Thread {
Stack<TileLoadDownloadRequest> requests = new Stack<TileLoadDownloadRequest>();
@ -170,182 +279,4 @@ public class ResourceManager {
requests.push(req);
}
};
private Bitmap getRequestedImageTile(TileLoadDownloadRequest req){
if(req.fileToLoad == null || req.dirWithTiles == null){
return null;
}
File en = new File(req.dirWithTiles, req.fileToLoad);
if (cacheOfImages.size() > maxImgCacheSize) {
onLowMemory();
}
if (!downloader.isFileCurrentlyDownloaded(en) && req.dirWithTiles.canRead()) {
if (en.exists()) {
long time = System.currentTimeMillis();
cacheOfImages.put(req.fileToLoad, BitmapFactory.decodeFile(en.getAbsolutePath()));
if (log.isDebugEnabled()) {
log.debug("Loaded file : " + req.fileToLoad + " " + -(time - System.currentTimeMillis()) + " ms");
}
}
if(cacheOfImages.get(req.fileToLoad) == null && req.url != null){
// TODO we could check that network is available (context is required)
// ConnectivityManager mgr = (ConnectivityManager) getContext().getSystemService(Context.CONNECTIVITY_SERVICE);
// NetworkInfo info = mgr.getActiveNetworkInfo();
// if (info != null && info.isConnected()) {
// downloader.requestToDownload(req);
// }
downloader.requestToDownload(req);
}
}
return cacheOfImages.get(req.fileToLoad);
}
private interface IndexVisitor {
/**
* returns if entry was visited succesfully
*/
public boolean visitEntry(String entryName, InputStream stream) throws IOException, SAXException;
}
public void indexingFiles(String pathToIndex, String ext, IProgress progress, String objectToIndex, IndexVisitor visitor) {
File file = new File(Environment.getExternalStorageDirectory(), pathToIndex);
if (file.exists() && file.canRead()) {
for (File f : file.listFiles()) {
InputStream stream = null;
ZipFile zipFile = null;
Enumeration<? extends ZipEntry> entries = null;
try {
if (f.getName().endsWith(".zip")) {
zipFile = new ZipFile(f);
entries = zipFile.entries();
} else {
stream = new FileInputStream(f);
}
} catch (IOException e) {
log.error("Can't read file " + f.getAbsolutePath(), e);
continue;
}
String entryName = f.getName();
do {
try {
if (entries != null && entries.hasMoreElements()) {
ZipEntry entry = entries.nextElement();
entryName = entry.getName();
stream = zipFile.getInputStream(entry);
}
if (entryName != null && entryName.endsWith(ext)) {
long start = System.currentTimeMillis();
if (log.isDebugEnabled()) {
log.debug("Starting index " + objectToIndex + " " + f.getAbsolutePath());
}
if (progress != null) {
progress.startTask("Indexing " + objectToIndex + " " + f.getName(), stream.available());
}
visitor.visitEntry(f.getName(), stream);
if (log.isDebugEnabled()) {
log.debug("Finished index " + objectToIndex + " " + f.getAbsolutePath() + " "
+ (System.currentTimeMillis() - start) + "ms");
}
}
} catch (IOException e) {
log.error("Can't read file " + f.getAbsolutePath(), e);
} catch (SAXException e) {
log.error("Can't read file " + f.getAbsolutePath(), e);
} finally {
Algoritms.closeStream(stream);
}
} while (zipFile != null && entries.hasMoreElements());
}
}
}
// POI INDEX //
public void indexingPoi(final IProgress progress) {
if (poiIndex == null) {
poiIndex = new DataTileManager<Amenity>();
indexingFiles(POI_PATH, ".osmand", progress, "POI", new IndexVisitor() {
@Override
public boolean visitEntry(String entryName, InputStream stream) throws IOException, SAXException {
OsmIndexStorage storage = new OsmIndexStorage(new Region());
storage.parseOSM(stream, progress);
Region region = ((OsmIndexStorage) storage).getRegion();
for (Amenity a : region.getAmenityManager().getAllObjects()) {
LatLon location = a.getLocation();
poiIndex.registerObject(location.getLatitude(), location.getLongitude(), a);
}
return true;
}
});
File file = new File(Environment.getExternalStorageDirectory(), POI_PATH);
if (file.exists() && file.canRead()) {
for (File f : file.listFiles()) {
if(f.getName().endsWith(".db")){
progress.startTask("Indexing poi " + f.getName(), -1);
amenityRepository.initialize(progress, f);
}
}
}
}
}
public void indexingLucene(final IProgress progress){
// read index
File file = new File(Environment.getExternalStorageDirectory(), LUCENE_PATH);
if (file.exists() && file.canRead()) {
amenityIndexSearcher.indexing(progress, file);
}
}
public void indexingAddresses(final IProgress progress){
indexingFiles(ADDRESS_PATH, ".osmand", progress, "address", new IndexVisitor() {
@Override
public boolean visitEntry(String entryName, InputStream stream) throws IOException, SAXException {
String name = entryName.substring(0, entryName.indexOf('.'));
Region region = new Region();
region.setName(name);
addressMap.put(name, region);
OsmIndexStorage storage = new OsmIndexStorage(region);
storage.parseOSM(stream, progress);
return true;
}
});
}
public DataTileManager<Amenity> getPoiIndex() {
if(poiIndex == null){
indexingPoi(null);
}
return poiIndex;
}
public OsmSQLLiteRepository getAmenityRepository() {
return amenityRepository;
}
public OsmLuceneRepository getAmenityIndexSearcher(){
return amenityIndexSearcher;
}
/// On low memory method ///
public void onLowMemory() {
log.info("On low memory : cleaning tiles - size = " + cacheOfImages.size());
ArrayList<String> list = new ArrayList<String>(cacheOfImages.keySet());
// remove first images (as we think they are older)
for (int i = 0; i < list.size()/2; i ++) {
Bitmap bmp = cacheOfImages.remove(list.get(i));
if(bmp != null){
bmp.recycle();
}
}
System.gc();
}
}

1
OsmAnd/src/com/osmand/activities/MainMenuActivity.java
View file

@ -93,7 +93,6 @@ public class MainMenuActivity extends Activity {
try {
ResourceManager.getResourceManager().indexingPoi(impl);
ResourceManager.getResourceManager().indexingAddresses(impl);
ResourceManager.getResourceManager().indexingLucene(impl);
} finally {
dlg.dismiss();
}

3
OsmAnd/src/com/osmand/activities/MapActivity.java
View file

@ -22,11 +22,11 @@ import android.widget.ImageButton;
import android.widget.Toast;
import android.widget.ZoomControls;
import com.osmand.IMapLocationListener;
import com.osmand.OsmandSettings;
import com.osmand.R;
import com.osmand.ResourceManager;
import com.osmand.data.preparation.MapTileDownloader;
import com.osmand.map.IMapLocationListener;
import com.osmand.osm.LatLon;
import com.osmand.views.OsmandMapTileView;
import com.osmand.views.POIMapLayer;
@ -64,7 +64,6 @@ public class MapActivity extends Activity implements LocationListener, IMapLocat
MapTileDownloader.getInstance().addDownloaderCallback(mapView);
mapView.setMapLocationListener(this);
poiMapLayer = new POIMapLayer();
poiMapLayer.setNodeManager(ResourceManager.getResourceManager().getPoiIndex());
mapView.addLayer(poiMapLayer);
locationLayer = new PointLocationLayer();
mapView.addLayer(locationLayer);

7
OsmAnd/src/com/osmand/activities/SearchActivity.java
View file

@ -24,7 +24,6 @@ import com.osmand.OsmandSettings;
import com.osmand.R;
import com.osmand.ResourceManager;
import com.osmand.data.Amenity;
import com.osmand.data.DataTileManager;
import com.osmand.data.Amenity.AmenityType;
import com.osmand.osm.LatLon;
import com.osmand.osm.MapUtils;
@ -67,11 +66,11 @@ public class SearchActivity extends ListActivity {
}
private void createAmenityFilter() {
DataTileManager<Amenity> poiIndex = ResourceManager.getResourceManager().getPoiIndex();
ResourceManager resourceManager = ResourceManager.getResourceManager();
filter = new TreeMap<AmenityType, List<Amenity>>();
LatLon lastKnownMapLocation = OsmandSettings.getLastKnownMapLocation(this);
List<Amenity> closestAmenities = poiIndex.getClosestObjects(lastKnownMapLocation.getLatitude(),
lastKnownMapLocation.getLongitude(), 0, 5);
List<Amenity> closestAmenities = resourceManager.searchAmenities(lastKnownMapLocation.getLatitude(),
lastKnownMapLocation.getLongitude(), 13, 500);
MapUtils.sortListOfMapObject(closestAmenities, lastKnownMapLocation.getLatitude(), lastKnownMapLocation.getLongitude());
for (Amenity n : closestAmenities) {
AmenityType type = n.getType();

2
OsmAnd/src/com/osmand/views/OsmandMapTileView.java
View file

@ -22,13 +22,13 @@ import android.view.SurfaceHolder;
import android.view.SurfaceView;
import android.view.SurfaceHolder.Callback;
import com.osmand.IMapLocationListener;
import com.osmand.LogUtil;
import com.osmand.OsmandSettings;
import com.osmand.ResourceManager;
import com.osmand.data.preparation.MapTileDownloader;
import com.osmand.data.preparation.MapTileDownloader.DownloadRequest;
import com.osmand.data.preparation.MapTileDownloader.IMapDownloaderCallback;
import com.osmand.map.IMapLocationListener;
import com.osmand.map.ITileSource;
import com.osmand.osm.MapUtils;
import com.osmand.views.AnimateDraggingMapThread.AnimateDraggingCallback;

75
OsmAnd/src/com/osmand/views/POIMapLayer.java
View file

@ -1,5 +1,6 @@
package com.osmand.views;
import java.util.ArrayList;
import java.util.List;
import android.graphics.Canvas;
@ -8,21 +9,18 @@ import android.graphics.Paint;
import android.view.MotionEvent;
import android.widget.Toast;
import com.osmand.OsmSQLLiteRepository;
import com.osmand.ResourceManager;
import com.osmand.data.Amenity;
import com.osmand.data.DataTileManager;
import com.osmand.osm.MapUtils;
import com.osmand.osm.io.OsmLuceneRepository;
public class POIMapLayer implements OsmandMapLayer {
private static final int radiusClick = 2; // for 15 level zoom
private DataTileManager<Amenity> nodeManager = null;
private Paint pointUI;
private Paint pointAltUI;
private OsmandMapTileView view;
private List<Amenity> objects;
private List<Amenity> objects = new ArrayList<Amenity>();
private ResourceManager resourceManager;
@ -57,29 +55,15 @@ public class POIMapLayer implements OsmandMapLayer {
}
public void setNodeManager(DataTileManager<Amenity> nodeManager) {
this.nodeManager = nodeManager;
if(view != null){
view.prepareImage();
}
}
public DataTileManager<Amenity> getNodeManager() {
return nodeManager;
}
@Override
public void initLayer(OsmandMapTileView view) {
this.view = view;
pointUI = new Paint();
pointUI.setColor(Color.BLUE);
pointUI.setAlpha(150);
pointUI.setAntiAlias(true);
pointAltUI = new Paint();
pointAltUI.setColor(Color.GREEN);
pointAltUI.setAlpha(150);
pointAltUI.setAntiAlias(true);
resourceManager = ResourceManager.getResourceManager();
}
public int getRadiusPoi(int zoom){
@ -92,7 +76,6 @@ public class POIMapLayer implements OsmandMapLayer {
@Override
public void onDraw(Canvas canvas) {
if (view.getZoom() >= 15) {
double tileNumberX = MapUtils.getTileNumberX(view.getZoom(), view.getLongitude());
double tileNumberY = MapUtils.getTileNumberY(view.getZoom(), view.getLatitude());
@ -103,42 +86,18 @@ public class POIMapLayer implements OsmandMapLayer {
double topLatitude = MapUtils.getLatitudeFromTile(view.getZoom(), yTileUp);
double leftLongitude = MapUtils.getLongitudeFromTile(view.getZoom(), xTileLeft);
double bottomLatitude = MapUtils.getLatitudeFromTile(view.getZoom(), yTileDown);
double rightLongitude= MapUtils.getLongitudeFromTile(view.getZoom(), xTileRight);
OsmLuceneRepository searcher = ResourceManager.getResourceManager().getAmenityIndexSearcher();
if (searcher != null) {
objects = searcher.searchAmenities(topLatitude, leftLongitude, bottomLatitude, rightLongitude);
for (Amenity o : objects) {
double tileX = MapUtils.getTileNumberX(view.getZoom(), o.getLocation().getLongitude());
int x = (int) ((tileX - xTileLeft) * getTileSize());
double tileY = MapUtils.getTileNumberY(view.getZoom(), o.getLocation().getLatitude());
int y = (int) ((tileY - yTileUp) * getTileSize());
canvas.drawCircle(x, y, getRadiusPoi(view.getZoom()), pointAltUI);
}
}
OsmSQLLiteRepository sqlLite = ResourceManager.getResourceManager().getAmenityRepository();
if (sqlLite != null) {
objects = sqlLite.searchAmenities(topLatitude, leftLongitude, bottomLatitude, rightLongitude);
for (Amenity o : objects) {
double tileX = MapUtils.getTileNumberX(view.getZoom(), o.getLocation().getLongitude());
int x = (int) ((tileX - xTileLeft) * getTileSize());
double tileY = MapUtils.getTileNumberY(view.getZoom(), o.getLocation().getLatitude());
int y = (int) ((tileY - yTileUp) * getTileSize());
canvas.drawCircle(x, y, getRadiusPoi(view.getZoom()), pointAltUI);
}
double rightLongitude = MapUtils.getLongitudeFromTile(view.getZoom(), xTileRight);
objects.clear();
resourceManager.searchAmenitiesAsync(topLatitude, leftLongitude, bottomLatitude, rightLongitude, objects);
for (Amenity o : objects) {
double tileX = MapUtils.getTileNumberX(view.getZoom(), o.getLocation().getLongitude());
int x = (int) ((tileX - xTileLeft) * getTileSize());
double tileY = MapUtils.getTileNumberY(view.getZoom(), o.getLocation().getLatitude());
int y = (int) ((tileY - yTileUp) * getTileSize());
canvas.drawCircle(x, y, getRadiusPoi(view.getZoom()), pointAltUI);
}
if (nodeManager != null) {
List<Amenity> objects = nodeManager.getObjects(topLatitude, leftLongitude, bottomLatitude, rightLongitude);
for (Amenity o : objects) {
double tileX = MapUtils.getTileNumberX(view.getZoom(), o.getLocation().getLongitude());
int x = (int) ((tileX - xTileLeft) * getTileSize());
double tileY = MapUtils.getTileNumberY(view.getZoom(), o.getLocation().getLatitude());
int y = (int) ((tileY - yTileUp) * getTileSize());
canvas.drawCircle(x, y, getRadiusPoi(view.getZoom()), pointAltUI);
}
}
}
}