first cut using lucene

git-svn-id: https://osmand.googlecode.com/svn/trunk@79 e29c36b1-1cfa-d876-8d93-3434fc2bb7b8
This commit is contained in:
Victor Shcherb 2010-05-25 16:53:52 +00:00
parent 56befb372c
commit 870a012365
14 changed files with 324 additions and 41 deletions

View file

@ -4,5 +4,6 @@
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="lib" path="lib/bzip2-20090327.jar"/>
<classpathentry kind="lib" path="lib/commons-logging-1.1.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-core-3.0.1.jar" sourcepath="C:/docs/lucene/lucene-src.zip"/>
<classpathentry kind="output" path="bin"/>
</classpath>

Binary file not shown.

View file

@ -1,43 +1,39 @@
package com.osmand;
/**
* This class is designed to put all to do's and link them with code.
* The whole methods could be paste or just constants.
* Do not worry to put ugly code here (just a little piece)
*
*/
public class ToDoConstants {
// OsmandMapTileView.java have problem with class loading (LogFactory, MapTileDownloader) -
// it is not editable in editor ?
public int MAKE_MAP_PANEL_EDITABLE_IN_EDITOR = 4;
/**
* Write activity to show something about authors / donation ....
*/
public int DESCRIBE_ABOUT_AUTHORS = 8;
// TODO ANDROID
// 0. Minimize memory used for index & improve time for reading index
// 1. POI search near to map location (show categories & type). First cut. (implement incremental search)
// 0. Minimize memory used for index & improve time for reading index
// 3. Revise osmand UI. Preparing new icons.
// 2. Showing compass on the map : use device compass if exists(?)
// 5. Search for city/streets/buildings
// 9. Config file log & see log from file (when exception happened to see from device)
// 9. Configure file log & see log from file (when exception happened to see from device)
// 11. Print out additional info speed, altitude, number of satellites
// 8. Enable change POI directly on map (requires OSM login)
// 13. Save point as favourite & introduce favourite points dialog
// 13. Save point as favorite & introduce favorite points dialog
// 14. Show zoom level on map
// 15. Investigate interruption of any long running operation & implement where it is needed
// 16. Support open street bugs api.
// 17. Enable go to location specifying coordinates
// 17. Enable go to location by specifying coordinates
// 18. Implement go to point
// 19. Show how map is rotated where north/south on map (do not consider compass)
// 20. Implement save track/route to gpx (?)
// 21. Implement zooming tile (if tile doesn't exist local, we can zoom in previous tile).
// 23. Implement moving point from center to bottom (for rotating map).
// It is not very useful to see what was before.
// FIXME Bugs Androd :
// FIXME Bugs Android :
// 0. FIX TODO for partial loading rotated map
// 1. When firstly run osmand navigation (from notification bar) show map & go to menu shows desktop.
// No chance to close application
@ -45,23 +41,19 @@ public class ToDoConstants {
// 4. Fix when POI selected & enable button backToLocation
// TODO SWING:
// 1. download tiles without using dir tiles
// 2. Config file log & see log from file
// 1. Download tiles without using dir tiles
// 2. Configure file log & see log from file
// 3. Reinvent index mechanism (save in zip file with tile indexes, save city/town addresses separately, read partially !)
// 4. Invent different file extensions for poi.index, address.index,...
// Max letter :
// 1. Fix bug 1
// 2. Create for each screen activity
// 3. Implement incremental search (reduce first time display to 10 & depth 2)
// 4. Improve navigate back/forward between screens
// 5. Implement exit confirmation
// DONE ANDROID :
// 12. Show information of where are you going (the arrow on the map)
// 10. Specify auto-rotating map (bearing of your direction)
// 22. Investigate 3D tile view (how it is done in osmand). Looking not very good, because of
// angle of perspective (best perspective angle = 60) use
// android.graphics.Camera.rotateX(6), getMatrix(m), canvas.concat(m) (find example in internet)
// Problems : to calculate how to drag point on map, to calculate how many tiles are needed, is location visible ....
// DONE SWING

View file

@ -68,8 +68,8 @@ public class DataTileManager<T> {
public List<T> getObjects(double latitudeUp, double longitudeUp, double latitudeDown, double longitudeDown) {
int tileXUp = (int) MapUtils.getTileNumberX(zoom, longitudeUp);
int tileYUp = (int) MapUtils.getTileNumberY(zoom, latitudeUp);
int tileXDown = (int) MapUtils.getTileNumberX(zoom, longitudeDown);
int tileYDown = (int) MapUtils.getTileNumberY(zoom, latitudeDown);
int tileXDown = (int) MapUtils.getTileNumberX(zoom, longitudeDown) + 1;
int tileYDown = (int) MapUtils.getTileNumberY(zoom, latitudeDown) + 1;
List<T> result = new ArrayList<T>();
for (int i = tileXUp; i <= tileXDown; i++) {
for (int j = tileYUp; j <= tileYDown; j++) {

View file

@ -73,6 +73,8 @@ public class DataIndexBuilder {
} finally {
output.close();
}
OsmStorageWriter writer = new OsmStorageWriter();
writer.saveLuceneIndex(new File(workingDir, "lucene"), region);
return this;
}

View file

@ -64,7 +64,10 @@ public class MapUtils {
} else if(e instanceof Relation){
List<LatLon> list = new ArrayList<LatLon>();
for(Entity fe : ((Relation) e).getMembers(null)){
list.add(getCenter(fe));
LatLon c = getCenter(fe);
if(c != null){
list.add(c);
}
}
return getWeightCenter(list);
}

View file

@ -0,0 +1,168 @@
package com.osmand.osm.io;
import java.io.File;
import java.io.IOException;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import org.apache.commons.logging.Log;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
import com.osmand.IProgress;
import com.osmand.LogUtil;
import com.osmand.data.Amenity;
import com.osmand.data.Amenity.AmenityType;
public class OsmLuceneRepository {
private static final Log log = LogUtil.getLog(OsmLuceneRepository.class);
private static final int MAX_POI_HITS = 1000;
private static DecimalFormat fmtLatitude = new DecimalFormat("000.000000", new DecimalFormatSymbols(Locale.US));
private static DecimalFormat negFmtLatitude = new DecimalFormat("00.000000", new DecimalFormatSymbols(Locale.US));
private static DecimalFormat fmtLongitude = new DecimalFormat("0000.000000", new DecimalFormatSymbols(Locale.US));
private static DecimalFormat negFmtLongitude = new DecimalFormat("000.000000", new DecimalFormatSymbols(Locale.US));
public static String formatLatitude(double latitude){
if(latitude <0 ){
return negFmtLatitude.format(latitude);
} else {
return fmtLatitude.format(latitude);
}
}
public static String formatLongitude(double longitude){
if(longitude <0 ){
return negFmtLongitude.format(longitude);
} else {
return fmtLongitude.format(longitude);
}
}
private List<Amenity> internalSearch(double topLatitude, double leftLongitude, double bottomLatitude, double rightLongitude){
queryFormat.setLength(0);
queryFormat.append("latitude:[").append(formatLatitude(bottomLatitude)).append(" TO ").append(formatLatitude(topLatitude)).append(
"]").append(" AND longitude:[").append(formatLongitude(leftLongitude)).append(" TO ").append(
formatLongitude(rightLongitude)).append("]");
TopScoreDocCollector collector = TopScoreDocCollector.create(MAX_POI_HITS, true);
try {
Query q = new QueryParser(Version.LUCENE_30, "id", new StandardAnalyzer(Version.LUCENE_30)).parse(queryFormat.toString());
long now = System.currentTimeMillis();
amenityIndexSearcher.search(q, collector);
ScoreDoc[] hits = collector.topDocs().scoreDocs;
List<Amenity> result = new ArrayList<Amenity>(hits.length);
for (int i = 0; i < hits.length; i++) {
result.add(convertAmenity(amenityIndexSearcher.doc(hits[i].doc)));
}
if (log.isDebugEnabled()) {
log.debug(String.format("Search for %s done in %s ms found %s.", q, System.currentTimeMillis() - now, hits.length));
}
return result;
} catch (IOException e) {
log.error("Failed to search.", e);
throw new RuntimeException(e);
} catch (ParseException e) {
log.error("Invalid query.", e);
return new ArrayList<Amenity>();
}
}
private List<Amenity> cachedAmenities = null;
private double cTopLatitude;
private double cBottomLatitude;
private double cLeftLongitude;
private double cRightLongitude;
private final StringBuilder queryFormat = new StringBuilder();
private IndexSearcher amenityIndexSearcher;
private boolean isLoading = false;
protected synchronized void loadAmenitiesInAnotherThread(final double topLatitude, final double leftLongitude, final double bottomLatitude, final double rightLongitude){
isLoading = true;
new Thread(new Runnable(){
@Override
public void run() {
try {
cachedAmenities = internalSearch(topLatitude, leftLongitude, bottomLatitude, rightLongitude);
cTopLatitude = topLatitude;
cLeftLongitude = leftLongitude;
cBottomLatitude = bottomLatitude ;
cRightLongitude = rightLongitude;
} finally {
synchronized (this) {
isLoading = false;
}
}
}
}, "Searching in index...").start();
}
public synchronized List<Amenity> searchAmenities(double topLatitude, double leftLongitude, double bottomLatitude, double rightLongitude) {
if(amenityIndexSearcher == null){
return Collections.emptyList();
}
// TODO take into account that right could be -53 & left = 175 (normalized coordinates
if (cTopLatitude >= topLatitude && cLeftLongitude <= leftLongitude && cRightLongitude >= rightLongitude
&& cBottomLatitude <= bottomLatitude) {
return cachedAmenities;
}
if(!isLoading){
double h = (topLatitude - bottomLatitude);
double w = (rightLongitude - leftLongitude);
topLatitude += h;
leftLongitude -= w;
bottomLatitude -= h;
rightLongitude += w;
loadAmenitiesInAnotherThread(topLatitude, leftLongitude, bottomLatitude, rightLongitude);
}
return Collections.emptyList();
}
public void indexing(final IProgress progress, File dir) {
long start = System.currentTimeMillis();
progress.startTask("Indexing lucene", -1);
try {
amenityIndexSearcher = new IndexSearcher(FSDirectory.open(dir));
} catch (Exception t) {
log.error("Failed to initialize searcher.", t);
throw new RuntimeException(t);
}
if (log.isDebugEnabled()) {
log.debug("Finished index lucene " + dir.getAbsolutePath() + " " + (System.currentTimeMillis() - start) + "ms");
}
}
protected Amenity convertAmenity(Document document) {
try {
Amenity am = new Amenity();
am.setName(document.get("name"));
am.setId(Long.parseLong(document.get("id")));
am.setSubType(document.get("subtype"));
am.setType(AmenityType.fromString(document.get("type")));
double longitude = fmtLongitude.parse(document.get("longitude")).doubleValue();
double latitude = fmtLatitude.parse(document.get("latitude")).doubleValue();
am.setLocation(latitude, longitude);
return am;
} catch (java.text.ParseException e) {
return null;
}
}
}

View file

@ -16,8 +16,17 @@ import static com.osmand.osm.io.OsmBaseStorage.ELEM_OSM;
import static com.osmand.osm.io.OsmBaseStorage.ELEM_RELATION;
import static com.osmand.osm.io.OsmBaseStorage.ELEM_TAG;
import static com.osmand.osm.io.OsmBaseStorage.ELEM_WAY;
import static com.osmand.osm.io.OsmIndexStorage.*;
import static com.osmand.osm.io.OsmIndexStorage.ATTR_CITYTYPE;
import static com.osmand.osm.io.OsmIndexStorage.ATTR_NAME;
import static com.osmand.osm.io.OsmIndexStorage.ATTR_SUBTYPE;
import static com.osmand.osm.io.OsmIndexStorage.ELEM_AMENITY;
import static com.osmand.osm.io.OsmIndexStorage.ELEM_BUILDING;
import static com.osmand.osm.io.OsmIndexStorage.ELEM_CITY;
import static com.osmand.osm.io.OsmIndexStorage.ELEM_OSMAND;
import static com.osmand.osm.io.OsmIndexStorage.ELEM_STREET;
import static com.osmand.osm.io.OsmIndexStorage.OSMAND_VERSION;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
@ -30,7 +39,21 @@ import java.util.Map.Entry;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamWriter;
import org.apache.commons.logging.Log;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriter.MaxFieldLength;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.Version;
import com.osmand.Algoritms;
import com.osmand.LogUtil;
import com.osmand.data.Amenity;
import com.osmand.data.Building;
import com.osmand.data.City;
@ -52,6 +75,9 @@ public class OsmStorageWriter {
private final String INDENT = " ";
private final String INDENT2 = INDENT + INDENT;
private final String INDENT3 = INDENT + INDENT + INDENT;
private static final Log log = LogUtil.getLog(OsmStorageWriter.class);
private static final Version VERSION = Version.LUCENE_30;
public OsmStorageWriter(){
@ -151,6 +177,45 @@ public class OsmStorageWriter {
return "node";
}
public void saveLuceneIndex(File dir, Region region) throws CorruptIndexException, LockObtainFailedException, IOException{
long now = System.currentTimeMillis();
IndexWriter writer = null;
try {
// Make a lucene writer and create new Lucene index with arg3 = true
writer = new IndexWriter(FSDirectory.open(dir), new StandardAnalyzer(VERSION), true, MaxFieldLength.LIMITED);
for (Amenity a : region.getAmenityManager().getAllObjects()) {
index(a, writer);
}
writer.optimize();
} finally {
try {
writer.close();
} catch (Exception t) {
log.error("Failed to close index.", t);
throw new RuntimeException(t);
}
log.info(String.format("Indexing done in %s ms.", System.currentTimeMillis() - now));
}
}
// TODO externalize strings
protected void index(Amenity amenity, IndexWriter writer) throws CorruptIndexException, IOException {
Document document = new Document();
document.add(new Field("id",""+amenity.getEntity().getId(),Store.YES, Index.NOT_ANALYZED));
LatLon latLon = amenity.getEntity().getLatLon();
document.add(new Field("name",amenity.getName(),Store.YES, Index.NOT_ANALYZED));
document.add(new Field("longitude",OsmLuceneRepository.formatLongitude(latLon.getLongitude()),Store.YES, Index.ANALYZED));
document.add(new Field("latitude",OsmLuceneRepository.formatLatitude(latLon.getLatitude()),Store.YES, Index.ANALYZED));
document.add(new Field("type",amenity.getType().name(),Store.YES, Index.ANALYZED));
document.add(new Field("subtype",amenity.getSubType(),Store.YES, Index.ANALYZED));
//for (Entry<String, String> entry:amenity.getNode().getTags().entrySet()) {
// document.add(new Field(entry.getKey(),entry.getValue(),Store.YES, Index.NOT_ANALYZED));
//}
writer.addDocument(document);
}
public void savePOIIndex(OutputStream output, Region region) throws XMLStreamException, IOException {
PropertyManager propertyManager = new PropertyManager(PropertyManager.CONTEXT_WRITER);

View file

@ -1,9 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" path="src"/>
<classpathentry excluding="**/MapPanel*|com/osmand/LogUtil.java|com/osmand/osm/io/OSMStorageWriter.java|com/osmand/DataExtraction.java|com/osmand/swing/|com/osmand/data/preparation/DataExtraction.java|com/osmand/data/preparation/DataIndexBuilder.java|com/osmand/osm/io/OsmStorageWriter.java" kind="src" path="use"/>
<classpathentry excluding="**/MapPanel*|com/osmand/LogUtil.java|com/osmand/osm/io/OSMStorageWriter.java|com/osmand/DataExtraction.java|com/osmand/swing/|com/osmand/data/preparation/DataExtraction.java|com/osmand/data/preparation/DataIndexBuilder.java|com/osmand/osm/io/OsmStorageWriter.java|test/" kind="src" path="use"/>
<classpathentry kind="src" path="gen"/>
<classpathentry kind="con" path="com.android.ide.eclipse.adt.ANDROID_FRAMEWORK"/>
<classpathentry kind="lib" path="lib/bzip2-20090327.jar"/>
<classpathentry kind="lib" path="lib/lucene-core-3.0.1.jar" sourcepath="C:/docs/lucene/lucene-src.zip"/>
<classpathentry kind="output" path="bin"/>
</classpath>

View file

@ -1,5 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<resources>
<string name="map_view_3d_descr">Enable 3D view of the map</string>
<string name="map_view_3d">Map View 3D</string>
<string name="rotate_map_to_bearing_descr">Rotate map to bearing of your direction</string>
<string name="rotate_map_to_bearing">Rotate map</string>
<string name="show_poi_over_map_description">Show POI on map</string>

View file

@ -37,6 +37,13 @@ public class OsmandSettings {
return prefs.getBoolean(ROTATE_MAP_TO_BEARING, false);
}
// this value string is synchronized with android.xml preference name
public static final String MAP_VIEW_3D = "map_view_3d";
public static boolean isMapView3D(Context ctx){
SharedPreferences prefs = ctx.getSharedPreferences(SHARED_PREFERENCES_NAME, Context.MODE_WORLD_READABLE);
return prefs.getBoolean(MAP_VIEW_3D, false);
}
// this value string is synchronized with android.xml preference name
public static final String MAP_TILE_SOURCES = "map_tile_sources";

View file

@ -29,6 +29,7 @@ import com.osmand.data.preparation.MapTileDownloader.IMapDownloaderCallback;
import com.osmand.map.ITileSource;
import com.osmand.osm.LatLon;
import com.osmand.osm.io.OsmIndexStorage;
import com.osmand.osm.io.OsmLuceneRepository;
/**
* Resource manager is responsible to work with all resources
@ -43,6 +44,7 @@ public class ResourceManager {
private static final String POI_PATH = "osmand/poi/";
private static final String ADDRESS_PATH = "osmand/address/";
private static final String TILES_PATH = "osmand/tiles/";
private static final String LUCENE_PATH = "osmand/lucene/";
private static final Log log = LogUtil.getLog(ResourceManager.class);
@ -70,6 +72,8 @@ public class ResourceManager {
public AsyncLoadingThread asyncLoadingTiles = new AsyncLoadingThread();
protected OsmLuceneRepository amenityIndexSearcher = new OsmLuceneRepository();
public ResourceManager() {
// TODO start/stop this thread when needed?
@ -279,6 +283,14 @@ public class ResourceManager {
}
}
public void indexingLucene(final IProgress progress){
// read index
File file = new File(Environment.getExternalStorageDirectory(), LUCENE_PATH);
if (file.exists() && file.canRead()) {
amenityIndexSearcher.indexing(progress, file);
}
}
public void indexingAddresses(final IProgress progress){
indexingFiles(ADDRESS_PATH, ".osmand", progress, "address", new IndexVisitor() {
@Override
@ -302,6 +314,10 @@ public class ResourceManager {
}
public OsmLuceneRepository getAmenityIndexSearcher(){
return amenityIndexSearcher;
}
/// On low memory method ///
public void onLowMemory() {

View file

@ -93,6 +93,7 @@ public class MainMenuActivity extends Activity {
try {
ResourceManager.getResourceManager().indexingPoi(impl);
ResourceManager.getResourceManager().indexingAddresses(impl);
ResourceManager.getResourceManager().indexingLucene(impl);
} finally {
dlg.dismiss();
}

View file

@ -8,15 +8,18 @@ import android.graphics.Paint;
import android.view.MotionEvent;
import android.widget.Toast;
import com.osmand.ResourceManager;
import com.osmand.data.Amenity;
import com.osmand.data.DataTileManager;
import com.osmand.osm.MapUtils;
import com.osmand.osm.io.OsmLuceneRepository;
public class POIMapLayer implements OsmandMapLayer {
private static final int radiusClick = 2; // for 15 level zoom
private DataTileManager<Amenity> nodeManager = null;
private Paint pointUI;
private Paint pointAltUI;
private OsmandMapTileView view;
private List<Amenity> objects;
@ -71,6 +74,11 @@ public class POIMapLayer implements OsmandMapLayer {
pointUI.setColor(Color.BLUE);
pointUI.setAlpha(150);
pointUI.setAntiAlias(true);
pointAltUI = new Paint();
pointAltUI.setColor(Color.GREEN);
pointAltUI.setAlpha(150);
pointAltUI.setAntiAlias(true);
}
public int getRadiusPoi(int zoom){
@ -83,25 +91,42 @@ public class POIMapLayer implements OsmandMapLayer {
@Override
public void onDraw(Canvas canvas) {
if (nodeManager != null && view.getZoom() >= 15) {
if (view.getZoom() >= 15) {
double tileNumberX = MapUtils.getTileNumberX(view.getZoom(), view.getLongitude());
double tileNumberY = MapUtils.getTileNumberY(view.getZoom(), view.getLatitude());
double xTileLeft = tileNumberX - view.getWidth() / (2d * getTileSize());
double xTileRight = tileNumberX + view.getWidth() / (2d * getTileSize());
double yTileUp = tileNumberY - view.getHeight() / (2d * getTileSize());
double yTileDown = tileNumberY + view.getHeight() / (2d * getTileSize());
double topLatitude = MapUtils.getLatitudeFromTile(view.getZoom(), yTileUp);
double leftLongitude = MapUtils.getLongitudeFromTile(view.getZoom(), xTileLeft);
double bottomLatitude = MapUtils.getLatitudeFromTile(view.getZoom(), yTileDown);
double rightLongitude= MapUtils.getLongitudeFromTile(view.getZoom(), xTileRight);
objects = nodeManager.getObjects(MapUtils.getLatitudeFromTile(view.getZoom(), yTileUp), MapUtils
.getLongitudeFromTile(view.getZoom(), xTileLeft), MapUtils.getLatitudeFromTile(view.getZoom(), yTileDown), MapUtils
.getLongitudeFromTile(view.getZoom(), xTileRight));
OsmLuceneRepository searcher = ResourceManager.getResourceManager().getAmenityIndexSearcher();
if (searcher != null) {
objects = searcher.searchAmenities(topLatitude, leftLongitude, bottomLatitude, rightLongitude);
for (Amenity o : objects) {
double tileX = MapUtils.getTileNumberX(view.getZoom(), o.getLocation().getLongitude());
int x = (int) ((tileX - xTileLeft) * getTileSize());
double tileY = MapUtils.getTileNumberY(view.getZoom(), o.getLocation().getLatitude());
int y = (int) ((tileY - yTileUp) * getTileSize());
canvas.drawCircle(x, y, getRadiusPoi(view.getZoom()), pointUI);
canvas.drawCircle(x, y, getRadiusPoi(view.getZoom()), pointAltUI);
}
}
if (nodeManager != null) {
objects = nodeManager.getObjects(topLatitude, leftLongitude, bottomLatitude, rightLongitude);
for (Amenity o : objects) {
double tileX = MapUtils.getTileNumberX(view.getZoom(), o.getLocation().getLongitude());
int x = (int) ((tileX - xTileLeft) * getTileSize());
double tileY = MapUtils.getTileNumberY(view.getZoom(), o.getLocation().getLatitude());
int y = (int) ((tileY - yTileUp) * getTileSize());
canvas.drawCircle(x, y, getRadiusPoi(view.getZoom()), pointAltUI);
}
}
}
}
@Override