From dd95a862ee9a5ebdba70db98731e6f6ebbe574e6 Mon Sep 17 00:00:00 2001 From: Victor Shcherb Date: Thu, 29 Sep 2011 01:02:50 +0200 Subject: [PATCH] Align old poi db index usage with new index usage --- .../binary/BinaryMapPoiReaderAdapter.java | 60 +++++--- .../data/preparation/IndexPoiCreator.java | 2 - OsmAnd/res/values/strings.xml | 4 + .../plus/AmenityIndexRepositoryOdb.java | 81 +++++++---- .../plus/BaseLocationIndexRepository.java | 16 ++- OsmAnd/src/net/osmand/plus/IndexFileList.java | 136 +++++++++--------- .../src/net/osmand/plus/ResourceManager.java | 81 ++++++++--- .../plus/activities/EditingPOIActivity.java | 33 +++-- .../plus/activities/MapActivityActions.java | 15 +- 9 files changed, 264 insertions(+), 164 deletions(-) diff --git a/DataExtractionOSM/src/net/osmand/binary/BinaryMapPoiReaderAdapter.java b/DataExtractionOSM/src/net/osmand/binary/BinaryMapPoiReaderAdapter.java index 72bef23299..8613fe6e36 100644 --- a/DataExtractionOSM/src/net/osmand/binary/BinaryMapPoiReaderAdapter.java +++ b/DataExtractionOSM/src/net/osmand/binary/BinaryMapPoiReaderAdapter.java @@ -221,7 +221,8 @@ public class BinaryMapPoiReaderAdapter { case OsmandOdb.OsmAndPoiNameIndex.TABLE_FIELD_NUMBER : { int length = readInt(); int oldLimit = codedIS.pushLimit(length); - dataOffsets = readIndexedStringTable(instance, query); + dataOffsets = new TIntArrayList(); + readIndexedStringTable(instance, query, "", dataOffsets, 0); codedIS.popLimit(oldLimit); break; } case OsmandOdb.OsmAndPoiNameIndex.DATA_FIELD_NUMBER : { @@ -297,40 +298,59 @@ public class BinaryMapPoiReaderAdapter { } } - private TIntArrayList readIndexedStringTable(Collator instance, String query) throws IOException { - // TODO support fully functional indexed string table - TIntArrayList list = new TIntArrayList(); - int charMatches = 0; - boolean keyMatches = false; + private int readIndexedStringTable(Collator instance, String query, String prefix, TIntArrayList list, int charMatches) throws IOException { + String key = null; while(true){ int t = codedIS.readTag(); int tag = WireFormat.getTagFieldNumber(t); switch (tag) { case 0: - return list; + return charMatches; case OsmandOdb.IndexedStringTable.KEY_FIELD_NUMBER : - String key = codedIS.readString(); - keyMatches = false; - int i=0; - for(; i= key.length() || instance.compare(key.substring(i, i + 1), query.substring(i, i + 1)) != 0) { - break; - } + key = codedIS.readString(); + if(prefix.length() > 0){ + key = prefix + key; } - if(i >= charMatches && i > 0){ - if(i > charMatches){ - list.clear(); - charMatches = i; + // check query is part of key (the best matching) + if(CollatorStringMatcher.cmatches(instance, key, query, StringMatcherMode.CHECK_ONLY_STARTS_WITH)){ + if(query.length() >= charMatches){ + if(query.length() > charMatches){ + charMatches = key.length(); + list.clear(); + } + } else { + key = null; } - keyMatches = true; + // check key is part of query + } else if (CollatorStringMatcher.cmatches(instance, query, key, StringMatcherMode.CHECK_ONLY_STARTS_WITH)) { + if (key.length() >= charMatches) { + if (key.length() > charMatches) { + charMatches = key.length(); + list.clear(); + } + } else { + key = null; + } + } else { + key = null; } break; case OsmandOdb.IndexedStringTable.VAL_FIELD_NUMBER : int val = codedIS.readUInt32(); - if (keyMatches) { + if (key != null) { list.add(val); } break; + case OsmandOdb.IndexedStringTable.SUBTABLES_FIELD_NUMBER : + int len = codedIS.readRawVarint32(); + int oldLim = codedIS.pushLimit(len); + if (key != null) { + charMatches = readIndexedStringTable(instance, query, key, list, charMatches); + } else { + codedIS.skipRawBytes(codedIS.getBytesUntilLimit()); + } + codedIS.popLimit(oldLim); + break; default: skipUnknownField(t); break; diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/IndexPoiCreator.java b/DataExtractionOSM/src/net/osmand/data/preparation/IndexPoiCreator.java index fde7029a57..5678cb7401 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/IndexPoiCreator.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/IndexPoiCreator.java @@ -557,9 +557,7 @@ public class IndexPoiCreator extends AbstractIndexPartCreator { public static void main(String[] args) throws SQLException, FileNotFoundException, IOException { // TODO support multiple reading amenity types! +/- // TODO support proper POI editing - // TODO support string trigramms // TODO support cancelling poi search request! Do it in another thread (Check is cancelled()!!!) - // TODO support fully functional indexed string table and pass name matcher long time = System.currentTimeMillis(); IndexPoiCreator poiCreator = new IndexPoiCreator(); // String fileSqlte = "/home/victor/projects/OsmAnd/data/osm-gen/POI/Ru-mow.poi.odb"; diff --git a/OsmAnd/res/values/strings.xml b/OsmAnd/res/values/strings.xml index 1eca70f89d..718d56d554 100644 --- a/OsmAnd/res/values/strings.xml +++ b/OsmAnd/res/values/strings.xml @@ -2,6 +2,10 @@ Upgrade Osmand+ On server are map files incompatible with your current version of application. To download and use them, please upgrade the application to newer version. + The poi data file \'%1$s\' is deprecated and can be deleted. + Local file to maintain poi changes not found and could not be created. + Changing poi inside application doesn\'t affect map files you downloaded but collects all changes in local file. + Rename Show on map Online Nominatim diff --git a/OsmAnd/src/net/osmand/plus/AmenityIndexRepositoryOdb.java b/OsmAnd/src/net/osmand/plus/AmenityIndexRepositoryOdb.java index b13ef1309d..22bb4a06aa 100644 --- a/OsmAnd/src/net/osmand/plus/AmenityIndexRepositoryOdb.java +++ b/OsmAnd/src/net/osmand/plus/AmenityIndexRepositoryOdb.java @@ -29,6 +29,7 @@ import org.apache.commons.logging.Log; import org.xml.sax.SAXException; import android.database.Cursor; +import android.database.sqlite.SQLiteDatabase; import android.database.sqlite.SQLiteStatement; public class AmenityIndexRepositoryOdb extends BaseLocationIndexRepository implements AmenityIndexRepository { @@ -91,37 +92,6 @@ public class AmenityIndexRepositoryOdb extends BaseLocationIndexRepository amenities, double leftLon, double topLat, double righLon, double bottomLat) { try { // bbox=left,bottom,right,top diff --git a/OsmAnd/src/net/osmand/plus/BaseLocationIndexRepository.java b/OsmAnd/src/net/osmand/plus/BaseLocationIndexRepository.java index d02b2f6b97..3678156488 100644 --- a/OsmAnd/src/net/osmand/plus/BaseLocationIndexRepository.java +++ b/OsmAnd/src/net/osmand/plus/BaseLocationIndexRepository.java @@ -44,6 +44,10 @@ public class BaseLocationIndexRepository { cZoom = 0; } + protected String getMetaLocation(String tableLocation) { + return "loc_meta_" + tableLocation; + } + public boolean initialize(final IProgress progress, File file, int version, String tableLocation, boolean searchX31) { long start = System.currentTimeMillis(); if(db != null){ @@ -57,7 +61,7 @@ public class BaseLocationIndexRepository { db = null; return false; } - String metaTable = "loc_meta_"+tableLocation; //$NON-NLS-1$ + String metaTable = getMetaLocation(tableLocation); Cursor cursor = db.rawQuery("SELECT name FROM sqlite_master WHERE type='table' AND name='"+metaTable+"'", null); //$NON-NLS-1$ //$NON-NLS-2$ boolean dbExist = cursor.moveToFirst(); cursor.close(); @@ -109,7 +113,7 @@ public class BaseLocationIndexRepository { query.close(); } if (write) { - db.execSQL("INSERT INTO " + metaTable + " VALUES (?, ?, ? ,?)", new Double[]{dataTopLatitude, dataRightLongitude, dataBottomLatitude, dataLeftLongitude}); //$NON-NLS-1$ //$NON-NLS-2$ + updateMaxMinBoundaries(tableLocation); } } if (log.isDebugEnabled()) { @@ -118,6 +122,12 @@ public class BaseLocationIndexRepository { return true; } + protected void updateMaxMinBoundaries(String tableLocation){ + String metatable = getMetaLocation(tableLocation); + db.execSQL("DELETE FROM " + metatable + " WHERE 1= 1" ) ; + db.execSQL("INSERT INTO " + metatable + " VALUES (?, ?, ? ,?)", new Double[]{dataTopLatitude, dataRightLongitude, dataBottomLatitude, dataLeftLongitude}); //$NON-NLS-1$ //$NON-NLS-2$ + } + public synchronized void close() { if (db != null) { db.close(); @@ -164,7 +174,7 @@ public class BaseLocationIndexRepository { } public boolean checkContains(double latitude, double longitude){ - if(latitude < dataTopLatitude && latitude > dataBottomLatitude && longitude > dataLeftLongitude && longitude < dataRightLongitude){ + if(latitude <= dataTopLatitude && latitude >= dataBottomLatitude && longitude >= dataLeftLongitude && longitude <= dataRightLongitude){ return true; } return false; diff --git a/OsmAnd/src/net/osmand/plus/IndexFileList.java b/OsmAnd/src/net/osmand/plus/IndexFileList.java index 1d9367d3a0..4a2dafbd8c 100644 --- a/OsmAnd/src/net/osmand/plus/IndexFileList.java +++ b/OsmAnd/src/net/osmand/plus/IndexFileList.java @@ -1,68 +1,68 @@ -package net.osmand.plus; - -import java.io.Serializable; -import java.util.Comparator; -import java.util.Map; -import java.util.TreeMap; - -import net.osmand.data.IndexConstants; -import net.osmand.plus.DownloadOsmandIndexesHelper.IndexItem; - -/** - * @author Pavol Zibrita - */ -public class IndexFileList implements Serializable { - private static final long serialVersionUID = 1L; - - TreeMap indexFiles = new TreeMap(new Comparator(){ - @SuppressWarnings("unused") - private static final long serialVersionUID = 1L; - - @Override - public int compare(String object1, String object2) { - if(object1.endsWith(IndexConstants.ANYVOICE_INDEX_EXT_ZIP)){ - if(object2.endsWith(IndexConstants.ANYVOICE_INDEX_EXT_ZIP)){ - return object1.compareTo(object2); - } else { - return -1; - } - } else if(object2.endsWith(IndexConstants.ANYVOICE_INDEX_EXT_ZIP)){ - return 1; - } - return object1.compareTo(object2); - } - }); - - private String mapversion; - - public IndexFileList() { - } - - public void setMapVersion(String mapversion) { - this.mapversion = mapversion; - } - - public void add(String name, IndexItem indexItem) { - if (indexItem.isAccepted()) { - indexFiles.put(name, indexItem); - } - } - - public boolean isAcceptable() { - return (indexFiles != null && !indexFiles.isEmpty()) || (mapversion != null); - } - - public Map getIndexFiles() { - return indexFiles; - } - - public boolean isIncreasedMapVersion() { - try { - int mapVersionInList = Integer.parseInt(mapversion); - return IndexConstants.BINARY_MAP_VERSION < mapVersionInList; - } catch (NumberFormatException e) { - //ignore this... - } - return false; - } -} +package net.osmand.plus; + +import java.io.Serializable; +import java.util.Comparator; +import java.util.Map; +import java.util.TreeMap; + +import net.osmand.data.IndexConstants; +import net.osmand.plus.DownloadOsmandIndexesHelper.IndexItem; + +/** + * @author Pavol Zibrita + */ +public class IndexFileList implements Serializable { + private static final long serialVersionUID = 1L; + + TreeMap indexFiles = new TreeMap(new Comparator(){ + @SuppressWarnings("unused") + private static final long serialVersionUID = 1L; + + @Override + public int compare(String object1, String object2) { + if(object1.endsWith(IndexConstants.ANYVOICE_INDEX_EXT_ZIP)){ + if(object2.endsWith(IndexConstants.ANYVOICE_INDEX_EXT_ZIP)){ + return object1.compareTo(object2); + } else { + return -1; + } + } else if(object2.endsWith(IndexConstants.ANYVOICE_INDEX_EXT_ZIP)){ + return 1; + } + return object1.compareTo(object2); + } + }); + + private String mapversion; + + public IndexFileList() { + } + + public void setMapVersion(String mapversion) { + this.mapversion = mapversion; + } + + public void add(String name, IndexItem indexItem) { + if (indexItem.isAccepted()) { + indexFiles.put(name, indexItem); + } + } + + public boolean isAcceptable() { + return (indexFiles != null && !indexFiles.isEmpty()) || (mapversion != null); + } + + public Map getIndexFiles() { + return indexFiles; + } + + public boolean isIncreasedMapVersion() { + try { + int mapVersionInList = Integer.parseInt(mapversion); + return IndexConstants.BINARY_MAP_VERSION < mapVersionInList; + } catch (NumberFormatException e) { + //ignore this... + } + return false; + } +} diff --git a/OsmAnd/src/net/osmand/plus/ResourceManager.java b/OsmAnd/src/net/osmand/plus/ResourceManager.java index 129db0e4be..617e449d4d 100644 --- a/OsmAnd/src/net/osmand/plus/ResourceManager.java +++ b/OsmAnd/src/net/osmand/plus/ResourceManager.java @@ -55,7 +55,6 @@ import android.view.WindowManager; * Such as indexes, tiles. * Also it is responsible to create cache for that resources if they * can't be loaded fully into memory & clear them on request. - *SQLITE */ public class ResourceManager { @@ -72,6 +71,7 @@ public class ResourceManager { public static final int LIMIT_TRANSPORT = 200; private static final Log log = LogUtil.getLog(ResourceManager.class); + private static final String MINE_POI_DB = APP_DIR + "mine"+ IndexConstants.POI_INDEX_EXT; protected static ResourceManager manager = null; @@ -98,6 +98,7 @@ public class ResourceManager { protected final List transportRepositories = new ArrayList(); + protected final Map indexFileNames = new LinkedHashMap(); protected final Map routingMapFiles = new LinkedHashMap(); @@ -108,6 +109,8 @@ public class ResourceManager { protected boolean internetIsNotAccessible = false; + protected AmenityIndexRepositoryOdb updatablePoiDb = null; + public ResourceManager(OsmandApplication context) { this.context = context; @@ -355,8 +358,8 @@ public class ResourceManager { initRenderers(progress); // do it lazy // indexingImageTiles(progress); - warnings.addAll(indexingPoi(progress)); warnings.addAll(indexingMaps(progress)); + warnings.addAll(indexingPoi(progress)); return warnings; } @@ -499,29 +502,82 @@ public class ResourceManager { } // POI INDEX // - public List indexingPoi(final IProgress progress) { + private List indexingPoi(final IProgress progress) { File file = context.getSettings().extendOsmandPath(POI_PATH); file.mkdirs(); List warnings = new ArrayList(); - closeAmenities(); if (file.exists() && file.canRead()) { for (File f : file.listFiles()) { indexingPoi(progress, warnings, f); } } + File updatablePoiDbFile = context.getSettings().extendOsmandPath(MINE_POI_DB); + if(updatablePoiDbFile.exists() && file.canRead()){ + try { + AmenityIndexRepositoryOdb odb = new AmenityIndexRepositoryOdb(); + boolean initialize = odb.initialize(progress, updatablePoiDbFile); + if(initialize){ + this.updatablePoiDb = odb; + } + } catch (SQLiteException e) { + } + } return warnings; } + public AmenityIndexRepositoryOdb getUpdatablePoiDb() { + if (updatablePoiDb == null) { + File updatablePoiDbFile = context.getSettings().extendOsmandPath(MINE_POI_DB); + if (!tryToOpenUpdatablePoiDb(updatablePoiDbFile)) { + if (updatablePoiDbFile.exists()) { + updatablePoiDbFile.delete(); + } + AmenityIndexRepositoryOdb.createAmenityIndexRepository(updatablePoiDbFile); + tryToOpenUpdatablePoiDb(updatablePoiDbFile); + } + } + return updatablePoiDb; + } + + private boolean tryToOpenUpdatablePoiDb(File updatablePoiDbFile) { + try { + AmenityIndexRepositoryOdb odb = new AmenityIndexRepositoryOdb(); + boolean initialize = odb.initialize(IProgress.EMPTY_PROGRESS, updatablePoiDbFile); + if (initialize) { + amenityRepositories.add(odb); + this.updatablePoiDb = odb; + return true; + } + } catch (SQLiteException e) { + } + return false; + } + public void indexingPoi(final IProgress progress, List warnings, File f) { if (f.getName().endsWith(IndexConstants.POI_INDEX_EXT)) { AmenityIndexRepositoryOdb repository = new AmenityIndexRepositoryOdb(); - progress.startTask(context.getString(R.string.indexing_poi) + " " + f.getName(), -1); //$NON-NLS-1$ try { boolean initialized = repository.initialize(progress, f); if (initialized) { - amenityRepositories.add(repository); - indexFileNames.put(f.getName(), MessageFormat.format("{0,date,dd.MM.yyyy}", new Date(f.lastModified()))); //$NON-NLS-1$ + boolean covered = false; + for(AmenityIndexRepository r : amenityRepositories){ + if(r instanceof AmenityIndexRepositoryBinary){ + double latC = (repository.dataBottomLatitude + repository.dataTopLatitude )/ 2; + double lonC = (repository.dataLeftLongitude + repository.dataRightLongitude) / 2; + if(r.checkContains(latC, lonC)){ + covered = true; + break; + } + } + } + if(covered){ + repository.close(); + warnings.add(context.getString(R.string.old_poi_file_should_be_deleted, f.getName())); //$NON-NLS-1$ + } else { + amenityRepositories.add(repository); + indexFileNames.put(f.getName(), MessageFormat.format("{0,date,dd.MM.yyyy}", new Date(f.lastModified()))); //$NON-NLS-1$ + } } else { warnings.add(MessageFormat.format(context.getString(R.string.version_index_is_not_supported), f.getName())); //$NON-NLS-1$ } @@ -540,16 +596,6 @@ public class ResourceManager { ////////////////////////////////////////////// Working with amenities //////////////////////////////////////////////// - public List searchAmenityRepositories(double latitude, double longitude) { - List repos = new ArrayList(); - for (AmenityIndexRepository index : amenityRepositories) { - if (index.checkContains(latitude,longitude)) { - repos.add(index); - } - } - return repos; - } - public List searchAmenities(PoiFilter filter, double topLatitude, double leftLongitude, double bottomLatitude, double rightLongitude, double lat, double lon) { @@ -647,6 +693,7 @@ public class ResourceManager { r.close(); } amenityRepositories.clear(); + updatablePoiDb = null; } public void closeAddresses(){ diff --git a/OsmAnd/src/net/osmand/plus/activities/EditingPOIActivity.java b/OsmAnd/src/net/osmand/plus/activities/EditingPOIActivity.java index 11407c0375..b08368bba1 100644 --- a/OsmAnd/src/net/osmand/plus/activities/EditingPOIActivity.java +++ b/OsmAnd/src/net/osmand/plus/activities/EditingPOIActivity.java @@ -37,7 +37,6 @@ import net.osmand.osm.OSMSettings.OSMTagKey; import net.osmand.osm.OpeningHoursParser.BasicDayOpeningHourRule; import net.osmand.osm.OpeningHoursParser.OpeningHoursRule; import net.osmand.osm.io.OsmBaseStorage; -import net.osmand.plus.AmenityIndexRepository; import net.osmand.plus.AmenityIndexRepositoryOdb; import net.osmand.plus.OsmandSettings; import net.osmand.plus.R; @@ -110,6 +109,7 @@ public class EditingPOIActivity { private final static Log log = LogUtil.getLog(EditingPOIActivity.class); + @@ -608,26 +608,31 @@ public class EditingPOIActivity { } private void updateNodeInIndexes(String action, Node n) { - List repos = app.getResourceManager().searchAmenityRepositories(n.getLatitude(), n.getLongitude()); - // delete all amenities with same id - if (DELETE_ACTION.equals(action) || MODIFY_ACTION.equals(action)) { - for (AmenityIndexRepository r : repos) { - if (r instanceof AmenityIndexRepositoryOdb) { - ((AmenityIndexRepositoryOdb) r).deleteAmenities(n.getId() << 1); - ((AmenityIndexRepositoryOdb) r).clearCache(); + final AmenityIndexRepositoryOdb repo = app.getResourceManager().getUpdatablePoiDb(); + view.post(new Runnable() { + + @Override + public void run() { + if (repo == null) { + Toast.makeText(app, app.getString(R.string.update_poi_no_offline_poi_index), Toast.LENGTH_SHORT).show(); + return; + } else { + Toast.makeText(app, app.getString(R.string.update_poi_does_not_change_indexes), Toast.LENGTH_SHORT).show(); } } + }); + + // delete all amenities with same id + if (DELETE_ACTION.equals(action) || MODIFY_ACTION.equals(action)) { + repo.deleteAmenities(n.getId() << 1); + repo.clearCache(); } // add amenities if (!DELETE_ACTION.equals(action)) { List ams = Amenity.parseAmenities(n, new ArrayList()); for (Amenity a : ams) { - for (AmenityIndexRepository r : repos) { - if (r instanceof AmenityIndexRepositoryOdb) { - ((AmenityIndexRepositoryOdb) r).addAmenity(a); - ((AmenityIndexRepositoryOdb) r).clearCache(); - } - } + repo.addAmenity(a); + repo.clearCache(); } } diff --git a/OsmAnd/src/net/osmand/plus/activities/MapActivityActions.java b/OsmAnd/src/net/osmand/plus/activities/MapActivityActions.java index 7b33645c89..3bdcd5de3f 100644 --- a/OsmAnd/src/net/osmand/plus/activities/MapActivityActions.java +++ b/OsmAnd/src/net/osmand/plus/activities/MapActivityActions.java @@ -17,7 +17,6 @@ import net.osmand.data.Amenity; import net.osmand.map.ITileSource; import net.osmand.osm.LatLon; import net.osmand.osm.MapUtils; -import net.osmand.plus.AmenityIndexRepository; import net.osmand.plus.AmenityIndexRepositoryOdb; import net.osmand.plus.FavouritesDbHelper; import net.osmand.plus.OsmandSettings; @@ -208,11 +207,13 @@ public class MapActivityActions { Toast.makeText(mapActivity, getString(R.string.update_poi_is_not_available_for_zoom), Toast.LENGTH_SHORT).show(); return; } - final List repos = ((OsmandApplication) mapActivity.getApplication()). - getResourceManager().searchAmenityRepositories(latitude, longitude); - if(repos.isEmpty()){ + final AmenityIndexRepositoryOdb repo = ((OsmandApplication) mapActivity.getApplication()). + getResourceManager().getUpdatablePoiDb(); + if(repo == null){ Toast.makeText(mapActivity, getString(R.string.update_poi_no_offline_poi_index), Toast.LENGTH_SHORT).show(); return; + } else { + Toast.makeText(mapActivity, getString(R.string.update_poi_does_not_change_indexes), Toast.LENGTH_SHORT).show(); } final OsmandMapTileView mapView = mapActivity.getMapView(); Rect pixRect = new Rect(-mapView.getWidth()/2, -mapView.getHeight()/2, 3*mapView.getWidth()/2, 3*mapView.getHeight()/2); @@ -235,11 +236,7 @@ public class MapActivityActions { if(!loadingPOIs){ showToast(getString(R.string.update_poi_error_loading)); } else { - for(AmenityIndexRepository r : repos){ - if(r instanceof AmenityIndexRepositoryOdb){ - ((AmenityIndexRepositoryOdb) r).updateAmenities(amenities, leftLon, topLat, rightLon, bottomLat); - } - } + repo.updateAmenities(amenities, leftLon, topLat, rightLon, bottomLat); showToast(MessageFormat.format(getString(R.string.update_poi_success), amenities.size())); mapView.refreshMap(); }