Merge pull request #4418 from osmandapp/PaulsBranch

Improved search for OSM Live

OsmAnd-java/src/net/osmand/binary/BinaryMapIndexReader.java

@@ -26,6 +26,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.TreeMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
@@ -405,7 +407,24 @@ public class BinaryMapIndexReader {
 		}
 		String ls = rg.get(0);
 		if (ls.lastIndexOf('_') != -1) {
+			if (ls.matches(".*([0-9]+_*){3}\\.obf")) {
+				Pattern osmDiffDateEnding = Pattern.compile("_([0-9]+_*){3}\\.obf");
+				Matcher m = osmDiffDateEnding.matcher(ls);
+				if (m.find()) {
+					ls = ls.substring(0, m.start());
+				}
 				return ls.substring(0, ls.lastIndexOf('_')).replace('_', ' ');
+
+			} else {
+				if (ls.contains(".")) {
+					ls = ls.substring(0, ls.indexOf("."));
+				}
+				if (ls.endsWith("_2")) {
+					ls = ls.substring(0, ls.length() - "_2".length());
+				}
+				return ls.substring(0, ls.lastIndexOf('_')).replace('_', ' ');
+			}
+
 		}
 		return ls;
 	}

OsmAnd-java/src/net/osmand/data/Amenity.java

@@ -29,6 +29,8 @@ public class Amenity extends MapObject {
 	public static final String OPENING_HOURS = "opening_hours";
 	public static final String CONTENT = "content";
 	public static final String CUISINE = "cuisine";
+	public static final String OSM_DELETE_VALUE = "delete";
+	public static final String OSM_DELETE_TAG = "osmand_change";
 
 	private String subType;
 	private PoiCategory type;
@@ -307,4 +309,8 @@ public class Amenity extends MapObject {
 		}
 		return x;
 	}
+
+	public boolean isClosed() {
+		return OSM_DELETE_VALUE.equals(getAdditionalInfo(OSM_DELETE_TAG));
+	}
 }

OsmAnd-java/src/net/osmand/search/core/SearchCoreFactory.java

@@ -38,14 +38,18 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeSet;
 
 import gnu.trove.list.array.TIntArrayList;
+import gnu.trove.set.hash.TLongHashSet;
 
 
 public class SearchCoreFactory {
@@ -486,6 +490,7 @@ public class SearchCoreFactory {
 					SearchPhraseDataType.POI);
 			final NameStringMatcher nm = phrase.getNameStringMatcher();
 			QuadRect bbox = phrase.getRadiusBBoxToSearch(BBOX_RADIUS_INSIDE);
+			final Set<String> ids = new HashSet<String>();
 			SearchRequest<Amenity> req = BinaryMapIndexReader.buildSearchPoiRequest(
 					(int)bbox.centerX(), (int)bbox.centerY(),
 					phrase.getUnknownSearchWord(),
@@ -498,6 +503,10 @@ public class SearchCoreFactory {
 							if (limit ++ > LIMIT) {
 								return false;
 							}
+							String poiID = object.getType().getKeyName() + "_" + object.getId();
+							if (ids.contains(poiID)) {
+								return false;
+							}
 							SearchResult sr = new SearchResult(phrase);
 							sr.otherNames = object.getAllNames(true);
 							sr.localeName = object.getName(phrase.getSettings().getLang(), phrase.getSettings().isTransliterate());
@@ -523,6 +532,7 @@ public class SearchCoreFactory {
 							phrase.countUnknownWordsMatch(sr);
 							sr.objectType = ObjectType.POI;
 							resultMatcher.publish(sr);
+							ids.add(poiID);
 							return false;
 						}
 
@@ -726,8 +736,9 @@ public class SearchCoreFactory {
 
 				QuadRect bbox = phrase.getRadiusBBoxToSearch(10000);
 				List<BinaryMapIndexReader> oo = phrase.getOfflineIndexes();
+				Set<String> searchedPois = new TreeSet<>();
 				for (BinaryMapIndexReader o : oo) {
-					ResultMatcher<Amenity> rm = getResultMatcher(phrase, resultMatcher, o);
+					ResultMatcher<Amenity> rm = getResultMatcher(phrase, resultMatcher, o, searchedPois);
 					if (obj instanceof CustomSearchPoiFilter) {
 						rm = ((CustomSearchPoiFilter) obj).wrapResultMatcher(rm);
 					}
@@ -743,13 +754,20 @@ public class SearchCoreFactory {
 		}
 
 		private ResultMatcher<Amenity> getResultMatcher(final SearchPhrase phrase, final SearchResultMatcher resultMatcher,
-														final BinaryMapIndexReader selected) {
+														final BinaryMapIndexReader selected, final Set<String> searchedPois) {
 			final NameStringMatcher ns = phrase.getNameStringMatcher();
 			return new ResultMatcher<Amenity>() {
 
 				@Override
 				public boolean publish(Amenity object) {
 					SearchResult res = new SearchResult(phrase);
+					String poiID = object.getType().getKeyName() + "_" + object.getId();
+					if(!searchedPois.add(poiID)) {
+						return false;
+					}
+					if(object.isClosed()) {
+						return false;
+					}
 					res.localeName = object.getName(phrase.getSettings().getLang(), phrase.getSettings().isTransliterate());
 					res.otherNames = object.getAllNames(true);
 					if (Algorithms.isEmpty(res.localeName)) {

OsmAnd-java/src/net/osmand/search/core/SearchPhrase.java

@@ -13,6 +13,7 @@ import net.osmand.util.Algorithms;
 import net.osmand.util.MapUtils;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
@@ -535,6 +536,7 @@ public class SearchPhrase {
         if (indexes == null) {
             indexes = new ArrayList<>(getOfflineIndexes());
         }
+        Map<String, List<BinaryMapIndexReader>> diffsByRegion = getDiffsByRegion();
         final LatLon ll = getLastTokenLocation();
         if (ll != null) {
             Collections.sort(indexes, new Comparator<BinaryMapIndexReader>() {
@@ -563,8 +565,40 @@ public class SearchPhrase {
                     return rc1;
                 }
             });
+            if (!diffsByRegion.isEmpty()) {
+                List<BinaryMapIndexReader> finalSort = new ArrayList<>();
+                for (int i = 0; i < indexes.size(); i++) {
+                    BinaryMapIndexReader currFile = indexes.get(i);
+                    if (diffsByRegion.get(currFile.getRegionName()) != null) {
+                        finalSort.addAll(diffsByRegion.get(currFile.getRegionName()));
+                        finalSort.add(currFile);
+                    } else {
+                        finalSort.add(currFile);
                     }
                 }
+                indexes.clear();
+                indexes.addAll(finalSort);
+            }
+        }
+    }
+
+    private Map<String, List<BinaryMapIndexReader>> getDiffsByRegion() {
+        Map<String, List<BinaryMapIndexReader>> result = new HashMap<>();
+        Iterator<BinaryMapIndexReader> it = indexes.iterator();
+        while (it.hasNext()) {
+            BinaryMapIndexReader r = it.next();
+            if (r.getFile().getName().matches(".*([0-9]+_*){3}\\.obf")) {
+                String currRegionName = r.getRegionName();
+                if (result.containsKey(currRegionName)) {
+                    result.get(currRegionName).add(r);
+                } else {
+                    result.put(currRegionName, new ArrayList<>(Arrays.asList(r)));
+                }
+                it.remove();
+            }
+        }
+        return result;
+    }
 
 	public static class NameStringMatcher implements StringMatcher {
 

OsmAnd-java/src/net/osmand/util/Algorithms.java

@@ -2,6 +2,7 @@ package net.osmand.util;
 
 import net.osmand.IProgress;
 import net.osmand.PlatformUtil;
+import net.osmand.binary.BinaryMapIndexReader;
 
 import org.apache.commons.logging.Log;
 
@@ -113,7 +114,10 @@ public class Algorithms {
 				return -simplifyFileName(o1.getName()).compareTo(simplifyFileName(o2.getName()));
 			}
 
-			public String simplifyFileName(String fn) {
+		};
+	}
+
+    private static String simplifyFileName(String fn) {
         String lc = fn.toLowerCase();
         if (lc.contains(".")) {
             lc = lc.substring(0, lc.indexOf("."));
@@ -133,6 +137,13 @@ public class Algorithms {
         }
         return lc;
     }
+
+    public static Comparator<String> getStringVersionComparator() {
+        return new Comparator<String>() {
+            @Override
+            public int compare(String o1, String o2) {
+                return -simplifyFileName(o1).compareTo(simplifyFileName(o2));
+            }
         };
     }
 

OsmAnd/src/net/osmand/plus/poi/PoiUIFilter.java

@@ -421,12 +421,20 @@ public class PoiUIFilter implements SearchPoiTypeFilter, Comparable<PoiUIFilter>
 	@Override
 	public ResultMatcher<Amenity> wrapResultMatcher(final ResultMatcher<Amenity> matcher) {
 		final AmenityNameFilter nm = getNameFilter(filterByName);
+		final Set<String> searchedPois = new TreeSet<>();
 		return new ResultMatcher<Amenity>() {
 
 			@Override
 			public boolean publish(Amenity a) {
 				if (nm.accept(a)) {
 					if (matcher == null || matcher.publish(a)) {
+						String poiID = a.getType().getKeyName() + "_" + a.getId();
+						if (!searchedPois.add(poiID)) {
+							return false;
+						}
+						if (a.isClosed()) {
+							return false;
+						}
 						return true;
 					}
 				}

OsmAnd/src/net/osmand/plus/resources/ResourceManager.java

@@ -60,6 +60,7 @@ import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -715,7 +716,10 @@ public class ResourceManager {
 				int left31 = MapUtils.get31TileNumberX(leftLongitude);
 				int bottom31 = MapUtils.get31TileNumberY(bottomLatitude);
 				int right31 = MapUtils.get31TileNumberX(rightLongitude);
-				for (AmenityIndexRepository index : amenityRepositories.values()) {
+				List<String> fileNames = new ArrayList<String>(amenityRepositories.keySet());
+				Collections.sort(fileNames, Algorithms.getStringVersionComparator());
+				for (String name : fileNames) {
+					AmenityIndexRepository index = amenityRepositories.get(name);
 					if (matcher != null && matcher.isCancelled()) {
 						searchAmenitiesInProgress = false;
 						break;