Speedup native load

DataExtractionOSM/src/net/osmand/NativeLibrary.java

@@ -84,6 +84,10 @@ public class NativeLibrary {
 		return initBinaryMapFile(filePath);
 	}
 	
+	public boolean initCacheMapFile(String filePath) {
+		return initCacheMapFiles(filePath);
+	}
+	
 	public boolean closeMapFile(String filePath) {
 		return closeBinaryMapFile(filePath);
 	}
@@ -110,6 +114,8 @@ public class NativeLibrary {
 
 	protected static native boolean initBinaryMapFile(String filePath);
 	
+	protected static native boolean initCacheMapFiles(String filePath);
+	
 	protected static native boolean closeBinaryMapFile(String filePath);
 	
 	protected static native void initRenderingRulesStorage(RenderingRulesStorage storage);

DataExtractionOSM/src/net/osmand/binary/CachedOsmandIndexes.java

@@ -45,6 +45,11 @@ public class CachedOsmandIndexes {
 			storedIndexBuilder = OsmandIndex.OsmAndStoredIndex.newBuilder();
 			storedIndexBuilder.setVersion(VERSION);
 			storedIndexBuilder.setDateCreated(System.currentTimeMillis());
+			if(storedIndex != null) {
+				for(FileIndex ex : storedIndex.getFileIndexList()) {
+					storedIndexBuilder.addFileIndex(ex);
+				}
+			}
 		}
 		
 		FileIndex.Builder fileIndex = OsmandIndex.FileIndex.newBuilder();

DataExtractionOSM/src/osmand_index.proto

@@ -2,9 +2,10 @@
 option java_package = "net.osmand.binary";
 //protoc --java_out=DataExtractionOSM/src DataExtractionOSM/src/osmand_index.proto
 
-// C++ # cd OsmAnd
-// # protoc --proto_path=../DataExtractionOSM/src --cpp_out=jni/osmand/proto ../DataExtractionOSM/src/osmand_odb.proto
+// C++ # cd OsmAnd-Kernel
+// # protoc --proto_path=../DataExtractionOSM/src --cpp_out=osmand/src/proto ../DataExtractionOSM/src/osmand_index.proto
 
+option optimize_for = LITE_RUNTIME;
 message OsmAndStoredIndex {
    required uint32 version = 1;
 

OsmAnd/src/net/osmand/plus/ResourceManager.java

@@ -39,6 +39,7 @@ import net.osmand.plus.AsyncLoadingThread.MapLoadRequest;
 import net.osmand.plus.AsyncLoadingThread.TileLoadDownloadRequest;
 import net.osmand.plus.AsyncLoadingThread.TransportLoadRequest;
 import net.osmand.plus.render.MapRenderRepositories;
+import net.osmand.plus.render.NativeOsmandLibrary;
 import net.osmand.plus.views.OsmandMapLayer.DrawSettings;
 import net.osmand.render.RenderingRulesStorage;
 
@@ -533,6 +534,11 @@ public class ResourceManager {
 		if(indCache.exists()) {
 			try {
 				cachedOsmandIndexes.readFromFile(indCache, CachedOsmandIndexes.VERSION);
+				NativeOsmandLibrary nativeLib = context.getSettings().NATIVE_RENDERING.get() ? NativeOsmandLibrary.getLoadedLibrary()
+						: null;
+				if(nativeLib != null) {
+					nativeLib.initCacheMapFile(indCache.getAbsolutePath());
+				}
 			} catch (Exception e) {
 				log.error(e.getMessage(), e);
 			}
@@ -546,7 +552,6 @@ public class ResourceManager {
 						BinaryMapIndexReader index = null;
 						try {
 							index = cachedOsmandIndexes.getReader(f);
-							
 							if (index.getVersion() != IndexConstants.BINARY_MAP_VERSION) {
 								index = null;
 							}

OsmAnd/src/net/osmand/plus/render/MapRenderRepositories.java

@@ -114,13 +114,13 @@ public class MapRenderRepositories {
 		NativeOsmandLibrary nativeLib = prefs.NATIVE_RENDERING.get() ? NativeOsmandLibrary.getLoadedLibrary() : null;
 		if (nativeLib != null) {
 			if (!nativeLib.initMapFile(file.getAbsolutePath())) {
-				log.debug("Initializing native db " + file.getAbsolutePath() + " failed!"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+				log.error("Initializing native db " + file.getAbsolutePath() + " failed!"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
 			} else {
 				nativeFiles.add(file.getAbsolutePath());
-				long val = System.currentTimeMillis();
-				if (log.isDebugEnabled()) {
-					log.debug("Initializing native db " + file.getAbsolutePath() + " " + (val - start) + "ms"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
-				}
+//				long val = System.currentTimeMillis();
+//				if (log.isDebugEnabled()) {
+//					log.debug("Initializing native db " + file.getAbsolutePath() + " " + (val - start) + "ms"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+//				}
 			}
 		}
 	}

Osmand-kernel/osmand/Common.mk

@@ -21,7 +21,8 @@ LOCAL_SRC_FILES := \
 	src/multipolygons.cpp \
 	src/renderRules.cpp \
 	src/rendering.cpp \
-	src/binaryRead.cpp
+	src/binaryRead.cpp \
+	src/proto/osmand_index.pb.cpp
 	
 ifdef OSMAND_PROFILE_NATIVE_OPERATIONS
 LOCAL_CFLAGS += \

Osmand-kernel/osmand/src/binaryRead.cpp

@@ -9,6 +9,7 @@
 #include "google/protobuf/io/zero_copy_stream_impl.h"
 #include "google/protobuf/wire_format_lite.cc"
 #include "proto/osmand_odb.pb.h"
+#include "proto/osmand_index.pb.h"
 #include "osmand_log.h"
 
 using namespace std;
@@ -20,6 +21,7 @@ using google::protobuf::internal::WireFormatLite;
 
 
 std::map< std::string, BinaryMapFile* > openFiles;
+OsmAndStoredIndex* cache = NULL;
 
 inline bool readInt(CodedInputStream* input, uint32_t* sz ){
 	uint8_t buf[4];
@@ -1035,8 +1037,9 @@ ResultPublisher* searchObjectsForRendering(SearchQuery* q, bool skipDuplicates,
 	return q->publisher;
 }
 
+
 void searchRouteRegion(CodedInputStream* input, SearchQuery* q, RoutingIndex* ind, std::vector<RouteSubregion>& subregions,
-		std::vector<RouteSubregion>& toLoad) {
+		std::vector<RouteSubregion>& toLoad, BinaryMapFile* file) {
 	for (std::vector<RouteSubregion>::iterator subreg = subregions.begin();
 						subreg != subregions.end(); subreg++) {
 		if (subreg->right >= q->left && q->right >= subreg->left && subreg->bottom >= q->top
@@ -1045,11 +1048,11 @@ void searchRouteRegion(CodedInputStream* input, SearchQuery* q, RoutingIndex* in
 				bool contains = subreg->right <= q->right && q->left <= subreg->left && subreg->top <= q->top
 						&& subreg->bottom >= q->bottom;
 				input->Seek(subreg->filePointer);
-				uint32_t old = input -> PushLimit(subreg->length);
+				uint32_t old = input->PushLimit(subreg->length);
 				readRouteTree(input, &(*subreg), NULL, contains? -1 : 1, false);
 				input->PopLimit(old);
 			}
-			searchRouteRegion(input, q, ind, subreg->subregions, toLoad);
+			searchRouteRegion(input, q, ind, subreg->subregions, toLoad, file);
 			if(subreg->mapDataBlock != 0) {
 				toLoad.push_back(*subreg);
 			}
@@ -1312,11 +1315,6 @@ void searchRouteRegion(SearchQuery* q, std::vector<RouteDataObject*>& list, Rout
 	bool basemapExists = false;
 	for (; i != openFiles.end() && !q->publisher->isCancelled(); i++) {
 		BinaryMapFile* file = i->second;
-		lseek(file->routefd, 0, SEEK_SET);
-		FileInputStream input(file->routefd);
-		input.SetCloseOnDelete(false);
-		CodedInputStream cis(&input);
-		cis.SetTotalBytesLimit(INT_MAX, INT_MAX >> 2);
 		for (std::vector<RoutingIndex>::iterator routingIndex = file->routingIndexes.begin();
 				routingIndex != file->routingIndexes.end(); routingIndex++) {
 			if (q->publisher->isCancelled()) {
@@ -1325,10 +1323,44 @@ void searchRouteRegion(SearchQuery* q, std::vector<RouteDataObject*>& list, Rout
 			if(rs != NULL && (rs->name != routingIndex->name || rs->filePointer != routingIndex->filePointer)){
 				continue;
 			}
+			// check boundaries and init
+			if (routingIndex->decodingRules.size() == 0) {
+				bool contain = false;
+				for (std::vector<RouteSubregion>::iterator subreg = routingIndex->subregions.begin(); subreg != routingIndex->subregions.end();
+						subreg++) {
+					if (subreg->right >= q->left && q->right >= subreg->left && subreg->bottom >= q->top
+							&& q->bottom >= subreg->top) {
+						contain = true;
+					}
+				}
+				if (contain) {
+					routingIndex->subregions.clear();
+					osmand_log_print(LOG_INFO, "Init native index %s", routingIndex->name.c_str());
+					lseek(file->routefd, 0, SEEK_SET);
+					FileInputStream input(file->routefd);
+					input.SetCloseOnDelete(false);
+					CodedInputStream cis(&input);
+					cis.SetTotalBytesLimit(INT_MAX, INT_MAX >> 2);
+
+					cis.Seek(routingIndex->filePointer);
+					uint32_t old = cis.PushLimit(routingIndex->length);
+					readRoutingIndex(&cis, routingIndex);
+					cis.PopLimit(old);
+				} else {
+					continue;
+				}
+			}
+			// could be simplified but it will be concurrency with init block
+			lseek(file->routefd, 0, SEEK_SET);
+			FileInputStream input(file->routefd);
+			input.SetCloseOnDelete(false);
+			CodedInputStream cis(&input);
+			cis.SetTotalBytesLimit(INT_MAX, INT_MAX >> 2);
+
 			std::vector<RouteSubregion> toLoad;
-			searchRouteRegion(&cis, q, &(*routingIndex), routingIndex->subregions, toLoad);
+			searchRouteRegion(&cis, q, &(*routingIndex), routingIndex->subregions, toLoad, file);
 			sort(toLoad.begin(), toLoad.end(), sortRouteRegions);
-			;
+
 			std::vector<RouteDataObject*> iteration;
 			int cnt = 0;
 			for (std::vector<RouteSubregion>::iterator subreg = toLoad.begin(); subreg != toLoad.end(); subreg++) {
@@ -1357,6 +1389,41 @@ bool closeBinaryMapFile(std::string inputName) {
 	return false;
 }
 
+bool initMapFilesFromCache(std::string inputName) {
+	GOOGLE_PROTOBUF_VERIFY_VERSION;
+#if defined(_WIN32)
+	int fileDescriptor = open(inputName.c_str(), O_RDONLY | O_BINARY);
+#else
+	int fileDescriptor = open(inputName.c_str(), O_RDONLY);
+#endif
+	if (fileDescriptor < 0) {
+		osmand_log_print(LOG_ERROR, "Cache file could not be open to read : %s", inputName.c_str());
+		return false;
+	}
+	FileInputStream input(fileDescriptor);
+	CodedInputStream cis(&input);
+	cis.SetTotalBytesLimit(INT_MAX, INT_MAX);
+	OsmAndStoredIndex* c = new OsmAndStoredIndex();
+	if(c->MergeFromCodedStream(&cis)){
+		osmand_log_print(LOG_INFO, "Native Cache file initialized %s", inputName.c_str());
+		cache = c;
+		for (int i = 0; i < cache->fileindex_size(); i++) {
+			FileIndex fi = cache->fileindex(i);
+		}
+		return true;
+	}
+	return false;
+}
+
+bool hasEnding (std::string const &fullString, std::string const &ending)
+{
+    if (fullString.length() >= ending.length()) {
+        return (0 == fullString.compare (fullString.length() - ending.length(), ending.length(), ending));
+    } else {
+        return false;
+    }
+}
+
 BinaryMapFile* initBinaryMapFile(std::string inputName) {
 	GOOGLE_PROTOBUF_VERIFY_VERSION;
 	std::map<std::string, BinaryMapFile*>::iterator iterator;
@@ -1378,7 +1445,72 @@ BinaryMapFile* initBinaryMapFile(std::string inputName) {
 	}
 	BinaryMapFile* mapFile = new BinaryMapFile();
 	mapFile->fd = fileDescriptor;
+	struct stat stat;
+	fstat(fileDescriptor, &stat);
+
 	mapFile->routefd = routeDescriptor;
+	bool readFromCache = cache != NULL;
+	FileIndex* fo = NULL;
+	if (readFromCache) {
+		for (int i = 0; i < cache->fileindex_size(); i++) {
+			FileIndex fi = cache->fileindex(i);
+			if (hasEnding(inputName, fi.filename()) && fi.size() == stat.st_size) {
+				fo = cache->mutable_fileindex(i);
+				break;
+			}
+		}
+	}
+	if (fo != NULL) {
+		mapFile->version = fo->version();
+		mapFile->dateCreated = fo->datemodified();
+		for (int i = 0; i < fo->mapindex_size(); i++) {
+			MapIndex mi;
+			MapPart mp = fo->mapindex(i);
+			mi.filePointer = mp.offset();
+			mi.length = mp.size();
+			mi.name = mp.name();
+			for (int j = 0; j < mp.levels_size(); j++) {
+				MapLevel ml = mp.levels(j);
+				MapRoot mr;
+				mr.bottom = ml.bottom();
+				mr.left = ml.left();
+				mr.right = ml.right();
+				mr.top = ml.top();
+				mr.maxZoom = ml.maxzoom();
+				mr.minZoom = ml.minzoom();
+				mr.filePointer = ml.offset();
+				mr.length = ml.size();
+				mi.levels.push_back(mr);
+			}
+			mapFile->basemap = mapFile->basemap || mi.name.find("basemap") != string::npos;
+			mapFile->mapIndexes.push_back(mi);
+			mapFile->indexes.push_back(&mapFile->mapIndexes.back());
+		}
+
+		for (int i = 0; i < fo->routingindex_size(); i++) {
+			RoutingIndex mi;
+			RoutingPart mp = fo->routingindex(i);
+			mi.filePointer = mp.offset();
+			mi.length = mp.size();
+			mi.name = mp.name();
+			for (int j = 0; j < mp.subregions_size(); j++) {
+				RoutingSubregion ml = mp.subregions(j);
+				RouteSubregion mr;
+				mr.bottom = ml.bottom();
+				mr.left = ml.left();
+				mr.right = ml.right();
+				mr.top = ml.top();
+				mr.mapDataBlock = ml.shiftodata();
+				mr.filePointer = ml.offset();
+				mr.length = ml.size();
+				osmand_log_print(LOG_DEBUG, "Native file init subregion %d %d %d ", mr.filePointer, mr.length, mr.mapDataBlock);
+				mi.subregions.push_back(mr);
+			}
+			mapFile->routingIndexes.push_back(mi);
+			mapFile->indexes.push_back(&mapFile->routingIndexes.back());
+		}
+		osmand_log_print(LOG_DEBUG, "Native file initialized from cache %s", inputName.c_str());
+	} else {
 		FileInputStream input(fileDescriptor);
 		input.SetCloseOnDelete(false);
 		CodedInputStream cis(&input);
@@ -1389,6 +1521,7 @@ BinaryMapFile* initBinaryMapFile(std::string inputName) {
 			delete mapFile;
 			return NULL;
 		}
+	}
 	mapFile->inputName = inputName;
 	openFiles.insert(std::pair<std::string, BinaryMapFile*>(inputName, mapFile));
 	return mapFile;

Osmand-kernel/osmand/src/binaryRead.h

@@ -249,6 +249,8 @@ ResultPublisher* searchObjectsForRendering(SearchQuery* q, bool skipDuplicates,
 
 BinaryMapFile* initBinaryMapFile(std::string inputName);
 
+bool initMapFilesFromCache(std::string inputName) ;
+
 bool closeBinaryMapFile(std::string inputName);
 
 #endif

Osmand-kernel/osmand/src/java_wrap.cpp

@@ -52,6 +52,16 @@ extern "C" JNIEXPORT void JNICALL Java_net_osmand_NativeLibrary_closeBinaryMapFi
 	closeBinaryMapFile(inputName);
 }
 
+
+extern "C" JNIEXPORT jboolean JNICALL Java_net_osmand_NativeLibrary_initCacheMapFiles(JNIEnv* ienv,
+		jobject obj,
+		jobject path) {
+	const char* utf = ienv->GetStringUTFChars((jstring) path, NULL);
+	std::string inputName(utf);
+	ienv->ReleaseStringUTFChars((jstring) path, utf);
+	return initMapFilesFromCache(inputName);
+}
+
 extern "C" JNIEXPORT jboolean JNICALL Java_net_osmand_NativeLibrary_initBinaryMapFile(JNIEnv* ienv,
 		jobject obj, jobject path) {
 	// Verify that the version of the library that we linked against is