Added analytics helper

OsmAnd/src/net/osmand/AndroidNetworkUtils.java

@@ -4,6 +4,7 @@ import android.graphics.Bitmap;
 import android.graphics.BitmapFactory;
 import android.os.AsyncTask;
 
+import net.osmand.osm.io.Base64;
 import net.osmand.osm.io.NetworkUtils;
 import net.osmand.plus.OsmandApplication;
 import net.osmand.plus.R;
@@ -15,18 +16,23 @@ import org.apache.commons.logging.Log;
 import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
 import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
+import java.net.URL;
 import java.net.URLConnection;
 import java.net.URLEncoder;
 import java.net.UnknownHostException;
 import java.text.MessageFormat;
 import java.util.List;
 import java.util.Map;
+import java.util.zip.GZIPOutputStream;
 
 public class AndroidNetworkUtils {
 
@@ -215,6 +221,88 @@ public class AndroidNetworkUtils {
 		return res;
 	}
 
+	private static final String BOUNDARY = "CowMooCowMooCowCowCow";
+
+	public static String uploadFile(String urlText, File file, boolean gzip, Map<String, String> additionalParams) throws IOException {
+		return uploadFile(urlText, new FileInputStream(file), file.getName(), gzip, additionalParams);
+	}
+
+	public static String uploadFile(String urlText, InputStream inputStream, String fileName, boolean gzip, Map<String, String> additionalParams) {
+		URL url;
+		try {
+			boolean firstPrm = !urlText.contains("?");
+			StringBuilder sb = new StringBuilder(urlText);
+			for (String key : additionalParams.keySet()) {
+				sb.append(firstPrm ? "?" : "&").append(key).append("=").append(URLEncoder.encode(additionalParams.get(key), "UTF-8"));
+				firstPrm = false;
+			}
+			urlText = sb.toString();
+
+			LOG.info("Start uploading file to " + urlText + " " + fileName);
+			url = new URL(urlText);
+
+			HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+			conn.setDoInput(true);
+			conn.setDoOutput(true);
+			conn.setRequestMethod("POST");
+			conn.setRequestProperty("Content-Type", "multipart/form-data; boundary=" + BOUNDARY);
+			conn.setRequestProperty("User-Agent", "OsmAnd");
+
+			OutputStream ous = conn.getOutputStream();
+			ous.write(("--" + BOUNDARY + "\r\n").getBytes());
+			if (gzip) {
+				fileName += ".gz";
+			}
+			ous.write(("content-disposition: form-data; name=\"file\"; filename=\"" + fileName + "\"\r\n").getBytes());
+			ous.write(("Content-Type: application/octet-stream\r\n\r\n").getBytes());
+
+			BufferedInputStream bis = new BufferedInputStream(inputStream, 20 * 1024);
+			ous.flush();
+			if (gzip) {
+				GZIPOutputStream gous = new GZIPOutputStream(ous, 1024);
+				Algorithms.streamCopy(bis, gous);
+				gous.flush();
+				gous.finish();
+			} else {
+				Algorithms.streamCopy(bis, ous);
+			}
+
+			ous.write(("\r\n--" + BOUNDARY + "--\r\n").getBytes());
+			ous.flush();
+			Algorithms.closeStream(bis);
+			Algorithms.closeStream(ous);
+
+			LOG.info("Finish uploading file " + fileName);
+			LOG.info("Response code and message : " + conn.getResponseCode() + " " + conn.getResponseMessage());
+			if (conn.getResponseCode() != 200) {
+				return conn.getResponseMessage();
+			}
+			InputStream is = conn.getInputStream();
+			StringBuilder responseBody = new StringBuilder();
+			if (is != null) {
+				BufferedReader in = new BufferedReader(new InputStreamReader(is, "UTF-8"));
+				String s;
+				boolean first = true;
+				while ((s = in.readLine()) != null) {
+					if (first) {
+						first = false;
+					} else {
+						responseBody.append("\n");
+					}
+					responseBody.append(s);
+
+				}
+				is.close();
+			}
+			String response = responseBody.toString();
+			LOG.info("Response : " + response);
+			return null;
+		} catch (IOException e) {
+			LOG.error(e.getMessage(), e);
+			return e.getMessage();
+		}
+	}
+
 	private static void showToast(OsmandApplication ctx, String message) {
 		ctx.showToastMessage(message);
 	}

OsmAnd/src/net/osmand/plus/AnalyticsHelper.java

@@ -0,0 +1,268 @@
+package net.osmand.plus;
+
+import android.annotation.SuppressLint;
+import android.database.Cursor;
+import android.database.DatabaseUtils;
+import android.database.sqlite.SQLiteDatabase;
+import android.database.sqlite.SQLiteOpenHelper;
+import android.provider.Settings;
+
+import net.osmand.AndroidNetworkUtils;
+import net.osmand.PlatformUtil;
+
+import org.apache.commons.logging.Log;
+import org.json.JSONArray;
+import org.json.JSONObject;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+public class AnalyticsHelper extends SQLiteOpenHelper {
+
+	private final static Log LOG = PlatformUtil.getLog(AnalyticsHelper.class);
+
+	private final static String ANALYTICS_UPLOAD_URL = "https://test.osmand.net/api/submit_analytics";
+	private final static String ANALYTICS_FILE_NAME = "analytics.json";
+
+	private final static int DATA_PARCEL_SIZE = 500; // 500 events
+	private final static int SUBMIT_DATA_INTERVAL = 60 * 60 * 1000; // 1 hour
+
+	private final static String PARAM_START_DATE = "startDate";
+	private final static String PARAM_FINISH_DATE = "finishDate";
+	private final static String PARAM_FIRST_INSTALL_DAYS = "nd";
+	private final static String PARAM_NUMBER_OF_STARTS = "ns";
+	private final static String PARAM_USER_ID = "aid";
+	private final static String PARAM_VERSION = "version";
+	private final static String PARAM_LANG = "lang";
+	private final static String PARAM_EVENTS = "events";
+
+	private final static String JSON_DATE = "date";
+	private final static String JSON_EVENT = "event";
+
+	private final static String DATABASE_NAME = "analytics";
+	private final static int DATABASE_VERSION = 1;
+
+	private final static String TABLE_NAME = "events";
+	private final static String COL_DATE = "date";
+	private final static String COL_EVENT = "event";
+
+	private OsmandApplication ctx;
+	private String insertEventScript;
+	private long lastSubmittedTime;
+
+	private ExecutorService executor = Executors.newSingleThreadExecutor();
+	private Future<?> submittingTask;
+
+	private static class AnalyticsItem {
+		long date;
+		String event;
+	}
+
+	private static class AnalyticsData {
+		long startDate;
+		long finishDate;
+		List<AnalyticsItem> items;
+	}
+
+	AnalyticsHelper(OsmandApplication ctx) {
+		super(ctx, DATABASE_NAME, null, DATABASE_VERSION);
+		this.ctx = ctx;
+		insertEventScript = "INSERT INTO " + TABLE_NAME + " VALUES (?, ?)";
+		submitCollectedDataAsync();
+	}
+
+	@Override
+	public void onCreate(SQLiteDatabase db) {
+		createTable(db);
+	}
+
+	private void createTable(SQLiteDatabase db) {
+		db.execSQL("CREATE TABLE " + TABLE_NAME + " (" + COL_DATE + " long, " + COL_EVENT + " text )");
+	}
+
+	@Override
+	public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) {
+	}
+
+	private long getCollectedRowsCount() {
+		long res = -1;
+		try {
+			SQLiteDatabase db = getWritableDatabase();
+			if (db != null && db.isOpen()) {
+				try {
+					res = DatabaseUtils.queryNumEntries(db, TABLE_NAME);
+				} finally {
+					db.close();
+				}
+			}
+		} catch (RuntimeException e) {
+			// ignore
+		}
+		return res;
+	}
+
+	private void clearDB( long finishDate) {
+		SQLiteDatabase db = getWritableDatabase();
+		if (db != null && db.isOpen()) {
+			try {
+				db.execSQL("DELETE FROM " + TABLE_NAME + " WHERE " + COL_DATE + " <= ?", new Object[]{finishDate});
+			} finally {
+				db.close();
+			}
+		}
+	}
+
+	public boolean submitCollectedDataAsync() {
+		if (ctx.getSettings().isInternetConnectionAvailable()) {
+			long collectedRowsCount = getCollectedRowsCount();
+			if (collectedRowsCount > DATA_PARCEL_SIZE) {
+				if (submittingTask == null || submittingTask.isDone()) {
+					submittingTask = executor.submit(new Runnable() {
+						@Override
+						public void run() {
+							submitCollectedData();
+						}
+					});
+					return true;
+				}
+			}
+		}
+		return false;
+	}
+
+
+	@SuppressLint("HardwareIds")
+	private boolean submitCollectedData() {
+		List<AnalyticsData> data = collectRecordedData();
+		for (AnalyticsData d : data) {
+			if (d.items != null && d.items.size() > 0) {
+				try {
+					JSONArray jsonItemsArray = new JSONArray();
+					for (AnalyticsItem item : d.items) {
+						JSONObject jsonItem = new JSONObject();
+						jsonItem.put(JSON_DATE, item.date);
+						jsonItem.put(JSON_EVENT, item.event);
+						jsonItemsArray.put(jsonItem);
+					}
+
+					Map<String, String> additionalData = new LinkedHashMap<String, String>();
+					additionalData.put(PARAM_START_DATE, String.valueOf(d.startDate));
+					additionalData.put(PARAM_FINISH_DATE, String.valueOf(d.finishDate));
+					additionalData.put(PARAM_VERSION, Version.getFullVersion(ctx));
+					additionalData.put(PARAM_LANG, ctx.getLanguage() + "");
+					additionalData.put(PARAM_FIRST_INSTALL_DAYS, String.valueOf(ctx.getAppInitializer().getFirstInstalledDays()));
+					additionalData.put(PARAM_NUMBER_OF_STARTS, String.valueOf(ctx.getAppInitializer().getNumberOfStarts()));
+					try {
+						additionalData.put(PARAM_USER_ID, Settings.Secure.getString(ctx.getContentResolver(), Settings.Secure.ANDROID_ID));
+					} catch (Exception e) {
+						// ignore
+					}
+
+					JSONObject json = new JSONObject();
+					for (Map.Entry<String, String> entry : additionalData.entrySet()) {
+						json.put(entry.getKey(), entry.getValue());
+					}
+					json.put(PARAM_EVENTS, jsonItemsArray);
+
+					String jsonStr = json.toString();
+					InputStream inputStream = new ByteArrayInputStream(jsonStr.getBytes());
+					String res = AndroidNetworkUtils.uploadFile(ANALYTICS_UPLOAD_URL, inputStream, ANALYTICS_FILE_NAME, true, additionalData);
+					if (res != null) {
+						return false;
+					}
+				} catch (Exception e) {
+					LOG.error(e);
+					return false;
+				}
+				clearDB(d.finishDate);
+			}
+		}
+		return true;
+	}
+
+	private List<AnalyticsData> collectRecordedData() {
+		List<AnalyticsData> data = new ArrayList<>();
+		SQLiteDatabase db = getReadableDatabase();
+		if (db != null && db.isOpen()) {
+			try {
+				collectDBData(db, data);
+			} finally {
+				db.close();
+			}
+		}
+		return data;
+	}
+
+	private void collectDBData(SQLiteDatabase db, List<AnalyticsData> data) {
+		Cursor query = db.rawQuery("SELECT " + COL_DATE + "," + COL_EVENT + " FROM " + TABLE_NAME + " ORDER BY " + COL_DATE + " ASC", null);
+		List<AnalyticsItem> items = new ArrayList<>();
+		int itemsCounter = 0;
+		long startDate = Long.MAX_VALUE;
+		long finishDate = 0;
+		if (query.moveToFirst()) {
+			do {
+				AnalyticsItem item = new AnalyticsItem();
+				long date = query.getLong(0);
+				item.date = date;
+				item.event = query.getString(1);
+				items.add(item);
+				itemsCounter++;
+
+				if (startDate > date) {
+					startDate = date;
+				}
+				if (finishDate < date) {
+					finishDate = date;
+				}
+
+				if (itemsCounter >= DATA_PARCEL_SIZE) {
+					AnalyticsData d = new AnalyticsData();
+					d.startDate = startDate;
+					d.finishDate = finishDate;
+					d.items = items;
+					data.add(d);
+					items = new ArrayList<>();
+					itemsCounter = 0;
+					startDate = Long.MAX_VALUE;
+					finishDate = 0;
+				}
+
+			} while (query.moveToNext());
+
+			if (itemsCounter > 0) {
+				AnalyticsData d = new AnalyticsData();
+				d.startDate = startDate;
+				d.finishDate = finishDate;
+				d.items = items;
+				data.add(d);
+			}
+		}
+		query.close();
+	}
+
+	public void addEvent(String event) {
+		SQLiteDatabase db = getWritableDatabase();
+		if (db != null && db.isOpen()) {
+			try {
+				db.execSQL(insertEventScript, new Object[]{System.currentTimeMillis(), event});
+			} finally {
+				db.close();
+			}
+		}
+		long currentTimeMillis = System.currentTimeMillis();
+		if (lastSubmittedTime + SUBMIT_DATA_INTERVAL < currentTimeMillis) {
+			if (!submitCollectedDataAsync()) {
+				lastSubmittedTime = currentTimeMillis - SUBMIT_DATA_INTERVAL / 4;
+			} else {
+				lastSubmittedTime = currentTimeMillis;
+			}
+		}
+	}
+}

OsmAnd/src/net/osmand/plus/AppInitializer.java

@@ -467,6 +467,7 @@ public class AppInitializer implements IProgress {
 		app.locationProvider = startupInit(new OsmAndLocationProvider(app), OsmAndLocationProvider.class);
 		app.avoidSpecificRoads = startupInit(new AvoidSpecificRoads(app), AvoidSpecificRoads.class);
 		app.savingTrackHelper = startupInit(new SavingTrackHelper(app), SavingTrackHelper.class);
+		app.analyticsHelper = startupInit(new AnalyticsHelper(app), AnalyticsHelper.class);
 		app.notificationHelper = startupInit(new NotificationHelper(app), NotificationHelper.class);
 		app.liveMonitoringHelper = startupInit(new LiveMonitoringHelper(app), LiveMonitoringHelper.class);
 		app.selectedGpxHelper = startupInit(new GpxSelectionHelper(app, app.savingTrackHelper), GpxSelectionHelper.class);

OsmAnd/src/net/osmand/plus/OsmandApplication.java

@@ -109,6 +109,7 @@ public class OsmandApplication extends MultiDexApplication {
 	GpxSelectionHelper selectedGpxHelper;
 	GPXDatabase gpxDatabase;
 	SavingTrackHelper savingTrackHelper;
+	AnalyticsHelper analyticsHelper;
 	NotificationHelper notificationHelper;
 	LiveMonitoringHelper liveMonitoringHelper;
 	TargetPointsHelper targetPointsHelper;
@@ -913,10 +914,10 @@ public class OsmandApplication extends MultiDexApplication {
 		try {
 			if (Version.isGooglePlayEnabled(this) && !Version.isPaidVersion(this)
 					&& !osmandSettings.DO_NOT_SEND_ANONYMOUS_APP_USAGE.get()) {
-				// not implemented yet
+				analyticsHelper.addEvent(event);
 			}
 		} catch (Exception e) {
-			// ignore
+			LOG.error(e);
 		}
 	}