Allow custom "user-agent" for tile download

This commit is contained in:
Guillaume Gonnet 2020-08-17 18:35:12 +02:00
parent 6ce4c12842
commit 4406b60c6d
7 changed files with 43 additions and 5 deletions

View file

@ -32,6 +32,8 @@ public interface ITileSource {
public String getReferer(); public String getReferer();
public String getUserAgent();
public void deleteTiles(String path); public void deleteTiles(String path);
public int getAvgSize(); public int getAvgSize();

View file

@ -90,6 +90,7 @@ public class MapTileDownloader {
public final int yTile; public final int yTile;
public String url; public String url;
public String referer = null; public String referer = null;
public String userAgent = null;
public boolean error; public boolean error;
public DownloadRequest(String url, File fileToSave, int xTile, int yTile, int zoom) { public DownloadRequest(String url, File fileToSave, int xTile, int yTile, int zoom) {
@ -266,7 +267,7 @@ public class MapTileDownloader {
request.setError(false); request.setError(false);
try { try {
URLConnection connection = NetworkUtils.getHttpURLConnection(request.url); URLConnection connection = NetworkUtils.getHttpURLConnection(request.url);
connection.setRequestProperty("User-Agent", USER_AGENT); //$NON-NLS-1$ connection.setRequestProperty("User-Agent", Algorithms.isEmpty(request.userAgent) ? USER_AGENT : request.userAgent); //$NON-NLS-1$
if (request.referer != null) if (request.referer != null)
connection.setRequestProperty("Referer", request.referer); //$NON-NLS-1$ connection.setRequestProperty("Referer", request.referer); //$NON-NLS-1$
connection.setConnectTimeout(CONNECTION_TIMEOUT); connection.setConnectTimeout(CONNECTION_TIMEOUT);

View file

@ -68,6 +68,7 @@ public class TileSourceManager {
private String[] randomsArray; private String[] randomsArray;
private String rule; private String rule;
private String referer; private String referer;
private String userAgent;
private boolean hidden; // if hidden in configure map settings, for example mapillary sources private boolean hidden; // if hidden in configure map settings, for example mapillary sources
private boolean isRuleAcceptable = true; private boolean isRuleAcceptable = true;
@ -261,6 +262,14 @@ public class TileSourceManager {
this.referer = referer; this.referer = referer;
} }
public String getUserAgent() {
return userAgent;
}
public void setUserAgent(String userAgent) {
this.userAgent = userAgent;
}
@Override @Override
public int getTileSize() { public int getTileSize() {
return tileSize; return tileSize;
@ -484,6 +493,9 @@ public class TileSourceManager {
if (!Algorithms.isEmpty(tm.getReferer())) { if (!Algorithms.isEmpty(tm.getReferer())) {
properties.put("referer", tm.getReferer()); properties.put("referer", tm.getReferer());
} }
if (!Algorithms.isEmpty(tm.getUserAgent())) {
properties.put("user_agent", tm.getUserAgent());
}
properties.put("ext", tm.getTileFormat()); properties.put("ext", tm.getTileFormat());
properties.put("min_zoom", tm.getMinimumZoomSupported() + ""); properties.put("min_zoom", tm.getMinimumZoomSupported() + "");
@ -708,6 +720,12 @@ public class TileSourceManager {
} }
String randoms = attributes.get("randoms"); String randoms = attributes.get("randoms");
TileSourceTemplate templ = new TileSourceTemplate(name, urlTemplate, ext, maxZoom, minZoom, tileSize, bitDensity, avgTileSize); TileSourceTemplate templ = new TileSourceTemplate(name, urlTemplate, ext, maxZoom, minZoom, tileSize, bitDensity, avgTileSize);
if (attributes.get("referer") != null) {
templ.setReferer(attributes.get("referer"));
}
if (attributes.get("user_agent") != null) {
templ.setUserAgent(attributes.get("user_agent"));
}
if(expirationTime >= 0) { if(expirationTime >= 0) {
templ.setExpirationTimeMinutes(expirationTime); templ.setExpirationTimeMinutes(expirationTime);
} }

View file

@ -39,6 +39,7 @@ public class SQLiteTileSource implements ITileSource {
private static final String ELLIPSOID = "ellipsoid"; private static final String ELLIPSOID = "ellipsoid";
private static final String INVERTED_Y = "inverted_y"; private static final String INVERTED_Y = "inverted_y";
private static final String REFERER = "referer"; private static final String REFERER = "referer";
private static final String USER_AGENT = "useragent";
private static final String TIME_COLUMN = "timecolumn"; private static final String TIME_COLUMN = "timecolumn";
private static final String EXPIRE_MINUTES = "expireminutes"; private static final String EXPIRE_MINUTES = "expireminutes";
private static final String RULE = "rule"; private static final String RULE = "rule";
@ -62,6 +63,7 @@ public class SQLiteTileSource implements ITileSource {
private String[] randomsArray; private String[] randomsArray;
private String rule = null; private String rule = null;
private String referer = null; private String referer = null;
private String userAgent = null;
int tileSize = 256; int tileSize = 256;
boolean tileSizeSpecified = false; boolean tileSizeSpecified = false;
@ -93,7 +95,7 @@ public class SQLiteTileSource implements ITileSource {
} }
public SQLiteTileSource(OsmandApplication ctx, String name, int minZoom, int maxZoom, String urlTemplate, public SQLiteTileSource(OsmandApplication ctx, String name, int minZoom, int maxZoom, String urlTemplate,
String randoms, boolean isEllipsoid, boolean invertedY, String referer, String randoms, boolean isEllipsoid, boolean invertedY, String referer, String userAgent,
boolean timeSupported, long expirationTimeMillis, boolean inversiveZoom, String rule) { boolean timeSupported, long expirationTimeMillis, boolean inversiveZoom, String rule) {
this.ctx = ctx; this.ctx = ctx;
this.name = name; this.name = name;
@ -104,6 +106,7 @@ public class SQLiteTileSource implements ITileSource {
this.expirationTimeMillis = expirationTimeMillis; this.expirationTimeMillis = expirationTimeMillis;
this.randoms = randoms; this.randoms = randoms;
this.referer = referer; this.referer = referer;
this.userAgent = userAgent;
this.rule = rule; this.rule = rule;
this.invertedY = invertedY; this.invertedY = invertedY;
this.timeSupported = timeSupported; this.timeSupported = timeSupported;
@ -120,6 +123,7 @@ public class SQLiteTileSource implements ITileSource {
this.expirationTimeMillis = tileSource.getExpirationTimeMillis(); this.expirationTimeMillis = tileSource.getExpirationTimeMillis();
this.randoms = tileSource.getRandoms(); this.randoms = tileSource.getRandoms();
this.referer = tileSource.getReferer(); this.referer = tileSource.getReferer();
this.userAgent = tileSource.getUserAgent();
this.invertedY = tileSource.isInvertedYTile(); this.invertedY = tileSource.isInvertedYTile();
this.timeSupported = tileSource.isTimeSupported(); this.timeSupported = tileSource.isTimeSupported();
this.inversiveZoom = tileSource.getInversiveZoom(); this.inversiveZoom = tileSource.getInversiveZoom();
@ -139,6 +143,7 @@ public class SQLiteTileSource implements ITileSource {
addInfoColumn(db, ELLIPSOID, isEllipsoid ? "1" : "0"); addInfoColumn(db, ELLIPSOID, isEllipsoid ? "1" : "0");
addInfoColumn(db, INVERTED_Y, invertedY ? "1" : "0"); addInfoColumn(db, INVERTED_Y, invertedY ? "1" : "0");
addInfoColumn(db, REFERER, referer); addInfoColumn(db, REFERER, referer);
addInfoColumn(db, USER_AGENT, userAgent);
addInfoColumn(db, TIME_COLUMN, timeSupported ? "yes" : "no"); addInfoColumn(db, TIME_COLUMN, timeSupported ? "yes" : "no");
addInfoColumn(db, EXPIRE_MINUTES, String.valueOf(getExpirationTimeMinutes())); addInfoColumn(db, EXPIRE_MINUTES, String.valueOf(getExpirationTimeMinutes()));
@ -264,6 +269,10 @@ public class SQLiteTileSource implements ITileSource {
if(refererId != -1) { if(refererId != -1) {
referer = cursor.getString(refererId); referer = cursor.getString(refererId);
} }
int userAgentId = list.indexOf(USER_AGENT);
if(userAgentId != -1) {
userAgent = cursor.getString(userAgentId);
}
int tnumbering = list.indexOf(TILENUMBERING); int tnumbering = list.indexOf(TILENUMBERING);
if(tnumbering != -1) { if(tnumbering != -1) {
inversiveZoom = BIG_PLANET_TILE_NUMBERING.equalsIgnoreCase(cursor.getString(tnumbering)); inversiveZoom = BIG_PLANET_TILE_NUMBERING.equalsIgnoreCase(cursor.getString(tnumbering));
@ -673,4 +682,8 @@ public class SQLiteTileSource implements ITileSource {
return referer; return referer;
} }
public String getUserAgent() {
return userAgent;
}
} }

View file

@ -105,12 +105,13 @@ public class AsyncLoadingThread extends Thread {
} }
public TileLoadDownloadRequest(File dirWithTiles, String url, File fileToSave, String tileId, ITileSource source, int tileX, public TileLoadDownloadRequest(File dirWithTiles, String url, File fileToSave, String tileId, ITileSource source, int tileX,
int tileY, int zoom, String referer) { int tileY, int zoom, String referer, String userAgent) {
super(url, fileToSave, tileX, tileY, zoom); super(url, fileToSave, tileX, tileY, zoom);
this.dirWithTiles = dirWithTiles; this.dirWithTiles = dirWithTiles;
this.tileSource = source; this.tileSource = source;
this.tileId = tileId; this.tileId = tileId;
this.referer = referer; this.referer = referer;
this.userAgent = userAgent;
} }
public void saveTile(InputStream inputStream) throws IOException { public void saveTile(InputStream inputStream) throws IOException {

View file

@ -144,7 +144,7 @@ public abstract class TilesCache<T> {
} }
} }
TileLoadDownloadRequest req = new TileLoadDownloadRequest(dirWithTiles, url, toSave, TileLoadDownloadRequest req = new TileLoadDownloadRequest(dirWithTiles, url, toSave,
tileId, map, x, y, zoom, map.getReferer()); tileId, map, x, y, zoom, map.getReferer(), map.getUserAgent());
if (sync) { if (sync) {
return getRequestedTile(req); return getRequestedTile(req);
} else { } else {

View file

@ -2035,6 +2035,7 @@ public class SettingsHelper {
boolean ellipsoid = object.optBoolean("ellipsoid", false); boolean ellipsoid = object.optBoolean("ellipsoid", false);
boolean invertedY = object.optBoolean("inverted_y", false); boolean invertedY = object.optBoolean("inverted_y", false);
String referer = object.optString("referer"); String referer = object.optString("referer");
String userAgent = object.optString("userAgent");
boolean timeSupported = object.optBoolean("timesupported", false); boolean timeSupported = object.optBoolean("timesupported", false);
long expire = object.optLong("expire", -1); long expire = object.optLong("expire", -1);
boolean inversiveZoom = object.optBoolean("inversiveZoom", false); boolean inversiveZoom = object.optBoolean("inversiveZoom", false);
@ -2054,13 +2055,14 @@ public class SettingsHelper {
tileSourceTemplate.setRule(rule); tileSourceTemplate.setRule(rule);
tileSourceTemplate.setRandoms(randoms); tileSourceTemplate.setRandoms(randoms);
tileSourceTemplate.setReferer(referer); tileSourceTemplate.setReferer(referer);
tileSourceTemplate.setUserAgent(userAgent);
tileSourceTemplate.setEllipticYTile(ellipsoid); tileSourceTemplate.setEllipticYTile(ellipsoid);
tileSourceTemplate.setInvertedYTile(invertedY); tileSourceTemplate.setInvertedYTile(invertedY);
tileSourceTemplate.setExpirationTimeMillis(timeSupported ? expire : -1); tileSourceTemplate.setExpirationTimeMillis(timeSupported ? expire : -1);
template = tileSourceTemplate; template = tileSourceTemplate;
} else { } else {
template = new SQLiteTileSource(app, name, minZoom, maxZoom, url, randoms, ellipsoid, invertedY, referer, timeSupported, expire, inversiveZoom, rule); template = new SQLiteTileSource(app, name, minZoom, maxZoom, url, randoms, ellipsoid, invertedY, referer, userAgent, timeSupported, expire, inversiveZoom, rule);
} }
items.add(template); items.add(template);
} }
@ -2087,6 +2089,7 @@ public class SettingsHelper {
jsonObject.put("ellipsoid", template.isEllipticYTile()); jsonObject.put("ellipsoid", template.isEllipticYTile());
jsonObject.put("inverted_y", template.isInvertedYTile()); jsonObject.put("inverted_y", template.isInvertedYTile());
jsonObject.put("referer", template.getReferer()); jsonObject.put("referer", template.getReferer());
jsonObject.put("userAgent", template.getUserAgent());
jsonObject.put("timesupported", template.isTimeSupported()); jsonObject.put("timesupported", template.isTimeSupported());
jsonObject.put("expire", template.getExpirationTimeMinutes()); jsonObject.put("expire", template.getExpirationTimeMinutes());
jsonObject.put("inversiveZoom", template.getInversiveZoom()); jsonObject.put("inversiveZoom", template.getInversiveZoom());