Android-builders-club/OsmAnd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Replace IndexZipper with IndexUploader

Browse source
This commit is contained in:
Victor Shcherb 2012-01-02 21:17:25 +01:00
parent 50eaa9b337
commit 81cc30aa91
6 changed files with 437 additions and 644 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

396
DataExtractionOSM/src/net/osmand/data/index/IndexBatchCreator.java
View file

@ -10,26 +10,19 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLEncoder;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.logging.FileHandler;
import java.util.logging.Handler;
import java.util.logging.LogManager;
import java.util.logging.Logger;
import java.util.logging.Level;
import java.util.logging.SimpleFormatter;
@ -39,10 +32,7 @@ import javax.xml.parsers.ParserConfigurationException;
import net.osmand.Algoritms;
import net.osmand.LogUtil;
import net.osmand.Version;
import net.osmand.binary.BinaryMapIndexReader;
import net.osmand.data.IndexConstants;
import net.osmand.data.index.ExtractGooglecodeAuthorization.GooglecodeUploadTokens;
import net.osmand.data.index.IndexZipper.OneFileException;
import net.osmand.data.preparation.DBDialect;
import net.osmand.data.preparation.IndexCreator;
import net.osmand.data.preparation.MapZooms;
@ -51,25 +41,17 @@ import net.osmand.osm.MapRenderingTypes;
import net.osmand.swing.OsmExtractionUI;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Jdk14Logger;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
import rtree.RTree;
public class IndexBatchCreator {
protected static final Log log = LogUtil.getLog(IndexBatchCreator.class);
private final static double MIN_SIZE_TO_UPLOAD = 0.015d;
private final static double MAX_SIZE_TO_NOT_SPLIT = 190d;
private final static double MAX_UPLOAD_SIZE = 195d;
public static class RegionCountries {
@ -83,44 +65,25 @@ public class IndexBatchCreator {
public String cityAdminLevel;
}
private boolean uploadToOsmandGooglecode = true;
// process atributtes
boolean downloadFiles = false;
boolean generateIndexes = false;
boolean uploadIndexes = false;
boolean skipGeneratedIndexes = false;
MapZooms mapZooms = null;
Integer zoomWaySmoothness = null;
MapRenderingTypes types = MapRenderingTypes.getDefault();
boolean deleteFilesAfterUploading = true;
File osmDirFiles;
File indexDirFiles;
File workDir;
File backupUploadedFiles;
boolean indexPOI = false;
boolean indexTransport = false;
boolean indexAddress = false;
boolean indexMap = false;
String user;
String password;
private String wget;
String googlePassword = "";
String cookieHSID = "";
String cookieSID = "";
String pagegen = "";
String token = "";
public static void main(String[] args) {
IndexBatchCreator creator = new IndexBatchCreator();
OsmExtractionUI.configLogFile();
if(args == null || args.length == 0){
@ -174,11 +137,7 @@ public class IndexBatchCreator {
throw new IllegalArgumentException("You should specify exactly 1 process element!");
}
Element process = (Element) list.item(0);
downloadFiles = Boolean.parseBoolean(process.getAttribute("downloadOsmFiles"));
generateIndexes = Boolean.parseBoolean(process.getAttribute("generateIndexes"));
uploadIndexes = Boolean.parseBoolean(process.getAttribute("uploadIndexes"));
deleteFilesAfterUploading = Boolean.parseBoolean(process.getAttribute("deleteFilesAfterUploading"));
IndexCreator.REMOVE_POI_DB = !Boolean.parseBoolean(process.getAttribute("keepPoiOdb"));
IndexCreator.REMOVE_POI_DB = true;
skipGeneratedIndexes = Boolean.parseBoolean(process.getAttribute("skipGeneratedIndexes"));
wget = process.getAttribute("wget");
@ -209,55 +168,6 @@ public class IndexBatchCreator {
workDir = new File(dir);
}
if (downloadFiles) {
if (regions == null) {
throw new IllegalArgumentException("Please specify regions.xml file as 2nd parameter");
}
}
dir = process.getAttribute("directory_for_uploaded_files");
if (dir != null) {
File file = new File(dir);
file.mkdirs();
if (file.exists()) {
backupUploadedFiles = file;
}
}
if(uploadIndexes){
list = doc.getElementsByTagName("authorization_info");
if(list.getLength() != 1){
throw new IllegalArgumentException("You should specify exactly 1 authorization_info element to upload indexes!");
}
Element authorization = (Element) list.item(0);
uploadToOsmandGooglecode = Boolean.parseBoolean(process.getAttribute("upload_osmand_googlecode"));
if(uploadToOsmandGooglecode){
user = authorization.getAttribute("google_code_user");
password = authorization.getAttribute("google_code_password");
cookieHSID = authorization.getAttribute("cookieHSID");
cookieSID = authorization.getAttribute("cookieSID");
pagegen = authorization.getAttribute("pagegen");
token = authorization.getAttribute("token");
if(googlePassword.length() > 0){
ExtractGooglecodeAuthorization gca = new ExtractGooglecodeAuthorization();
try {
GooglecodeUploadTokens tokens = gca.getGooglecodeTokensForUpload(user, googlePassword);
cookieHSID = tokens.getHsid();
cookieSID = tokens.getSid();
pagegen = tokens.getPagegen();
token = tokens.getToken();
} catch (IOException e) {
log.error("Error retrieving google tokens", e);
}
}
} else {
user = authorization.getAttribute("osmand_download_user");
password = authorization.getAttribute("osmand_download_password");
}
}
List<RegionCountries> countriesToDownload = new ArrayList<RegionCountries>();
parseCountriesToDownload(doc, countriesToDownload);
if (regions != null) {
@ -265,8 +175,6 @@ public class IndexBatchCreator {
}
runBatch(countriesToDownload);
}
private void parseCountriesToDownload(Document doc, List<RegionCountries> countriesToDownload) {
@ -340,22 +248,16 @@ public class IndexBatchCreator {
}
public void runBatch(List<RegionCountries> countriesToDownload ){
Set<String> alreadyUploadedFiles = new LinkedHashSet<String>();
Set<String> alreadyGeneratedFiles = new LinkedHashSet<String>();
if(downloadFiles){
downloadFilesAndGenerateIndex(countriesToDownload, alreadyGeneratedFiles, alreadyUploadedFiles);
}
if(generateIndexes && !downloadFiles){
generatedIndexes(alreadyGeneratedFiles, alreadyUploadedFiles);
}
if(uploadIndexes){
uploadIndexes(alreadyUploadedFiles);
if(countriesToDownload.isEmpty()){
downloadFilesAndGenerateIndex(countriesToDownload, alreadyGeneratedFiles);
}
generatedIndexes(alreadyGeneratedFiles);
}
protected void downloadFilesAndGenerateIndex(List<RegionCountries> countriesToDownload, Set<String> alreadyGeneratedFiles, Set<String> alreadyUploadedFiles){
protected void downloadFilesAndGenerateIndex(List<RegionCountries> countriesToDownload, Set<String> alreadyGeneratedFiles){
// clean before downloading
// for(File f : osmDirFiles.listFiles()){
// log.info("Delete old file " + f.getName()); //$NON-NLS-1$
@ -377,15 +279,15 @@ public class IndexBatchCreator {
if(skipGeneratedIndexes && bmif.exists()){
continue;
}
File toSave = downloadFile(url, fileName, alreadyGeneratedFiles, alreadyUploadedFiles);
if (toSave != null && generateIndexes) {
generateIndex(toSave, regionName, regionSpecificData, alreadyGeneratedFiles, alreadyUploadedFiles);
File toSave = downloadFile(url, fileName, alreadyGeneratedFiles);
if (toSave != null) {
generateIndex(toSave, regionName, regionSpecificData, alreadyGeneratedFiles);
}
}
}
}
protected File downloadFile(String url, String regionName, Set<String> alreadyGeneratedFiles, Set<String> alreadyUploadedFiles) {
protected File downloadFile(String url, String regionName, Set<String> alreadyGeneratedFiles) {
String ext = ".osm";
if(url.endsWith(".osm.bz2")){
ext = ".osm.bz2";
@ -440,7 +342,6 @@ public class IndexBatchCreator {
}
private final static int DOWNLOAD_DEBUG = 1 << 20;
private final static int MB = 1 << 20;
private final static int BUFFER_SIZE = 1 << 15;
private File internalDownload(String url, File toSave) {
int count = 0;
@ -482,13 +383,13 @@ public class IndexBatchCreator {
}
}
protected void generatedIndexes(Set<String> alreadyGeneratedFiles, Set<String> alreadyUploadedFiles) {
protected void generatedIndexes(Set<String> alreadyGeneratedFiles) {
for (File f : getSortedFiles(osmDirFiles)) {
if (alreadyGeneratedFiles.contains(f.getName())) {
continue;
}
if (f.getName().endsWith(".osm.bz2") || f.getName().endsWith(".osm") || f.getName().endsWith(".osm.pbf")) {
generateIndex(f, null, null, alreadyGeneratedFiles, alreadyUploadedFiles);
generateIndex(f, null, null, alreadyGeneratedFiles);
}
}
log.info("GENERATING INDEXES FINISHED ");
@ -496,10 +397,7 @@ public class IndexBatchCreator {
protected void generateIndex(File f, String rName, RegionSpecificData regionSpecificData, Set<String> alreadyGeneratedFiles, Set<String> alreadyUploadedFiles) {
if (!generateIndexes) {
return;
}
protected void generateIndex(File f, String rName, RegionSpecificData regionSpecificData, Set<String> alreadyGeneratedFiles) {
try {
// be independent of previous results
RTree.clearCache();
@ -543,19 +441,19 @@ public class IndexBatchCreator {
FileHandler fh = null;
// configure log path
try {
File fs = new File(workDir, mapFileName+".gen.log");
File fs = new File(workDir, mapFileName + ".gen.log");
FileOutputStream fout = new FileOutputStream(fs);
fout.write((new Date() + "\n").getBytes());
fout.write((Version.APP_MAP_CREATOR_FULL_NAME+"\n").getBytes());
fout.write((Version.APP_MAP_CREATOR_FULL_NAME + "\n").getBytes());
fout.close();
fh = new FileHandler(fs.getAbsolutePath(), 5000000, 1, false);
fh.setFormatter(new SimpleFormatter());
fh.setLevel(Level.ALL);
Jdk14Logger jdk14Logger = new Jdk14Logger("tempLogger");
jdk14Logger.getLogger().setLevel(Level.ALL);
jdk14Logger.getLogger().setUseParentHandlers(false);
jdk14Logger.getLogger().addHandler(fh);
warningsAboutMapData = jdk14Logger;
jdk14Logger.getLogger().setLevel(Level.ALL);
jdk14Logger.getLogger().setUseParentHandlers(false);
jdk14Logger.getLogger().addHandler(fh);
warningsAboutMapData = jdk14Logger;
} catch (SecurityException e1) {
e1.printStackTrace();
} catch (IOException e1) {
@ -570,10 +468,6 @@ public class IndexBatchCreator {
File generated = new File(workDir, mapFileName);
File ready = new File(indexDirFiles, mapFileName);
generated.renameTo(ready);
// Do not upload poi files any more
if (indexMap || indexAddress || indexTransport || indexPOI) {
uploadIndex(ready, alreadyUploadedFiles);
}
if(fh != null) {
LogManager.getLogManager().getLogger("").removeHandler(fh);
fh.close();
@ -599,258 +493,4 @@ public class IndexBatchCreator {
});
return listFiles;
}
protected void uploadIndexes(Set<String> alreadyUploadedFiles){
for(File f : getSortedFiles(indexDirFiles)){
if(!alreadyUploadedFiles.contains(f.getName()) && !f.isDirectory()){
uploadIndex(f, alreadyUploadedFiles);
if(!alreadyUploadedFiles.contains(f.getName())){
log.warn("! NOT UPLOADED " + f.getName());
}
}
}
log.info("UPLOADING INDEXES FINISHED ");
}
protected void uploadIndex(final File f, Set<String> alreadyUploadedFiles){
if(!uploadIndexes){
return;
}
String fileName = f.getName();
log.info("Upload index " + fileName);
String summary;
double mbLengh = (double)f.length() / MB;
boolean zip = true;
if(fileName.endsWith(IndexConstants.POI_INDEX_EXT) || fileName.endsWith(IndexConstants.POI_INDEX_EXT_ZIP)){
summary = "POI index for " ;
} else if(fileName.endsWith(IndexConstants.BINARY_MAP_INDEX_EXT) || fileName.endsWith(IndexConstants.BINARY_MAP_INDEX_EXT_ZIP)){
boolean addr = indexAddress;
boolean trans = indexTransport;
boolean map = indexMap;
boolean poi = indexPOI;
RandomAccessFile raf = null;
if (fileName.endsWith(IndexConstants.BINARY_MAP_INDEX_EXT)) {
try {
raf = new RandomAccessFile(f, "r");
BinaryMapIndexReader reader = new BinaryMapIndexReader(raf);
trans = reader.hasTransportData();
map = reader.containsMapData();
addr = reader.containsAddressData();
poi = reader.containsPoiData();
reader.close();
} catch (Exception e) {
log.info("File with not be uploaded! Exception", e);
if (raf != null) {
try {
raf.close();
} catch (IOException e1) {
}
}
//do not upload probably corrupted file
return;
}
}
summary = " index for ";
boolean fir = true;
if (addr) {
summary = "Address" + (fir ? "" : ", ") + summary;
fir = false;
}
if (trans) {
summary = "Transport" + (fir ? "" : ", ") + summary;
fir = false;
}
if (poi) {
summary = "POI" + (fir ? "" : ", ") + summary;
fir = false;
}
if (map) {
summary = "Map" + (fir ? "" : ", ") + summary;
fir = false;
}
} else {
return;
}
if(mbLengh < MIN_SIZE_TO_UPLOAD){
// do not upload small files
return;
}
String regionName = fileName.substring(0, fileName.lastIndexOf('_', fileName.indexOf('.')));
summary += regionName;
summary = summary.replace('_', ' ');
File toUpload = f;
if((fileName.endsWith(".odb") || fileName.endsWith(".obf")) && zip){
String n = fileName;
if(fileName.endsWith(".odb")){
n = fileName.substring(0, fileName.length() - 4);
}
String zipFileName = n+".zip";
log.info("Zipping file " + fileName);
try {
toUpload = IndexZipper.zip(f, zipFileName, summary);
} catch (OneFileException e) {
log.error("Exception while zipping file", e);
return ; //do not continue if error
}
if(f.delete()){
log.info("Source odb file was deleted");
}
}
List<File> splittedFiles = Collections.emptyList();
try {
splittedFiles = splitFiles(toUpload);
boolean uploaded = true;
for (File fs : splittedFiles) {
uploaded &= uploadFileToServer(fs, toUpload, summary);
}
// remove source file if file was splitted
if (uploaded) {
if (deleteFilesAfterUploading && toUpload.exists()) {
toUpload.delete();
} else if (backupUploadedFiles != null) {
File toBackup = new File(backupUploadedFiles, toUpload.getName());
if(toBackup.exists()){
toBackup.delete();
}
toUpload.renameTo(toBackup);
}
}
alreadyUploadedFiles.add(fileName);
} catch (IOException e) {
log.error("Input/output exception uploading " + fileName, e);
} finally {
// remove all splitted files
for(File fs : splittedFiles){
if(!fs.equals(toUpload)){
fs.delete();
}
}
}
}
private List<File> splitFiles(File f) throws IOException {
double mbLengh = (double)f.length() / MB;
if(mbLengh < MAX_SIZE_TO_NOT_SPLIT) {
return Collections.singletonList(f);
} else {
ArrayList<File> arrayList = new ArrayList<File>();
FileInputStream in = new FileInputStream(f);
byte[] buffer = new byte[BUFFER_SIZE];
int i = 1;
int read = 0;
while(read != -1){
File fout = new File(f.getParent(), f.getName() + "-"+i);
arrayList.add(fout);
FileOutputStream fo = new FileOutputStream(fout);
int limit = (int) (MAX_SIZE_TO_NOT_SPLIT * MB);
while(limit > 0 && ((read = in.read(buffer)) != -1)){
fo.write(buffer, 0, read);
limit -= read;
}
fo.flush();
fo.close();
i++;
}
in.close();
return arrayList;
}
}
private boolean uploadFileToServer(File f, File original, String summary) throws IOException {
if (f.length() / MB > MAX_UPLOAD_SIZE && uploadToOsmandGooglecode) {
System.err.println("ERROR : file " + f.getName() + " exceeded 200 mb!!! Could not be uploaded.");
return false; // restriction for google code
}
double originalLength = (double) original.length() / MB;
if (uploadToOsmandGooglecode) {
try {
DownloaderIndexFromGoogleCode.deleteFileFromGoogleDownloads(f.getName(), token, pagegen, cookieHSID, cookieSID);
if(f.getName().endsWith("obf.zip") && f.length() / MB < 5){
// try to delete without .zip part
DownloaderIndexFromGoogleCode.deleteFileFromGoogleDownloads(f.getName().substring(0, f.getName().length() - 4),
token, pagegen, cookieHSID, cookieSID);
} else if(f.getName().endsWith("poi.zip") && f.length() / MB < 5){
// try to delete without .zip part
DownloaderIndexFromGoogleCode.deleteFileFromGoogleDownloads(f.getName().substring(0, f.getName().length() - 3) +"odb",
token, pagegen, cookieHSID, cookieSID);
} else if(f.getName().endsWith(".zip-1") && f.length() / MB < 10){
DownloaderIndexFromGoogleCode.deleteFileFromGoogleDownloads(f.getName().substring(0, f.getName().length() - 2),
token, pagegen, cookieHSID, cookieSID);
}
try {
Thread.sleep(4000);
} catch (InterruptedException e) {
// wait 5 seconds
}
} catch (IOException e) {
log.warn("Deleting file from downloads" + f.getName() + " " + e.getMessage());
}
}
MessageFormat dateFormat = new MessageFormat("{0,date,dd.MM.yyyy}", Locale.US);
MessageFormat numberFormat = new MessageFormat("{0,number,##.#}", Locale.US);
String size = numberFormat.format(new Object[] { originalLength });
String date = dateFormat.format(new Object[] { new Date(original.lastModified()) });
if (!uploadToOsmandGooglecode) {
uploadToDownloadOsmandNet(f, summary, size, date);
} else {
String descriptionFile = "{" + date + " : " + size + " MB}";
summary += " " + descriptionFile;
GoogleCodeUploadIndex uploader = new GoogleCodeUploadIndex();
uploader.setFileName(f.getAbsolutePath());
uploader.setTargetFileName(f.getName());
uploader.setProjectName("osmand");
uploader.setUserName(user);
uploader.setPassword(password);
uploader.setLabels("Type-Archive, Testdata");
uploader.setSummary(summary);
uploader.setDescription(summary);
uploader.upload();
}
return true;
}
@SuppressWarnings("deprecation")
private void uploadToDownloadOsmandNet(File f, String description, String size, String date) throws IOException{
log.info("Uploading file " + f.getName() + " " + size + " MB " + date + " of " + description);
// Upload to ftp
FTPFileUpload upload = new FTPFileUpload();
upload.upload("download.osmand.net", user, password, "indexes/" + f.getName(), f, 1 << 15);
String url = "http://download.osmand.net/xml_update.php?";
url += "index="+URLEncoder.encode(f.getName());
url += "&description="+URLEncoder.encode(description);
url += "&date="+URLEncoder.encode(date);
url += "&size="+URLEncoder.encode(size);
url += "&action=update";
log.info("Updating index " + url); //$NON-NLS-1$//$NON-NLS-2$
HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();
connection.setRequestMethod("POST");
connection.connect();
if(connection.getResponseCode() != HttpURLConnection.HTTP_OK){
log.error("Error updating indexes " + connection.getResponseMessage());
}
InputStream is = connection.getInputStream();
while(is.read() != -1);
connection.disconnect();
log.info("Finish updating index");
}
}

414
DataExtractionOSM/src/net/osmand/data/index/IndexUploader.java Normal file
View file

@ -0,0 +1,414 @@
package net.osmand.data.index;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLEncoder;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.Enumeration;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipException;
import java.util.zip.ZipFile;
import java.util.zip.ZipOutputStream;
import org.apache.commons.logging.Log;
import net.osmand.Algoritms;
import net.osmand.LogUtil;
import net.osmand.binary.BinaryMapIndexReader;
import net.osmand.data.IndexConstants;
/**
* This helper will find obf and zip files, create description for them, and zip them, or update the description. This helper also can
* upload files through ssh.
*
* @author Pavol Zibrita <pavol.zibrita@gmail.com>
*/
public class IndexUploader {
protected static final Log log = LogUtil.getLog(IndexUploader.class);
private final static double MIN_SIZE_TO_UPLOAD = 0.015d;
private final static double MAX_SIZE_TO_NOT_SPLIT = 190d;
private final static double MAX_UPLOAD_SIZE = 195d;
private final static int BUFFER_SIZE = 1 << 15;
private final static int MB = 1 << 20;
/**
* Something bad have happend
*/
public static class IndexZipperException extends Exception {
private static final long serialVersionUID = 2343219168909577070L;
public IndexZipperException(String message) {
super(message);
}
}
/**
* Processing of one file failed, but other files could continue
*/
public static class OneFileException extends Exception {
private static final long serialVersionUID = 6463200194419498979L;
public OneFileException(String message) {
super(message);
}
}
private File directory;
private File targetDirectory;
public IndexUploader(String path, String targetPath) throws IndexZipperException {
directory = new File(path);
if (!directory.isDirectory()) {
throw new IndexZipperException("Not a directory:" + path);
}
}
private void run() {
for (File f : directory.listFiles()) {
try {
if (!f.isFile()) {
continue;
}
File unzipped = unzip(f);
String description = getDescription(unzipped);
zip(unzipped, getZipfileName(unzipped), description);
unzipped.delete(); // delete the unzipped file
} catch (OneFileException e) {
log.error(f.getName() + ": " + e.getMessage());
}
}
}
public static File zip(File f, String zipFileName, String description) throws OneFileException {
File zFile = new File(f.getParentFile(), zipFileName);
try {
log.info("Zipping to file: " + zipFileName + " file:" + f.getName() + " with desc:" + description);
ZipOutputStream zout = new ZipOutputStream(new FileOutputStream(zFile));
zout.setLevel(9);
ZipEntry zEntry = new ZipEntry(f.getName());
zEntry.setSize(f.length());
zEntry.setComment(description);
zout.putNextEntry(zEntry);
FileInputStream is = new FileInputStream(f);
copyAndClose(is, zout);
} catch (IOException e) {
throw new OneFileException("cannot zip file:" + e.getMessage());
}
return zFile;
}
private String getZipfileName(File unzipped) {
String fileName = unzipped.getName();
String n = fileName;
if (fileName.endsWith(".odb")) {
throw new UnsupportedOperationException("Odb is not supported any more");
// n = fileName.substring(0, fileName.length() - 4);
}
return n + ".zip";
}
private String getDescription(File f) throws OneFileException {
String fileName = f.getName();
String summary = null;
if (fileName.endsWith(IndexConstants.POI_INDEX_EXT) || fileName.endsWith(IndexConstants.POI_INDEX_EXT_ZIP)) {
summary = "POI index for ";
} else if (fileName.endsWith(IndexConstants.BINARY_MAP_INDEX_EXT) || fileName.endsWith(IndexConstants.BINARY_MAP_INDEX_EXT_ZIP)) {
RandomAccessFile raf = null;
try {
raf = new RandomAccessFile(f, "r");
BinaryMapIndexReader reader = new BinaryMapIndexReader(raf);
summary = " index for ";
boolean fir = true;
if (reader.containsAddressData()) {
summary = "Address" + (fir ? "" : ", ") + summary;
fir = false;
}
if (reader.hasTransportData()) {
summary = "Transport" + (fir ? "" : ", ") + summary;
fir = false;
}
if (reader.containsPoiData()) {
summary = "POI" + (fir ? "" : ", ") + summary;
fir = false;
}
if (reader.containsMapData()) {
summary = "Map" + (fir ? "" : ", ") + summary;
fir = false;
}
reader.close();
} catch (IOException e) {
if (raf != null) {
try {
raf.close();
} catch (IOException e1) {
}
}
throw new OneFileException("Reader could not read the index: " + e.getMessage());
}
} else {
throw new OneFileException("Not a processable file.");
}
String regionName = fileName.substring(0, fileName.lastIndexOf('_', fileName.indexOf('.')));
summary += regionName;
summary = summary.replace('_', ' ');
return summary;
}
private File unzip(File f) throws OneFileException {
try {
if (!Algoritms.isZipFile(f)) {
return f;
}
log.info("Unzipping file: " + f.getName());
ZipFile zipFile;
zipFile = new ZipFile(f);
Enumeration<? extends ZipEntry> entries = zipFile.entries();
while (entries.hasMoreElements()) {
ZipEntry entry = entries.nextElement();
File tempFile = new File(f.getParentFile(), entry.getName());
copyAndClose(zipFile.getInputStream(entry), new FileOutputStream(tempFile));
return tempFile;
}
return null;
} catch (ZipException e) {
throw new OneFileException("cannot unzip:" + e.getMessage());
} catch (IOException e) {
throw new OneFileException("cannot unzip:" + e.getMessage());
}
}
public static void copyAndClose(InputStream in, OutputStream out) throws IOException {
Algoritms.streamCopy(in, out);
Algoritms.closeStream(in);
Algoritms.closeStream(out);
}
private List<File> splitFiles(File f) throws IOException {
double mbLengh = (double) f.length() / MB;
if (mbLengh < MAX_SIZE_TO_NOT_SPLIT) {
return Collections.singletonList(f);
} else {
ArrayList<File> arrayList = new ArrayList<File>();
FileInputStream in = new FileInputStream(f);
byte[] buffer = new byte[BUFFER_SIZE];
int i = 1;
int read = 0;
while (read != -1) {
File fout = new File(f.getParent(), f.getName() + "-" + i);
arrayList.add(fout);
FileOutputStream fo = new FileOutputStream(fout);
int limit = (int) (MAX_SIZE_TO_NOT_SPLIT * MB);
while (limit > 0 && ((read = in.read(buffer)) != -1)) {
fo.write(buffer, 0, read);
limit -= read;
}
fo.flush();
fo.close();
i++;
}
in.close();
return arrayList;
}
}
public void uploadIndex(File indexFile, File dirToBackup, Set<String> alreadyUploadedFiles, String user, String password) {
double mbLengh = (double) indexFile.length() / MB;
String fileName = indexFile.getName();
if (mbLengh < MIN_SIZE_TO_UPLOAD) {
log.info("Skip uploading index due to size " + fileName);
// do not upload small files
return;
}
try {
log.info("Upload index " + fileName);
String summary = getDescription(indexFile);
File toUpload = indexFile;
if (fileName.endsWith(".obf")) {
String zipFileName = fileName + ".zip";
log.info("Zipping file " + fileName);
toUpload = IndexUploader.zip(indexFile, zipFileName, summary);
if (indexFile.delete()) {
log.info("Source obf file was deleted.");
}
}
boolean uploaded = uploadFileToServer(toUpload, summary, user, password, false, null, null, null, null);
// remove source file if file was splitted
if (uploaded && dirToBackup != null) {
File toBackup = new File(dirToBackup, toUpload.getName());
if (toBackup.exists()) {
toBackup.delete();
}
toUpload.renameTo(toBackup);
}
alreadyUploadedFiles.add(toUpload.getName());
} catch (OneFileException e) {
log.error("Exception ", e);
return; // do not continue if error
} catch (IOException e) {
log.error("Input/output exception uploading " + fileName, e);
}
}
public static void main(String[] args) {
try {
String srcPath = extractDirectory(args, 0);
String targetPath = srcPath;
if (args.length > 1) {
targetPath = extractDirectory(args, 1);
}
IndexUploader indexZipper = new IndexUploader(srcPath, targetPath);
indexZipper.run();
} catch (IndexZipperException e) {
log.error(e.getMessage());
}
}
private static String extractDirectory(String[] args, int ind) throws IndexZipperException {
if (args.length > ind) {
if ("-h".equals(args[0])) {
throw new IndexZipperException("Usage: IndexZipper [directory] (if not specified, the current one will be taken)");
} else {
return args[ind];
}
}
return ".";
}
@Deprecated
public void uploadToGoogleCode(File f, String summary, String user, String password, String token, String pagegen, String cookieHSID,
String cookieSID) throws IOException {
if (f.length() / MB > MAX_UPLOAD_SIZE) {
System.err.println("ERROR : file " + f.getName() + " exceeded 200 mb!!! Could not be uploaded.");
throw new IOException("ERROR : file " + f.getName() + " exceeded 200 mb!!! Could not be uploaded.");
// restriction for google code
}
try {
DownloaderIndexFromGoogleCode.deleteFileFromGoogleDownloads(f.getName(), token, pagegen, cookieHSID, cookieSID);
if (f.getName().endsWith("obf.zip") && f.length() / MB < 5) {
// try to delete without .zip part
DownloaderIndexFromGoogleCode.deleteFileFromGoogleDownloads(f.getName().substring(0, f.getName().length() - 4), token,
pagegen, cookieHSID, cookieSID);
} else if (f.getName().endsWith("poi.zip") && f.length() / MB < 5) {
// try to delete without .zip part
DownloaderIndexFromGoogleCode.deleteFileFromGoogleDownloads(f.getName().substring(0, f.getName().length() - 3) + "odb",
token, pagegen, cookieHSID, cookieSID);
} else if (f.getName().endsWith(".zip-1")) {
DownloaderIndexFromGoogleCode.deleteFileFromGoogleDownloads(f.getName().substring(0, f.getName().length() - 2), token,
pagegen, cookieHSID, cookieSID);
}
try {
Thread.sleep(4000);
} catch (InterruptedException e) {
// wait 5 seconds
}
} catch (IOException e) {
log.warn("Deleting file from downloads" + f.getName() + " " + e.getMessage());
}
GoogleCodeUploadIndex uploader = new GoogleCodeUploadIndex();
uploader.setFileName(f.getAbsolutePath());
uploader.setTargetFileName(f.getName());
uploader.setProjectName("osmand");
uploader.setUserName(user);
uploader.setPassword(password);
uploader.setLabels("Type-Archive, Testdata");
uploader.setSummary(summary);
uploader.setDescription(summary);
uploader.upload();
}
@Deprecated
public boolean uploadFileToServer(File original, String summary, String user, String password, boolean uploadToOsmandGooglecode,
String token, String pagegen, String cookieHSID, String cookieSID) throws IOException {
double originalLength = (double) original.length() / MB;
MessageFormat dateFormat = new MessageFormat("{0,date,dd.MM.yyyy}", Locale.US);
MessageFormat numberFormat = new MessageFormat("{0,number,##.#}", Locale.US);
String size = numberFormat.format(new Object[] { originalLength });
String date = dateFormat.format(new Object[] { new Date(original.lastModified()) });
try {
if (uploadToOsmandGooglecode) {
uploadToDownloadOsmandNet(original, summary, size, date, user, password);
} else {
String descriptionFile = "{" + date + " : " + size + " MB}";
summary += " " + descriptionFile;
List<File> splittedFiles = Collections.emptyList();
try {
splittedFiles = splitFiles(original);
for (File fs : splittedFiles) {
uploadToGoogleCode(fs, summary, user, password, token, pagegen, cookieHSID, cookieSID);
}
} finally {
// remove all splitted files
for (File fs : splittedFiles) {
if (!fs.equals(original)) {
fs.delete();
}
}
}
}
} catch (IOException e) {
log.error("Input/output exception uploading " + original.getName(), e);
return false;
}
return true;
}
@Deprecated
public void uploadToDownloadOsmandNet(File f, String description, String size, String date, String user, String password)
throws IOException {
log.info("Uploading file " + f.getName() + " " + size + " MB " + date + " of " + description);
// Upload to ftp
FTPFileUpload upload = new FTPFileUpload();
upload.upload("download.osmand.net", user, password, "indexes/" + f.getName(), f, 1 << 15);
String url = "http://download.osmand.net/xml_update.php?";
url += "index=" + URLEncoder.encode(f.getName());
url += "&description=" + URLEncoder.encode(description);
url += "&date=" + URLEncoder.encode(date);
url += "&size=" + URLEncoder.encode(size);
url += "&action=update";
log.info("Updating index " + url); //$NON-NLS-1$//$NON-NLS-2$
HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();
connection.setRequestMethod("POST");
connection.connect();
if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) {
log.error("Error updating indexes " + connection.getResponseMessage());
}
InputStream is = connection.getInputStream();
while (is.read() != -1)
;
connection.disconnect();
log.info("Finish updating index");
}
}

227
DataExtractionOSM/src/net/osmand/data/index/IndexZipper.java
View file

@ -1,227 +0,0 @@
package net.osmand.data.index;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.util.Enumeration;
import java.util.zip.ZipEntry;
import java.util.zip.ZipException;
import java.util.zip.ZipFile;
import java.util.zip.ZipOutputStream;
import org.apache.commons.logging.Log;
import net.osmand.Algoritms;
import net.osmand.LogUtil;
import net.osmand.binary.BinaryMapIndexReader;
import net.osmand.data.IndexConstants;
/**
* This helper will find obf and zip files, create description for them, and zip
* them, or update the description.
*
* @author Pavol Zibrita <pavol.zibrita@gmail.com>
*/
public class IndexZipper {
protected static final Log log = LogUtil.getLog(IndexZipper.class);
/**
* Something bad have happend
*/
public static class IndexZipperException extends Exception {
private static final long serialVersionUID = 2343219168909577070L;
public IndexZipperException(String message) {
super(message);
}
}
/**
* Processing of one file failed, but other files could continue
*/
public static class OneFileException extends Exception {
private static final long serialVersionUID = 6463200194419498979L;
public OneFileException(String message) {
super(message);
}
}
private File directory;
private File targetDirectory;
public IndexZipper(String path, String targetPath) throws IndexZipperException {
directory = new File(path);
if (!directory.isDirectory()) {
throw new IndexZipperException("Not a directory:" + path);
}
}
private void run() {
for (File f : directory.listFiles()) {
try {
if (!f.isFile()) {
continue;
}
File unzipped = unzip(f);
String description = getDescription(unzipped);
zip(unzipped, getZipfileName(unzipped), description);
unzipped.delete(); // delete the unzipped file
} catch (OneFileException e) {
log.error(f.getName() + ": " + e.getMessage());
}
}
}
public static File zip(File f, String zipFileName, String description)
throws OneFileException {
File zFile = new File(f.getParentFile(), zipFileName);
try {
log.info("Zipping to file: " + zipFileName + " file:" + f.getName() + " with desc:" + description);
ZipOutputStream zout = new ZipOutputStream(new FileOutputStream(
zFile));
zout.setLevel(9);
ZipEntry zEntry = new ZipEntry(f.getName());
zEntry.setSize(f.length());
zEntry.setComment(description);
zout.putNextEntry(zEntry);
FileInputStream is = new FileInputStream(f);
copyAndClose(is, zout);
} catch (IOException e) {
throw new OneFileException("cannot zip file:" + e.getMessage());
}
return zFile;
}
private String getZipfileName(File unzipped) {
String fileName = unzipped.getName();
String n = fileName;
if (fileName.endsWith(".odb")) {
throw new UnsupportedOperationException("Odb is not supported any more");
// n = fileName.substring(0, fileName.length() - 4);
}
return n + ".zip";
}
private String getDescription(File f) throws OneFileException {
String fileName = f.getName();
String summary = null;
if (fileName.endsWith(IndexConstants.POI_INDEX_EXT)
|| fileName.endsWith(IndexConstants.POI_INDEX_EXT_ZIP)) {
summary = "POI index for ";
} else if (fileName.endsWith(IndexConstants.BINARY_MAP_INDEX_EXT)
|| fileName.endsWith(IndexConstants.BINARY_MAP_INDEX_EXT_ZIP)) {
RandomAccessFile raf = null;
try {
raf = new RandomAccessFile(f, "r");
BinaryMapIndexReader reader = new BinaryMapIndexReader(raf);
summary = " index for ";
boolean fir = true;
if (reader.containsAddressData()) {
summary = "Address" + (fir ? "" : ", ") + summary;
fir = false;
}
if (reader.hasTransportData()) {
summary = "Transport" + (fir ? "" : ", ") + summary;
fir = false;
}
if (reader.containsPoiData()) {
summary = "POI" + (fir ? "" : ", ") + summary;
fir = false;
}
if (reader.containsMapData()) {
summary = "Map" + (fir ? "" : ", ") + summary;
fir = false;
}
reader.close();
} catch (IOException e) {
if (raf != null) {
try {
raf.close();
} catch (IOException e1) {
}
}
throw new OneFileException("Reader could not read the index: "
+ e.getMessage());
}
} else {
throw new OneFileException("Not a processable file.");
}
String regionName = fileName.substring(0,
fileName.lastIndexOf('_', fileName.indexOf('.')));
summary += regionName;
summary = summary.replace('_', ' ');
return summary;
}
private File unzip(File f) throws OneFileException {
try {
if (!Algoritms.isZipFile(f)) {
return f;
}
log.info("Unzipping file: " + f.getName());
ZipFile zipFile;
zipFile = new ZipFile(f);
Enumeration<? extends ZipEntry> entries = zipFile.entries();
while (entries.hasMoreElements()) {
ZipEntry entry = entries.nextElement();
File tempFile = new File(f.getParentFile(), entry.getName());
copyAndClose(zipFile.getInputStream(entry),
new FileOutputStream(tempFile));
return tempFile;
}
return null;
} catch (ZipException e) {
throw new OneFileException("cannot unzip:" + e.getMessage());
} catch (IOException e) {
throw new OneFileException("cannot unzip:" + e.getMessage());
}
}
public static void copyAndClose(InputStream in, OutputStream out)
throws IOException {
Algoritms.streamCopy(in, out);
Algoritms.closeStream(in);
Algoritms.closeStream(out);
}
public static void main(String[] args) {
try {
String srcPath = extractDirectory(args, 0);
String targetPath = srcPath;
if(args.length > 1) {
targetPath = extractDirectory(args, 1);
}
IndexZipper indexZipper = new IndexZipper(srcPath, targetPath);
indexZipper.run();
} catch (IndexZipperException e) {
log.error(e.getMessage());
}
}
private static String extractDirectory(String[] args, int ind)
throws IndexZipperException {
if (args.length > ind) {
if ("-h".equals(args[0])) {
throw new IndexZipperException(
"Usage: IndexZipper [directory] (if not specified, the current one will be taken)");
} else {
return args[ind];
}
}
return ".";
}
}

21
DataExtractionOSM/src/net/osmand/data/index/batch.xml
View file

@ -1,19 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
<batch_process>
<!-- These attributes are required to upload/delete downloads
* cookieHSID - cookie HSID is required for delete index (could be taken from source (!) of page when you are about delete index)
* cookieSID - cookie SID is required for delete index (could be taken from source (!) of page when you are about delete index)
* pagegen - pagegen attribute is required for delete index (could be taken from source (!) of page when you are about delete index)
* token - token attribute is required for delete index (could be taken from source (!) of page when you are about delete index)
* google_code_user - is required to upload index
* google_code_password - is required to upload index (is not gmail password!) - could be found in profile
-->
<authorization_info cookieHSID=""
cookieSID=""
pagegen="" token=""
google_code_user="" google_code_password=""
osmand_download_user="" osmand_download_password=""/>
<process_attributes mapZooms="" renderingTypesFile="" zoomWaySmoothness=""
osmDbDialect="sqlite" mapDbDialect="sqlite"/>
@ -26,11 +12,8 @@
or you can upload any file you have to googlecode (just put into 'directory_for_index_files')
-->
<!-- zoomWaySmoothness - 1-4, typical mapZooms - 8-10;11-12;13-14;15 -->
<process directory_for_osm_files="/home/..." directory_for_index_files="/home/..."
directory_for_uploaded_files="" keepPoiOdb="false"
downloadOsmFiles="true" generateIndexes="true" uploadIndexes="true"
upload_osmand_googlecode="true" deleteFilesAfterUploading="true" indexPOI="true" indexMap="true"
indexTransport="true" indexAddress="true">
<process directory_for_osm_files="/home/..." directory_for_index_files="/home/..." directory_for_generation="/home/..."
skipGeneratedIndexes="true" indexPOI="true" indexMap="true" indexTransport="true" indexAddress="true">
<!-- Add wget="C:/Program Files/GNUWin32/bin/wget.exe" to process, to use wget for download.
On linux systems if wget is in your path it can be wget="wget" or you can make own script with wget command:
wget="/path/to/script/wget.sh"

2
DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java
View file

@ -50,7 +50,7 @@ public class IndexCreator {
// Sqlite better to use only for 32-bit machines
public static DBDialect dialect = DBDialect.SQLITE;
public static DBDialect mapDBDialect = DBDialect.SQLITE;
public static boolean REMOVE_POI_DB = false;
public static boolean REMOVE_POI_DB = true;
public static final int BATCH_SIZE = 5000;
public static final int BATCH_SIZE_OSM = 10000;

21
build-scripts/indexes-batch-generate.xml
View file

@ -1,19 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
<batch_process>
<!-- These attributes are required to upload/delete downloads
* cookieHSID - cookie HSID is required for delete index (could be taken from source (!) of page when you are about delete index)
* cookieSID - cookie SID is required for delete index (could be taken from source (!) of page when you are about delete index)
* pagegen - pagegen attribute is required for delete index (could be taken from source (!) of page when you are about delete index)
* token - token attribute is required for delete index (could be taken from source (!) of page when you are about delete index)
* google_code_user - is required to upload index
* google_code_password - is required to upload index (is not gmail password!) - could be found in profile
-->
<authorization_info cookieHSID=""
cookieSID=""
pagegen="" token=""
google_code_user="" google_code_password=""
osmand_download_user="" osmand_download_password=""/>
<process_attributes mapZooms="" renderingTypesFile="" zoomWaySmoothness="2"
osmDbDialect="sqlite" mapDbDialect="sqlite"/>
@ -26,11 +12,8 @@
or you can upload any file you have to googlecode (just put into 'directory_for_index_files')
-->
<!-- zoomWaySmoothness - 1-4, typical mapZooms - 8-10;11-12;13-14;15 -->
<process directory_for_osm_files=".work/osm" directory_for_index_files="/var/lib/jenkins/indexes"
directory_for_uploaded_files="/var/lib/jenkins/indexes/uploaded" keepPoiOdb="false" directory_for_generation=".work"
downloadOsmFiles="true" generateIndexes="true" uploadIndexes="false" skipGeneratedIndexes="true"
upload_osmand_googlecode="false" deleteFilesAfterUploading="false" indexPOI="true" indexMap="true"
indexTransport="true" indexAddress="true">
<process directory_for_osm_files=".work/osm" directory_for_index_files="/var/lib/jenkins/indexes" directory_for_generation=".work"
skipGeneratedIndexes="true" indexPOI="true" indexMap="true" indexTransport="true" indexAddress="true">
<!-- Add wget="C:/Program Files/GNUWin32/bin/wget.exe" to process, to use wget for download.
On linux systems if wget is in your path it can be wget="wget" or you can make own script with wget command:
wget="/path/to/script/wget.sh"