Replace IndexZipper with IndexUploader
This commit is contained in:
Victor Shcherb
parent
50eaa9b337
commit
81cc30aa91
6 changed files with 437 additions and 644 deletions
|
|
@ -10,26 +10,19 @@ import java.io.IOException;
|
|||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.OutputStream;
|
||||
import java.io.RandomAccessFile;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URL;
|
||||
import java.net.URLEncoder;
|
||||
import java.text.MessageFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.Date;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.logging.FileHandler;
|
||||
import java.util.logging.Handler;
|
||||
import java.util.logging.LogManager;
|
||||
import java.util.logging.Logger;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.SimpleFormatter;
|
||||
|
||||
|
|
@ -39,10 +32,7 @@ import javax.xml.parsers.ParserConfigurationException;
|
|||
import net.osmand.Algoritms;
|
||||
import net.osmand.LogUtil;
|
||||
import net.osmand.Version;
|
||||
import net.osmand.binary.BinaryMapIndexReader;
|
||||
import net.osmand.data.IndexConstants;
|
||||
import net.osmand.data.index.ExtractGooglecodeAuthorization.GooglecodeUploadTokens;
|
||||
import net.osmand.data.index.IndexZipper.OneFileException;
|
||||
import net.osmand.data.preparation.DBDialect;
|
||||
import net.osmand.data.preparation.IndexCreator;
|
||||
import net.osmand.data.preparation.MapZooms;
|
||||
|
|
@ -51,25 +41,17 @@ import net.osmand.osm.MapRenderingTypes;
|
|||
import net.osmand.swing.OsmExtractionUI;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.commons.logging.impl.Jdk14Logger;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
import org.w3c.dom.NodeList;
|
||||
import org.xml.sax.SAXException;
|
||||
|
||||
|
||||
import rtree.RTree;
|
||||
|
||||
|
||||
|
||||
public class IndexBatchCreator {
|
||||
|
||||
|
||||
protected static final Log log = LogUtil.getLog(IndexBatchCreator.class);
|
||||
private final static double MIN_SIZE_TO_UPLOAD = 0.015d;
|
||||
private final static double MAX_SIZE_TO_NOT_SPLIT = 190d;
|
||||
private final static double MAX_UPLOAD_SIZE = 195d;
|
||||
|
||||
|
||||
public static class RegionCountries {
|
||||
|
|
@ -83,44 +65,25 @@ public class IndexBatchCreator {
|
|||
public String cityAdminLevel;
|
||||
}
|
||||
|
||||
private boolean uploadToOsmandGooglecode = true;
|
||||
|
||||
|
||||
// process atributtes
|
||||
boolean downloadFiles = false;
|
||||
boolean generateIndexes = false;
|
||||
boolean uploadIndexes = false;
|
||||
boolean skipGeneratedIndexes = false;
|
||||
MapZooms mapZooms = null;
|
||||
Integer zoomWaySmoothness = null;
|
||||
MapRenderingTypes types = MapRenderingTypes.getDefault();
|
||||
boolean deleteFilesAfterUploading = true;
|
||||
|
||||
File osmDirFiles;
|
||||
File indexDirFiles;
|
||||
File workDir;
|
||||
File backupUploadedFiles;
|
||||
|
||||
boolean indexPOI = false;
|
||||
boolean indexTransport = false;
|
||||
boolean indexAddress = false;
|
||||
boolean indexMap = false;
|
||||
|
||||
|
||||
String user;
|
||||
String password;
|
||||
|
||||
private String wget;
|
||||
|
||||
String googlePassword = "";
|
||||
String cookieHSID = "";
|
||||
String cookieSID = "";
|
||||
String pagegen = "";
|
||||
String token = "";
|
||||
|
||||
|
||||
public static void main(String[] args) {
|
||||
|
||||
IndexBatchCreator creator = new IndexBatchCreator();
|
||||
OsmExtractionUI.configLogFile();
|
||||
if(args == null || args.length == 0){
|
||||
|
|
@ -174,11 +137,7 @@ public class IndexBatchCreator {
|
|||
throw new IllegalArgumentException("You should specify exactly 1 process element!");
|
||||
}
|
||||
Element process = (Element) list.item(0);
|
||||
downloadFiles = Boolean.parseBoolean(process.getAttribute("downloadOsmFiles"));
|
||||
generateIndexes = Boolean.parseBoolean(process.getAttribute("generateIndexes"));
|
||||
uploadIndexes = Boolean.parseBoolean(process.getAttribute("uploadIndexes"));
|
||||
deleteFilesAfterUploading = Boolean.parseBoolean(process.getAttribute("deleteFilesAfterUploading"));
|
||||
IndexCreator.REMOVE_POI_DB = !Boolean.parseBoolean(process.getAttribute("keepPoiOdb"));
|
||||
IndexCreator.REMOVE_POI_DB = true;
|
||||
skipGeneratedIndexes = Boolean.parseBoolean(process.getAttribute("skipGeneratedIndexes"));
|
||||
wget = process.getAttribute("wget");
|
||||
|
||||
|
|
@ -209,55 +168,6 @@ public class IndexBatchCreator {
|
|||
workDir = new File(dir);
|
||||
}
|
||||
|
||||
if (downloadFiles) {
|
||||
if (regions == null) {
|
||||
throw new IllegalArgumentException("Please specify regions.xml file as 2nd parameter");
|
||||
}
|
||||
}
|
||||
|
||||
dir = process.getAttribute("directory_for_uploaded_files");
|
||||
if (dir != null) {
|
||||
File file = new File(dir);
|
||||
file.mkdirs();
|
||||
if (file.exists()) {
|
||||
backupUploadedFiles = file;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(uploadIndexes){
|
||||
list = doc.getElementsByTagName("authorization_info");
|
||||
if(list.getLength() != 1){
|
||||
throw new IllegalArgumentException("You should specify exactly 1 authorization_info element to upload indexes!");
|
||||
}
|
||||
Element authorization = (Element) list.item(0);
|
||||
uploadToOsmandGooglecode = Boolean.parseBoolean(process.getAttribute("upload_osmand_googlecode"));
|
||||
if(uploadToOsmandGooglecode){
|
||||
user = authorization.getAttribute("google_code_user");
|
||||
password = authorization.getAttribute("google_code_password");
|
||||
|
||||
cookieHSID = authorization.getAttribute("cookieHSID");
|
||||
cookieSID = authorization.getAttribute("cookieSID");
|
||||
pagegen = authorization.getAttribute("pagegen");
|
||||
token = authorization.getAttribute("token");
|
||||
if(googlePassword.length() > 0){
|
||||
ExtractGooglecodeAuthorization gca = new ExtractGooglecodeAuthorization();
|
||||
try {
|
||||
GooglecodeUploadTokens tokens = gca.getGooglecodeTokensForUpload(user, googlePassword);
|
||||
cookieHSID = tokens.getHsid();
|
||||
cookieSID = tokens.getSid();
|
||||
pagegen = tokens.getPagegen();
|
||||
token = tokens.getToken();
|
||||
} catch (IOException e) {
|
||||
log.error("Error retrieving google tokens", e);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
user = authorization.getAttribute("osmand_download_user");
|
||||
password = authorization.getAttribute("osmand_download_password");
|
||||
}
|
||||
}
|
||||
|
||||
List<RegionCountries> countriesToDownload = new ArrayList<RegionCountries>();
|
||||
parseCountriesToDownload(doc, countriesToDownload);
|
||||
if (regions != null) {
|
||||
|
|
@ -265,8 +175,6 @@ public class IndexBatchCreator {
|
|||
}
|
||||
|
||||
runBatch(countriesToDownload);
|
||||
|
||||
|
||||
}
|
||||
|
||||
private void parseCountriesToDownload(Document doc, List<RegionCountries> countriesToDownload) {
|
||||
|
|
@ -340,22 +248,16 @@ public class IndexBatchCreator {
|
|||
}
|
||||
|
||||
public void runBatch(List<RegionCountries> countriesToDownload ){
|
||||
Set<String> alreadyUploadedFiles = new LinkedHashSet<String>();
|
||||
Set<String> alreadyGeneratedFiles = new LinkedHashSet<String>();
|
||||
if(downloadFiles){
|
||||
downloadFilesAndGenerateIndex(countriesToDownload, alreadyGeneratedFiles, alreadyUploadedFiles);
|
||||
}
|
||||
if(generateIndexes && !downloadFiles){
|
||||
generatedIndexes(alreadyGeneratedFiles, alreadyUploadedFiles);
|
||||
}
|
||||
if(uploadIndexes){
|
||||
uploadIndexes(alreadyUploadedFiles);
|
||||
if(countriesToDownload.isEmpty()){
|
||||
downloadFilesAndGenerateIndex(countriesToDownload, alreadyGeneratedFiles);
|
||||
}
|
||||
generatedIndexes(alreadyGeneratedFiles);
|
||||
}
|
||||
|
||||
|
||||
|
||||
protected void downloadFilesAndGenerateIndex(List<RegionCountries> countriesToDownload, Set<String> alreadyGeneratedFiles, Set<String> alreadyUploadedFiles){
|
||||
protected void downloadFilesAndGenerateIndex(List<RegionCountries> countriesToDownload, Set<String> alreadyGeneratedFiles){
|
||||
// clean before downloading
|
||||
// for(File f : osmDirFiles.listFiles()){
|
||||
// log.info("Delete old file " + f.getName()); //$NON-NLS-1$
|
||||
|
|
@ -377,15 +279,15 @@ public class IndexBatchCreator {
|
|||
if(skipGeneratedIndexes && bmif.exists()){
|
||||
continue;
|
||||
}
|
||||
File toSave = downloadFile(url, fileName, alreadyGeneratedFiles, alreadyUploadedFiles);
|
||||
if (toSave != null && generateIndexes) {
|
||||
generateIndex(toSave, regionName, regionSpecificData, alreadyGeneratedFiles, alreadyUploadedFiles);
|
||||
File toSave = downloadFile(url, fileName, alreadyGeneratedFiles);
|
||||
if (toSave != null) {
|
||||
generateIndex(toSave, regionName, regionSpecificData, alreadyGeneratedFiles);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected File downloadFile(String url, String regionName, Set<String> alreadyGeneratedFiles, Set<String> alreadyUploadedFiles) {
|
||||
protected File downloadFile(String url, String regionName, Set<String> alreadyGeneratedFiles) {
|
||||
String ext = ".osm";
|
||||
if(url.endsWith(".osm.bz2")){
|
||||
ext = ".osm.bz2";
|
||||
|
|
@ -440,7 +342,6 @@ public class IndexBatchCreator {
|
|||
}
|
||||
|
||||
private final static int DOWNLOAD_DEBUG = 1 << 20;
|
||||
private final static int MB = 1 << 20;
|
||||
private final static int BUFFER_SIZE = 1 << 15;
|
||||
private File internalDownload(String url, File toSave) {
|
||||
int count = 0;
|
||||
|
|
@ -482,13 +383,13 @@ public class IndexBatchCreator {
|
|||
}
|
||||
}
|
||||
|
||||
protected void generatedIndexes(Set<String> alreadyGeneratedFiles, Set<String> alreadyUploadedFiles) {
|
||||
protected void generatedIndexes(Set<String> alreadyGeneratedFiles) {
|
||||
for (File f : getSortedFiles(osmDirFiles)) {
|
||||
if (alreadyGeneratedFiles.contains(f.getName())) {
|
||||
continue;
|
||||
}
|
||||
if (f.getName().endsWith(".osm.bz2") || f.getName().endsWith(".osm") || f.getName().endsWith(".osm.pbf")) {
|
||||
generateIndex(f, null, null, alreadyGeneratedFiles, alreadyUploadedFiles);
|
||||
generateIndex(f, null, null, alreadyGeneratedFiles);
|
||||
}
|
||||
}
|
||||
log.info("GENERATING INDEXES FINISHED ");
|
||||
|
|
@ -496,10 +397,7 @@ public class IndexBatchCreator {
|
|||
|
||||
|
||||
|
||||
protected void generateIndex(File f, String rName, RegionSpecificData regionSpecificData, Set<String> alreadyGeneratedFiles, Set<String> alreadyUploadedFiles) {
|
||||
if (!generateIndexes) {
|
||||
return;
|
||||
}
|
||||
protected void generateIndex(File f, String rName, RegionSpecificData regionSpecificData, Set<String> alreadyGeneratedFiles) {
|
||||
try {
|
||||
// be independent of previous results
|
||||
RTree.clearCache();
|
||||
|
|
@ -543,19 +441,19 @@ public class IndexBatchCreator {
|
|||
FileHandler fh = null;
|
||||
// configure log path
|
||||
try {
|
||||
File fs = new File(workDir, mapFileName+".gen.log");
|
||||
File fs = new File(workDir, mapFileName + ".gen.log");
|
||||
FileOutputStream fout = new FileOutputStream(fs);
|
||||
fout.write((new Date() + "\n").getBytes());
|
||||
fout.write((Version.APP_MAP_CREATOR_FULL_NAME+"\n").getBytes());
|
||||
fout.write((Version.APP_MAP_CREATOR_FULL_NAME + "\n").getBytes());
|
||||
fout.close();
|
||||
fh = new FileHandler(fs.getAbsolutePath(), 5000000, 1, false);
|
||||
fh.setFormatter(new SimpleFormatter());
|
||||
fh.setLevel(Level.ALL);
|
||||
Jdk14Logger jdk14Logger = new Jdk14Logger("tempLogger");
|
||||
jdk14Logger.getLogger().setLevel(Level.ALL);
|
||||
jdk14Logger.getLogger().setUseParentHandlers(false);
|
||||
jdk14Logger.getLogger().addHandler(fh);
|
||||
warningsAboutMapData = jdk14Logger;
|
||||
jdk14Logger.getLogger().setLevel(Level.ALL);
|
||||
jdk14Logger.getLogger().setUseParentHandlers(false);
|
||||
jdk14Logger.getLogger().addHandler(fh);
|
||||
warningsAboutMapData = jdk14Logger;
|
||||
} catch (SecurityException e1) {
|
||||
e1.printStackTrace();
|
||||
} catch (IOException e1) {
|
||||
|
|
@ -570,10 +468,6 @@ public class IndexBatchCreator {
|
|||
File generated = new File(workDir, mapFileName);
|
||||
File ready = new File(indexDirFiles, mapFileName);
|
||||
generated.renameTo(ready);
|
||||
// Do not upload poi files any more
|
||||
if (indexMap || indexAddress || indexTransport || indexPOI) {
|
||||
uploadIndex(ready, alreadyUploadedFiles);
|
||||
}
|
||||
if(fh != null) {
|
||||
LogManager.getLogManager().getLogger("").removeHandler(fh);
|
||||
fh.close();
|
||||
|
|
@ -599,258 +493,4 @@ public class IndexBatchCreator {
|
|||
});
|
||||
return listFiles;
|
||||
}
|
||||
|
||||
protected void uploadIndexes(Set<String> alreadyUploadedFiles){
|
||||
for(File f : getSortedFiles(indexDirFiles)){
|
||||
if(!alreadyUploadedFiles.contains(f.getName()) && !f.isDirectory()){
|
||||
uploadIndex(f, alreadyUploadedFiles);
|
||||
if(!alreadyUploadedFiles.contains(f.getName())){
|
||||
log.warn("! NOT UPLOADED " + f.getName());
|
||||
}
|
||||
}
|
||||
}
|
||||
log.info("UPLOADING INDEXES FINISHED ");
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
protected void uploadIndex(final File f, Set<String> alreadyUploadedFiles){
|
||||
if(!uploadIndexes){
|
||||
return;
|
||||
}
|
||||
|
||||
String fileName = f.getName();
|
||||
log.info("Upload index " + fileName);
|
||||
String summary;
|
||||
double mbLengh = (double)f.length() / MB;
|
||||
boolean zip = true;
|
||||
if(fileName.endsWith(IndexConstants.POI_INDEX_EXT) || fileName.endsWith(IndexConstants.POI_INDEX_EXT_ZIP)){
|
||||
summary = "POI index for " ;
|
||||
} else if(fileName.endsWith(IndexConstants.BINARY_MAP_INDEX_EXT) || fileName.endsWith(IndexConstants.BINARY_MAP_INDEX_EXT_ZIP)){
|
||||
boolean addr = indexAddress;
|
||||
boolean trans = indexTransport;
|
||||
boolean map = indexMap;
|
||||
boolean poi = indexPOI;
|
||||
RandomAccessFile raf = null;
|
||||
if (fileName.endsWith(IndexConstants.BINARY_MAP_INDEX_EXT)) {
|
||||
try {
|
||||
raf = new RandomAccessFile(f, "r");
|
||||
BinaryMapIndexReader reader = new BinaryMapIndexReader(raf);
|
||||
trans = reader.hasTransportData();
|
||||
map = reader.containsMapData();
|
||||
addr = reader.containsAddressData();
|
||||
poi = reader.containsPoiData();
|
||||
reader.close();
|
||||
} catch (Exception e) {
|
||||
log.info("File with not be uploaded! Exception", e);
|
||||
if (raf != null) {
|
||||
try {
|
||||
raf.close();
|
||||
} catch (IOException e1) {
|
||||
}
|
||||
}
|
||||
//do not upload probably corrupted file
|
||||
return;
|
||||
}
|
||||
}
|
||||
summary = " index for ";
|
||||
boolean fir = true;
|
||||
if (addr) {
|
||||
summary = "Address" + (fir ? "" : ", ") + summary;
|
||||
fir = false;
|
||||
}
|
||||
if (trans) {
|
||||
summary = "Transport" + (fir ? "" : ", ") + summary;
|
||||
fir = false;
|
||||
}
|
||||
if (poi) {
|
||||
summary = "POI" + (fir ? "" : ", ") + summary;
|
||||
fir = false;
|
||||
}
|
||||
if (map) {
|
||||
summary = "Map" + (fir ? "" : ", ") + summary;
|
||||
fir = false;
|
||||
}
|
||||
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
|
||||
if(mbLengh < MIN_SIZE_TO_UPLOAD){
|
||||
// do not upload small files
|
||||
return;
|
||||
}
|
||||
|
||||
String regionName = fileName.substring(0, fileName.lastIndexOf('_', fileName.indexOf('.')));
|
||||
summary += regionName;
|
||||
summary = summary.replace('_', ' ');
|
||||
|
||||
File toUpload = f;
|
||||
if((fileName.endsWith(".odb") || fileName.endsWith(".obf")) && zip){
|
||||
String n = fileName;
|
||||
if(fileName.endsWith(".odb")){
|
||||
n = fileName.substring(0, fileName.length() - 4);
|
||||
}
|
||||
String zipFileName = n+".zip";
|
||||
log.info("Zipping file " + fileName);
|
||||
try {
|
||||
toUpload = IndexZipper.zip(f, zipFileName, summary);
|
||||
} catch (OneFileException e) {
|
||||
log.error("Exception while zipping file", e);
|
||||
return ; //do not continue if error
|
||||
}
|
||||
if(f.delete()){
|
||||
log.info("Source odb file was deleted");
|
||||
}
|
||||
}
|
||||
|
||||
List<File> splittedFiles = Collections.emptyList();
|
||||
try {
|
||||
splittedFiles = splitFiles(toUpload);
|
||||
boolean uploaded = true;
|
||||
for (File fs : splittedFiles) {
|
||||
uploaded &= uploadFileToServer(fs, toUpload, summary);
|
||||
}
|
||||
// remove source file if file was splitted
|
||||
if (uploaded) {
|
||||
if (deleteFilesAfterUploading && toUpload.exists()) {
|
||||
toUpload.delete();
|
||||
} else if (backupUploadedFiles != null) {
|
||||
File toBackup = new File(backupUploadedFiles, toUpload.getName());
|
||||
if(toBackup.exists()){
|
||||
toBackup.delete();
|
||||
}
|
||||
toUpload.renameTo(toBackup);
|
||||
}
|
||||
}
|
||||
alreadyUploadedFiles.add(fileName);
|
||||
} catch (IOException e) {
|
||||
log.error("Input/output exception uploading " + fileName, e);
|
||||
} finally {
|
||||
// remove all splitted files
|
||||
for(File fs : splittedFiles){
|
||||
if(!fs.equals(toUpload)){
|
||||
fs.delete();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private List<File> splitFiles(File f) throws IOException {
|
||||
double mbLengh = (double)f.length() / MB;
|
||||
if(mbLengh < MAX_SIZE_TO_NOT_SPLIT) {
|
||||
return Collections.singletonList(f);
|
||||
} else {
|
||||
ArrayList<File> arrayList = new ArrayList<File>();
|
||||
FileInputStream in = new FileInputStream(f);
|
||||
byte[] buffer = new byte[BUFFER_SIZE];
|
||||
|
||||
int i = 1;
|
||||
int read = 0;
|
||||
while(read != -1){
|
||||
File fout = new File(f.getParent(), f.getName() + "-"+i);
|
||||
arrayList.add(fout);
|
||||
FileOutputStream fo = new FileOutputStream(fout);
|
||||
int limit = (int) (MAX_SIZE_TO_NOT_SPLIT * MB);
|
||||
while(limit > 0 && ((read = in.read(buffer)) != -1)){
|
||||
fo.write(buffer, 0, read);
|
||||
limit -= read;
|
||||
}
|
||||
fo.flush();
|
||||
fo.close();
|
||||
i++;
|
||||
}
|
||||
|
||||
in.close();
|
||||
|
||||
return arrayList;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private boolean uploadFileToServer(File f, File original, String summary) throws IOException {
|
||||
if (f.length() / MB > MAX_UPLOAD_SIZE && uploadToOsmandGooglecode) {
|
||||
System.err.println("ERROR : file " + f.getName() + " exceeded 200 mb!!! Could not be uploaded.");
|
||||
return false; // restriction for google code
|
||||
}
|
||||
double originalLength = (double) original.length() / MB;
|
||||
if (uploadToOsmandGooglecode) {
|
||||
try {
|
||||
DownloaderIndexFromGoogleCode.deleteFileFromGoogleDownloads(f.getName(), token, pagegen, cookieHSID, cookieSID);
|
||||
if(f.getName().endsWith("obf.zip") && f.length() / MB < 5){
|
||||
// try to delete without .zip part
|
||||
DownloaderIndexFromGoogleCode.deleteFileFromGoogleDownloads(f.getName().substring(0, f.getName().length() - 4),
|
||||
token, pagegen, cookieHSID, cookieSID);
|
||||
} else if(f.getName().endsWith("poi.zip") && f.length() / MB < 5){
|
||||
// try to delete without .zip part
|
||||
DownloaderIndexFromGoogleCode.deleteFileFromGoogleDownloads(f.getName().substring(0, f.getName().length() - 3) +"odb",
|
||||
token, pagegen, cookieHSID, cookieSID);
|
||||
} else if(f.getName().endsWith(".zip-1") && f.length() / MB < 10){
|
||||
DownloaderIndexFromGoogleCode.deleteFileFromGoogleDownloads(f.getName().substring(0, f.getName().length() - 2),
|
||||
token, pagegen, cookieHSID, cookieSID);
|
||||
}
|
||||
try {
|
||||
Thread.sleep(4000);
|
||||
} catch (InterruptedException e) {
|
||||
// wait 5 seconds
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.warn("Deleting file from downloads" + f.getName() + " " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
MessageFormat dateFormat = new MessageFormat("{0,date,dd.MM.yyyy}", Locale.US);
|
||||
MessageFormat numberFormat = new MessageFormat("{0,number,##.#}", Locale.US);
|
||||
String size = numberFormat.format(new Object[] { originalLength });
|
||||
String date = dateFormat.format(new Object[] { new Date(original.lastModified()) });
|
||||
if (!uploadToOsmandGooglecode) {
|
||||
uploadToDownloadOsmandNet(f, summary, size, date);
|
||||
} else {
|
||||
String descriptionFile = "{" + date + " : " + size + " MB}";
|
||||
summary += " " + descriptionFile;
|
||||
GoogleCodeUploadIndex uploader = new GoogleCodeUploadIndex();
|
||||
uploader.setFileName(f.getAbsolutePath());
|
||||
uploader.setTargetFileName(f.getName());
|
||||
uploader.setProjectName("osmand");
|
||||
uploader.setUserName(user);
|
||||
uploader.setPassword(password);
|
||||
uploader.setLabels("Type-Archive, Testdata");
|
||||
uploader.setSummary(summary);
|
||||
uploader.setDescription(summary);
|
||||
uploader.upload();
|
||||
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
private void uploadToDownloadOsmandNet(File f, String description, String size, String date) throws IOException{
|
||||
log.info("Uploading file " + f.getName() + " " + size + " MB " + date + " of " + description);
|
||||
// Upload to ftp
|
||||
FTPFileUpload upload = new FTPFileUpload();
|
||||
upload.upload("download.osmand.net", user, password, "indexes/" + f.getName(), f, 1 << 15);
|
||||
|
||||
|
||||
String url = "http://download.osmand.net/xml_update.php?";
|
||||
url += "index="+URLEncoder.encode(f.getName());
|
||||
url += "&description="+URLEncoder.encode(description);
|
||||
url += "&date="+URLEncoder.encode(date);
|
||||
url += "&size="+URLEncoder.encode(size);
|
||||
url += "&action=update";
|
||||
log.info("Updating index " + url); //$NON-NLS-1$//$NON-NLS-2$
|
||||
HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();
|
||||
connection.setRequestMethod("POST");
|
||||
connection.connect();
|
||||
if(connection.getResponseCode() != HttpURLConnection.HTTP_OK){
|
||||
log.error("Error updating indexes " + connection.getResponseMessage());
|
||||
}
|
||||
InputStream is = connection.getInputStream();
|
||||
while(is.read() != -1);
|
||||
connection.disconnect();
|
||||
log.info("Finish updating index");
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
414
DataExtractionOSM/src/net/osmand/data/index/IndexUploader.java
Normal file
414
DataExtractionOSM/src/net/osmand/data/index/IndexUploader.java
Normal file
|
|
@ -0,0 +1,414 @@
|
|||
package net.osmand.data.index;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.io.RandomAccessFile;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URL;
|
||||
import java.net.URLEncoder;
|
||||
import java.text.MessageFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Date;
|
||||
import java.util.Enumeration;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipException;
|
||||
import java.util.zip.ZipFile;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
|
||||
import net.osmand.Algoritms;
|
||||
import net.osmand.LogUtil;
|
||||
import net.osmand.binary.BinaryMapIndexReader;
|
||||
import net.osmand.data.IndexConstants;
|
||||
|
||||
/**
|
||||
* This helper will find obf and zip files, create description for them, and zip them, or update the description. This helper also can
|
||||
* upload files through ssh.
|
||||
*
|
||||
* @author Pavol Zibrita <pavol.zibrita@gmail.com>
|
||||
*/
|
||||
public class IndexUploader {
|
||||
|
||||
protected static final Log log = LogUtil.getLog(IndexUploader.class);
|
||||
private final static double MIN_SIZE_TO_UPLOAD = 0.015d;
|
||||
private final static double MAX_SIZE_TO_NOT_SPLIT = 190d;
|
||||
private final static double MAX_UPLOAD_SIZE = 195d;
|
||||
|
||||
private final static int BUFFER_SIZE = 1 << 15;
|
||||
private final static int MB = 1 << 20;
|
||||
|
||||
/**
|
||||
* Something bad have happend
|
||||
*/
|
||||
public static class IndexZipperException extends Exception {
|
||||
|
||||
private static final long serialVersionUID = 2343219168909577070L;
|
||||
|
||||
public IndexZipperException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Processing of one file failed, but other files could continue
|
||||
*/
|
||||
public static class OneFileException extends Exception {
|
||||
|
||||
private static final long serialVersionUID = 6463200194419498979L;
|
||||
|
||||
public OneFileException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private File directory;
|
||||
private File targetDirectory;
|
||||
|
||||
public IndexUploader(String path, String targetPath) throws IndexZipperException {
|
||||
directory = new File(path);
|
||||
if (!directory.isDirectory()) {
|
||||
throw new IndexZipperException("Not a directory:" + path);
|
||||
}
|
||||
}
|
||||
|
||||
private void run() {
|
||||
for (File f : directory.listFiles()) {
|
||||
try {
|
||||
if (!f.isFile()) {
|
||||
continue;
|
||||
}
|
||||
File unzipped = unzip(f);
|
||||
String description = getDescription(unzipped);
|
||||
zip(unzipped, getZipfileName(unzipped), description);
|
||||
unzipped.delete(); // delete the unzipped file
|
||||
} catch (OneFileException e) {
|
||||
log.error(f.getName() + ": " + e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static File zip(File f, String zipFileName, String description) throws OneFileException {
|
||||
File zFile = new File(f.getParentFile(), zipFileName);
|
||||
try {
|
||||
log.info("Zipping to file: " + zipFileName + " file:" + f.getName() + " with desc:" + description);
|
||||
ZipOutputStream zout = new ZipOutputStream(new FileOutputStream(zFile));
|
||||
zout.setLevel(9);
|
||||
ZipEntry zEntry = new ZipEntry(f.getName());
|
||||
zEntry.setSize(f.length());
|
||||
zEntry.setComment(description);
|
||||
zout.putNextEntry(zEntry);
|
||||
FileInputStream is = new FileInputStream(f);
|
||||
copyAndClose(is, zout);
|
||||
} catch (IOException e) {
|
||||
throw new OneFileException("cannot zip file:" + e.getMessage());
|
||||
}
|
||||
return zFile;
|
||||
}
|
||||
|
||||
private String getZipfileName(File unzipped) {
|
||||
String fileName = unzipped.getName();
|
||||
String n = fileName;
|
||||
if (fileName.endsWith(".odb")) {
|
||||
throw new UnsupportedOperationException("Odb is not supported any more");
|
||||
// n = fileName.substring(0, fileName.length() - 4);
|
||||
}
|
||||
return n + ".zip";
|
||||
}
|
||||
|
||||
private String getDescription(File f) throws OneFileException {
|
||||
String fileName = f.getName();
|
||||
String summary = null;
|
||||
if (fileName.endsWith(IndexConstants.POI_INDEX_EXT) || fileName.endsWith(IndexConstants.POI_INDEX_EXT_ZIP)) {
|
||||
summary = "POI index for ";
|
||||
} else if (fileName.endsWith(IndexConstants.BINARY_MAP_INDEX_EXT) || fileName.endsWith(IndexConstants.BINARY_MAP_INDEX_EXT_ZIP)) {
|
||||
RandomAccessFile raf = null;
|
||||
try {
|
||||
raf = new RandomAccessFile(f, "r");
|
||||
BinaryMapIndexReader reader = new BinaryMapIndexReader(raf);
|
||||
summary = " index for ";
|
||||
boolean fir = true;
|
||||
if (reader.containsAddressData()) {
|
||||
summary = "Address" + (fir ? "" : ", ") + summary;
|
||||
fir = false;
|
||||
}
|
||||
if (reader.hasTransportData()) {
|
||||
summary = "Transport" + (fir ? "" : ", ") + summary;
|
||||
fir = false;
|
||||
}
|
||||
if (reader.containsPoiData()) {
|
||||
summary = "POI" + (fir ? "" : ", ") + summary;
|
||||
fir = false;
|
||||
}
|
||||
if (reader.containsMapData()) {
|
||||
summary = "Map" + (fir ? "" : ", ") + summary;
|
||||
fir = false;
|
||||
}
|
||||
reader.close();
|
||||
} catch (IOException e) {
|
||||
if (raf != null) {
|
||||
try {
|
||||
raf.close();
|
||||
} catch (IOException e1) {
|
||||
}
|
||||
}
|
||||
throw new OneFileException("Reader could not read the index: " + e.getMessage());
|
||||
}
|
||||
} else {
|
||||
throw new OneFileException("Not a processable file.");
|
||||
}
|
||||
|
||||
String regionName = fileName.substring(0, fileName.lastIndexOf('_', fileName.indexOf('.')));
|
||||
summary += regionName;
|
||||
summary = summary.replace('_', ' ');
|
||||
|
||||
return summary;
|
||||
}
|
||||
|
||||
private File unzip(File f) throws OneFileException {
|
||||
try {
|
||||
if (!Algoritms.isZipFile(f)) {
|
||||
return f;
|
||||
}
|
||||
|
||||
log.info("Unzipping file: " + f.getName());
|
||||
ZipFile zipFile;
|
||||
zipFile = new ZipFile(f);
|
||||
Enumeration<? extends ZipEntry> entries = zipFile.entries();
|
||||
while (entries.hasMoreElements()) {
|
||||
ZipEntry entry = entries.nextElement();
|
||||
File tempFile = new File(f.getParentFile(), entry.getName());
|
||||
copyAndClose(zipFile.getInputStream(entry), new FileOutputStream(tempFile));
|
||||
return tempFile;
|
||||
}
|
||||
return null;
|
||||
} catch (ZipException e) {
|
||||
throw new OneFileException("cannot unzip:" + e.getMessage());
|
||||
} catch (IOException e) {
|
||||
throw new OneFileException("cannot unzip:" + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
public static void copyAndClose(InputStream in, OutputStream out) throws IOException {
|
||||
Algoritms.streamCopy(in, out);
|
||||
Algoritms.closeStream(in);
|
||||
Algoritms.closeStream(out);
|
||||
}
|
||||
|
||||
private List<File> splitFiles(File f) throws IOException {
|
||||
double mbLengh = (double) f.length() / MB;
|
||||
if (mbLengh < MAX_SIZE_TO_NOT_SPLIT) {
|
||||
return Collections.singletonList(f);
|
||||
} else {
|
||||
ArrayList<File> arrayList = new ArrayList<File>();
|
||||
FileInputStream in = new FileInputStream(f);
|
||||
byte[] buffer = new byte[BUFFER_SIZE];
|
||||
|
||||
int i = 1;
|
||||
int read = 0;
|
||||
while (read != -1) {
|
||||
File fout = new File(f.getParent(), f.getName() + "-" + i);
|
||||
arrayList.add(fout);
|
||||
FileOutputStream fo = new FileOutputStream(fout);
|
||||
int limit = (int) (MAX_SIZE_TO_NOT_SPLIT * MB);
|
||||
while (limit > 0 && ((read = in.read(buffer)) != -1)) {
|
||||
fo.write(buffer, 0, read);
|
||||
limit -= read;
|
||||
}
|
||||
fo.flush();
|
||||
fo.close();
|
||||
i++;
|
||||
}
|
||||
|
||||
in.close();
|
||||
|
||||
return arrayList;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public void uploadIndex(File indexFile, File dirToBackup, Set<String> alreadyUploadedFiles, String user, String password) {
|
||||
double mbLengh = (double) indexFile.length() / MB;
|
||||
String fileName = indexFile.getName();
|
||||
if (mbLengh < MIN_SIZE_TO_UPLOAD) {
|
||||
log.info("Skip uploading index due to size " + fileName);
|
||||
// do not upload small files
|
||||
return;
|
||||
}
|
||||
try {
|
||||
log.info("Upload index " + fileName);
|
||||
String summary = getDescription(indexFile);
|
||||
File toUpload = indexFile;
|
||||
if (fileName.endsWith(".obf")) {
|
||||
String zipFileName = fileName + ".zip";
|
||||
log.info("Zipping file " + fileName);
|
||||
|
||||
toUpload = IndexUploader.zip(indexFile, zipFileName, summary);
|
||||
if (indexFile.delete()) {
|
||||
log.info("Source obf file was deleted.");
|
||||
}
|
||||
}
|
||||
|
||||
boolean uploaded = uploadFileToServer(toUpload, summary, user, password, false, null, null, null, null);
|
||||
// remove source file if file was splitted
|
||||
if (uploaded && dirToBackup != null) {
|
||||
File toBackup = new File(dirToBackup, toUpload.getName());
|
||||
if (toBackup.exists()) {
|
||||
toBackup.delete();
|
||||
}
|
||||
toUpload.renameTo(toBackup);
|
||||
}
|
||||
alreadyUploadedFiles.add(toUpload.getName());
|
||||
} catch (OneFileException e) {
|
||||
log.error("Exception ", e);
|
||||
return; // do not continue if error
|
||||
} catch (IOException e) {
|
||||
log.error("Input/output exception uploading " + fileName, e);
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
try {
|
||||
String srcPath = extractDirectory(args, 0);
|
||||
String targetPath = srcPath;
|
||||
if (args.length > 1) {
|
||||
targetPath = extractDirectory(args, 1);
|
||||
}
|
||||
IndexUploader indexZipper = new IndexUploader(srcPath, targetPath);
|
||||
indexZipper.run();
|
||||
} catch (IndexZipperException e) {
|
||||
log.error(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static String extractDirectory(String[] args, int ind) throws IndexZipperException {
|
||||
if (args.length > ind) {
|
||||
if ("-h".equals(args[0])) {
|
||||
throw new IndexZipperException("Usage: IndexZipper [directory] (if not specified, the current one will be taken)");
|
||||
} else {
|
||||
return args[ind];
|
||||
}
|
||||
}
|
||||
return ".";
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
public void uploadToGoogleCode(File f, String summary, String user, String password, String token, String pagegen, String cookieHSID,
|
||||
String cookieSID) throws IOException {
|
||||
if (f.length() / MB > MAX_UPLOAD_SIZE) {
|
||||
System.err.println("ERROR : file " + f.getName() + " exceeded 200 mb!!! Could not be uploaded.");
|
||||
throw new IOException("ERROR : file " + f.getName() + " exceeded 200 mb!!! Could not be uploaded.");
|
||||
// restriction for google code
|
||||
}
|
||||
try {
|
||||
DownloaderIndexFromGoogleCode.deleteFileFromGoogleDownloads(f.getName(), token, pagegen, cookieHSID, cookieSID);
|
||||
if (f.getName().endsWith("obf.zip") && f.length() / MB < 5) {
|
||||
// try to delete without .zip part
|
||||
DownloaderIndexFromGoogleCode.deleteFileFromGoogleDownloads(f.getName().substring(0, f.getName().length() - 4), token,
|
||||
pagegen, cookieHSID, cookieSID);
|
||||
} else if (f.getName().endsWith("poi.zip") && f.length() / MB < 5) {
|
||||
// try to delete without .zip part
|
||||
DownloaderIndexFromGoogleCode.deleteFileFromGoogleDownloads(f.getName().substring(0, f.getName().length() - 3) + "odb",
|
||||
token, pagegen, cookieHSID, cookieSID);
|
||||
} else if (f.getName().endsWith(".zip-1")) {
|
||||
DownloaderIndexFromGoogleCode.deleteFileFromGoogleDownloads(f.getName().substring(0, f.getName().length() - 2), token,
|
||||
pagegen, cookieHSID, cookieSID);
|
||||
}
|
||||
try {
|
||||
Thread.sleep(4000);
|
||||
} catch (InterruptedException e) {
|
||||
// wait 5 seconds
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.warn("Deleting file from downloads" + f.getName() + " " + e.getMessage());
|
||||
}
|
||||
|
||||
GoogleCodeUploadIndex uploader = new GoogleCodeUploadIndex();
|
||||
uploader.setFileName(f.getAbsolutePath());
|
||||
uploader.setTargetFileName(f.getName());
|
||||
uploader.setProjectName("osmand");
|
||||
uploader.setUserName(user);
|
||||
uploader.setPassword(password);
|
||||
uploader.setLabels("Type-Archive, Testdata");
|
||||
uploader.setSummary(summary);
|
||||
uploader.setDescription(summary);
|
||||
uploader.upload();
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
public boolean uploadFileToServer(File original, String summary, String user, String password, boolean uploadToOsmandGooglecode,
|
||||
String token, String pagegen, String cookieHSID, String cookieSID) throws IOException {
|
||||
double originalLength = (double) original.length() / MB;
|
||||
MessageFormat dateFormat = new MessageFormat("{0,date,dd.MM.yyyy}", Locale.US);
|
||||
MessageFormat numberFormat = new MessageFormat("{0,number,##.#}", Locale.US);
|
||||
String size = numberFormat.format(new Object[] { originalLength });
|
||||
String date = dateFormat.format(new Object[] { new Date(original.lastModified()) });
|
||||
try {
|
||||
if (uploadToOsmandGooglecode) {
|
||||
uploadToDownloadOsmandNet(original, summary, size, date, user, password);
|
||||
} else {
|
||||
String descriptionFile = "{" + date + " : " + size + " MB}";
|
||||
summary += " " + descriptionFile;
|
||||
|
||||
List<File> splittedFiles = Collections.emptyList();
|
||||
try {
|
||||
splittedFiles = splitFiles(original);
|
||||
for (File fs : splittedFiles) {
|
||||
uploadToGoogleCode(fs, summary, user, password, token, pagegen, cookieHSID, cookieSID);
|
||||
}
|
||||
|
||||
} finally {
|
||||
// remove all splitted files
|
||||
for (File fs : splittedFiles) {
|
||||
if (!fs.equals(original)) {
|
||||
fs.delete();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.error("Input/output exception uploading " + original.getName(), e);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
public void uploadToDownloadOsmandNet(File f, String description, String size, String date, String user, String password)
|
||||
throws IOException {
|
||||
log.info("Uploading file " + f.getName() + " " + size + " MB " + date + " of " + description);
|
||||
// Upload to ftp
|
||||
FTPFileUpload upload = new FTPFileUpload();
|
||||
upload.upload("download.osmand.net", user, password, "indexes/" + f.getName(), f, 1 << 15);
|
||||
|
||||
String url = "http://download.osmand.net/xml_update.php?";
|
||||
url += "index=" + URLEncoder.encode(f.getName());
|
||||
url += "&description=" + URLEncoder.encode(description);
|
||||
url += "&date=" + URLEncoder.encode(date);
|
||||
url += "&size=" + URLEncoder.encode(size);
|
||||
url += "&action=update";
|
||||
log.info("Updating index " + url); //$NON-NLS-1$//$NON-NLS-2$
|
||||
HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();
|
||||
connection.setRequestMethod("POST");
|
||||
connection.connect();
|
||||
if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) {
|
||||
log.error("Error updating indexes " + connection.getResponseMessage());
|
||||
}
|
||||
InputStream is = connection.getInputStream();
|
||||
while (is.read() != -1)
|
||||
;
|
||||
connection.disconnect();
|
||||
log.info("Finish updating index");
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,227 +0,0 @@
|
|||
package net.osmand.data.index;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.io.RandomAccessFile;
|
||||
import java.util.Enumeration;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipException;
|
||||
import java.util.zip.ZipFile;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
|
||||
import net.osmand.Algoritms;
|
||||
import net.osmand.LogUtil;
|
||||
import net.osmand.binary.BinaryMapIndexReader;
|
||||
import net.osmand.data.IndexConstants;
|
||||
|
||||
/**
|
||||
* This helper will find obf and zip files, create description for them, and zip
|
||||
* them, or update the description.
|
||||
*
|
||||
* @author Pavol Zibrita <pavol.zibrita@gmail.com>
|
||||
*/
|
||||
public class IndexZipper {
|
||||
|
||||
protected static final Log log = LogUtil.getLog(IndexZipper.class);
|
||||
|
||||
/**
|
||||
* Something bad have happend
|
||||
*/
|
||||
public static class IndexZipperException extends Exception {
|
||||
|
||||
private static final long serialVersionUID = 2343219168909577070L;
|
||||
|
||||
public IndexZipperException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Processing of one file failed, but other files could continue
|
||||
*/
|
||||
public static class OneFileException extends Exception {
|
||||
|
||||
private static final long serialVersionUID = 6463200194419498979L;
|
||||
|
||||
public OneFileException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private File directory;
|
||||
private File targetDirectory;
|
||||
|
||||
public IndexZipper(String path, String targetPath) throws IndexZipperException {
|
||||
directory = new File(path);
|
||||
if (!directory.isDirectory()) {
|
||||
throw new IndexZipperException("Not a directory:" + path);
|
||||
}
|
||||
}
|
||||
|
||||
private void run() {
|
||||
for (File f : directory.listFiles()) {
|
||||
try {
|
||||
if (!f.isFile()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
File unzipped = unzip(f);
|
||||
String description = getDescription(unzipped);
|
||||
zip(unzipped, getZipfileName(unzipped), description);
|
||||
unzipped.delete(); // delete the unzipped file
|
||||
} catch (OneFileException e) {
|
||||
log.error(f.getName() + ": " + e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static File zip(File f, String zipFileName, String description)
|
||||
throws OneFileException {
|
||||
File zFile = new File(f.getParentFile(), zipFileName);
|
||||
try {
|
||||
log.info("Zipping to file: " + zipFileName + " file:" + f.getName() + " with desc:" + description);
|
||||
ZipOutputStream zout = new ZipOutputStream(new FileOutputStream(
|
||||
zFile));
|
||||
zout.setLevel(9);
|
||||
ZipEntry zEntry = new ZipEntry(f.getName());
|
||||
zEntry.setSize(f.length());
|
||||
zEntry.setComment(description);
|
||||
zout.putNextEntry(zEntry);
|
||||
FileInputStream is = new FileInputStream(f);
|
||||
copyAndClose(is, zout);
|
||||
} catch (IOException e) {
|
||||
throw new OneFileException("cannot zip file:" + e.getMessage());
|
||||
}
|
||||
return zFile;
|
||||
}
|
||||
|
||||
private String getZipfileName(File unzipped) {
|
||||
String fileName = unzipped.getName();
|
||||
String n = fileName;
|
||||
if (fileName.endsWith(".odb")) {
|
||||
throw new UnsupportedOperationException("Odb is not supported any more");
|
||||
// n = fileName.substring(0, fileName.length() - 4);
|
||||
}
|
||||
return n + ".zip";
|
||||
}
|
||||
|
||||
private String getDescription(File f) throws OneFileException {
|
||||
String fileName = f.getName();
|
||||
String summary = null;
|
||||
if (fileName.endsWith(IndexConstants.POI_INDEX_EXT)
|
||||
|| fileName.endsWith(IndexConstants.POI_INDEX_EXT_ZIP)) {
|
||||
summary = "POI index for ";
|
||||
} else if (fileName.endsWith(IndexConstants.BINARY_MAP_INDEX_EXT)
|
||||
|| fileName.endsWith(IndexConstants.BINARY_MAP_INDEX_EXT_ZIP)) {
|
||||
RandomAccessFile raf = null;
|
||||
try {
|
||||
raf = new RandomAccessFile(f, "r");
|
||||
BinaryMapIndexReader reader = new BinaryMapIndexReader(raf);
|
||||
summary = " index for ";
|
||||
boolean fir = true;
|
||||
if (reader.containsAddressData()) {
|
||||
summary = "Address" + (fir ? "" : ", ") + summary;
|
||||
fir = false;
|
||||
}
|
||||
if (reader.hasTransportData()) {
|
||||
summary = "Transport" + (fir ? "" : ", ") + summary;
|
||||
fir = false;
|
||||
}
|
||||
if (reader.containsPoiData()) {
|
||||
summary = "POI" + (fir ? "" : ", ") + summary;
|
||||
fir = false;
|
||||
}
|
||||
if (reader.containsMapData()) {
|
||||
summary = "Map" + (fir ? "" : ", ") + summary;
|
||||
fir = false;
|
||||
}
|
||||
reader.close();
|
||||
} catch (IOException e) {
|
||||
if (raf != null) {
|
||||
try {
|
||||
raf.close();
|
||||
} catch (IOException e1) {
|
||||
}
|
||||
}
|
||||
throw new OneFileException("Reader could not read the index: "
|
||||
+ e.getMessage());
|
||||
}
|
||||
} else {
|
||||
throw new OneFileException("Not a processable file.");
|
||||
}
|
||||
|
||||
String regionName = fileName.substring(0,
|
||||
fileName.lastIndexOf('_', fileName.indexOf('.')));
|
||||
summary += regionName;
|
||||
summary = summary.replace('_', ' ');
|
||||
|
||||
return summary;
|
||||
}
|
||||
|
||||
private File unzip(File f) throws OneFileException {
|
||||
try {
|
||||
if (!Algoritms.isZipFile(f)) {
|
||||
return f;
|
||||
}
|
||||
|
||||
log.info("Unzipping file: " + f.getName());
|
||||
ZipFile zipFile;
|
||||
zipFile = new ZipFile(f);
|
||||
Enumeration<? extends ZipEntry> entries = zipFile.entries();
|
||||
while (entries.hasMoreElements()) {
|
||||
ZipEntry entry = entries.nextElement();
|
||||
File tempFile = new File(f.getParentFile(), entry.getName());
|
||||
copyAndClose(zipFile.getInputStream(entry),
|
||||
new FileOutputStream(tempFile));
|
||||
return tempFile;
|
||||
}
|
||||
return null;
|
||||
} catch (ZipException e) {
|
||||
throw new OneFileException("cannot unzip:" + e.getMessage());
|
||||
} catch (IOException e) {
|
||||
throw new OneFileException("cannot unzip:" + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
public static void copyAndClose(InputStream in, OutputStream out)
|
||||
throws IOException {
|
||||
Algoritms.streamCopy(in, out);
|
||||
Algoritms.closeStream(in);
|
||||
Algoritms.closeStream(out);
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
try {
|
||||
String srcPath = extractDirectory(args, 0);
|
||||
String targetPath = srcPath;
|
||||
if(args.length > 1) {
|
||||
targetPath = extractDirectory(args, 1);
|
||||
}
|
||||
IndexZipper indexZipper = new IndexZipper(srcPath, targetPath);
|
||||
indexZipper.run();
|
||||
} catch (IndexZipperException e) {
|
||||
log.error(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static String extractDirectory(String[] args, int ind)
|
||||
throws IndexZipperException {
|
||||
if (args.length > ind) {
|
||||
if ("-h".equals(args[0])) {
|
||||
throw new IndexZipperException(
|
||||
"Usage: IndexZipper [directory] (if not specified, the current one will be taken)");
|
||||
} else {
|
||||
return args[ind];
|
||||
}
|
||||
}
|
||||
return ".";
|
||||
}
|
||||
}
|
||||
|
|
@ -1,19 +1,5 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<batch_process>
|
||||
<!-- These attributes are required to upload/delete downloads
|
||||
* cookieHSID - cookie HSID is required for delete index (could be taken from source (!) of page when you are about delete index)
|
||||
* cookieSID - cookie SID is required for delete index (could be taken from source (!) of page when you are about delete index)
|
||||
* pagegen - pagegen attribute is required for delete index (could be taken from source (!) of page when you are about delete index)
|
||||
* token - token attribute is required for delete index (could be taken from source (!) of page when you are about delete index)
|
||||
* google_code_user - is required to upload index
|
||||
* google_code_password - is required to upload index (is not gmail password!) - could be found in profile
|
||||
-->
|
||||
<authorization_info cookieHSID=""
|
||||
cookieSID=""
|
||||
pagegen="" token=""
|
||||
google_code_user="" google_code_password=""
|
||||
osmand_download_user="" osmand_download_password=""/>
|
||||
|
||||
<process_attributes mapZooms="" renderingTypesFile="" zoomWaySmoothness=""
|
||||
osmDbDialect="sqlite" mapDbDialect="sqlite"/>
|
||||
|
||||
|
|
@ -26,11 +12,8 @@
|
|||
or you can upload any file you have to googlecode (just put into 'directory_for_index_files')
|
||||
-->
|
||||
<!-- zoomWaySmoothness - 1-4, typical mapZooms - 8-10;11-12;13-14;15 -->
|
||||
<process directory_for_osm_files="/home/..." directory_for_index_files="/home/..."
|
||||
directory_for_uploaded_files="" keepPoiOdb="false"
|
||||
downloadOsmFiles="true" generateIndexes="true" uploadIndexes="true"
|
||||
upload_osmand_googlecode="true" deleteFilesAfterUploading="true" indexPOI="true" indexMap="true"
|
||||
indexTransport="true" indexAddress="true">
|
||||
<process directory_for_osm_files="/home/..." directory_for_index_files="/home/..." directory_for_generation="/home/..."
|
||||
skipGeneratedIndexes="true" indexPOI="true" indexMap="true" indexTransport="true" indexAddress="true">
|
||||
<!-- Add wget="C:/Program Files/GNUWin32/bin/wget.exe" to process, to use wget for download.
|
||||
On linux systems if wget is in your path it can be wget="wget" or you can make own script with wget command:
|
||||
wget="/path/to/script/wget.sh"
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ public class IndexCreator {
|
|||
// Sqlite better to use only for 32-bit machines
|
||||
public static DBDialect dialect = DBDialect.SQLITE;
|
||||
public static DBDialect mapDBDialect = DBDialect.SQLITE;
|
||||
public static boolean REMOVE_POI_DB = false;
|
||||
public static boolean REMOVE_POI_DB = true;
|
||||
|
||||
public static final int BATCH_SIZE = 5000;
|
||||
public static final int BATCH_SIZE_OSM = 10000;
|
||||
|
|
|
|||
|
|
@ -1,19 +1,5 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<batch_process>
|
||||
<!-- These attributes are required to upload/delete downloads
|
||||
* cookieHSID - cookie HSID is required for delete index (could be taken from source (!) of page when you are about delete index)
|
||||
* cookieSID - cookie SID is required for delete index (could be taken from source (!) of page when you are about delete index)
|
||||
* pagegen - pagegen attribute is required for delete index (could be taken from source (!) of page when you are about delete index)
|
||||
* token - token attribute is required for delete index (could be taken from source (!) of page when you are about delete index)
|
||||
* google_code_user - is required to upload index
|
||||
* google_code_password - is required to upload index (is not gmail password!) - could be found in profile
|
||||
-->
|
||||
<authorization_info cookieHSID=""
|
||||
cookieSID=""
|
||||
pagegen="" token=""
|
||||
google_code_user="" google_code_password=""
|
||||
osmand_download_user="" osmand_download_password=""/>
|
||||
|
||||
<process_attributes mapZooms="" renderingTypesFile="" zoomWaySmoothness="2"
|
||||
osmDbDialect="sqlite" mapDbDialect="sqlite"/>
|
||||
|
||||
|
|
@ -26,11 +12,8 @@
|
|||
or you can upload any file you have to googlecode (just put into 'directory_for_index_files')
|
||||
-->
|
||||
<!-- zoomWaySmoothness - 1-4, typical mapZooms - 8-10;11-12;13-14;15 -->
|
||||
<process directory_for_osm_files=".work/osm" directory_for_index_files="/var/lib/jenkins/indexes"
|
||||
directory_for_uploaded_files="/var/lib/jenkins/indexes/uploaded" keepPoiOdb="false" directory_for_generation=".work"
|
||||
downloadOsmFiles="true" generateIndexes="true" uploadIndexes="false" skipGeneratedIndexes="true"
|
||||
upload_osmand_googlecode="false" deleteFilesAfterUploading="false" indexPOI="true" indexMap="true"
|
||||
indexTransport="true" indexAddress="true">
|
||||
<process directory_for_osm_files=".work/osm" directory_for_index_files="/var/lib/jenkins/indexes" directory_for_generation=".work"
|
||||
skipGeneratedIndexes="true" indexPOI="true" indexMap="true" indexTransport="true" indexAddress="true">
|
||||
<!-- Add wget="C:/Program Files/GNUWin32/bin/wget.exe" to process, to use wget for download.
|
||||
On linux systems if wget is in your path it can be wget="wget" or you can make own script with wget command:
|
||||
wget="/path/to/script/wget.sh"
|
||||
|
|
|
|||
Loading…
Reference in a new issue