Android-builders-club/OsmAnd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

update batch creator to support download.osmand.net

Browse source 
git-svn-id: https://osmand.googlecode.com/svn/trunk@823 e29c36b1-1cfa-d876-8d93-3434fc2bb7b8
This commit is contained in:
Olga.shcherb 2010-12-16 12:49:12 +00:00
parent caa6b1dd58
commit b291d3518f
3 changed files with 261 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

183
DataExtractionOSM/src/net/osmand/data/index/FTPFileUpload.java Normal file
View file

@ -0,0 +1,183 @@
package net.osmand.data.index;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
/**
* This class is used to upload a file to a FTP server.
*
* @author Muthu
*/
public class FTPFileUpload
{
/**
* Upload a file to a FTP server. A FTP URL is generated with the
* following syntax:
* ftp://user:password@host:port/filePath;type=i.
*
* @param ftpServer , FTP server address (optional port ':portNumber').
* @param user , Optional user name to login.
* @param password , Optional password for user.
* @param fileName , Destination file name on FTP server (with optional
* preceding relative path, e.g. "myDir/myFile.txt").
* @param source , Source file to upload.
* @throws MalformedURLException, IOException on error.
*/
public void upload( String ftpServer, String user, String password,
String fileName, File source, int bufferSize ) throws MalformedURLException,
IOException
{
if (ftpServer != null && fileName != null && source != null)
{
StringBuffer sb = new StringBuffer( "ftp://" );
// check for authentication else assume its anonymous access.
if (user != null && password != null)
{
sb.append( user );
sb.append( ':' );
sb.append( password );
sb.append( '@' );
}
sb.append( ftpServer );
sb.append( '/' );
sb.append( fileName );
/*
* type ==> a=ASCII mode, i=image (binary) mode, d= file directory
* listing
*/
sb.append( ";type=i" );
BufferedInputStream bis = null;
BufferedOutputStream bos = null;
try
{
URL url = new URL( sb.toString() );
URLConnection urlc = url.openConnection();
bos = new BufferedOutputStream( urlc.getOutputStream(), bufferSize);
bis = new BufferedInputStream( new FileInputStream( source ), bufferSize);
int i;
// read byte by byte until end of stream
while ((i = bis.read()) != -1)
{
bos.write( i );
}
}
finally
{
if (bis != null)
try
{
bis.close();
}
catch (IOException ioe)
{
ioe.printStackTrace();
}
if (bos != null)
try
{
bos.close();
}
catch (IOException ioe)
{
ioe.printStackTrace();
}
}
}
else
{
System.out.println( "Input not available." );
}
}
/**
* Download a file from a FTP server. A FTP URL is generated with the
* following syntax:
* ftp://user:password@host:port/filePath;type=i.
*
* @param ftpServer , FTP server address (optional port ':portNumber').
* @param user , Optional user name to login.
* @param password , Optional password for user.
* @param fileName , Name of file to download (with optional preceeding
* relative path, e.g. one/two/three.txt).
* @param destination , Destination file to save.
* @throws MalformedURLException, IOException on error.
*/
public void download( String ftpServer, String user, String password,
String fileName, File destination ) throws MalformedURLException,
IOException
{
if (ftpServer != null && fileName != null && destination != null)
{
StringBuffer sb = new StringBuffer( "ftp://" );
// check for authentication else assume its anonymous access.
if (user != null && password != null)
{
sb.append( user );
sb.append( ':' );
sb.append( password );
sb.append( '@' );
}
sb.append( ftpServer );
sb.append( '/' );
sb.append( fileName );
/*
* type ==> a=ASCII mode, i=image (binary) mode, d= file directory
* listing
*/
sb.append( ";type=i" );
BufferedInputStream bis = null;
BufferedOutputStream bos = null;
try
{
URL url = new URL( sb.toString() );
URLConnection urlc = url.openConnection();
bis = new BufferedInputStream( urlc.getInputStream() );
bos = new BufferedOutputStream( new FileOutputStream(
destination.getName() ) );
int i;
while ((i = bis.read()) != -1)
{
bos.write( i );
}
}
finally
{
if (bis != null)
try
{
bis.close();
}
catch (IOException ioe)
{
ioe.printStackTrace();
}
if (bos != null)
try
{
bos.close();
}
catch (IOException ioe)
{
ioe.printStackTrace();
}
}
}
else
{
System.out.println( "Input not available" );
}
}
}

93
DataExtractionOSM/src/net/osmand/data/index/IndexBatchCreator.java
View file

@ -12,6 +12,7 @@ import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.net.URL;
import java.net.URLEncoder;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Arrays;
@ -57,6 +58,8 @@ public class IndexBatchCreator {
String siteToDownload = "";
}
private boolean uploadToOsmandDownloads = true;
// process atributtes
boolean downloadFiles = false;
@ -85,6 +88,7 @@ public class IndexBatchCreator {
public static void main(String[] args) {
IndexBatchCreator creator = new IndexBatchCreator();
if(args == null || args.length == 0){
System.out.println("Please specify -local parameter or path to batch.xml configuration file as 1 argument.");
@ -174,8 +178,14 @@ public class IndexBatchCreator {
cookieSID = authorization.getAttribute("cookieSID");
pagegen = authorization.getAttribute("pagegen");
token = authorization.getAttribute("token");
user = authorization.getAttribute("google_code_user");
password = authorization.getAttribute("google_code_password");
uploadToOsmandDownloads = Boolean.parseBoolean(process.getAttribute("upload_osmand_download"));
if(uploadToOsmandDownloads){
user = authorization.getAttribute("osmand_download_user");
password = authorization.getAttribute("osmand_download_password");
} else {
user = authorization.getAttribute("google_code_user");
password = authorization.getAttribute("google_code_password");
}
}
List<RegionCountries> countriesToDownload = new ArrayList<RegionCountries>();
@ -446,7 +456,7 @@ public class IndexBatchCreator {
if(!uploadIndexes){
return;
}
MessageFormat format = new MessageFormat("{0,date,dd.MM.yyyy} : {1, number,##.#} MB", Locale.US);
String summary;
double mbLengh = (double)f.length() / MB;
boolean zip = true;
@ -533,37 +543,53 @@ public class IndexBatchCreator {
f = zFile;
}
try {
DownloaderIndexFromGoogleCode.deleteFileFromGoogleDownloads(f.getName(), token, pagegen, cookieHSID, cookieSID);
if (!uploadToOsmandDownloads) {
try {
Thread.sleep(4000);
} catch (InterruptedException e) {
// wait 5 seconds
DownloaderIndexFromGoogleCode.deleteFileFromGoogleDownloads(f.getName(), token, pagegen, cookieHSID, cookieSID);
try {
Thread.sleep(4000);
} catch (InterruptedException e) {
// wait 5 seconds
}
} catch (IOException e) {
log.warn("Deleting file from downloads" + f.getName() + " " + e.getMessage());
}
} catch (IOException e) {
log.warn("Deleting file from downloads" + f.getName() + " " + e.getMessage());
}
mbLengh = (double)f.length() / MB;
if(mbLengh > 100){
if(mbLengh > 100 && !uploadToOsmandDownloads){
System.err.println("ERROR : file " + f.getName() + " exceeded 100 mb!!! Could not be uploaded.");
return; // restriction for google code
}
String descriptionFile = "{"+format.format(new Object[]{new Date(f.lastModified()), mbLengh})+"}";
summary += regionName + " " + descriptionFile;
GoogleCodeUploadIndex uploader = new GoogleCodeUploadIndex();
uploader.setFileName(f.getAbsolutePath());
uploader.setTargetFileName(f.getName());
uploader.setProjectName("osmand");
uploader.setUserName(user);
uploader.setPassword(password);
uploader.setLabels("Type-Archive, Testdata");
uploader.setSummary(summary.replace('_', ' '));
summary += regionName;
summary = summary.replace('_', ' ');
try {
uploader.upload();
if(deleteFilesAfterUploading){
MessageFormat dateFormat = new MessageFormat("{0,date,dd.MM.yyyy}", Locale.US);
MessageFormat numberFormat = new MessageFormat("{0,number,##.#}", Locale.US);
String size = numberFormat.format(new Object[] {mbLengh});
String date = dateFormat.format(new Object[] {new Date(f.lastModified())});
if (uploadToOsmandDownloads) {
uploadToDownloadOsmandNet(f, summary, size,date);
} else {
String descriptionFile = "{" + date + " : " + size + "}";
summary += " " + descriptionFile;
GoogleCodeUploadIndex uploader = new GoogleCodeUploadIndex();
uploader.setFileName(f.getAbsolutePath());
uploader.setTargetFileName(f.getName());
uploader.setProjectName("osmand");
uploader.setUserName(user);
uploader.setPassword(password);
uploader.setLabels("Type-Archive, Testdata");
uploader.setSummary(summary);
uploader.upload();
}
if (deleteFilesAfterUploading) {
f.delete();
}
alreadyUploadedFiles.add(f.getName());
@ -572,6 +598,25 @@ public class IndexBatchCreator {
}
}
@SuppressWarnings("deprecation")
private void uploadToDownloadOsmandNet(File f, String description, String size, String date) throws IOException{
log.info("Uploading file " + f.getName() + " " + size + " MB " + date + " of " + description);
// Upload to ftp
FTPFileUpload upload = new FTPFileUpload();
upload.upload("download.osmand.net", user, password, "indexes/" + f.getName(), f, 1 << 15);
String url = "http://download.osmand.net/xml_update.php?";
url += "index="+URLEncoder.encode(f.getName());
url += "&description="+URLEncoder.encode(description);
url += "&date="+URLEncoder.encode(date);
url += "&size="+URLEncoder.encode(size);
url += "&action=update";
log.info("Updating index " + url); //$NON-NLS-1$//$NON-NLS-2$
URL ourl = new URL(url);
InputStream is = ourl.openStream();
safeClose(is, "close file"); //$NON-NLS-1$
log.info("Finish updating index");
}
}

14
DataExtractionOSM/src/net/osmand/data/index/batch.xml
View file

@ -11,7 +11,8 @@
<authorization_info cookieHSID=""
cookieSID=""
pagegen="" token=""
google_code_user="" google_code_password="" />
google_code_user="" google_code_password=""
osmand_download_user="" osmand_download_password=""/>
<!-- There are 3 subprocess :
1. Download fresh osm files from servers to 'directory_for_osm_files' (override existings).
2. Generate index files from all files in 'directory_for_osm_files' and put all indexes into 'directory_for_index_files'
@ -19,16 +20,19 @@
All these subprocess could be ran independently ! So you can create some files check them and after that try to upload on googlecode,
or you can upload any file you have to googlecode (just put into 'directory_for_index_files')
-->
<process directory_for_osm_files="E:/Information/OSM maps/osm_batch" directory_for_index_files="E:/Information/OSM maps/osm_batch_ind"
downloadOsmFiles="true" generateIndexes="true" uploadIndexes="true"
<process directory_for_osm_files="D:/android/batch_gen_osm" directory_for_index_files="D:/android/batch_gen_index"
downloadOsmFiles="true" generateIndexes="true" uploadIndexes="true" upload_osmand_download="true"
deleteFilesAfterUploading="true" indexPOI="true" indexMap="true"
indexTransport="true" indexAddress="true" mapZooms="" renderingTypesFile="">
<!-- Add wget="C:/Program Files/GNUWin32/bin/wget.exe" to process, to use wget for download.
On linux systems if wget is in your path it can be wget="wget" or you can make own script with wget command:
wget="/path/to/script/wget.sh"
Defaultly enabled parameter of wget is: --read-timeout=5 that prevents hanging of download from cloudmade/geofabrik server
Defaultly enabled parameter of wget is: &-&-read-timeout=5 that prevents hanging of download from cloudmade/geofabrik server
-->
<regions siteToDownload="http://download.geofabrik.de/osm/europe/{0}.osm.pbf" region_prefix="" region_suffix="_Europe">
<region name="andorra" esize="6" />
</regions>
<!-- Countries to download from osm server -->
<!-- EUROPE -->
<regions siteToDownload="http://download.geofabrik.de/osm/europe/{0}.osm.pbf" region_prefix="" skip="true" region_suffix="_Europe">