Android-builders-club/OsmAnd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Update site

Browse source
This commit is contained in:
Victor Shcherb 2012-11-15 17:23:14 +01:00
parent 01e49ed1db
commit d90e77c510
10 changed files with 74 additions and 328 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

132
DataExtractionOSM/src/net/osmand/data/index/IndexUploader.java
View file

@ -108,6 +108,7 @@ public class IndexUploader {
private UploadCredentials uploadCredentials = new DummyCredentials();
private FileFilter fileFilter = new FileFilter();
private FileFilter deleteFileFilter = null;
private boolean roadProcess;
public IndexUploader(String path, String targetPath) throws IndexUploadException {
directory = new File(path);
@ -146,37 +147,39 @@ public class IndexUploader {
private int parseFilter(String[] args, int start) throws IndexUploadException {
FileFilter fileFilter = null;
if (args[start].startsWith("--ff=")) {
try {
if(fileFilter == null) {
int p;
do {
p = start;
if (args[start].startsWith("--ff=")) {
try {
if (fileFilter == null) {
fileFilter = new FileFilter();
}
fileFilter.parseFile(args[start].substring("--ff=".length()));
} catch (IOException e) {
throw new IndexUploadException(e.getMessage());
}
start++;
} else if (args[start].startsWith("--fp=")) {
if (fileFilter == null) {
fileFilter = new FileFilter();
}
fileFilter.parseFile(args[start].substring("--ff=".length()));
} catch (IOException e) {
throw new IndexUploadException(e.getMessage());
fileFilter.parseCSV(args[start].substring("--fp=".length()));
start++;
} else if (args[start].startsWith("--ep=")) {
if (fileFilter == null) {
throw new NullPointerException();
}
fileFilter.parseExcludeCSV(args[start].substring("--ep=".length()));
start++;
} else if (args[start].startsWith("--dp=")) {
deleteFileFilter = new FileFilter();
deleteFileFilter.parseCSV(args[start].substring("--dp=".length()));
start++;
} else if (args[start].startsWith("--roads")) {
roadProcess = true;
}
start++;
}
if (args[start].startsWith("--fp=")) {
if(fileFilter == null) {
fileFilter = new FileFilter();
}
fileFilter.parseCSV(args[start].substring("--fp=".length()));
start++;
}
if (args[start].startsWith("--ep=")) {
if(fileFilter == null) {
throw new NullPointerException();
}
fileFilter.parseExcludeCSV(args[start].substring("--ep=".length()));
start++;
}
if (args[start].startsWith("--dp=")) {
deleteFileFilter = new FileFilter();
deleteFileFilter.parseCSV(args[start].substring("--dp=".length()));
start++;
}
} while(p != start);
if(fileFilter != null) {
this.fileFilter = fileFilter;
}
@ -295,17 +298,19 @@ public class IndexUploader {
File unzipped = unzip(f);
File logFile = new File(f.getParentFile(), unzipped.getName() + IndexBatchCreator.GEN_LOG_EXT);
try {
String description = getDescription(unzipped);
List<File> files = new ArrayList<File>();
files.add(unzipped);
if (logFile.exists()) {
files.add(logFile);
String description = checkfileAndGetDescription(unzipped);
if(description == null) {
log.info("Skip file " + f.getName());
} else {
List<File> files = new ArrayList<File>();
files.add(unzipped);
if (logFile.exists()) {
files.add(logFile);
}
File zFile = new File(f.getParentFile(), unzipped.getName() + ".zip");
zip(files, zFile, description, timestampCreated);
uploadIndex(f, zFile, description, uploadCredentials);
}
File zFile = new File(f.getParentFile(), unzipped.getName() + ".zip");
zip(files, zFile, description, timestampCreated);
uploadIndex(f, zFile, description, uploadCredentials);
} finally {
if (!f.getName().equals(unzipped.getName()) ||
(targetDirectory != null && !targetDirectory.equals(directory))) {
@ -368,9 +373,8 @@ public class IndexUploader {
return zFile;
}
private String getDescription(File f) throws OneFileException {
private String checkfileAndGetDescription(File f) throws OneFileException {
String fileName = f.getName();
String summary = null;
if (fileName.endsWith(IndexConstants.BINARY_MAP_INDEX_EXT) || fileName.endsWith(IndexConstants.BINARY_MAP_INDEX_EXT_ZIP)) {
RandomAccessFile raf = null;
try {
@ -379,25 +383,13 @@ public class IndexUploader {
if(reader.getVersion() != IndexConstants.BINARY_MAP_VERSION) {
throw new OneFileException("Uploader version is not compatible " + reader.getVersion() + " to current " + IndexConstants.BINARY_MAP_VERSION);
}
summary = " index for ";
boolean fir = true;
if (reader.containsAddressData()) {
summary = "Address" + (fir ? "" : ", ") + summary;
fir = false;
}
if (reader.hasTransportData()) {
summary = "Transport" + (fir ? "" : ", ") + summary;
fir = false;
}
if (reader.containsPoiData()) {
summary = "POI" + (fir ? "" : ", ") + summary;
fir = false;
}
if (reader.containsMapData()) {
summary = "Map" + (fir ? "" : ", ") + summary;
fir = false;
boolean roadFile = reader.containsRouteData() && !reader.containsMapData();
if(roadFile != this.roadProcess) {
return null;
}
String summary = getDescription(reader, fileName);
reader.close();
return summary;
} catch (IOException e) {
if (raf != null) {
try {
@ -410,7 +402,32 @@ public class IndexUploader {
} else {
throw new OneFileException("Not supported file format " + fileName);
}
}
private String getDescription(BinaryMapIndexReader reader, String fileName) {
String summary;
summary = " index for ";
boolean fir = true;
if (reader.containsAddressData()) {
summary = "Address" + (fir ? "" : ", ") + summary;
fir = false;
}
if (reader.hasTransportData()) {
summary = "Transport" + (fir ? "" : ", ") + summary;
fir = false;
}
if (reader.containsPoiData()) {
summary = "POI" + (fir ? "" : ", ") + summary;
fir = false;
}
if (reader.containsRouteData()) {
summary = "Roads" + (fir ? "" : ", ") + summary;
fir = false;
}
if (reader.containsMapData()) {
summary = "Map" + (fir ? "" : ", ") + summary;
fir = false;
}
int last = fileName.lastIndexOf('_', fileName.indexOf('.'));
if (last == -1) {
last = fileName.indexOf('.');
@ -418,7 +435,6 @@ public class IndexUploader {
String regionName = fileName.substring(0, last);
summary += regionName;
summary = summary.replace('_', ' ');
return summary;
}

96
build-scripts/build_branches.sh
View file

@ -1,96 +0,0 @@
#!/bin/sh
DIRECTORY=$(cd `dirname $0` && pwd)
BRANCHES_CHANGED=0
GIT_DIR="$DIRECTORY"/osmand-git
GIT_ORIGIN_NAME=origin
BUILD_DIR="$DIRECTORY"/builds
LATESTS_DIR="$DIRECTORY"/latests
VERSION_FILE=./DataExtractionOSM/src/net/osmand/Version.java
DATE=$(date +%Y-%m-%d)
SHORT_DATE=$(date +%d-%m)
# clean all files in build directory
rm -r "$BUILD_DIR"
mkdir "$BUILD_DIR"
rm -r "$LATESTS_DIR"
mkdir "$LATESTS_DIR"
cd "${GIT_DIR}"
# clear old branches
git remote prune origin
git branch -r | grep origin | while read i
do
cd "${GIT_DIR}"
ch=$(expr index "$i" ">")
if [ $ch = 0 ]; then
BRANCH=${i#"$GIT_ORIGIN_NAME/"}
#clear success status
# what is it?
rm -f "$DIRECTORY/$BRANCH.fixed"
mv -f "$DIRECTORY/$BRANCH.failed" "$DIRECTORY/$BRANCH.stillfailing" > /dev/null
git diff --exit-code "$BRANCH" "$GIT_ORIGIN_NAME/$BRANCH" --quiet
RES_DIFF=$?
if [ $RES_DIFF != 0 ]; then
echo "Checkouting branch and create build for $BRANCH"
## reset all previous changes in working tree
BRANCHES_CHANGED=1
BUILD_LOG="$DIRECTORY"/build.log
echo "Branch:" $BRANCH > $BUILD_LOG
git checkout .
git reset HEAD --hard
git checkout -f $BRANCH
git reset $GIT_ORIGIN_NAME/$BRANCH --hard
sed -e "s/\(APP_DESCRIPTION.*=.*\"\).*\(\".*\)/\1$SHORT_DATE $BRANCH\2/g" $VERSION_FILE > ${VERSION_FILE}.bak
mv ${VERSION_FILE}.bak ${VERSION_FILE}
## build map creator
cd ./DataExtractionOSM/
ant clean compile build -logfile $BUILD_LOG
if [ "$BRANCH" = "release" ]; then
cp build.zip "$LATESTS_DIR/OsmAndMapCreator-stable.zip"
elif [ "$BRANCH" = "master" ]; then
cp build.zip "$LATESTS_DIR/OsmAndMapCreator-development.zip"
fi
mv build.zip "$BUILD_DIR/OsmAndMapCreator-$BRANCH-nb-$DATE.zip"
## build osmand app
cd ../OsmAnd/
cp "$DIRECTORY"/local.properties local.properties
rm -r bin
mkdir bin
if [ ! -d assets ]; then
mkdir assets
fi
ant clean debug -logfile $BUILD_LOG
if [ "$BRANCH" = "release" ]; then
cp bin/OsmAnd-debug.apk "$LATESTS_DIR/OsmAnd-stable.apk"
elif [ "$BRANCH" = "master" ]; then
cp bin/OsmAnd-debug.apk "$LATESTS_DIR/OsmAnd-development.apk"
fi
mv bin/OsmAnd-debug.apk "$BUILD_DIR/OsmAnd-$BRANCH-nb-$DATE.apk"
#put the log to std out
BUILD=`grep FAILED $DIRECTORY/build.log | wc -l`
if [ ! $BUILD -eq 0 ]; then
mv -f "$DIRECTORY/$BRANCH.stillfailing" "$DIRECTORY/$BRANCH.failed" > /dev/null
java -version
cat $BUILD_LOG
touch "$DIRECTORY/$BRANCH.failed"
else
if [ -f "$DIRECTORY/$BRANCH.stillfailing" ]; then
echo "Build fixed"
rm -f $DIRECTORY/$BRANCH.failed
rm -f $DIRECTORY/$BRANCH.stillfailing
touch $DIRECTORY/$BRANCH.fixed
fi
fi
fi
fi
done
exit $BRANCHES_CHANGED

29
build-scripts/copyto_dir.sh
View file

@ -1,29 +0,0 @@
#!/bin/sh
DIRECTORY=$(cd `dirname $0` && pwd)
FTP_SITE=download.osmand.net
FTP_FOLDER=/var/www-download/night-builds
FTP_LATEST=/var/www-download/latest-night-build
# FTP_USER in local.properties
# FTP_PWD= in local.properties
BUILD_DIR="$DIRECTORY"/builds
LATEST_DIR="$DIRECTORY"/latests
#. "$DIRECTORY"/local.properties
# 3. upload to ftp server
#lftp -c "set net:timeout 45;open -u $FTP_USER,$FTP_PWD $FTP_SITE;mirror -R $BUILD_DIR $FTP_FOLDER;mirror -R $LATEST_DIR $FTP_LATEST"
#rm $FTP_FOLDER/*
#rm $FTP_LATEST/*
cp -f $BUILD_DIR/* $FTP_FOLDER 2> /dev/null
cp -f $LATEST_DIR/* $FTP_LATEST 2> /dev/null
#ftp -n -v $FTP_SITE <<SCRIPT 2>&1
#quote USER $FTP_USER
#quote PASS $FTP_PWD
#cd $FTP_FOLDER
#ls
#quit

59
build-scripts/daily_build.sh
View file

@ -1,59 +0,0 @@
#!/bin/sh
DIRECTORY=$(cd `dirname $0` && pwd)
## VARIABLES ###
LOG_DIR="$DIRECTORY"/logs
DATE=$(date +%d-%m-%y)
CLOG_FILE="$LOG_DIR/${DATE}.log"
LOG_FILE="$LOG_DIR/tmp.log"
LOCK_FILE="$DIRECTORY"/build.lock
if [ -f $LOCK_FILE ]; then
exit
fi
touch $LOCK_FILE
mkdir -p $LOG_DIR
echo > $LOG_FILE
touch $CLOG_FILE
cd "${DIRECTORY}"
git pull --rebase >>$LOG_FILE 2>&1
# 1. Update git directory
"${DIRECTORY}/update_git.sh" >>$LOG_FILE 2>&1
# 2. Go through branches and generates builds
"${DIRECTORY}/build_branches.sh" >>$LOG_FILE 2>&1
# exit if nothing was changed
# if [ $? = 0 ]; then
# exit 0
# fi
# 3. upload to ftp server
#"${DIRECTORY}/upload_ftp.sh" 2>&1 >>$LOG_FILE
# 3. upload to ftp server
"${DIRECTORY}/copyto_dir.sh" >>$LOG_FILE 2>&1
# 4. Synchronize github with googlecode mercurial
"${DIRECTORY}/sync_git_google.sh" >>$LOG_FILE 2>&1
# 5. update site files
"${DIRECTORY}/update_site.sh" >>$LOG_FILE 2>&1
cat $LOG_FILE >> $CLOG_FILE
BUILD=`ls *.fixed *.failed 2> /dev/null | wc -l`
if [ ! $BUILD -eq 0 ]; then
# if some status change, print out complete log
echo "Builds status changed"
echo "-------------"
echo `ls *.fixed *.failed 2> /dev/null`
echo "-------------"
echo "Complete log file:"
echo "-------------"
cat $LOG_FILE
fi
rm -f $LOCK_FILE

14
build-scripts/indexes-generate-inmem.sh
View file

@ -1,14 +0,0 @@
# remove backup and create new backup
# we should not rm, just do incremental updates for now! rm -rf backup
# remove all previous files
mkdir ~/indexes
mkdir ~/indexes/uploaded
rm -rf .work
mkdir .work
mkdir .work/osm
if [ -z $INDEXES_FILE ]; then INDEXES_FILE="build-scripts/regions/indexes.xml"; echo "$INDEXES_FILE"; fi
echo 'Running java net.osmand.data.index.IndexBatchCreator with $INDEXES_FILE'
java -XX:+UseParallelGC -Xmx8096M -Xmn512M -Djava.util.logging.config.file=build-scripts/batch-logging.properties -cp "DataExtractionOSM/OsmAndMapCreator.jar:DataExtractionOSM/lib/*.jar" net.osmand.data.index.IndexBatchCreator build-scripts/indexes-batch-generate-inmem.xml "$INDEXES_FILE"

13
build-scripts/indexes-generate-roads.sh
View file

@ -1,13 +0,0 @@
# remove backup and create new backup
# we should not rm, just do incremental updates for now! rm -rf backup
# remove all previous files
mkdir /var/www-download/road-indexes
rm -rf .work
mkdir .work
mkdir .work/osm
if [ -z $INDEXES_FILE ]; then INDEXES_FILE="build-scripts/regions/indexes.xml"; echo "$INDEXES_FILE"; fi
echo 'Running java net.osmand.data.index.IndexBatchCreator with $INDEXES_FILE'
java -XX:+UseParallelGC -Xmx4096M -Xmn512M -Djava.util.logging.config.file=build-scripts/batch-logging.properties -cp "DataExtractionOSM/OsmAndMapCreator.jar:DataExtractionOSM/lib/*.jar" net.osmand.data.index.IndexBatchCreator build-scripts/indexes-roads-batch-generate.xml "$INDEXES_FILE"

6
build-scripts/indexes-generate-srtm.sh
View file

@ -1,6 +0,0 @@
rm -rf .work
mkdir .work
if [ -z $INDEXES_FILE ]; then INDEXES_FILE="build-scripts/regions/srtm_indexes.xml"; echo "$INDEXES_FILE"; fi
echo "Running java net.osmand.data.index.IndexBatchCreator with $INDEXES_FILE"
java -XX:+UseParallelGC -Xmx8096M -Xmn512M -Djava.util.logging.config.file=build-scripts/batch-logging.properties -cp "DataExtractionOSM/OsmAndMapCreator.jar:DataExtractionOSM/lib/*.jar" net.osmand.data.index.IndexBatchCreator build-scripts/srtm-batch-generate.xml "$INDEXES_FILE"

12
build-scripts/local.properties.sample
View file

@ -1,12 +0,0 @@
# This file is automatically generated by Android Tools.
# Do not modify this file -- YOUR CHANGES WILL BE ERASED!
#
# This file must *NOT* be checked in Version Control Systems,
# as it contains information specific to your local configuration.
# location of the SDK. This is only used by Ant
# For customization when using a Version Control System, please read the
# header note.
sdk.dir=/home/zibrita/osmand-build/android-sdk-linux_86
FTP_USER=osmand
FTP_PWD=

17
build-scripts/update_git.sh
View file

@ -1,17 +0,0 @@
#!/bin/sh
DIRECTORY=$(cd `dirname $0` && pwd)
GIT_DIR="$DIRECTORY"/osmand-git
GIT_URL=git://github.com/osmandapp/Osmand.git
GIT_ORIGIN_NAME=origin
#rm -rf "${GIT_DIR}"
# initialize git if it is not present (clone it)
if [ ! -d "$GIT_DIR" ]; then
git clone ${GIT_URL} "${GIT_DIR}"
fi
# update git
cd "${GIT_DIR}"
git reset --hard
git checkout -q master
git fetch ${GIT_ORIGIN_NAME}

24
build-scripts/upload_ftp.sh
View file

@ -1,24 +0,0 @@
#!/bin/sh
DIRECTORY=$(cd `dirname $0` && pwd)
FTP_SITE=download.osmand.net
FTP_FOLDER=night-builds
FTP_LATEST=latest-night-build
# FTP_USER in local.properties
# FTP_PWD= in local.properties
BUILD_DIR="$DIRECTORY"/builds
LATEST_DIR="$DIRECTORY"/latests
. "$DIRECTORY"/local.properties
# 3. upload to ftp server
lftp -c "set net:timeout 45;open -u $FTP_USER,$FTP_PWD $FTP_SITE;mirror -R $BUILD_DIR $FTP_FOLDER;mirror -R $LATEST_DIR $FTP_LATEST"
#ftp -n -v $FTP_SITE <<SCRIPT 2>&1
#quote USER $FTP_USER
#quote PASS $FTP_PWD
#cd $FTP_FOLDER
#ls
#quit