Android-builders-club/OsmAnd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

fix bug with rtree cache

Browse source 
git-svn-id: https://osmand.googlecode.com/svn/trunk@737 e29c36b1-1cfa-d876-8d93-3434fc2bb7b8
This commit is contained in:
Victor Shcherb 2010-12-01 09:49:26 +00:00
parent 250bcd4544
commit a1a66cb627
6 changed files with 45 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
DataExtractionOSM/src/net/osmand/data/index/IndexBatchCreator.java
View file

@ -37,6 +37,9 @@ import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
import rtree.RTree;
public class IndexBatchCreator {
@ -234,7 +237,7 @@ public class IndexBatchCreator {
for(String name : regionCountries.regionNames){
name = name.toLowerCase();
String url = MessageFormat.format(site, name);
downloadFile(url, prefix+name+suffix, alreadyGeneratedFiles, alreadyUploadedFiles);
downloadFile(url, prefix+name, suffix,alreadyGeneratedFiles, alreadyUploadedFiles);
}
}
@ -244,7 +247,7 @@ public class IndexBatchCreator {
private final static int DOWNLOAD_DEBUG = 1 << 20;
private final static int MB = 1 << 20;
private final static int BUFFER_SIZE = 1 << 15;
protected void downloadFile(String url, String country, Set<String> alreadyGeneratedFiles, Set<String> alreadyUploadedFiles) {
protected void downloadFile(String url, String country, String suffix, Set<String> alreadyGeneratedFiles, Set<String> alreadyUploadedFiles) {
byte[] buffer = new byte[BUFFER_SIZE];
int count = 0;
int downloaded = 0;
@ -255,7 +258,7 @@ public class IndexBatchCreator {
} else if(url.endsWith(".osm.pbf")){
ext = ".osm.pbf";
}
File toSave = new File(osmDirFiles, country+ext);
File toSave = new File(osmDirFiles, country + suffix + ext);
try {
log.info("Downloading country " + country + " from " + url); //$NON-NLS-1$//$NON-NLS-2$
FileOutputStream ostream = new FileOutputStream(toSave);
@ -271,7 +274,7 @@ public class IndexBatchCreator {
}
ostream.close();
stream.close();
generateIndex(toSave, alreadyGeneratedFiles, alreadyUploadedFiles);
generateIndex(toSave, country, alreadyGeneratedFiles, alreadyUploadedFiles);
} catch (IOException e) {
log.error("Input/output exception " + toSave.getName() + " downloading from " + url, e); //$NON-NLS-1$ //$NON-NLS-2$
}
@ -283,8 +286,8 @@ public class IndexBatchCreator {
if (alreadyGeneratedFiles.contains(f.getName())) {
continue;
}
if (f.getName().endsWith(".osm.bz2") || f.getName().endsWith(".osm")) {
generateIndex(f, alreadyGeneratedFiles, alreadyUploadedFiles);
if (f.getName().endsWith(".osm.bz2") || f.getName().endsWith(".osm") || f.getName().endsWith(".osm.pbf")) {
generateIndex(f, null, alreadyGeneratedFiles, alreadyUploadedFiles);
}
}
System.out.println("GENERATING INDEXES FINISHED ");
@ -292,16 +295,24 @@ public class IndexBatchCreator {
protected void generateIndex(File f, Set<String> alreadyGeneratedFiles, Set<String> alreadyUploadedFiles) {
protected void generateIndex(File f, String rName, Set<String> alreadyGeneratedFiles, Set<String> alreadyUploadedFiles) {
if (!generateIndexes) {
return;
}
try {
// be independent of previous results
RTree.clearCache();
String regionName = f.getName();
int i = f.getName().indexOf('.');
if (i > -1) {
regionName = Algoritms.capitalizeFirstLetterAndLowercase(f.getName().substring(0, i));
}
if(Algoritms.isEmpty(rName)){
rName = regionName;
} else {
rName = Algoritms.capitalizeFirstLetterAndLowercase(rName);
}
IndexCreator indexCreator = new IndexCreator(indexDirFiles);
indexCreator.setIndexAddress(indexAddress);
@ -311,6 +322,7 @@ public class IndexBatchCreator {
indexCreator.setLastModifiedDate(f.lastModified());
indexCreator.setNormalizeStreets(true);
indexCreator.setSaveAddressWays(true);
indexCreator.setRegionName(rName);
String poiFileName = regionName + "_" + IndexConstants.POI_TABLE_VERSION + IndexConstants.POI_INDEX_EXT;
indexCreator.setPoiFileName(poiFileName);

26
DataExtractionOSM/src/net/osmand/data/index/batch.xml
View file

@ -25,9 +25,8 @@
indexTransport="true" indexAddress="true" mapZooms="" renderingTypesFile="">
<!-- Countries to download from osm server -->
<!-- EUROPE -->
<regions siteToDownload="http://download.geofabrik.de/osm/europe/{0}.osm.pbf" region_prefix="" region_suffix="_Europe">
<regions siteToDownload="http://download.geofabrik.de/osm/europe/{0}.osm.pbf" region_prefix="" skip="true" region_suffix="_Europe">
<region name="albania" esize="6" />
<region name="andorra" esize="1" />
<region name="austria" esize="110" />
@ -78,7 +77,8 @@
</regions>
<!-- FRANCE -->
<regions siteToDownload="http://download.geofabrik.de/osm/europe/france/{0}.osm.bz2" region_prefix="France_" region_suffix="_Europe">
<regions siteToDownload="http://download.geofabrik.de/osm/europe/france/{0}.osm.bz2" region_prefix="France_"
skip="true" region_suffix="_Europe">
<region name="alsace" esize="" />
<region name="aquitaine" esize="" />
<region name="auvergne" esize="" />
@ -104,7 +104,8 @@
</regions>
<!-- GERMANY -->
<regions siteToDownload="http://download.geofabrik.de/osm/europe/germany/{0}.osm.bz2" region_prefix="Germany_" region_suffix="_Europe">
<regions siteToDownload="http://download.geofabrik.de/osm/europe/germany/{0}.osm.bz2" region_prefix="Germany_" skip="true"
region_suffix="_Europe">
<region name="baden-wuerttemberg" esize="" />
<region name="bayern" esize="" />
<region name="berlin" esize="" />
@ -124,7 +125,7 @@
</regions>
<!-- CENTRAL AMERICA -->
<regions siteToDownload="http://downloads.cloudmade.com/north_america/{0}/{0}.osm.bz2" region_prefix="" region_suffix="_NA">
<regions siteToDownload="http://downloads.cloudmade.com/north_america/{0}/{0}.osm.bz2" region_prefix="" skip="true" region_suffix="_NA">
<region name="bahamas" esize="" />
<region name="costa_rica" esize="" />
<region name="cuba" esize="" />
@ -163,7 +164,8 @@
<!-- SOUTH AMERICA -->
<regions siteToDownload="http://downloads.cloudmade.com/south_america/{0}/{0}.osm.bz2" region_prefix="" region_suffix="_SA">
<regions siteToDownload="http://downloads.cloudmade.com/south_america/{0}/{0}.osm.bz2" region_prefix="" skip="true"
region_suffix="_SA">
<region name="argentina" esize=""/>
<region name="bolivia" esize=""/>
<region name="brazil" esize=""/>
@ -182,7 +184,7 @@
<!-- ASIA -->
<regions siteToDownload="http://downloads.cloudmade.com/asia/{0}/{0}.osm.bz2" region_prefix="" region_suffix="_Asia">
<regions siteToDownload="http://downloads.cloudmade.com/asia/{0}/{0}.osm.bz2" region_prefix="" skip="true" region_suffix="_Asia">
<region name="afghanistan" esize=""/>
<region name="bahrain" esize=""/>
<region name="bangladesh" esize=""/>
@ -235,7 +237,7 @@
</regions>
<!-- USA -->
<regions siteToDownload="http://downloads.cloudmade.com/north_america/united_states/{0}/{0}.osm.bz2" region_prefix="US_" region_suffix="_NA">
<regions siteToDownload="http://downloads.cloudmade.com/north_america/united_states/{0}/{0}.osm.bz2" skip="true" region_prefix="US_" region_suffix="_NA">
<region name="alabama" esize=""/>
<region name="alaska" esize=""/>
<region name="arizona" esize=""/>
@ -291,7 +293,7 @@
</regions>
<!-- Canada -->
<regions siteToDownload="http://downloads.cloudmade.com/north_america/canada/{0}/{0}.osm.bz2" region_prefix="Canada_" region_suffix="_NA">
<regions siteToDownload="http://downloads.cloudmade.com/north_america/canada/{0}/{0}.osm.bz2" skip="true" region_prefix="Canada_" region_suffix="_NA">
<region name="alberta" esize=""/>
<region name="british_columbia" esize=""/>
<region name="manitoba" esize=""/>
@ -308,7 +310,7 @@
</regions>
<!-- Russia -->
<regions siteToDownload="http://gis-lab.info/data/osm/{0}/{0}.osm.bz2" region_prefix="Russia_" region_suffix="_Asia">
<regions siteToDownload="http://gis-lab.info/data/osm/{0}/{0}.osm.bz2" region_prefix="Russia_" skip="true" region_suffix="_Asia">
<region name="adygeya" esize=""/>
<region name="altay" esize=""/>
<region name="altayskiy" esize=""/>
@ -395,7 +397,7 @@
</regions>
<!-- AFRICA -->
<regions siteToDownload="http://downloads.cloudmade.com/africa/{0}/{0}.osm.bz2" region_prefix="" region_suffix="_Africa">
<regions siteToDownload="http://downloads.cloudmade.com/africa/{0}/{0}.osm.bz2" region_prefix="" skip="true" region_suffix="_Africa">
<region name="algeria" esize=""/>
<region name="egypt" esize=""/>
<region name="madagascar" esize=""/>
@ -461,7 +463,7 @@
<!-- OCEANIA -->
<regions siteToDownload="http://downloads.cloudmade.com/oceania/{0}/{0}.osm.bz2" region_prefix="" region_suffix="_Australia">
<regions siteToDownload="http://downloads.cloudmade.com/oceania/{0}/{0}.osm.bz2" region_prefix="" skip="true" region_suffix="_Australia">
<region name="australia" esize=""/>
<region name="new_zealand" esize=""/>
<!-- NOT GENERATED

1
DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java
View file

@ -2076,6 +2076,7 @@ public class IndexCreator {
public void writeBinaryTransportIndex(BinaryMapIndexWriter writer) throws IOException, SQLException {
try {
transportStopsTree.flush();
visitedStops = null; // allow gc to collect it
PreparedStatement selectTransportRouteData = mapConnection.prepareStatement(
"SELECT id, dist, name, name_en, ref, operator, type FROM transport_route"); //$NON-NLS-1$

3
DataExtractionOSM/src/net/osmand/swing/OsmExtractionUI.java
View file

@ -52,6 +52,8 @@ import org.apache.commons.logging.LogFactory;
import org.apache.tools.bzip2.CBZip2OutputStream;
import org.xml.sax.SAXException;
import rtree.RTree;
public class OsmExtractionUI implements IMapLocationListener {
@ -419,6 +421,7 @@ public class OsmExtractionUI implements IMapLocationListener {
} else {
types = new MapRenderingTypes(fn);
}
RTree.clearCache();
creator.generateIndexes(f, dlg, filter, DataExtractionSettings.getSettings().getMapZooms(), types);
} catch (IOException e) {
throw new IllegalArgumentException(e);

3
DataExtractionOSM/src/rtree/FileHdr.java
View file

@ -36,6 +36,7 @@ public class FileHdr
/**Index of the top most element*/
private int topIdx;
private RandomAccessFile file;
private String fileName;
private boolean dirty = false;/*Tells whethet this is a dirty filehdr or not*/
/**If any write thread is interested then increment this. This variable
results in the fact that writes will always have the preference.
@ -66,6 +67,7 @@ public class FileHdr
{
try{
this.file = new RandomAccessFile(fileName,"rw");
this.fileName = fileName;
this.writeThr = false;
this.stkLimit = stkLimit;
waiters = new Vector();
@ -315,6 +317,7 @@ public class FileHdr
flush();
file.close();
}catch (Exception e) {
System.err.println(fileName);
e.printStackTrace();
}
}

5
DataExtractionOSM/src/rtree/RTree.java
View file

@ -68,6 +68,11 @@ public class RTree //the tree that would be made
flHdr = flH;
}
}
public static void clearCache(){
chdNodes = new CachedNodes();
fileList = new HashMap();
}
public RTree(String fileName)
throws RTreeException
{