Android-builders-club/OsmAnd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

fix bug with rtree cache

Browse source 
git-svn-id: https://osmand.googlecode.com/svn/trunk@737 e29c36b1-1cfa-d876-8d93-3434fc2bb7b8
This commit is contained in:
Victor Shcherb 2010-12-01 09:49:26 +00:00
parent 250bcd4544
commit a1a66cb627
6 changed files with 45 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
DataExtractionOSM/src/net/osmand/data/index/IndexBatchCreator.java
View file

@ -37,6 +37,9 @@ import org.w3c.dom.Element;
import org.w3c.dom.NodeList; import org.w3c.dom.NodeList;
import org.xml.sax.SAXException; import org.xml.sax.SAXException;
import rtree.RTree;
public class IndexBatchCreator { public class IndexBatchCreator {
@ -234,7 +237,7 @@ public class IndexBatchCreator {
for(String name : regionCountries.regionNames){ for(String name : regionCountries.regionNames){
name = name.toLowerCase(); name = name.toLowerCase();
String url = MessageFormat.format(site, name); String url = MessageFormat.format(site, name);
downloadFile(url, prefix+name+suffix, alreadyGeneratedFiles, alreadyUploadedFiles); downloadFile(url, prefix+name, suffix,alreadyGeneratedFiles, alreadyUploadedFiles);
} }
} }
@ -244,7 +247,7 @@ public class IndexBatchCreator {
private final static int DOWNLOAD_DEBUG = 1 << 20; private final static int DOWNLOAD_DEBUG = 1 << 20;
private final static int MB = 1 << 20; private final static int MB = 1 << 20;
private final static int BUFFER_SIZE = 1 << 15; private final static int BUFFER_SIZE = 1 << 15;
protected void downloadFile(String url, String country, Set<String> alreadyGeneratedFiles, Set<String> alreadyUploadedFiles) { protected void downloadFile(String url, String country, String suffix, Set<String> alreadyGeneratedFiles, Set<String> alreadyUploadedFiles) {
byte[] buffer = new byte[BUFFER_SIZE]; byte[] buffer = new byte[BUFFER_SIZE];
int count = 0; int count = 0;
int downloaded = 0; int downloaded = 0;
@ -255,7 +258,7 @@ public class IndexBatchCreator {
} else if(url.endsWith(".osm.pbf")){ } else if(url.endsWith(".osm.pbf")){
ext = ".osm.pbf"; ext = ".osm.pbf";
} }
File toSave = new File(osmDirFiles, country+ext); File toSave = new File(osmDirFiles, country + suffix + ext);
try { try {
log.info("Downloading country " + country + " from " + url); //$NON-NLS-1$//$NON-NLS-2$ log.info("Downloading country " + country + " from " + url); //$NON-NLS-1$//$NON-NLS-2$
FileOutputStream ostream = new FileOutputStream(toSave); FileOutputStream ostream = new FileOutputStream(toSave);
@ -271,7 +274,7 @@ public class IndexBatchCreator {
} }
ostream.close(); ostream.close();
stream.close(); stream.close();
generateIndex(toSave, alreadyGeneratedFiles, alreadyUploadedFiles); generateIndex(toSave, country, alreadyGeneratedFiles, alreadyUploadedFiles);
} catch (IOException e) { } catch (IOException e) {
log.error("Input/output exception " + toSave.getName() + " downloading from " + url, e); //$NON-NLS-1$ //$NON-NLS-2$ log.error("Input/output exception " + toSave.getName() + " downloading from " + url, e); //$NON-NLS-1$ //$NON-NLS-2$
} }
@ -283,8 +286,8 @@ public class IndexBatchCreator {
if (alreadyGeneratedFiles.contains(f.getName())) { if (alreadyGeneratedFiles.contains(f.getName())) {
continue; continue;
} }
if (f.getName().endsWith(".osm.bz2") || f.getName().endsWith(".osm")) { if (f.getName().endsWith(".osm.bz2") || f.getName().endsWith(".osm") || f.getName().endsWith(".osm.pbf")) {
generateIndex(f, alreadyGeneratedFiles, alreadyUploadedFiles); generateIndex(f, null, alreadyGeneratedFiles, alreadyUploadedFiles);
} }
} }
System.out.println("GENERATING INDEXES FINISHED "); System.out.println("GENERATING INDEXES FINISHED ");
@ -292,16 +295,24 @@ public class IndexBatchCreator {
protected void generateIndex(File f, Set<String> alreadyGeneratedFiles, Set<String> alreadyUploadedFiles) { protected void generateIndex(File f, String rName, Set<String> alreadyGeneratedFiles, Set<String> alreadyUploadedFiles) {
if (!generateIndexes) { if (!generateIndexes) {
return; return;
} }
try { try {
// be independent of previous results
RTree.clearCache();
String regionName = f.getName(); String regionName = f.getName();
int i = f.getName().indexOf('.'); int i = f.getName().indexOf('.');
if (i > -1) { if (i > -1) {
regionName = Algoritms.capitalizeFirstLetterAndLowercase(f.getName().substring(0, i)); regionName = Algoritms.capitalizeFirstLetterAndLowercase(f.getName().substring(0, i));
} }
if(Algoritms.isEmpty(rName)){
rName = regionName;
} else {
rName = Algoritms.capitalizeFirstLetterAndLowercase(rName);
}
IndexCreator indexCreator = new IndexCreator(indexDirFiles); IndexCreator indexCreator = new IndexCreator(indexDirFiles);
indexCreator.setIndexAddress(indexAddress); indexCreator.setIndexAddress(indexAddress);
@ -311,6 +322,7 @@ public class IndexBatchCreator {
indexCreator.setLastModifiedDate(f.lastModified()); indexCreator.setLastModifiedDate(f.lastModified());
indexCreator.setNormalizeStreets(true); indexCreator.setNormalizeStreets(true);
indexCreator.setSaveAddressWays(true); indexCreator.setSaveAddressWays(true);
indexCreator.setRegionName(rName);
String poiFileName = regionName + "_" + IndexConstants.POI_TABLE_VERSION + IndexConstants.POI_INDEX_EXT; String poiFileName = regionName + "_" + IndexConstants.POI_TABLE_VERSION + IndexConstants.POI_INDEX_EXT;
indexCreator.setPoiFileName(poiFileName); indexCreator.setPoiFileName(poiFileName);

26
DataExtractionOSM/src/net/osmand/data/index/batch.xml
View file

@ -25,9 +25,8 @@
indexTransport="true" indexAddress="true" mapZooms="" renderingTypesFile=""> indexTransport="true" indexAddress="true" mapZooms="" renderingTypesFile="">
<!-- Countries to download from osm server --> <!-- Countries to download from osm server -->
<!-- EUROPE --> <!-- EUROPE -->
<regions siteToDownload="http://download.geofabrik.de/osm/europe/{0}.osm.pbf" region_prefix="" region_suffix="_Europe"> <regions siteToDownload="http://download.geofabrik.de/osm/europe/{0}.osm.pbf" region_prefix="" skip="true" region_suffix="_Europe">
<region name="albania" esize="6" /> <region name="albania" esize="6" />
<region name="andorra" esize="1" /> <region name="andorra" esize="1" />
<region name="austria" esize="110" /> <region name="austria" esize="110" />
@ -78,7 +77,8 @@
</regions> </regions>
<!-- FRANCE --> <!-- FRANCE -->
<regions siteToDownload="http://download.geofabrik.de/osm/europe/france/{0}.osm.bz2" region_prefix="France_" region_suffix="_Europe"> <regions siteToDownload="http://download.geofabrik.de/osm/europe/france/{0}.osm.bz2" region_prefix="France_"
skip="true" region_suffix="_Europe">
<region name="alsace" esize="" /> <region name="alsace" esize="" />
<region name="aquitaine" esize="" /> <region name="aquitaine" esize="" />
<region name="auvergne" esize="" /> <region name="auvergne" esize="" />
@ -104,7 +104,8 @@
</regions> </regions>
<!-- GERMANY --> <!-- GERMANY -->
<regions siteToDownload="http://download.geofabrik.de/osm/europe/germany/{0}.osm.bz2" region_prefix="Germany_" region_suffix="_Europe"> <regions siteToDownload="http://download.geofabrik.de/osm/europe/germany/{0}.osm.bz2" region_prefix="Germany_" skip="true"
region_suffix="_Europe">
<region name="baden-wuerttemberg" esize="" /> <region name="baden-wuerttemberg" esize="" />
<region name="bayern" esize="" /> <region name="bayern" esize="" />
<region name="berlin" esize="" /> <region name="berlin" esize="" />
@ -124,7 +125,7 @@
</regions> </regions>
<!-- CENTRAL AMERICA --> <!-- CENTRAL AMERICA -->
<regions siteToDownload="http://downloads.cloudmade.com/north_america/{0}/{0}.osm.bz2" region_prefix="" region_suffix="_NA"> <regions siteToDownload="http://downloads.cloudmade.com/north_america/{0}/{0}.osm.bz2" region_prefix="" skip="true" region_suffix="_NA">
<region name="bahamas" esize="" /> <region name="bahamas" esize="" />
<region name="costa_rica" esize="" /> <region name="costa_rica" esize="" />
<region name="cuba" esize="" /> <region name="cuba" esize="" />
@ -163,7 +164,8 @@
<!-- SOUTH AMERICA --> <!-- SOUTH AMERICA -->
<regions siteToDownload="http://downloads.cloudmade.com/south_america/{0}/{0}.osm.bz2" region_prefix="" region_suffix="_SA"> <regions siteToDownload="http://downloads.cloudmade.com/south_america/{0}/{0}.osm.bz2" region_prefix="" skip="true"
region_suffix="_SA">
<region name="argentina" esize=""/> <region name="argentina" esize=""/>
<region name="bolivia" esize=""/> <region name="bolivia" esize=""/>
<region name="brazil" esize=""/> <region name="brazil" esize=""/>
@ -182,7 +184,7 @@
<!-- ASIA --> <!-- ASIA -->
<regions siteToDownload="http://downloads.cloudmade.com/asia/{0}/{0}.osm.bz2" region_prefix="" region_suffix="_Asia"> <regions siteToDownload="http://downloads.cloudmade.com/asia/{0}/{0}.osm.bz2" region_prefix="" skip="true" region_suffix="_Asia">
<region name="afghanistan" esize=""/> <region name="afghanistan" esize=""/>
<region name="bahrain" esize=""/> <region name="bahrain" esize=""/>
<region name="bangladesh" esize=""/> <region name="bangladesh" esize=""/>
@ -235,7 +237,7 @@
</regions> </regions>
<!-- USA --> <!-- USA -->
<regions siteToDownload="http://downloads.cloudmade.com/north_america/united_states/{0}/{0}.osm.bz2" region_prefix="US_" region_suffix="_NA"> <regions siteToDownload="http://downloads.cloudmade.com/north_america/united_states/{0}/{0}.osm.bz2" skip="true" region_prefix="US_" region_suffix="_NA">
<region name="alabama" esize=""/> <region name="alabama" esize=""/>
<region name="alaska" esize=""/> <region name="alaska" esize=""/>
<region name="arizona" esize=""/> <region name="arizona" esize=""/>
@ -291,7 +293,7 @@
</regions> </regions>
<!-- Canada --> <!-- Canada -->
<regions siteToDownload="http://downloads.cloudmade.com/north_america/canada/{0}/{0}.osm.bz2" region_prefix="Canada_" region_suffix="_NA"> <regions siteToDownload="http://downloads.cloudmade.com/north_america/canada/{0}/{0}.osm.bz2" skip="true" region_prefix="Canada_" region_suffix="_NA">
<region name="alberta" esize=""/> <region name="alberta" esize=""/>
<region name="british_columbia" esize=""/> <region name="british_columbia" esize=""/>
<region name="manitoba" esize=""/> <region name="manitoba" esize=""/>
@ -308,7 +310,7 @@
</regions> </regions>
<!-- Russia --> <!-- Russia -->
<regions siteToDownload="http://gis-lab.info/data/osm/{0}/{0}.osm.bz2" region_prefix="Russia_" region_suffix="_Asia"> <regions siteToDownload="http://gis-lab.info/data/osm/{0}/{0}.osm.bz2" region_prefix="Russia_" skip="true" region_suffix="_Asia">
<region name="adygeya" esize=""/> <region name="adygeya" esize=""/>
<region name="altay" esize=""/> <region name="altay" esize=""/>
<region name="altayskiy" esize=""/> <region name="altayskiy" esize=""/>
@ -395,7 +397,7 @@
</regions> </regions>
<!-- AFRICA --> <!-- AFRICA -->
<regions siteToDownload="http://downloads.cloudmade.com/africa/{0}/{0}.osm.bz2" region_prefix="" region_suffix="_Africa"> <regions siteToDownload="http://downloads.cloudmade.com/africa/{0}/{0}.osm.bz2" region_prefix="" skip="true" region_suffix="_Africa">
<region name="algeria" esize=""/> <region name="algeria" esize=""/>
<region name="egypt" esize=""/> <region name="egypt" esize=""/>
<region name="madagascar" esize=""/> <region name="madagascar" esize=""/>
@ -461,7 +463,7 @@
<!-- OCEANIA --> <!-- OCEANIA -->
<regions siteToDownload="http://downloads.cloudmade.com/oceania/{0}/{0}.osm.bz2" region_prefix="" region_suffix="_Australia"> <regions siteToDownload="http://downloads.cloudmade.com/oceania/{0}/{0}.osm.bz2" region_prefix="" skip="true" region_suffix="_Australia">
<region name="australia" esize=""/> <region name="australia" esize=""/>
<region name="new_zealand" esize=""/> <region name="new_zealand" esize=""/>
<!-- NOT GENERATED <!-- NOT GENERATED

1
DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java
View file

@ -2076,6 +2076,7 @@ public class IndexCreator {
public void writeBinaryTransportIndex(BinaryMapIndexWriter writer) throws IOException, SQLException { public void writeBinaryTransportIndex(BinaryMapIndexWriter writer) throws IOException, SQLException {
try { try {
transportStopsTree.flush();
visitedStops = null; // allow gc to collect it visitedStops = null; // allow gc to collect it
PreparedStatement selectTransportRouteData = mapConnection.prepareStatement( PreparedStatement selectTransportRouteData = mapConnection.prepareStatement(
"SELECT id, dist, name, name_en, ref, operator, type FROM transport_route"); //$NON-NLS-1$ "SELECT id, dist, name, name_en, ref, operator, type FROM transport_route"); //$NON-NLS-1$

3
DataExtractionOSM/src/net/osmand/swing/OsmExtractionUI.java
View file

@ -52,6 +52,8 @@ import org.apache.commons.logging.LogFactory;
import org.apache.tools.bzip2.CBZip2OutputStream; import org.apache.tools.bzip2.CBZip2OutputStream;
import org.xml.sax.SAXException; import org.xml.sax.SAXException;
import rtree.RTree;
public class OsmExtractionUI implements IMapLocationListener { public class OsmExtractionUI implements IMapLocationListener {
@ -419,6 +421,7 @@ public class OsmExtractionUI implements IMapLocationListener {
} else { } else {
types = new MapRenderingTypes(fn); types = new MapRenderingTypes(fn);
} }
RTree.clearCache();
creator.generateIndexes(f, dlg, filter, DataExtractionSettings.getSettings().getMapZooms(), types); creator.generateIndexes(f, dlg, filter, DataExtractionSettings.getSettings().getMapZooms(), types);
} catch (IOException e) { } catch (IOException e) {
throw new IllegalArgumentException(e); throw new IllegalArgumentException(e);

3
DataExtractionOSM/src/rtree/FileHdr.java
View file

@ -36,6 +36,7 @@ public class FileHdr
/**Index of the top most element*/ /**Index of the top most element*/
private int topIdx; private int topIdx;
private RandomAccessFile file; private RandomAccessFile file;
private String fileName;
private boolean dirty = false;/*Tells whethet this is a dirty filehdr or not*/ private boolean dirty = false;/*Tells whethet this is a dirty filehdr or not*/
/**If any write thread is interested then increment this. This variable /**If any write thread is interested then increment this. This variable
results in the fact that writes will always have the preference. results in the fact that writes will always have the preference.
@ -66,6 +67,7 @@ public class FileHdr
{ {
try{ try{
this.file = new RandomAccessFile(fileName,"rw"); this.file = new RandomAccessFile(fileName,"rw");
this.fileName = fileName;
this.writeThr = false; this.writeThr = false;
this.stkLimit = stkLimit; this.stkLimit = stkLimit;
waiters = new Vector(); waiters = new Vector();
@ -315,6 +317,7 @@ public class FileHdr
flush(); flush();
file.close(); file.close();
}catch (Exception e) { }catch (Exception e) {
System.err.println(fileName);
e.printStackTrace(); e.printStackTrace();
} }
} }

5
DataExtractionOSM/src/rtree/RTree.java
View file

@ -68,6 +68,11 @@ public class RTree //the tree that would be made
flHdr = flH; flHdr = flH;
} }
} }
public static void clearCache(){
chdNodes = new CachedNodes();
fileList = new HashMap();
}
public RTree(String fileName) public RTree(String fileName)
throws RTreeException throws RTreeException
{ {