Android-builders-club/OsmAnd
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

change index format (to .zip)

Browse source 
git-svn-id: https://osmand.googlecode.com/svn/trunk@65 e29c36b1-1cfa-d876-8d93-3434fc2bb7b8
This commit is contained in:
Victor Shcherb 2010-05-17 19:38:20 +00:00
parent 8292016d10
commit 2b9afd0abb
2 changed files with 105 additions and 128 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
DataExtractionOSM/src/com/osmand/data/preparation/DataIndexBuilder.java
View file

@ -6,11 +6,11 @@ import java.io.IOException;
import java.io.OutputStream; import java.io.OutputStream;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamException;
import org.apache.tools.bzip2.CBZip2OutputStream;
import com.osmand.data.Amenity; import com.osmand.data.Amenity;
import com.osmand.data.Region; import com.osmand.data.Region;
import com.osmand.osm.io.OsmStorageWriter; import com.osmand.osm.io.OsmStorageWriter;
@ -36,20 +36,24 @@ public class DataIndexBuilder {
} }
protected OutputStream checkFile(String name) throws IOException { protected OutputStream checkFile(String name) throws IOException {
if (zipped && !name.endsWith(".bz2")) { String fileName = name;
name += ".bz2"; if (zipped) {
// change name
name = new File(name).getName();
fileName += ".zip";
} }
File f = new File(workingDir, name); File f = new File(workingDir, fileName);
f.mkdirs(); f.mkdirs();
// remove existing file // remove existing file
if (f.exists()) { if (f.exists()) {
f.delete(); f.delete();
} }
OutputStream output = new FileOutputStream(f); OutputStream output = new FileOutputStream(f);
if (name.endsWith(".bz2")) { if(zipped){
output.write('B'); ZipOutputStream zipStream = new ZipOutputStream(output);
output.write('Z'); zipStream.setLevel(5);
output = new CBZip2OutputStream(output); zipStream.putNextEntry(new ZipEntry(name));
output = zipStream;
} }
return output; return output;
} }
@ -62,14 +66,7 @@ public class DataIndexBuilder {
for(Amenity a : list) { for(Amenity a : list) {
interestedObjects.add(a.getEntity().getId()); interestedObjects.add(a.getEntity().getId());
} }
OutputStream output = checkFile("POI/"+region.getName()+".osm"); OutputStream output = checkFile("POI/"+region.getName()+".osmand");
try {
OsmStorageWriter writer = new OsmStorageWriter();
writer.saveStorage(output, region.getStorage(), interestedObjects, false);
} finally {
output.close();
}
output = checkFile("POI/"+region.getName()+".osmand");
try { try {
OsmStorageWriter writer = new OsmStorageWriter(); OsmStorageWriter writer = new OsmStorageWriter();
writer.savePOIIndex(output, region); writer.savePOIIndex(output, region);

172
OsmAnd/src/com/osmand/ResourceManager.java
View file

@ -5,13 +5,15 @@ import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Enumeration;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.Map; import java.util.Map;
import java.util.Stack; import java.util.Stack;
import java.util.TreeMap; import java.util.TreeMap;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.tools.bzip2.CBZip2InputStream;
import org.xml.sax.SAXException; import org.xml.sax.SAXException;
import android.graphics.Bitmap; import android.graphics.Bitmap;
@ -24,10 +26,7 @@ import com.osmand.data.Region;
import com.osmand.data.preparation.MapTileDownloader; import com.osmand.data.preparation.MapTileDownloader;
import com.osmand.data.preparation.MapTileDownloader.DownloadRequest; import com.osmand.data.preparation.MapTileDownloader.DownloadRequest;
import com.osmand.map.ITileSource; import com.osmand.map.ITileSource;
import com.osmand.osm.Entity;
import com.osmand.osm.LatLon; import com.osmand.osm.LatLon;
import com.osmand.osm.Node;
import com.osmand.osm.io.OsmBaseStorage;
import com.osmand.osm.io.OsmIndexStorage; import com.osmand.osm.io.OsmIndexStorage;
/** /**
@ -196,121 +195,102 @@ public class ResourceManager {
return cacheOfImages.get(req.fileToLoad); return cacheOfImages.get(req.fileToLoad);
} }
private interface IndexVisitor {
/**
* returns if entry was visited succesfully
*/
public boolean visitEntry(String entryName, InputStream stream) throws IOException, SAXException;
}
// POI INDEX //
public void indexingPoi(IProgress progress){ public void indexingFiles(String pathToIndex, String ext, IProgress progress, String objectToIndex, IndexVisitor visitor) {
if (poiIndex == null) { File file = new File(Environment.getExternalStorageDirectory(), pathToIndex);
File file = new File(Environment.getExternalStorageDirectory(), POI_PATH);
poiIndex = new DataTileManager<Amenity>();
if (file.exists() && file.canRead()) { if (file.exists() && file.canRead()) {
for (File f : file.listFiles()) { for (File f : file.listFiles()) {
if (f.getName().endsWith(".bz2") || f.getName().endsWith(".osm") || f.getName().endsWith(".osmand")) { InputStream stream = null;
ZipFile zipFile = null;
Enumeration<? extends ZipEntry> entries = null;
try {
if (f.getName().endsWith(".zip")) {
zipFile = new ZipFile(f);
entries = zipFile.entries();
} else {
stream = new FileInputStream(f);
}
} catch (IOException e) {
log.error("Can't read file " + f.getAbsolutePath(), e);
continue;
}
String entryName = f.getName();
do {
try {
if (entries != null && entries.hasMoreElements()) {
ZipEntry entry = entries.nextElement();
entryName = entry.getName();
stream = zipFile.getInputStream(entry);
}
if (entryName != null && entryName.endsWith(ext)) {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
if (log.isDebugEnabled()) { if (log.isDebugEnabled()) {
log.debug("Starting index POI " + f.getAbsolutePath()); log.debug("Starting index " + objectToIndex + " " + f.getAbsolutePath());
} }
boolean zipped = f.getName().endsWith(".bz2");
InputStream stream = null; if (progress != null) {
try { progress.startTask("Indexing " + objectToIndex + " " + f.getName(), stream.available());
OsmBaseStorage storage;
boolean indexStorage = false;
if(f.getName().contains(".osmand")){
storage = new OsmIndexStorage(new Region());
indexStorage = true;
} else {
storage = new OsmBaseStorage();
} }
stream = new FileInputStream(f); visitor.visitEntry(f.getName(), stream);
// stream = new BufferedInputStream(stream); if (log.isDebugEnabled()) {
InputStream streamForProgress = stream; log.debug("Finished index " + objectToIndex + " " + f.getAbsolutePath() + " "
if (zipped) { + (System.currentTimeMillis() - start) + "ms");
if (stream.read() != 'B' || stream.read() != 'Z') {
log.error("Can't read poi file " + f.getAbsolutePath()
+ "The source stream must start with the characters BZ if it is to be read as a BZip2 stream.");
continue;
} else {
stream = new CBZip2InputStream(stream);
} }
} }
if(progress != null){ } catch (IOException e) {
progress.startTask("Indexing poi " + f.getName(), stream.available()); log.error("Can't read file " + f.getAbsolutePath(), e);
} catch (SAXException e) {
log.error("Can't read file " + f.getAbsolutePath(), e);
} finally {
Algoritms.closeStream(stream);
} }
storage.parseOSM(stream, progress, streamForProgress); } while (zipFile != null && entries.hasMoreElements());
if(indexStorage){ }
}
}
// POI INDEX //
public void indexingPoi(final IProgress progress) {
if (poiIndex == null) {
poiIndex = new DataTileManager<Amenity>();
indexingFiles(POI_PATH, ".osmand", progress, "POI", new IndexVisitor() {
@Override
public boolean visitEntry(String entryName, InputStream stream) throws IOException, SAXException {
OsmIndexStorage storage = new OsmIndexStorage(new Region());
storage.parseOSM(stream, progress);
Region region = ((OsmIndexStorage) storage).getRegion(); Region region = ((OsmIndexStorage) storage).getRegion();
for(Amenity a : region.getAmenityManager().getAllObjects()){ for (Amenity a : region.getAmenityManager().getAllObjects()) {
LatLon location = a.getLocation(); LatLon location = a.getLocation();
poiIndex.registerObject(location.getLatitude(), location.getLongitude(), a); poiIndex.registerObject(location.getLatitude(), location.getLongitude(), a);
} }
} else { return true;
for (Entity e : storage.getRegisteredEntities().values()) {
if (e instanceof Node && Amenity.isAmenity((Node) e)) {
poiIndex.registerObject(((Node) e).getLatitude(), ((Node) e).getLongitude(), new Amenity((Node) e));
}
}
}
if (log.isDebugEnabled()) {
log.debug("Finishing index POI " + f.getAbsolutePath() + " " +(System.currentTimeMillis() - start)+"ms");
}
} catch (IOException e) {
log.error("Can't read poi file " + f.getAbsolutePath(), e);
} catch (SAXException e) {
log.error("Can't read poi file " + f.getAbsolutePath(), e);
} finally {
Algoritms.closeStream(stream);
}
}
}
} }
});
} }
} }
public void indexingAddresses(IProgress progress){ public void indexingAddresses(final IProgress progress){
File file = new File(Environment.getExternalStorageDirectory(), ADDRESS_PATH); indexingFiles(ADDRESS_PATH, ".osmand", progress, "address", new IndexVisitor() {
if (file.exists() && file.canRead()) { @Override
for (File f : file.listFiles()) { public boolean visitEntry(String entryName, InputStream stream) throws IOException, SAXException {
if (f.getName().endsWith(".osmand.bz2") || f.getName().endsWith(".osmand")) { String name = entryName.substring(0, entryName.indexOf('.'));
long start = System.currentTimeMillis();
if (log.isDebugEnabled()) {
log.debug("Starting index address " + f.getAbsolutePath());
}
boolean zipped = f.getName().endsWith(".bz2");
InputStream stream = null;
String name = f.getName().substring(0, f.getName().indexOf('.'));
Region region = new Region(); Region region = new Region();
region.setName(name); region.setName(name);
addressMap.put(name, region); addressMap.put(name, region);
try {
OsmIndexStorage storage = new OsmIndexStorage(region); OsmIndexStorage storage = new OsmIndexStorage(region);
stream = new FileInputStream(f); storage.parseOSM(stream, progress);
// stream = new BufferedInputStream(stream); return true;
InputStream streamForProgress = stream;
if (zipped) {
if (stream.read() != 'B' || stream.read() != 'Z') {
log.error("Can't read index file " + f.getAbsolutePath()
+ "The source stream must start with the characters BZ if it is to be read as a BZip2 stream.");
continue;
} else {
stream = new CBZip2InputStream(stream);
}
}
if(progress != null){
progress.startTask("Indexing address " + f.getName(), stream.available());
}
storage.parseOSM(stream, progress, streamForProgress);
if (log.isDebugEnabled()) {
log.debug("Finishing index address " + f.getAbsolutePath() + " " +(System.currentTimeMillis() - start)+"ms");
}
} catch (IOException e) {
log.error("Can't read index file " + f.getAbsolutePath(), e);
} catch (SAXException e) {
log.error("Can't read index file " + f.getAbsolutePath(), e);
} finally {
Algoritms.closeStream(stream);
}
}
}
} }
});
} }
public DataTileManager<Amenity> getPoiIndex() { public DataTileManager<Amenity> getPoiIndex() {