From 7d905ead02a143f1804f383be0595b5f1f2fbd24 Mon Sep 17 00:00:00 2001 From: Victor Shcherb Date: Sat, 7 Apr 2012 13:05:02 +0200 Subject: [PATCH] Add basemap generator --- .../net/osmand/binary/BinaryInspector.java | 2 +- ...neProcessor.java => BasemapProcessor.java} | 294 +----------------- .../osmand/data/preparation/IndexCreator.java | 92 +++++- .../preparation/IndexVectorMapCreator.java | 14 +- 4 files changed, 108 insertions(+), 294 deletions(-) rename DataExtractionOSM/src/net/osmand/data/preparation/{CoastlineProcessor.java => BasemapProcessor.java} (60%) diff --git a/DataExtractionOSM/src/net/osmand/binary/BinaryInspector.java b/DataExtractionOSM/src/net/osmand/binary/BinaryInspector.java index 0fd92bd223..e62fcf604d 100644 --- a/DataExtractionOSM/src/net/osmand/binary/BinaryInspector.java +++ b/DataExtractionOSM/src/net/osmand/binary/BinaryInspector.java @@ -46,7 +46,7 @@ public class BinaryInspector { public static void main(String[] args) throws IOException { inspector(args); // test cases show info - inspector(new String[]{"/home/victor/projects/OsmAnd/data/osm-gen/basemap_coastlines"}); + inspector(new String[]{"/home/victor/projects/OsmAnd/data/osm-gen/basemap_coastlines.obf"}); // test case extract parts diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/CoastlineProcessor.java b/DataExtractionOSM/src/net/osmand/data/preparation/BasemapProcessor.java similarity index 60% rename from DataExtractionOSM/src/net/osmand/data/preparation/CoastlineProcessor.java rename to DataExtractionOSM/src/net/osmand/data/preparation/BasemapProcessor.java index 8d6ce75926..1d5afd4b2a 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/CoastlineProcessor.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/BasemapProcessor.java @@ -2,11 +2,8 @@ package net.osmand.data.preparation; import gnu.trove.list.array.TIntArrayList; import gnu.trove.map.hash.TLongObjectHashMap; -import gnu.trove.procedure.TObjectProcedure; import java.io.ByteArrayOutputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; @@ -17,7 +14,6 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -import javax.xml.stream.XMLStreamException; import net.osmand.Algoritms; import net.osmand.binary.OsmandOdb.MapData; @@ -25,20 +21,17 @@ import net.osmand.binary.OsmandOdb.MapDataBlock; import net.osmand.data.MapAlgorithms; import net.osmand.data.preparation.MapZooms.MapZoomPair; import net.osmand.osm.Entity; -import net.osmand.osm.Entity.EntityId; -import net.osmand.osm.EntityInfo; import net.osmand.osm.MapRenderingTypes; import net.osmand.osm.MapUtils; import net.osmand.osm.Node; import net.osmand.osm.Way; import net.osmand.osm.WayChain; -import net.osmand.osm.io.OsmBaseStorage; -import net.osmand.osm.io.OsmStorageWriter; +import net.osmand.osm.OSMSettings.OSMTagKey; import org.apache.commons.logging.Log; import org.apache.tools.bzip2.CBZip2InputStream; -public class CoastlineProcessor { +public class BasemapProcessor { TLongObjectHashMap coastlinesEndPoint = new TLongObjectHashMap(); TLongObjectHashMap coastlinesStartPoint = new TLongObjectHashMap(); @@ -133,7 +126,7 @@ public class CoastlineProcessor { } - public CoastlineProcessor(Log logMapDataWarn, MapZooms mapZooms, MapRenderingTypes renderingTypes, int zoomWaySmothness) { + public BasemapProcessor(Log logMapDataWarn, MapZooms mapZooms, MapRenderingTypes renderingTypes, int zoomWaySmothness) { this.logMapDataWarn = logMapDataWarn; this.mapZooms = mapZooms; this.renderingTypes = renderingTypes; @@ -142,14 +135,14 @@ public class CoastlineProcessor { quadTrees = new SimplisticQuadTree[mapZooms.getLevels().size()]; for (int i=0;i< mapZooms.getLevels().size(); i++) { MapZoomPair p = mapZooms.getLevels().get(i); - quadTrees[i] = constructTilesQuadTree(Math.min(p.getMaxZoom() - 1, 1)); + quadTrees[i] = constructTilesQuadTree(Math.min(p.getMaxZoom() - 1, 12)); } } private void constructBitSetInfo() { try { - InputStream stream = CoastlineProcessor.class.getResourceAsStream("oceantiles_12.dat.bz2"); + InputStream stream = BasemapProcessor.class.getResourceAsStream("oceantiles_12.dat.bz2"); if (stream.read() != 'B' || stream.read() != 'Z') { throw new RuntimeException("The source stream must start with the characters BZ if it is to be read as a BZip2 stream."); //$NON-NLS-1$ } @@ -282,7 +275,7 @@ public class CoastlineProcessor { // write map levels and map index writer.startWriteMapLevelIndex(p.getMinZoom(), p.getMaxZoom(), 0, (1 << 31) - 1, 0, (1 << 31) - 1); - Map refs = new LinkedHashMap(); + Map refs = new LinkedHashMap(); writeBinaryMapTree(quadTrees[i], writer, refs, p); // without data blocks @@ -310,7 +303,7 @@ public class CoastlineProcessor { for (Way w : quad.getCoastlines(p)) { dataBlock.setBaseId(w.getId()); List res = new ArrayList(); - MapAlgorithms.simplifyDouglasPeucker(w.getNodes(), p.getMaxZoom() - 2 + 8 + zoomWaySmothness, 3, res); + MapAlgorithms.simplifyDouglasPeucker(w.getNodes(), p.getMaxZoom() - 1 + 8 + zoomWaySmothness, 3, res); ByteArrayOutputStream bcoordinates = new ByteArrayOutputStream(); for (Node n : res) { if (n != null) { @@ -361,12 +354,18 @@ public class CoastlineProcessor { } + public void processEntity(Entity e) { + if(e instanceof Way && "coastline".equals(e.getTag(OSMTagKey.NATURAL))) { + processCoastline((Way) e); + } + } + public void processCoastline(Way e) { renderingTypes.getCoastlineRuleType().updateFreq(); int ij = 0; - for(MapZoomPair p : mapZooms.getLevels()) { + for(MapZoomPair zoomPair : mapZooms.getLevels()) { SimplisticQuadTree quadTree = quadTrees[ij++]; - int z = Math.min((p.getMinZoom() + p.getMaxZoom()) / 2, p.getMinZoom() + 3); + int z = Math.min((zoomPair.getMinZoom() + zoomPair.getMaxZoom()) / 2 - 1, zoomPair.getMinZoom() + 1); List ns = e.getNodes(); if(ns.size() < 2) { return; @@ -434,272 +433,11 @@ public class CoastlineProcessor { System.err.println("Tile " + tilex + " / " + tiley + " at " + z + " can not be found"); } } - quad.addCoastline(p, w); + quad.addCoastline(zoomPair, w); } } } - - - - ///////////////////////////// NOT USED CODE TO DELETE /////////////////////////////// - - public void processCoastlineOld(Way e) { - WayChain chain = null; - if(coastlinesEndPoint.contains(e.getFirstNodeId())){ - chain = coastlinesEndPoint.remove(e.getFirstNodeId()); - chain.append(e); - coastlinesEndPoint.put(chain.getLastNode(), chain); - } - if(coastlinesStartPoint.contains(e.getLastNodeId())) { - WayChain chain2 = coastlinesStartPoint.remove(e.getLastNodeId()); - if(chain == null) { - chain = chain2; - chain.prepend(e); - coastlinesStartPoint.put(chain.getFistNode(), chain); - } else if(chain2 != chain) { - // remove chain 2 - coastlinesEndPoint.remove(chain2.getLastNode()); - chain.append(chain2); - coastlinesEndPoint.put(chain.getLastNode(), chain); - } else { - // cycle detected : skip it - } - } - if(chain == null) { - chain = new WayChain(e); - coastlinesEndPoint.put(chain.getLastNode(), chain); - coastlinesStartPoint.put(chain.getFistNode(), chain); - } - } - - private long nodeId = 1 << 100; - - private class CoastlineTile { - List> chains = new ArrayList>(); - List> islands = new ArrayList>(); - - double lleft = 0; - double lright = 0; - double ltop = 0; - double lbottom = 0; - - private CoastlineTile(List chain) { - ltop = lbottom = chain.get(0).getLatitude(); - lleft = lright = chain.get(0).getLongitude(); - addChain(chain); - } - - public boolean intersect(CoastlineTile t) { - if (lleft > t.lright || lright < t.lleft || ltop < t.lbottom || lbottom > t.ltop) { - return false; - } - return true; - } - - public void combineTiles(CoastlineTile t) { - for(List chain : t.chains) { - addSimpleChain(chain); - } - for(List island : t.islands) { - addIsland(island); - } - } - - public boolean contains(List chain) { - for (int i = 0; i < chain.size(); i++) { - if (lleft <= chain.get(i).getLongitude() && lright >= chain.get(i).getLongitude()) { - if (lbottom <= chain.get(i).getLatitude() && ltop >= chain.get(i).getLatitude()) { - return true; - } - } - } - return false; - } - - private void addSimpleChain(List chain) { - updateBoundaries(chain); - chains.add(chain); - } - - public void addChain(List chain) { - updateBoundaries(chain); - int ind = 0; - while (ind < chain.size() - 1) { - List subChain = new ArrayList(); - Node first = chain.get(ind); - boolean directionToRight = chain.get(ind + 1).getLongitude() > first.getLongitude(); - int nextLonMaximum = ind + 1; - double lonEnd = first.getLongitude(); - double latPeek = first.getLatitude(); - int latPeekInd = ind; - int latLocPeekInd = ind; - for (int j = ind + 1; j < chain.size(); j++) { - if (directionToRight) { - if (chain.get(j).getLatitude() <= latPeek) { - latPeek = chain.get(j).getLatitude(); - latPeekInd = j; - } - if (chain.get(j).getLongitude() >= lonEnd) { - nextLonMaximum = j; - lonEnd = chain.get(j).getLongitude(); - latLocPeekInd = latPeekInd; - } else if (chain.get(j).getLongitude() < first.getLongitude()) { - break; - } - } else { - if (chain.get(j).getLatitude() >= latPeek) { - latPeek = chain.get(j).getLatitude(); - latPeekInd = j; - } - if (chain.get(j).getLongitude() <= lonEnd) { - nextLonMaximum = j; - lonEnd = chain.get(j).getLongitude(); - latLocPeekInd = latPeekInd; - } else if (chain.get(j).getLongitude() > first.getLongitude()) { - break; - } - } - } - if(latLocPeekInd > ind) { - for (int i = ind; i <= latLocPeekInd; i++) { - subChain.add(chain.get(i)); - } - Node ned = new Node(chain.get(latLocPeekInd).getLatitude(), first.getLongitude(), nodeId++); - subChain.add(ned); - subChain.add(first); - chains.add(subChain); - subChain= new ArrayList(); - } - - for (int i = latLocPeekInd; i <= nextLonMaximum; i++) { - subChain.add(chain.get(i)); - } - Node ned = new Node(chain.get(latLocPeekInd).getLatitude(), lonEnd, nodeId++); - subChain.add(ned); - ind = nextLonMaximum; - subChain.add(chain.get(latLocPeekInd)); - chains.add(subChain); - - } - } - - public void addIsland(List chain){ - updateBoundaries(chain); - islands.add(chain); - } - - public void updateBoundaries(List chain) { - for (int i = 0; i < chain.size(); i++) { - lleft = Math.min(lleft, chain.get(i).getLongitude()); - lright = Math.max(lright, chain.get(i).getLongitude()); - ltop = Math.max(ltop, chain.get(i).getLatitude()); - lbottom = Math.min(lbottom, chain.get(i).getLatitude()); - } - } - - } - - public void processCoastlines() { - System.out.println("Way chains " + coastlinesStartPoint.size()); - final List processed = new ArrayList(); - final List> islands = new ArrayList>(); - coastlinesStartPoint.forEachValue(new TObjectProcedure() { - @Override - public boolean execute(WayChain object) { - boolean closed = object.getFistNode() == object.getLastNode(); - if (!closed) { - List ns = object.getChainNodes(); - boolean update = true; - CoastlineTile tile = new CoastlineTile(ns); - while (update) { - Iterator it = processed.iterator(); - update = false; - while (it.hasNext()) { - CoastlineTile newTile = it.next(); - if (newTile.intersect(tile)) { - it.remove(); - newTile.combineTiles(tile); - tile = newTile; - update = true; - break; - } - } - } - processed.add(0, tile); - - System.out.println((closed ? "Closed " : "Not closed ") + "way sizes " + object.getWays().size() + " ids " - + object.getWays()); - } else { - List nodes = object.getChainNodes(); - Way w = new Way(-1, nodes); - if(w.getFirstNodeId() != w.getLastNodeId()) { - w.addNode(w.getNodes().get(0)); - } - if(MapAlgorithms.isClockwiseWay(w)) { - if(!object.isIncomplete()) { - System.out.println("??? Lake " + object.getWays()); - } - } else { - islands.add(w.getNodes()); - } - } - return true; - } - }); - for(List island : islands) { - boolean log = true; - for(CoastlineTile ts : processed) { - if(ts.contains(island)){ - ts.addIsland(island); - log = false; - break; - } - } - if(log) { - System.out.println("Island missed"); - } - } - OsmBaseStorage st = new OsmBaseStorage(); - OsmStorageWriter writer = new OsmStorageWriter(); - Map entities = st.getRegisteredEntities(); - for(CoastlineTile ts : processed) { - System.out.println("Coastline Tile left,top,right,bottom : " +((float)ts.lleft)+"," - +((float)ts.ltop)+","+((float)ts.lright)+","+((float)ts.lbottom)); - System.out.println(" Chains " + ts.chains.size() + " islands " + ts.islands.size()); - for(List ns : ts.chains) { - registerWay(entities, st.getRegisteredEntityInfo(), ns); - } - for(List ns : ts.islands) { - registerWay(entities, st.getRegisteredEntityInfo(), ns); - } - } - try { - writer.saveStorage(new FileOutputStream("/home/victor/projects/OsmAnd/data/osm-maps/check_coastline.osm"), st, null, true); - } catch (FileNotFoundException e) { - e.printStackTrace(); - } catch (XMLStreamException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } - - } - - - - long id = 10000; - private void registerWay(Map entities, Map map, List ns) { - Way w = new Way(id++, ns); - for(Node n : ns) { - entities.put(EntityId.valueOf(n), n); - map.put(EntityId.valueOf(n), new EntityInfo("1")); - } - entities.put(EntityId.valueOf(w), w); - map.put(EntityId.valueOf(w), new EntityInfo("1")); - } - - } diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java b/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java index 16c23e15aa..481f77c200 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/IndexCreator.java @@ -338,7 +338,93 @@ public class IndexCreator { indexTransportCreator.createDatabaseStructure(mapConnection, mapDBDialect, getRTreeTransportStopsFileName()); } } + + public void generateBasemapIndex(File readFile, IProgress progress, IOsmStorageFilter addFilter, MapZooms mapZooms, + MapRenderingTypes renderingTypes, Log logMapDataWarn) throws IOException, SAXException, SQLException, InterruptedException { + if (logMapDataWarn == null) { + logMapDataWarn = log; + } + if (logMapDataWarn == null) { + logMapDataWarn = log; + } + if (renderingTypes == null) { + renderingTypes = MapRenderingTypes.getDefault(); + } + if (mapZooms == null) { + mapZooms = MapZooms.getDefault(); + } + + // clear previous results and setting variables + if (readFile != null && regionName == null) { + int i = readFile.getName().indexOf('.'); + if (i > -1) { + regionName = Algoritms.capitalizeFirstLetterAndLowercase(readFile.getName().substring(0, i)); + } + } + try { + this.accessor = new OsmDbAccessor(); + createPlainOsmDb(progress, readFile, addFilter); + // 2. Create index connections and index structure + + final BasemapProcessor processor = new BasemapProcessor(logMapDataWarn, mapZooms, renderingTypes, zoomWaySmothness); + + progress.setGeneralProgress("[50 / 100]"); + progress.startTask(Messages.getString("IndexCreator.PROCESS_OSM_NODES"), accessor.getAllNodes()); + accessor.iterateOverEntities(progress, EntityType.NODE, new OsmDbVisitor() { + @Override + public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException { + ctx.loadEntityData(e); + processor.processEntity(e); + } + }); + progress.setGeneralProgress("[70 / 100]"); + progress.startTask(Messages.getString("IndexCreator.PROCESS_OSM_WAYS"), accessor.getAllWays()); + accessor.iterateOverEntities(progress, EntityType.WAY, new OsmDbVisitor() { + @Override + public void iterateEntity(Entity e, OsmDbAccessorContext ctx) throws SQLException { + ctx.loadEntityData(e); + processor.processEntity(e); + } + }); + mapFile = new File(workingDir, getMapFileName()); + // to save space + mapFile.getParentFile().mkdirs(); + if (mapFile.exists()) { + mapFile.delete(); + } + mapRAFile = new RandomAccessFile(mapFile, "rw"); + BinaryMapIndexWriter writer = new BinaryMapIndexWriter(mapRAFile); + + progress.setGeneralProgress("[95 of 100]"); + progress.startTask("Writing map index to binary file...", -1); + processor.writeCoastlinesFile(writer, regionName); + progress.finishTask(); + writer.close(); + mapRAFile.close(); + log.info("Finish writing binary file"); //$NON-NLS-1$ + } catch (RuntimeException e) { + log.error("Log exception", e); //$NON-NLS-1$ + throw e; + } catch (SQLException e) { + log.error("Log exception", e); //$NON-NLS-1$ + throw e; + } catch (IOException e) { + log.error("Log exception", e); //$NON-NLS-1$ + throw e; + } catch (SAXException e) { + log.error("Log exception", e); //$NON-NLS-1$ + throw e; + } finally { + try { + accessor.closeReadingConnection(); + } catch (SQLException e) { + e.printStackTrace(); + } catch (RuntimeException e) { + e.printStackTrace(); + } + } + } public void generateIndexes(File readFile, IProgress progress, IOsmStorageFilter addFilter, MapZooms mapZooms, MapRenderingTypes renderingTypes, Log logMapDataWarn) throws IOException, SAXException, SQLException, InterruptedException { @@ -366,7 +452,7 @@ public class IndexCreator { this.indexTransportCreator = new IndexTransportCreator(); this.indexPoiCreator = new IndexPoiCreator(renderingTypes); this.indexAddressCreator = new IndexAddressCreator(logMapDataWarn); - this.indexMapCreator = new IndexVectorMapCreator(logMapDataWarn,mapZooms, renderingTypes, zoomWaySmothness); + this.indexMapCreator = new IndexVectorMapCreator(logMapDataWarn, mapZooms, renderingTypes, zoomWaySmothness); this.accessor = new OsmDbAccessor(); // init address @@ -642,10 +728,10 @@ public class IndexCreator { // creator.generateIndexes(new File("/home/victor/projects/OsmAnd/data/osm-maps/cuba2.osm.bz2"), // new ConsoleProgressImplementation(1), null, zooms, rt, log); // ;6-8;9-14 - zooms = MapZooms.parseZooms("1-3;4-6;7-9;10-14"); + zooms = MapZooms.parseZooms("1-3;4-6;7-9;10-"); creator.setRegionName("basemap"); creator.setMapFileName("basemap_coastlines.obf"); - creator.generateIndexes(new File("/home/victor/projects/OsmAnd/data/basemap/10m_coastline_out.osm"), + creator.generateBasemapIndex(new File("/home/victor/projects/OsmAnd/data/basemap/10m_coastline_out.osm"), new ConsoleProgressImplementation(1), null, zooms, rt, log); diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/IndexVectorMapCreator.java b/DataExtractionOSM/src/net/osmand/data/preparation/IndexVectorMapCreator.java index f86521978d..9e02f7e428 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/IndexVectorMapCreator.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/IndexVectorMapCreator.java @@ -60,8 +60,6 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator { private static final int MAP_LEVELS_MAX = 1 << MAP_LEVELS_POWER; private MapRenderingTypes renderingTypes; private MapZooms mapZooms; - - private boolean COASTLINE_PROCESS = true; Map multiPolygonsWays = new LinkedHashMap(); @@ -76,7 +74,7 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator { TIntArrayList addtypeUse = new TIntArrayList(8); List restrictionsUse = new ArrayList(8); - private CoastlineProcessor coastlineProcessor; + private BasemapProcessor basemapProcessor; private PreparedStatement mapBinaryStat; private PreparedStatement mapLowLevelBinaryStat; private int lowLevelWays = -1; @@ -93,7 +91,7 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator { this.mapZooms = mapZooms; this.zoomWaySmothness = zoomWaySmothness; this.renderingTypes = renderingTypes; - this.coastlineProcessor = new CoastlineProcessor(logMapDataWarn, mapZooms, renderingTypes, zoomWaySmothness); + this.basemapProcessor = new BasemapProcessor(logMapDataWarn, mapZooms, renderingTypes, zoomWaySmothness); lowLevelWays = -1; } @@ -504,10 +502,6 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator { if (e instanceof Way || e instanceof Node) { // manipulate what kind of way to load ctx.loadEntityData(e); - if (e instanceof Way && "coastline".equals(e.getTag(OSMTagKey.NATURAL)) && COASTLINE_PROCESS) { - coastlineProcessor.processCoastline((Way) e); - return; - } for (int level = 0; level < mapZooms.size(); level++) { boolean area = renderingTypes.encodeEntityWithType(e, mapZooms.getLevel(level).getMaxZoom(), typeUse, addtypeUse, namesUse, tempNameUse); @@ -556,10 +550,6 @@ public class IndexVectorMapCreator extends AbstractIndexPartCreator { public void writeBinaryMapIndex(BinaryMapIndexWriter writer, String regionName) throws IOException, SQLException { closePreparedStatements(mapBinaryStat, mapLowLevelBinaryStat); mapConnection.commit(); - if(COASTLINE_PROCESS) { - coastlineProcessor.writeCoastlinesFile(writer, regionName); - return; - } try { writer.startWriteMapIndex(regionName); // write map encoding rules