diff --git a/DataExtractionOSM/src/com/osmand/ToDoConstants.java b/DataExtractionOSM/src/com/osmand/ToDoConstants.java index 0c9d52b7cc..ab0759a3a5 100644 --- a/DataExtractionOSM/src/com/osmand/ToDoConstants.java +++ b/DataExtractionOSM/src/com/osmand/ToDoConstants.java @@ -46,5 +46,6 @@ public class ToDoConstants { // 4. Config file log & see log from file // 5. specify area to load map (filter for osm loading) // 6. Predefine what should be extracted from osm (building, poi or roads) + // 7. Fix TODO in files (accept amenity - way) } diff --git a/DataExtractionOSM/src/com/osmand/data/Amenity.java b/DataExtractionOSM/src/com/osmand/data/Amenity.java index 6b46e2bbff..8211f64afb 100644 --- a/DataExtractionOSM/src/com/osmand/data/Amenity.java +++ b/DataExtractionOSM/src/com/osmand/data/Amenity.java @@ -4,6 +4,7 @@ import java.util.LinkedHashMap; import java.util.Map; import com.osmand.Algoritms; +import com.osmand.osm.Entity; import com.osmand.osm.Node; import com.osmand.osm.OSMSettings.OSMTagKey; @@ -108,7 +109,10 @@ public class Amenity extends MapObject { return AmenityType.OTHER; } - public static boolean isAmenity(Node n){ + public static boolean isAmenity(Entity n){ + if(!(n instanceof Node)){ + return false; + } if(n.getTag(OSMTagKey.AMENITY) != null){ return true; } else if(n.getTag(OSMTagKey.SHOP) != null){ diff --git a/DataExtractionOSM/src/com/osmand/data/Street.java b/DataExtractionOSM/src/com/osmand/data/Street.java index 198115ce76..64a3eb5029 100644 --- a/DataExtractionOSM/src/com/osmand/data/Street.java +++ b/DataExtractionOSM/src/com/osmand/data/Street.java @@ -40,7 +40,7 @@ public class Street extends MapObject { if(center == null){ calculateCenter(); } - return center.getLatLon(); + return center == null ? null : center.getLatLon(); } protected void calculateCenter(){ @@ -51,10 +51,12 @@ public class Street extends MapObject { LatLon c = MapUtils.getWeightCenterForNodes(wayNodes); double dist = Double.POSITIVE_INFINITY; for(Node n : wayNodes){ - double nd = MapUtils.getDistance(n, c); - if(nd < dist){ - center = n; - dist = nd; + if (n != null) { + double nd = MapUtils.getDistance(n, c); + if (nd < dist) { + center = n; + dist = nd; + } } } } diff --git a/DataExtractionOSM/src/com/osmand/data/preparation/DataExtraction.java b/DataExtractionOSM/src/com/osmand/data/preparation/DataExtraction.java index 385e39f29f..ae8168a83c 100644 --- a/DataExtractionOSM/src/com/osmand/data/preparation/DataExtraction.java +++ b/DataExtractionOSM/src/com/osmand/data/preparation/DataExtraction.java @@ -23,15 +23,16 @@ import com.osmand.data.City; import com.osmand.data.DataTileManager; import com.osmand.data.Region; import com.osmand.data.Street; +import com.osmand.data.City.CityType; import com.osmand.impl.ConsoleProgressImplementation; import com.osmand.osm.Entity; import com.osmand.osm.LatLon; import com.osmand.osm.MapUtils; import com.osmand.osm.Node; import com.osmand.osm.OSMSettings; -import com.osmand.osm.Relation; import com.osmand.osm.Way; import com.osmand.osm.OSMSettings.OSMTagKey; +import com.osmand.osm.io.IOsmStorageFilter; import com.osmand.osm.io.OSMStorageWriter; import com.osmand.osm.io.OsmBaseStorage; import com.osmand.swing.OsmExtractionUI; @@ -140,7 +141,7 @@ public class DataExtraction { } - public Region readCountry(String path, IProgress progress) throws IOException, SAXException{ + public Region readCountry(String path, IProgress progress, IOsmStorageFilter... filters) throws IOException, SAXException{ InputStream stream = new FileInputStream(path); InputStream streamFile = stream; long st = System.currentTimeMillis(); @@ -163,45 +164,40 @@ public class DataExtraction { // highways count ways = new ArrayList(); - OsmBaseStorage storage = new OsmBaseStorage(){ + IOsmStorageFilter filter = new IOsmStorageFilter(){ @Override - public boolean acceptEntityToLoad(Entity e) { + public boolean acceptEntityToLoad(OsmBaseStorage storage, Entity e) { if ("yes".equals(e.getTag(OSMTagKey.BUILDING))) { if (e.getTag(OSMTagKey.ADDR_HOUSE_NUMBER) != null && e.getTag(OSMTagKey.ADDR_STREET) != null) { buildings.add(e); return true; } } - return super.acceptEntityToLoad(e); - } - - @Override - public boolean acceptNodeToLoad(Node n) { - // TODO accept amenity for way! hospital, university, parking, fast_food... - if(Amenity.isAmenity(n)){ - amenities.add(new Amenity(n)); - } - if (n.getTag(OSMTagKey.PLACE) != null) { - places.add(n); - } - - return true; - } - - @Override - public boolean acceptRelationToLoad(Relation w) { - return false; - } - @Override - public boolean acceptWayToLoad(Way w) { - if (OSMSettings.wayForCar(w.getTag(OSMTagKey.HIGHWAY))) { - ways.add(w); + if(Amenity.isAmenity(e)){ + amenities.add(new Amenity((Node) e)); return true; } - return false; + if (e instanceof Node && e.getTag(OSMTagKey.PLACE) != null) { + places.add((Node) e); + return true; + } + if (e instanceof Way && OSMSettings.wayForCar(e.getTag(OSMTagKey.HIGHWAY))) { + ways.add((Way) e); + return true; + } + return e instanceof Node; } }; + OsmBaseStorage storage = new OsmBaseStorage(); + if(filters != null){ + for(IOsmStorageFilter f : filters){ + if(f != null){ + storage.getFilters().add(f); + } + } + } + storage.getFilters().add(filter); storage.parseOSM(stream, progress, streamFile); if (log.isDebugEnabled()) { @@ -236,14 +232,24 @@ public class DataExtraction { if (w.getTag(OSMTagKey.NAME) != null) { String street = w.getTag(OSMTagKey.NAME); LatLon center = MapUtils.getWeightCenterForNodes(w.getNodes()); - City city = country.getClosestCity(center); - if (city != null) { - Street str = city.registerStreet(street); - for(Node n : w.getNodes()){ - str.getWayNodes().add(n); + if (center != null) { + City city = country.getClosestCity(center); + if(city == null){ + Node n = new Node(center.getLatitude(), center.getLongitude(), -1); + n.putTag(OSMTagKey.PLACE.getValue(), CityType.TOWN.name()); + n.putTag(OSMTagKey.NAME.getValue(), "Uknown city"); + country.registerCity(n); + city = country.getClosestCity(center); } + + if (city != null) { + Street str = city.registerStreet(street); + for (Node n : w.getNodes()) { + str.getWayNodes().add(n); + } + } + waysManager.registerObject(center.getLatitude(), center.getLongitude(), w); } - waysManager.registerObject(center.getLatitude(), center.getLongitude(), w); } } progress.finishTask(); @@ -260,6 +266,13 @@ public class DataExtraction { // no nodes where loaded for this way } else { City city = country.getClosestCity(center); + if(city == null){ + Node n = new Node(center.getLatitude(), center.getLongitude(), -1); + n.putTag(OSMTagKey.PLACE.getValue(), CityType.TOWN.name()); + n.putTag(OSMTagKey.NAME.getValue(), "Uknown city"); + country.registerCity(n); + city = country.getClosestCity(center); + } if (city != null) { city.registerBuilding(b); } @@ -271,4 +284,5 @@ public class DataExtraction { return country; } + } diff --git a/DataExtractionOSM/src/com/osmand/osm/MapUtils.java b/DataExtractionOSM/src/com/osmand/osm/MapUtils.java index e519d3faae..cd6f42188c 100644 --- a/DataExtractionOSM/src/com/osmand/osm/MapUtils.java +++ b/DataExtractionOSM/src/com/osmand/osm/MapUtils.java @@ -81,16 +81,23 @@ public class MapUtils { } public static LatLon getWeightCenterForNodes(Collection nodes){ - if(nodes.isEmpty()){ + if (nodes.isEmpty()) { return null; } double longitude = 0; double latitude = 0; - for(Node n : nodes){ - longitude += n.getLongitude(); - latitude += n.getLatitude(); + int count = 0; + for (Node n : nodes) { + if (n != null) { + count++; + longitude += n.getLongitude(); + latitude += n.getLatitude(); + } } - return new LatLon(latitude/nodes.size(), longitude/nodes.size()); + if (count == 0) { + return null; + } + return new LatLon(latitude/count, longitude/count); } diff --git a/DataExtractionOSM/src/com/osmand/osm/io/IOsmStorageFilter.java b/DataExtractionOSM/src/com/osmand/osm/io/IOsmStorageFilter.java new file mode 100644 index 0000000000..6fdba04432 --- /dev/null +++ b/DataExtractionOSM/src/com/osmand/osm/io/IOsmStorageFilter.java @@ -0,0 +1,9 @@ +package com.osmand.osm.io; + +import com.osmand.osm.Entity; + +public interface IOsmStorageFilter { + + public boolean acceptEntityToLoad(OsmBaseStorage storage, Entity entity); + +} diff --git a/DataExtractionOSM/src/com/osmand/osm/io/OsmBaseStorage.java b/DataExtractionOSM/src/com/osmand/osm/io/OsmBaseStorage.java index 27b3bfa684..1194679e05 100644 --- a/DataExtractionOSM/src/com/osmand/osm/io/OsmBaseStorage.java +++ b/DataExtractionOSM/src/com/osmand/osm/io/OsmBaseStorage.java @@ -2,8 +2,10 @@ package com.osmand.osm.io; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; import java.util.HashSet; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Set; @@ -53,7 +55,7 @@ public class OsmBaseStorage extends DefaultHandler { protected IProgress progress; protected InputStream inputStream; protected InputStream streamForProgress; - + protected List filters = new ArrayList(); public synchronized void parseOSM(InputStream stream, IProgress progress, InputStream streamForProgress) throws IOException, SAXException { @@ -203,6 +205,15 @@ public class OsmBaseStorage extends DefaultHandler { } + protected boolean acceptEntityToLoad(Entity entity) { + for(IOsmStorageFilter f : filters){ + if(!f.acceptEntityToLoad(this, entity)){ + return false; + } + } + return true; + } + public void completeReading(){ for(Entity e : entities.values()){ e.initializeLinks(entities); @@ -213,32 +224,14 @@ public class OsmBaseStorage extends DefaultHandler { - public boolean acceptEntityToLoad(Entity e){ - if(e instanceof Way){ - return acceptWayToLoad((Way) e); - } else if(e instanceof Relation){ - return acceptRelationToLoad((Relation) e); - } else if(e instanceof Node){ - return acceptNodeToLoad((Node) e); - } - return false; - } - - public boolean acceptWayToLoad(Way w){ - return true; - } - - public boolean acceptRelationToLoad(Relation w){ - return true; - } - - public boolean acceptNodeToLoad(Node n){ - return true; - } + public Map getRegisteredEntities() { return entities; } - + + public List getFilters() { + return filters; + } } diff --git a/DataExtractionOSM/src/com/osmand/osm/io/OsmBoundsFilter.java b/DataExtractionOSM/src/com/osmand/osm/io/OsmBoundsFilter.java new file mode 100644 index 0000000000..b527e4c6d8 --- /dev/null +++ b/DataExtractionOSM/src/com/osmand/osm/io/OsmBoundsFilter.java @@ -0,0 +1,53 @@ +package com.osmand.osm.io; + +import com.osmand.osm.Entity; +import com.osmand.osm.Node; +import com.osmand.osm.Relation; +import com.osmand.osm.Way; + +public class OsmBoundsFilter implements IOsmStorageFilter { + + private final double lonEnd; + private final double latDown; + private final double latUp; + private final double lonStart; + + public OsmBoundsFilter(double latStart, double lonStart, double latEnd, double lonEnd){ + this.latUp = latStart; + this.lonStart = lonStart; + this.latDown = latEnd; + this.lonEnd = lonEnd; + + } + + @Override + public boolean acceptEntityToLoad(OsmBaseStorage storage, Entity entity) { + if(entity instanceof Node){ + double lon = ((Node) entity).getLongitude(); + double lat = ((Node) entity).getLatitude(); + if (latDown <= lat && lat <= latUp && lonStart <= lon && lon <= lonEnd) { + return true; + } + return false; + } + // IMPORTANT : The main assumption is that order is preserved in osm file (first are node, way, relation)!!! + if(entity instanceof Way){ + for(Long l : ((Way) entity).getNodeIds()){ + if(!storage.getRegisteredEntities().containsKey(l)){ + return false; + } + } + return true; + } + if(entity instanceof Relation){ + for(Long l : ((Relation) entity).getMemberIds()){ + if(!storage.getRegisteredEntities().containsKey(l)){ + return false; + } + } + return true; + } + return false; + } + +} diff --git a/DataExtractionOSM/src/com/osmand/swing/OsmExtractionUI.java b/DataExtractionOSM/src/com/osmand/swing/OsmExtractionUI.java index b9aed6ae6b..a0b06d442d 100644 --- a/DataExtractionOSM/src/com/osmand/swing/OsmExtractionUI.java +++ b/DataExtractionOSM/src/com/osmand/swing/OsmExtractionUI.java @@ -9,7 +9,9 @@ import java.awt.event.ActionListener; import java.awt.event.WindowAdapter; import java.awt.event.WindowEvent; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.Collection; @@ -56,6 +58,7 @@ import javax.xml.stream.XMLStreamException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.tools.bzip2.CBZip2OutputStream; import org.xml.sax.SAXException; import com.osmand.Algoritms; @@ -75,6 +78,10 @@ import com.osmand.osm.Entity; import com.osmand.osm.LatLon; import com.osmand.osm.MapUtils; import com.osmand.osm.Node; +import com.osmand.osm.io.IOsmStorageFilter; +import com.osmand.osm.io.OSMStorageWriter; +import com.osmand.osm.io.OsmBoundsFilter; +import com.osmand.swing.MapPanel.MapSelectionArea; public class OsmExtractionUI implements IMapLocationListener { @@ -101,9 +108,11 @@ public class OsmExtractionUI implements IMapLocationListener { private JButton generateDataButton; private JCheckBox buildPoiIndex; private JCheckBox buildAddressIndex; + private JCheckBox filterAreaToLoad; private TreeModelListener treeModelListener; + public OsmExtractionUI(final Region r){ this.region = r; mapPanel = new MapPanel(DataExtractionSettings.getSettings().getTilesDirectory()); @@ -315,6 +324,11 @@ public class OsmExtractionUI implements IMapLocationListener { panel.add(buildAddressIndex); buildAddressIndex.setSelected(true); + filterAreaToLoad = new JCheckBox(); + filterAreaToLoad.setText("Filtering area when load file"); + panel.add(filterAreaToLoad); + filterAreaToLoad.setSelected(false); + updateButtonsBar(); } @@ -387,6 +401,8 @@ public class OsmExtractionUI implements IMapLocationListener { bar.add(menu); JMenuItem loadFile = new JMenuItem("Load osm file..."); menu.add(loadFile); + JMenuItem saveOsmFile = new JMenuItem("Save data to osm file..."); + menu.add(saveOsmFile); JMenuItem specifyWorkingDir = new JMenuItem("Specify working directory..."); menu.add(specifyWorkingDir); menu.addSeparator(); @@ -431,7 +447,7 @@ public class OsmExtractionUI implements IMapLocationListener { fc.setDialogTitle("Choose osm file"); fc.setFileSelectionMode(JFileChooser.FILES_ONLY); fc.setAcceptAllFileFilterUsed(true); - fc.setCurrentDirectory(DataExtractionSettings.getSettings().getDefaultWorkingDir()); + fc.setCurrentDirectory(DataExtractionSettings.getSettings().getDefaultWorkingDir().getParentFile()); //System.out.println("opening fc for extension " + extension); fc.setFileFilter(new FileFilter(){ @@ -453,9 +469,49 @@ public class OsmExtractionUI implements IMapLocationListener { } }); + saveOsmFile.addActionListener(new ActionListener(){ + + @Override + public void actionPerformed(ActionEvent e) { + if(region == null){ + return; + } + JFileChooser fc = new JFileChooser(); + fc.setDialogTitle("Choose osm file to save"); + fc.setFileSelectionMode(JFileChooser.FILES_ONLY); + fc.setAcceptAllFileFilterUsed(true); + fc.setCurrentDirectory(DataExtractionSettings.getSettings().getDefaultWorkingDir().getParentFile()); + //System.out.println("opening fc for extension " + extension); + fc.setFileFilter(new FileFilter(){ + + @Override + public boolean accept(File f) { + return f.isDirectory() || f.getName().endsWith(".bz2") || f.getName().endsWith(".osm"); + } + + @Override + public String getDescription() { + return "Osm Files (*.bz2, *.osm)"; + } + }); + + int answer = fc.showSaveDialog(frame); + if (answer == JFileChooser.APPROVE_OPTION && fc.getSelectedFile() != null){ + saveCountry(fc.getSelectedFile()); + } + } + + }); } public void loadCountry(final File f){ + final IOsmStorageFilter filter; + if(filterAreaToLoad.isSelected() && mapPanel.getSelectionArea().isVisible()){ + MapSelectionArea area = mapPanel.getSelectionArea(); + filter = new OsmBoundsFilter(area.getLat1(), area.getLon1(), area.getLat2(), area.getLon2()); + } else { + filter = null; + } try { final ProgressDialog dlg = new ProgressDialog(frame, "Loading osm file"); dlg.setRunnable(new Runnable(){ @@ -464,7 +520,7 @@ public class OsmExtractionUI implements IMapLocationListener { public void run() { Region res; try { - res = new DataExtraction().readCountry(f.getAbsolutePath(), dlg); + res = new DataExtraction().readCountry(f.getAbsolutePath(), dlg, filter); } catch (IOException e) { throw new IllegalArgumentException(e); } catch (SAXException e) { @@ -491,6 +547,40 @@ public class OsmExtractionUI implements IMapLocationListener { } } + public void saveCountry(final File f){ + final OSMStorageWriter writer = new OSMStorageWriter(region.getStorage().getRegisteredEntities()); + try { + final ProgressDialog dlg = new ProgressDialog(frame, "Saving osm file"); + dlg.setRunnable(new Runnable() { + @Override + public void run() { + try { + OutputStream output = new FileOutputStream(f); + try { + if (f.getName().endsWith(".bz2")) { + output.write('B'); + output.write('Z'); + output = new CBZip2OutputStream(output); + } + writer.saveStorage(output, null, false); + } finally { + output.close(); + } + } catch (IOException e) { + throw new IllegalArgumentException(e); + } catch (XMLStreamException e) { + throw new IllegalArgumentException(e); + } + } + }); + dlg.run(); + } catch (InterruptedException e1) { + log.error("Interrupted", e1); + } catch (InvocationTargetException e1) { + log.error("Exception during operation", e1.getCause()); + } + } + @Override public void locationChanged(final double newLatitude, final double newLongitude, Object source){ if (amenitiesTree != null) { diff --git a/DataExtractionOSM/src/com/osmand/swing/ProgressDialog.java b/DataExtractionOSM/src/com/osmand/swing/ProgressDialog.java index 75c85404fb..1026b470a0 100644 --- a/DataExtractionOSM/src/com/osmand/swing/ProgressDialog.java +++ b/DataExtractionOSM/src/com/osmand/swing/ProgressDialog.java @@ -88,6 +88,9 @@ public class ProgressDialog extends JDialog implements IProgress { label.setText("Please waiting..."); progressBar.setIndeterminate(true); setSize(550, 100); + double x = getParent().getBounds().getCenterX(); + double y = getParent().getBounds().getCenterY(); + setLocation((int) x - getWidth() / 2, (int) y - getHeight() / 2); } public Object getResult() {