diff --git a/DataExtractionOSM/src/com/osmand/osm/EntityInfo.java b/DataExtractionOSM/src/com/osmand/osm/EntityInfo.java index 1ff7b983a2..a75b363261 100644 --- a/DataExtractionOSM/src/com/osmand/osm/EntityInfo.java +++ b/DataExtractionOSM/src/com/osmand/osm/EntityInfo.java @@ -10,8 +10,15 @@ public class EntityInfo { String visible; String version; String changeset; + String action; + public String getAction() { + return action; + } + public void setAction(String action) { + this.action = action; + } public String getTimestamp() { return timestamp; } diff --git a/DataExtractionOSM/src/com/osmand/osm/MapUtils.java b/DataExtractionOSM/src/com/osmand/osm/MapUtils.java index 4c89399a34..609b3a7a11 100644 --- a/DataExtractionOSM/src/com/osmand/osm/MapUtils.java +++ b/DataExtractionOSM/src/com/osmand/osm/MapUtils.java @@ -195,6 +195,16 @@ public class MapUtils { }); } + public static void sortListOfEntities(List list, final double lat, final double lon){ + Collections.sort(list, new Comparator() { + @Override + public int compare(Entity o1, Entity o2) { + return Double.compare(MapUtils.getDistance(o1.getLatLon(), lat, lon), MapUtils.getDistance(o2.getLatLon(), + lat, lon)); + } + }); + } + public static String getFormattedDistance(int meters){ if(meters >= 100000){ diff --git a/DataExtractionOSM/src/com/osmand/osm/io/OsmStorageWriter.java b/DataExtractionOSM/src/com/osmand/osm/io/OsmStorageWriter.java index 2658155a68..142f84bd04 100644 --- a/DataExtractionOSM/src/com/osmand/osm/io/OsmStorageWriter.java +++ b/DataExtractionOSM/src/com/osmand/osm/io/OsmStorageWriter.java @@ -62,7 +62,6 @@ public class OsmStorageWriter { // String indent = "{http://xml.apache.org/xslt}indent-amount"; // transformer.setOutputProperty(indent, "4"); - XMLStreamWriter streamWriter = new XMLStreamWriterImpl(output, propertyManager); List nodes = new ArrayList(); List ways = new ArrayList(); @@ -99,7 +98,7 @@ public class OsmStorageWriter { streamWriter.writeAttribute(ATTR_LAT, n.getLatitude()+""); streamWriter.writeAttribute(ATTR_LON, n.getLongitude()+""); streamWriter.writeAttribute(ATTR_ID, n.getId()+""); - writeEntityAttributes(streamWriter, entityInfo.get(n.getId())); + writeEntityAttributes(streamWriter, n, entityInfo.get(n.getId())); writeTags(streamWriter, n); writeEndElement(streamWriter, INDENT); } @@ -107,7 +106,7 @@ public class OsmStorageWriter { for(Way w : ways){ writeStartElement(streamWriter, ELEM_WAY, INDENT); streamWriter.writeAttribute(ATTR_ID, w.getId()+""); - writeEntityAttributes(streamWriter, entityInfo.get(w.getId())); + writeEntityAttributes(streamWriter, w, entityInfo.get(w.getId())); for(Long r : w.getNodeIds()){ writeStartElement(streamWriter, ELEM_ND, INDENT2); streamWriter.writeAttribute(ATTR_REF, r+""); @@ -120,7 +119,7 @@ public class OsmStorageWriter { for(Relation r : relations){ writeStartElement(streamWriter, ELEM_RELATION, INDENT); streamWriter.writeAttribute(ATTR_ID, r.getId()+""); - writeEntityAttributes(streamWriter, entityInfo.get(r.getId())); + writeEntityAttributes(streamWriter, r, entityInfo.get(r.getId())); for(Entry e : r.getMembersMap().entrySet()){ writeStartElement(streamWriter, ELEM_MEMBER, INDENT2); streamWriter.writeAttribute(ATTR_REF, e.getKey()+""); @@ -141,8 +140,15 @@ public class OsmStorageWriter { streamWriter.flush(); } - private void writeEntityAttributes(XMLStreamWriter writer, EntityInfo info) throws XMLStreamException{ + private void writeEntityAttributes(XMLStreamWriter writer, Entity i, EntityInfo info) throws XMLStreamException{ + if(i.getId() < 0 && (info == null || info.getAction() == null)){ + writer.writeAttribute("action", "modify"); + } if(info != null){ + // for josm editor + if(info.getAction() != null){ + writer.writeAttribute("action", info.getAction()); + } if(info.getChangeset() != null){ writer.writeAttribute(ATTR_CHANGESET, info.getChangeset()); } diff --git a/DataExtractionOSM/src/com/osmand/osm/util/MinskTransReader.java b/DataExtractionOSM/src/com/osmand/osm/util/MinskTransReader.java index 295e35d5f1..c2562df14c 100644 --- a/DataExtractionOSM/src/com/osmand/osm/util/MinskTransReader.java +++ b/DataExtractionOSM/src/com/osmand/osm/util/MinskTransReader.java @@ -1,15 +1,48 @@ package com.osmand.osm.util; +import java.awt.BorderLayout; +import java.awt.Container; +import java.awt.event.WindowAdapter; +import java.awt.event.WindowEvent; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStreamReader; -import java.io.OutputStreamWriter; import java.nio.charset.Charset; import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.swing.JFrame; +import javax.swing.JMenuBar; +import javax.swing.UIManager; +import javax.xml.stream.XMLStreamException; + +import org.xml.sax.SAXException; + +import com.osmand.data.DataTileManager; +import com.osmand.impl.ConsoleProgressImplementation; +import com.osmand.osm.Entity; +import com.osmand.osm.EntityInfo; +import com.osmand.osm.LatLon; +import com.osmand.osm.MapUtils; +import com.osmand.osm.Node; +import com.osmand.osm.Relation; +import com.osmand.osm.Way; +import com.osmand.osm.OSMSettings.OSMTagKey; +import com.osmand.osm.io.IOsmStorageFilter; +import com.osmand.osm.io.OsmBaseStorage; +import com.osmand.osm.io.OsmStorageWriter; +import com.osmand.swing.DataExtractionSettings; +import com.osmand.swing.MapPanel; public class MinskTransReader { // Routes RouteNum; Authority; City; Transport; Operator; ValidityPeriods; SpecialDates;RouteTag;RouteType;Commercial;RouteName;Weekdays;RouteID;Entry;RouteStops;Datestart @@ -30,21 +63,283 @@ public class MinskTransReader { public String name ; //4 } - public static void main(String[] args) throws IOException { - FileInputStream fis = new FileInputStream(new File("E:/routes.txt")); - BufferedReader reader = new BufferedReader(new InputStreamReader(fis, Charset.forName("cp1251"))); + public static final int default_dist_to_stop = 60; + + public static final String pathToRoutes = "E:/routes.txt"; + public static final String pathToStops = "E:/stops.txt"; + public static final String pathToMinsk = "E:\\Information\\OSM maps\\data.osm"; + public static final String pathToSave = "E:\\Information\\OSM maps\\data_edit.osm"; + + public static void main(String[] args) throws IOException, SAXException, XMLStreamException { + FileInputStream fis = new FileInputStream(new File(pathToRoutes)); + BufferedReader reader = new BufferedReader(new InputStreamReader(fis, Charset.forName("UTF-8"))); List routes = readRoutes(reader); - fis = new FileInputStream(new File("E:/stops.txt")); - reader = new BufferedReader(new InputStreamReader(fis, Charset.forName("cp1251"))); + fis = new FileInputStream(new File(pathToStops)); + reader = new BufferedReader(new InputStreamReader(fis, Charset.forName("UTF-8"))); List stops = readStopes(reader); - OutputStreamWriter writer = new OutputStreamWriter(new FileOutputStream("E:/routes_out.txt"), Charset.forName("cp1251")); - - for(TransportStop r : stops){ - writer.write(r.stopId + " lat : " + r.latitude + " lon : " + r.longitude +" " + r.name + "\n"); + Map stopsMap = new LinkedHashMap(); + for(TransportStop s : stops){ + stopsMap.put(s.stopId, s); } + // checking that stops are good for(TransportRoute r : routes){ - writer.write(r.routeId +" " + r.routeNum +" " + r.routeType + " " + r.routeName+"\n"); + for(String string : r.routeStops){ + if(!stopsMap.containsKey(string)){ + throw new IllegalArgumentException("Check stop " + string + " of route " + r.routeName); + } + } + } +// showMapPanelWithCorrelatedBusStops(stopsMap, busStops); + + OsmBaseStorage storage = filterBusStops(stopsMap, routes); + OsmStorageWriter writer = new OsmStorageWriter(); + writer.saveStorage(new FileOutputStream(pathToSave), storage, null, true); + + } + + public static void showMapPanelWithCorrelatedBusStops(Map stopsMap, DataTileManager busStops) { + Map result = correlateExistingBusStopsWithImported(busStops, stopsMap); + DataTileManager nodes = new DataTileManager(); + for (String trId : result.keySet()) { + TransportStop r = stopsMap.get(trId); + Way way = new Way(-1); + way.addNode(result.get(trId)); + way.addNode(new Node(r.latitude, r.longitude, -1)); + nodes.registerObject(r.latitude, r.longitude, way); + } + for(String trId : stopsMap.keySet()){ + if(!result.containsKey(trId)){ + TransportStop r = stopsMap.get(trId); + nodes.registerObject(r.latitude, r.longitude, new Node(r.latitude, r.longitude, -1)); + } + } + showMapPanel(nodes); + } + + public static Map correlateExistingBusStopsWithImported(DataTileManager busStops, Map stopsMap){ + Map correlated = new LinkedHashMap(); + Map reverse = new LinkedHashMap(); + List stopsToCheck = new ArrayList(stopsMap.values()); + for(int k =0; k closestObjects = busStops.getClosestObjects(r.latitude, r.longitude, 0, 1); + // filter closest objects + for(int i=0; i default_dist_to_stop){ + closestObjects.remove(i); + } else{ + i++; + } + } + MapUtils.sortListOfEntities(closestObjects, r.latitude, r.longitude); + int ind = 0; + boolean ccorrelated = false; + while(ind < closestObjects.size() && !ccorrelated){ + Node foundNode = closestObjects.get(ind); + if(!reverse.containsKey(foundNode)){ + // all is good no one registered to that stop + reverse.put(foundNode, r.stopId); + correlated.put(r.stopId, foundNode); + ccorrelated = true; + } else { + // recorrelate existing node and add to todo list + String stopId = reverse.get(foundNode); + TransportStop st = stopsMap.get(stopId); + if(MapUtils.getDistance(foundNode, r.latitude, r.longitude) < MapUtils.getDistance(foundNode, st.latitude, st.longitude)){ + // check that stop again + stopsToCheck.add(st); + reverse.put(foundNode, r.stopId); + correlated.put(r.stopId, foundNode); + correlated.remove(st.stopId); + ccorrelated = true; + } + } + ind++; + } + } + return correlated; + + } + + public static void showMapPanel(DataTileManager points){ + JFrame frame = new JFrame("Map view"); + try { + UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName()); + } catch (Exception e) { + e.printStackTrace(); + } + + final MapPanel panel = new MapPanel(DataExtractionSettings.getSettings().getTilesDirectory()); + panel.setPoints(points); + frame.addWindowListener(new WindowAdapter(){ + @Override + public void windowClosing(WindowEvent e) { + DataExtractionSettings settings = DataExtractionSettings.getSettings(); + settings.saveDefaultLocation(panel.getLatitude(), panel.getLongitude()); + settings.saveDefaultZoom(panel.getZoom()); + System.exit(0); + } + }); + Container content = frame.getContentPane(); + content.add(panel, BorderLayout.CENTER); + + JMenuBar bar = new JMenuBar(); + bar.add(MapPanel.getMenuToChooseSource(panel)); + frame.setJMenuBar(bar); + frame.setSize(512, 512); + frame.setVisible(true); + + } + + protected static void removeGeneratedNotUsedBusStops(Map correlated, + Map definedRoutes, DataTileManager busStops, OsmBaseStorage storage){ + Set usedNodes = new LinkedHashSet(correlated.values()); + for(Relation r : definedRoutes.values()){ + for(Entity e : r.getMembers(null)){ + if(e instanceof Node){ + usedNodes.add((Node) e); + } + } + } + + for(Node stop : busStops.getAllObjects()){ + if(!usedNodes.contains(stop) && "yes".equals(stop.getTag("generated"))){ + EntityInfo info = storage.getRegisteredEntityInfo().get(stop.getId()); + info.setAction("delete"); + System.out.println("[DEL] Remove generated not used stop " + stop.getId() + " " + stop.getTag("name")); + } + } + + } + + protected static OsmBaseStorage filterBusStops(Map stopsMap, List routes) throws FileNotFoundException, IOException, SAXException{ + long time = System.currentTimeMillis(); + System.out.println("Start : "); + OsmBaseStorage storage = new OsmBaseStorage(); + + final Map definedRoutes = new HashMap(); + final DataTileManager busStops = new DataTileManager(); + busStops.setZoom(17); + storage.getFilters().add(new IOsmStorageFilter(){ + + @Override + public boolean acceptEntityToLoad(OsmBaseStorage storage, Entity entity) { + if(entity.getTag("route") != null){ + definedRoutes.put(entity.getTag("route") + "_" + entity.getTag("ref"), (Relation) entity); + return true; + } + if(entity.getTag(OSMTagKey.HIGHWAY) != null && entity.getTag(OSMTagKey.HIGHWAY).equals("bus_stop")){ + LatLon e = entity.getLatLon(); + busStops.registerObject(e.getLatitude(), e.getLongitude(), (Node) entity); + } + return entity instanceof Node; + } + + }); + storage.parseOSM(new FileInputStream(pathToMinsk), new ConsoleProgressImplementation()); + + Map correlated = correlateExistingBusStopsWithImported(busStops, stopsMap); + removeGeneratedNotUsedBusStops(correlated, definedRoutes, busStops, storage); + + + + registerNewRoutesAndEditExisting(stopsMap, routes, storage, definedRoutes, correlated); + System.out.println("End time : " + (System.currentTimeMillis() - time)); + return storage; + } + + protected static long id = -55000; + protected static void registerNewRoutesAndEditExisting(Map stopsMap, List routes, + OsmBaseStorage storage, final Map definedRoutes, Map correlated) { + Map checkedRoutes = new LinkedHashMap(); + // because routes can changed on schedule that's why for 1 relation many routes. + Set visitedRoutes = new HashSet(); + + for (TransportRoute r : routes) { + // register only bus/trolleybus + if (!r.transport.equals("bus") && !r.transport.equals("trolleybus")) { + continue; + } + String s = r.transport + "_" + r.routeNum; + + boolean reverse = r.routeType.equals("B>A"); + boolean direct = r.routeType.equals("A>B"); + if (!reverse && !direct) { + // that's additinal route skip it + continue; + } + if (!visitedRoutes.add(s + "_" + direct)) { + // skip it : duplicated route (schedule changed) + continue; + } + + if (definedRoutes.containsKey(s)) { + checkedRoutes.put(s, definedRoutes.get(s)); + // System.out.println("Already registered " + s); + } else { + if (!checkedRoutes.containsKey(s)) { + if(reverse){ + System.err.println("Strange route skipped : " + s); + continue; + } + Relation relation = new Relation(id--); + relation.putTag("route", r.transport); + relation.putTag("ref", r.routeNum); + relation.putTag("name", r.routeName); + relation.putTag("operator", "КУП \"Минсктранс\""); + relation.putTag("type", "route"); + relation.putTag("generated", "yes"); + checkedRoutes.put(s, relation); + storage.getRegisteredEntities().put(relation.getId(), relation); + System.out.println("[ADD] Registered new route " + s); + } + Relation relation = checkedRoutes.get(s); + + // correlating stops + for (int i = 0; i < r.routeStops.size(); i++) { + String stop = r.routeStops.get(i); + if (!stopsMap.containsKey(stop)) { + throw new IllegalArgumentException("Stops file is not corresponded to routes file"); + } + if (!correlated.containsKey(stop)) { + TransportStop st = stopsMap.get(stop); + Node node = new Node(st.latitude, st.longitude, id--); + node.putTag("highway", "bus_stop"); + if (st.name != null) { + node.putTag("name", st.name); + } else { + throw new IllegalArgumentException("Something wrong check " + st.stopId); + } + node.putTag("generated", "yes"); + storage.getRegisteredEntities().put(node.getId(), node); + System.out.println("[ADD] Added new bus_stop : " + node.getId() + " " + st.name + " minsktrans_stop_id " + st.stopId); + correlated.put(stop, node); + } + if (i == 0 || i == r.routeStops.size() - 1) { + if (direct) { + relation.addMember(correlated.get(stop).getId(), "stop"); + } + } else { + if (direct) { + relation.addMember(correlated.get(stop).getId(), "forward:stop"); + } else { + relation.addMember(correlated.get(stop).getId(), "backward:stop"); + } + } + + } + } + } + + // check relations that are not exist + for(String s : definedRoutes.keySet()){ + if(!checkedRoutes.containsKey(s)){ + Relation rel = definedRoutes.get(s); + storage.getRegisteredEntityInfo().get(rel.getId()).setAction("delete"); + System.out.println("[DEL] Route is deprecated : " + rel.getTag("route")+"_"+rel.getTag("ref") + " " + rel.getTag("name")); + + } } } @@ -112,7 +407,8 @@ public class MinskTransReader { protected static List readStopes(BufferedReader reader) throws IOException { String st = null; int line = 0; - List stopes = new ArrayList(); + List stopes = new ArrayList(); + TransportStop previous = null; while((st = reader.readLine()) != null){ if(line++ == 0){ continue; @@ -128,15 +424,20 @@ public class MinskTransReader { if(i==0){ current.stopId = newS.trim(); } else if(i==4){ - current.name = newS; + if(newS.length() == 0 && previous != null){ + current.name = previous.name; + } else { + current.name = newS; + } } else if(i==5){ - current.latitude = Double.parseDouble(newS)/1e5; + current.longitude = Double.parseDouble(newS)/1e5; } else if(i==6){ - current.longitude = Double.parseDouble(newS)/1e5; + current.latitude = Double.parseDouble(newS)/1e5; } stI = endI + 1; i++; } + previous = current; stopes.add(current); } return stopes;