diff --git a/DataExtractionOSM/src/net/osmand/Algoritms.java b/DataExtractionOSM/src/net/osmand/Algoritms.java index 6a5e852a05..e82defb338 100644 --- a/DataExtractionOSM/src/net/osmand/Algoritms.java +++ b/DataExtractionOSM/src/net/osmand/Algoritms.java @@ -121,6 +121,9 @@ public class Algoritms { } public static boolean removeAllFiles(File f) { + if (f == null) { + return false; + } if (f.isDirectory()) { for (File c : f.listFiles()) { removeAllFiles(c); diff --git a/DataExtractionOSM/src/net/osmand/data/Amenity.java b/DataExtractionOSM/src/net/osmand/data/Amenity.java index 66c9b5216a..017371c131 100644 --- a/DataExtractionOSM/src/net/osmand/data/Amenity.java +++ b/DataExtractionOSM/src/net/osmand/data/Amenity.java @@ -17,6 +17,7 @@ public class Amenity extends MapObject { private AmenityType type; private String openingHours; private String phone; + private String description; private String site; public Amenity(Entity entity, AmenityType type, String subtype){ @@ -47,6 +48,7 @@ public class Amenity extends MapObject { } } } + this.description = entity.getTag(OSMTagKey.DESCRIPTION); } @Override @@ -151,6 +153,14 @@ public class Amenity extends MapObject { this.phone = phone; } + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + @Override public void doDataPreparation() { diff --git a/DataExtractionOSM/src/net/osmand/data/AmenityType.java b/DataExtractionOSM/src/net/osmand/data/AmenityType.java index b82d92d1c0..46a9e7089c 100644 --- a/DataExtractionOSM/src/net/osmand/data/AmenityType.java +++ b/DataExtractionOSM/src/net/osmand/data/AmenityType.java @@ -30,7 +30,9 @@ public enum AmenityType { EMERGENCY("emergency"), // [TAG] emergency //$NON-NLS-1$ //$NON-NLS-2$ MILITARY("military"), // [TAG] military //$NON-NLS-1$ //$NON-NLS-2$ ADMINISTRATIVE("administrative"), // [TAG] administrative //$NON-NLS-1$ //$NON-NLS-2$ - GEOCACHE("geocache"), //$NON-NLS-1$//$NON-NLS-2$ + GEOCACHE("geocache"), //$NON-NLS-1$ + OSMWIKI("osmwiki"), //$NON-NLS-1$ + USER_DEFINED("user_defined"), //$NON-NLS-1$ OTHER("amenity"), // grave-yard, police, post-office [+Internet_access] //$NON-NLS-1$ //$NON-NLS-2$ ; diff --git a/DataExtractionOSM/src/net/osmand/data/index/WikiIndexer.java b/DataExtractionOSM/src/net/osmand/data/index/WikiIndexer.java new file mode 100644 index 0000000000..3d6bbcd97d --- /dev/null +++ b/DataExtractionOSM/src/net/osmand/data/index/WikiIndexer.java @@ -0,0 +1,510 @@ +package net.osmand.data.index; + +import java.io.BufferedInputStream; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStreamWriter; +import java.sql.SQLException; + +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.parsers.SAXParser; +import javax.xml.parsers.SAXParserFactory; +import javax.xml.stream.XMLOutputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamWriter; + +import org.apache.commons.logging.Log; +import org.apache.tools.bzip2.CBZip2InputStream; +import org.xml.sax.Attributes; +import org.xml.sax.SAXException; +import org.xml.sax.helpers.DefaultHandler; + + + +import net.osmand.Algoritms; +import net.osmand.LogUtil; +import net.osmand.Version; +import net.osmand.data.preparation.IndexCreator; +import net.osmand.impl.ConsoleProgressImplementation; + +public class WikiIndexer { + private static final Log log = LogUtil.getLog(WikiIndexer.class); + private final File srcPath; + private final File workPath; + private final File targetPath; + public static class WikiIndexerException extends Exception { + private static final long serialVersionUID = 1L; + public WikiIndexerException(String name) { + super(name); + } + public WikiIndexerException(String string, Exception e) { + super(string, e); + } + + } + + public WikiIndexer(File srcPath, File targetPath, File workPath) { + this.srcPath = srcPath; + this.targetPath = targetPath; + this.workPath = workPath; + } + + public static void main(String[] args) { + try { + File srcPath = extractDirectory(args, 0); + File targetPath = extractDirectory(args, 1); + File workPath = extractDirectory(args, 2); + + WikiIndexer wikiIndexer = new WikiIndexer(srcPath, targetPath, workPath); + wikiIndexer.run(); + + } catch (WikiIndexerException e) { + log.error(e.getMessage()); + } + } + + private static File extractDirectory(String[] args, int ind) throws WikiIndexerException { + if (args.length <= ind) { + throw new WikiIndexerException("Usage: WikiIndexer src_directory target_directory work_directory [--description={full|normal|minimum}]" + " missing " + (ind + 1)); + } else { + File fs = new File(args[ind]); + fs.mkdir(); + if(!fs.exists() || !fs.isDirectory()) { + throw new WikiIndexerException("Specified directory doesn't exist : " + args[ind]); + } + return fs; + } + } + + public void run() { + File[] listFiles = srcPath.listFiles(); + for(File f : listFiles) { + try { + if (f.isFile() && (f.getName().endsWith(".xml") || f.getName().endsWith(".xml.bz2"))) { + log.info("About to process " + f.getName()); + File outFile = process(f); + if (outFile != null) { + + IndexCreator ic = new IndexCreator(workPath); + ic.setIndexPOI(true); + ic.setIndexMap(false); + ic.setIndexTransport(false); + ic.setIndexAddress(false); + ic.generateIndexes(outFile, new ConsoleProgressImplementation(3), null, null, null, log); + // final step + new File(workPath, ic.getMapFileName()).renameTo(new File(targetPath, ic.getMapFileName())); + } + } + } catch (WikiIndexerException e) { + log.error("Error processing "+f.getName(), e); + } catch (RuntimeException e) { + log.error("Error processing "+f.getName(), e); + } catch (IOException e) { + log.error("Error processing "+f.getName(), e); + } catch (SAXException e) { + log.error("Error processing "+f.getName(), e); + } catch (SQLException e) { + log.error("Error processing "+f.getName(), e); + } catch (InterruptedException e) { + log.error("Error processing "+f.getName(), e); + } + } + } + + protected File process(File f) throws WikiIndexerException { + InputStream fi = null; + BufferedWriter out = null; + try { + int in = f.getName().indexOf('.'); + File osmOut = new File(workPath, f.getName().substring(0, in) + ".osm"); + fi = new BufferedInputStream(new FileInputStream(f)); + InputStream progressStream = fi; + if(f.getName().endsWith(".bz2")){ + if (fi.read() != 'B' || fi.read() != 'Z') { + throw new RuntimeException("The source stream must start with the characters BZ if it is to be read as a BZip2 stream."); //$NON-NLS-1$ + } else { + fi = new CBZip2InputStream(fi); + } + } + ConsoleProgressImplementation progress = new ConsoleProgressImplementation(); + out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(osmOut), "UTF-8")); + SAXParser saxParser = SAXParserFactory.newInstance().newSAXParser(); + WikiOsmHandler wikiOsmHandler = new WikiOsmHandler(saxParser, out, progress, progressStream); + saxParser.parse(fi, wikiOsmHandler); + + if(wikiOsmHandler.getCount() < 1){ + return null; + } + return osmOut; + } catch (ParserConfigurationException e) { + throw new WikiIndexerException("Parse exception", e); + } catch (SAXException e) { + throw new WikiIndexerException("Parse exception", e); + } catch (IOException e) { + throw new WikiIndexerException("Parse exception", e); + } catch (XMLStreamException e) { + throw new WikiIndexerException("Parse exception", e); + } finally { + Algoritms.closeStream(out); + Algoritms.closeStream(fi); + } + } + + + public class WikiOsmHandler extends DefaultHandler { + long id = 1; + private final SAXParser saxParser; + private boolean page = false; + private StringBuilder ctext = null; + + private StringBuilder title = new StringBuilder(); + private StringBuilder text = new StringBuilder(); + + private final ConsoleProgressImplementation progress; + private final InputStream progIS; + private XMLStreamWriter streamWriter; + + WikiOsmHandler(SAXParser saxParser, BufferedWriter outOsm, ConsoleProgressImplementation progress, InputStream progIS) throws IOException, XMLStreamException { + this.saxParser = saxParser; + this.progress = progress; + this.progIS = progIS; + XMLOutputFactory xof = XMLOutputFactory.newInstance(); + streamWriter = xof.createXMLStreamWriter(outOsm); + streamWriter.writeStartDocument(); + streamWriter.writeCharacters("\n"); + streamWriter.writeStartElement("osm"); + streamWriter.writeAttribute("version", "0.6"); + streamWriter.writeAttribute("generator", Version.APP_MAP_CREATOR_VERSION); + + + progress.startTask("Parse wiki xml", progIS.available()); + } + + public int getCount() { + return (int) (id - 1); + } + + @Override + public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException { + String name = saxParser.isNamespaceAware() ? localName : qName; + if (!page) { + page = name.equals("page"); + } else { + if(name.equals("title")) { + title.setLength(0); + ctext = title; + } else if(name.equals("text")) { + text.setLength(0); + ctext = text; + } + } + } + + + @Override + public void characters(char[] ch, int start, int length) throws SAXException { + if (page) { + if(ctext != null) { + ctext.append(ch, start, length); + } + } + } + + @Override + public void endElement(String uri, String localName, String qName) throws SAXException { + String name = saxParser.isNamespaceAware() ? localName : qName; + try { + if (page) { + if(name.equals("page")) { + page = false; + progress.remaining(progIS.available()); + } else if(name.equals("title")) { + ctext = null; + } else if(name.equals("text")) { + analyzeTextForGeoInfo(); + ctext = null; + } + } + } catch (IOException e) { + throw new SAXException(e); + } catch (XMLStreamException e) { + throw new SAXException(e); + } + } + + private String readProperty(String prop, int s, int e){ + int res = -1; + for (int i = s; i < e - prop.length(); i++) { + if(prop.charAt(0) == text.charAt(i)) { + boolean neq = false; + for (int j = 0; j < prop.length(); j++) { + if(prop.charAt(j) != text.charAt(i + j)) { + neq = true; + break; + } + } + if(!neq){ + res = i; + break; + } + } + } + if(res == -1){ + return null; + } + int sr = -1; + int se = e; + for(int i = res + prop.length() + 1; i", startComment); + h = text.indexOf("{{", i); + check = true; + } + } + return h; + } + + private int findClosedBrackets(int i){ + if(i == -1){ + return -1; + } + int stack = 1; + int h = text.indexOf("{{", i+2); + int e = text.indexOf("}}", i+2); + while(stack != 0 && e != -1) { + if(h!= -1 && h 0) { + writeNode(lat, lon, subcategory, description); + } + System.out.println("Point " + title + " " + subcategory + " Lon:" + lat + " Lon:" + lon + " " ); + System.out.println("Description : " + description.toString().trim()); + System.out.println("---------------------"); + } + } + + private int checkAndParse(int i, String start, String end, StringBuilder d, boolean add){ + if(text.charAt(i) != start.charAt(0)) { + return -1; + } + for (int j = 1 ; j < start.length(); j++) { + if(text.charAt(i + j) != start.charAt(j)){ + return -1; + } + } + int st = i+start.length(); + int en = text.length(); + boolean colon = false; + for (int j = i + start.length(); j < text.length(); j++) { + if (text.charAt(j) == '|') { + st = j + 1; + if(colon){ + // Special case to prevent adding + // [[File:av.png|thumb|220|220]] + add = false; + } + } else if (j + end.length() <= text.length()) { + boolean eq = true; + if (text.charAt(j) == ':') { + colon = true; + } + for (int k = 0; k < end.length(); k++) { + if (text.charAt(j + k) != end.charAt(k)) { + eq = false; + break; + } + } + if (eq) { + en = j; + break; + } + + } + } + if(add){ + d.append(text, st, en); + } + return en + end.length(); + } + + + private void processDescription(StringBuilder description, int start) { + for (int j = start ; j < text.length();) { + if (text.charAt(j) == '=' && text.charAt(j+1) == '=') { + break; + } else if (text.charAt(j) == '\n' && j - start > 2048) { + break; + } else { + int r = -1; + if(r == -1) { + r = checkAndParse(j, "", description, false); + } + if (r == -1) { + r = checkAndParse(j, "[[", "]]", description, true); + } + if(r == -1) { + r = checkAndParse(j, "{{", "}}", description, true); + } + if(r == -1) { + r = checkAndParse(j, "''", "''", description,true); + } + + if(r == -1) { + description.append(text.charAt(j)); + j++; + } else { + j = r; + } + } + } + } + + private void writeNode(double lat, double lon, String subcategory, StringBuilder description) throws XMLStreamException { + streamWriter.writeCharacters("\n"); + streamWriter.writeStartElement("node"); + streamWriter.writeAttribute("id", "-" + id++); + streamWriter.writeAttribute("lat", lat+""); + streamWriter.writeAttribute("lon", lon+""); + + streamWriter.writeCharacters("\n "); + streamWriter.writeStartElement("tag"); + streamWriter.writeAttribute("k", "name"); + streamWriter.writeAttribute("v", title.toString()); + streamWriter.writeEndElement(); + + streamWriter.writeCharacters("\n "); + streamWriter.writeStartElement("tag"); + streamWriter.writeAttribute("k", "osmwiki"); + streamWriter.writeAttribute("v", subcategory); + streamWriter.writeEndElement(); + + streamWriter.writeCharacters("\n "); + streamWriter.writeStartElement("tag"); + streamWriter.writeAttribute("k", "description"); + streamWriter.writeAttribute("v", description.toString().trim()); + streamWriter.writeEndElement(); + + streamWriter.writeEndElement(); + streamWriter.writeCharacters("\n"); + } + + @Override + public void endDocument() throws SAXException { + try { + streamWriter.writeEndElement(); + streamWriter.writeCharacters("\n"); + streamWriter.writeEndDocument(); + } catch (XMLStreamException e) { + throw new SAXException(e); + } + } + + } +} diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/BinaryMapIndexWriter.java b/DataExtractionOSM/src/net/osmand/data/preparation/BinaryMapIndexWriter.java index c1147ef3af..05010a7141 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/BinaryMapIndexWriter.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/BinaryMapIndexWriter.java @@ -893,7 +893,7 @@ public class BinaryMapIndexWriter { } public void writePoiDataAtom(long id, int x24shift, int y24shift, String nameEn, String name, TIntArrayList types, String openingHours, - String site, String phone) throws IOException { + String site, String phone, String description) throws IOException { checkPeekState(POI_DATA); OsmAndPoiBoxDataAtom.Builder builder = OsmandOdb.OsmAndPoiBoxDataAtom.newBuilder(); @@ -920,6 +920,9 @@ public class BinaryMapIndexWriter { if(!Algoritms.isEmpty(phone)){ builder.setPhone(phone); } + if(!Algoritms.isEmpty(description)){ + builder.setNote(description); + } codedOutStream.writeMessage(OsmandOdb.OsmAndPoiBoxData.POIDATA_FIELD_NUMBER, builder.build()); diff --git a/DataExtractionOSM/src/net/osmand/data/preparation/IndexPoiCreator.java b/DataExtractionOSM/src/net/osmand/data/preparation/IndexPoiCreator.java index 5c974d6c7a..39abca6522 100644 --- a/DataExtractionOSM/src/net/osmand/data/preparation/IndexPoiCreator.java +++ b/DataExtractionOSM/src/net/osmand/data/preparation/IndexPoiCreator.java @@ -131,6 +131,7 @@ public class IndexPoiCreator extends AbstractIndexPartCreator { poiPreparedStatement.setString(8, amenity.getOpeningHours()); poiPreparedStatement.setString(9, amenity.getSite()); poiPreparedStatement.setString(10, amenity.getPhone()); + poiPreparedStatement.setString(11, amenity.getDescription()); addBatch(poiPreparedStatement); } @@ -147,8 +148,8 @@ public class IndexPoiCreator extends AbstractIndexPartCreator { // create database structure Statement stat = poiConnection.createStatement(); stat.executeUpdate("create table " + IndexConstants.POI_TABLE + //$NON-NLS-1$ - "(id bigint, x int, y int, name_en varchar(1024), name varchar(1024), " - + "type varchar(1024), subtype varchar(1024), opening_hours varchar(1024), phone varchar(1024), site varchar(1024)," + " (id bigint, x int, y int, name_en varchar(1024), name varchar(1024), " + + "type varchar(1024), subtype varchar(1024), opening_hours varchar(1024), phone varchar(1024), site varchar(1024), description varchar(4096), " + "primary key(id, type, subtype))"); stat.executeUpdate("create index poi_loc on poi (x, y, type, subtype)"); stat.executeUpdate("create index poi_id on poi (id, type, subtype)"); @@ -157,8 +158,8 @@ public class IndexPoiCreator extends AbstractIndexPartCreator { // create prepared statment poiPreparedStatement = poiConnection - .prepareStatement("INSERT INTO " + IndexConstants.POI_TABLE + "(id, x, y, name_en, name, type, subtype, opening_hours, site, phone) " + //$NON-NLS-1$//$NON-NLS-2$ - "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); + .prepareStatement("INSERT INTO " + IndexConstants.POI_TABLE + "(id, x, y, name_en, name, type, subtype, opening_hours, site, phone, description) " + //$NON-NLS-1$//$NON-NLS-2$ + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); pStatements.put(poiPreparedStatement, 0); poiConnection.setAutoCommit(false); @@ -204,7 +205,7 @@ public class IndexPoiCreator extends AbstractIndexPartCreator { // 0. process all entities ResultSet rs; if(useInMemoryCreator) { - rs = poiConnection.createStatement().executeQuery("SELECT x,y,name,name_en,type,subtype, id, opening_hours, site, phone from poi"); + rs = poiConnection.createStatement().executeQuery("SELECT x,y,name,name_en,type,subtype,id,opening_hours,site, phone, description from poi"); } else { rs = poiConnection.createStatement().executeQuery("SELECT x,y,name,name_en,type,subtype from poi"); } @@ -276,6 +277,7 @@ public class IndexPoiCreator extends AbstractIndexPartCreator { poiData.openingHours = rs.getString(8); poiData.site = rs.getString(9); poiData.phone = rs.getString(10); + poiData.description = rs.getString(11); prevTree.getNode().poiData.add(poiData); } @@ -342,7 +344,7 @@ public class IndexPoiCreator extends AbstractIndexPartCreator { buildTypeIds(type, subtype, categories, catIndexes, types); int x24shift = (x31 >> 7) - (x << (24 - z)); int y24shift = (y31 >> 7) - (y << (24 - z)); - writer.writePoiDataAtom(poi.id, x24shift, y24shift, poi.nameEn, poi.name, types, poi.openingHours, poi.site, poi.phone); + writer.writePoiDataAtom(poi.id, x24shift, y24shift, poi.nameEn, poi.name, types, poi.openingHours, poi.site, poi.phone, poi.description); } } else { @@ -368,8 +370,9 @@ public class IndexPoiCreator extends AbstractIndexPartCreator { String openingHours = rset.getString(8); String site = rset.getString(9); String phone = rset.getString(10); + String description = rset.getString(11); - writer.writePoiDataAtom(id, x24shift, y24shift, nameEn, name, types, openingHours, site, phone); + writer.writePoiDataAtom(id, x24shift, y24shift, nameEn, name, types, openingHours, site, phone, description); } rset.close(); } @@ -458,6 +461,7 @@ public class IndexPoiCreator extends AbstractIndexPartCreator { String openingHours; String phone; String site; + String description; } public static class PoiTileBox { diff --git a/DataExtractionOSM/src/net/osmand/osm/MapRenderingTypes.java b/DataExtractionOSM/src/net/osmand/osm/MapRenderingTypes.java index 43833d3ef8..0557a73975 100644 --- a/DataExtractionOSM/src/net/osmand/osm/MapRenderingTypes.java +++ b/DataExtractionOSM/src/net/osmand/osm/MapRenderingTypes.java @@ -969,6 +969,9 @@ public class MapRenderingTypes { amenityTagValToType.put("amenity", AmenityType.OTHER); //$NON-NLS-1$ amenityTagValToType.put("place", AmenityType.ADMINISTRATIVE); //$NON-NLS-1$ amenityTagValToType.put("sport", AmenityType.SPORT); //$NON-NLS-1$ + + amenityTagValToType.put("osmwiki", AmenityType.OSMWIKI); //$NON-NLS-1$ + amenityTagValToType.put("user_defined", AmenityType.USER_DEFINED); //$NON-NLS-1$ } diff --git a/DataExtractionOSM/src/net/osmand/osm/OSMSettings.java b/DataExtractionOSM/src/net/osmand/osm/OSMSettings.java index 3942d0d8bc..163236fe41 100644 --- a/DataExtractionOSM/src/net/osmand/osm/OSMSettings.java +++ b/DataExtractionOSM/src/net/osmand/osm/OSMSettings.java @@ -53,6 +53,7 @@ public class OSMSettings { OPENING_HOURS("opening_hours"), //$NON-NLS-1$ PHONE("phone"), //$NON-NLS-1$ + DESCRIPTION("description"), //$NON-NLS-1$ WEBSITE("website"), //$NON-NLS-1$ URL("url"), //$NON-NLS-1$ WIKIPEDIA("wikipedia"), //$NON-NLS-1$