001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.io;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.InputStream;
007import java.util.Collection;
008import java.util.Objects;
009import java.util.regex.Matcher;
010import java.util.regex.Pattern;
011
012import javax.xml.stream.Location;
013import javax.xml.stream.XMLStreamConstants;
014import javax.xml.stream.XMLStreamException;
015import javax.xml.stream.XMLStreamReader;
016
017import org.openstreetmap.josm.data.osm.Changeset;
018import org.openstreetmap.josm.data.osm.DataSet;
019import org.openstreetmap.josm.data.osm.Node;
020import org.openstreetmap.josm.data.osm.PrimitiveData;
021import org.openstreetmap.josm.data.osm.Relation;
022import org.openstreetmap.josm.data.osm.RelationMemberData;
023import org.openstreetmap.josm.data.osm.Tagged;
024import org.openstreetmap.josm.data.osm.Way;
025import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
026import org.openstreetmap.josm.gui.progress.ProgressMonitor;
027import org.openstreetmap.josm.tools.Logging;
028import org.openstreetmap.josm.tools.UncheckedParseException;
029import org.openstreetmap.josm.tools.XmlUtils;
030
031/**
032 * Parser for the Osm API (XML output). Read from an input stream and construct a dataset out of it.
033 *
034 * For each xml element, there is a dedicated method.
035 * The XMLStreamReader cursor points to the start of the element, when the method is
036 * entered, and it must point to the end of the same element, when it is exited.
037 */
038public class OsmReader extends AbstractReader {
039
040    protected XMLStreamReader parser;
041
042    /**
043     * constructor (for private and subclasses use only)
044     *
045     * @see #parseDataSet(InputStream, ProgressMonitor)
046     */
047    protected OsmReader() {
048        // Restricts visibility
049    }
050
051    protected void setParser(XMLStreamReader parser) {
052        this.parser = parser;
053    }
054
055    protected void throwException(Throwable th) throws XMLStreamException {
056        throw new XmlStreamParsingException(th.getMessage(), parser.getLocation(), th);
057    }
058
059    protected void throwException(String msg, Throwable th) throws XMLStreamException {
060        throw new XmlStreamParsingException(msg, parser.getLocation(), th);
061    }
062
063    protected void throwException(String msg) throws XMLStreamException {
064        throw new XmlStreamParsingException(msg, parser.getLocation());
065    }
066
067    protected void parse() throws XMLStreamException {
068        int event = parser.getEventType();
069        while (true) {
070            if (event == XMLStreamConstants.START_ELEMENT) {
071                parseRoot();
072            } else if (event == XMLStreamConstants.END_ELEMENT)
073                return;
074            if (parser.hasNext()) {
075                event = parser.next();
076            } else {
077                break;
078            }
079        }
080        parser.close();
081    }
082
083    protected void parseRoot() throws XMLStreamException {
084        if ("osm".equals(parser.getLocalName())) {
085            parseOsm();
086        } else {
087            parseUnknown();
088        }
089    }
090
091    private void parseOsm() throws XMLStreamException {
092        try {
093            parseVersion(parser.getAttributeValue(null, "version"));
094            parseDownloadPolicy("download", parser.getAttributeValue(null, "download"));
095            parseUploadPolicy("upload", parser.getAttributeValue(null, "upload"));
096            parseLocked(parser.getAttributeValue(null, "locked"));
097        } catch (IllegalDataException e) {
098            throwException(e);
099        }
100        String generator = parser.getAttributeValue(null, "generator");
101        Long uploadChangesetId = null;
102        if (parser.getAttributeValue(null, "upload-changeset") != null) {
103            uploadChangesetId = getLong("upload-changeset");
104        }
105        while (parser.hasNext()) {
106            int event = parser.next();
107
108            if (cancel) {
109                cancel = false;
110                throw new OsmParsingCanceledException(tr("Reading was canceled"), parser.getLocation());
111            }
112
113            if (event == XMLStreamConstants.START_ELEMENT) {
114                switch (parser.getLocalName()) {
115                case "bounds":
116                    parseBounds(generator);
117                    break;
118                case "node":
119                    parseNode();
120                    break;
121                case "way":
122                    parseWay();
123                    break;
124                case "relation":
125                    parseRelation();
126                    break;
127                case "changeset":
128                    parseChangeset(uploadChangesetId);
129                    break;
130                case "remark": // Used by Overpass API
131                    parseRemark();
132                    break;
133                default:
134                    parseUnknown();
135                }
136            } else if (event == XMLStreamConstants.END_ELEMENT) {
137                return;
138            }
139        }
140    }
141
142    private void handleIllegalDataException(IllegalDataException e) throws XMLStreamException {
143        Throwable cause = e.getCause();
144        if (cause instanceof XMLStreamException) {
145            throw (XMLStreamException) cause;
146        } else {
147            throwException(e);
148        }
149    }
150
151    private void parseRemark() throws XMLStreamException {
152        while (parser.hasNext()) {
153            int event = parser.next();
154            if (event == XMLStreamConstants.CHARACTERS) {
155                ds.setRemark(parser.getText());
156            } else if (event == XMLStreamConstants.END_ELEMENT) {
157                return;
158            }
159        }
160    }
161
162    private void parseBounds(String generator) throws XMLStreamException {
163        String minlon = parser.getAttributeValue(null, "minlon");
164        String minlat = parser.getAttributeValue(null, "minlat");
165        String maxlon = parser.getAttributeValue(null, "maxlon");
166        String maxlat = parser.getAttributeValue(null, "maxlat");
167        String origin = parser.getAttributeValue(null, "origin");
168        try {
169            parseBounds(generator, minlon, minlat, maxlon, maxlat, origin);
170        } catch (IllegalDataException e) {
171            handleIllegalDataException(e);
172        }
173        jumpToEnd();
174    }
175
176    protected Node parseNode() throws XMLStreamException {
177        String lat = parser.getAttributeValue(null, "lat");
178        String lon = parser.getAttributeValue(null, "lon");
179        try {
180            return parseNode(lat, lon, this::readCommon, this::parseNodeTags);
181        } catch (IllegalDataException e) {
182            handleIllegalDataException(e);
183        }
184        return null;
185    }
186
187    private void parseNodeTags(Node n) throws IllegalDataException {
188        try {
189            while (parser.hasNext()) {
190                int event = parser.next();
191                if (event == XMLStreamConstants.START_ELEMENT) {
192                    if ("tag".equals(parser.getLocalName())) {
193                        parseTag(n);
194                    } else {
195                        parseUnknown();
196                    }
197                } else if (event == XMLStreamConstants.END_ELEMENT) {
198                    return;
199                }
200            }
201        } catch (XMLStreamException e) {
202            throw new IllegalDataException(e);
203        }
204    }
205
206    protected Way parseWay() throws XMLStreamException {
207        try {
208            return parseWay(this::readCommon, this::parseWayNodesAndTags);
209        } catch (IllegalDataException e) {
210            handleIllegalDataException(e);
211        }
212        return null;
213    }
214
215    private void parseWayNodesAndTags(Way w, Collection<Long> nodeIds) throws IllegalDataException {
216        try {
217            while (parser.hasNext()) {
218                int event = parser.next();
219                if (event == XMLStreamConstants.START_ELEMENT) {
220                    switch (parser.getLocalName()) {
221                    case "nd":
222                        nodeIds.add(parseWayNode(w));
223                        break;
224                    case "tag":
225                        parseTag(w);
226                        break;
227                    default:
228                        parseUnknown();
229                    }
230                } else if (event == XMLStreamConstants.END_ELEMENT) {
231                    break;
232                }
233            }
234        } catch (XMLStreamException e) {
235            throw new IllegalDataException(e);
236        }
237    }
238
239    private long parseWayNode(Way w) throws XMLStreamException {
240        if (parser.getAttributeValue(null, "ref") == null) {
241            throwException(
242                    tr("Missing mandatory attribute ''{0}'' on <nd> of way {1}.", "ref", Long.toString(w.getUniqueId()))
243            );
244        }
245        long id = getLong("ref");
246        if (id == 0) {
247            throwException(
248                    tr("Illegal value of attribute ''ref'' of element <nd>. Got {0}.", Long.toString(id))
249            );
250        }
251        jumpToEnd();
252        return id;
253    }
254
255    protected Relation parseRelation() throws XMLStreamException {
256        try {
257            return parseRelation(this::readCommon, this::parseRelationMembersAndTags);
258        } catch (IllegalDataException e) {
259            handleIllegalDataException(e);
260        }
261        return null;
262    }
263
264    private void parseRelationMembersAndTags(Relation r, Collection<RelationMemberData> members) throws IllegalDataException {
265        try {
266            while (parser.hasNext()) {
267                int event = parser.next();
268                if (event == XMLStreamConstants.START_ELEMENT) {
269                    switch (parser.getLocalName()) {
270                    case "member":
271                        members.add(parseRelationMember(r));
272                        break;
273                    case "tag":
274                        parseTag(r);
275                        break;
276                    default:
277                        parseUnknown();
278                    }
279                } else if (event == XMLStreamConstants.END_ELEMENT) {
280                    break;
281                }
282            }
283        } catch (XMLStreamException e) {
284            throw new IllegalDataException(e);
285        }
286    }
287
288    private RelationMemberData parseRelationMember(Relation r) throws XMLStreamException {
289        RelationMemberData result = null;
290        try {
291            String ref = parser.getAttributeValue(null, "ref");
292            String type = parser.getAttributeValue(null, "type");
293            String role = parser.getAttributeValue(null, "role");
294            result = parseRelationMember(r, ref, type, role);
295            jumpToEnd();
296        } catch (IllegalDataException e) {
297            handleIllegalDataException(e);
298        }
299        return result;
300    }
301
302    private void parseChangeset(Long uploadChangesetId) throws XMLStreamException {
303
304        Long id = null;
305        if (parser.getAttributeValue(null, "id") != null) {
306            id = getLong("id");
307        }
308        // Read changeset info if neither upload-changeset nor id are set, or if they are both set to the same value
309        if (Objects.equals(id, uploadChangesetId)) {
310            uploadChangeset = new Changeset(id != null ? id.intValue() : 0);
311            while (true) {
312                int event = parser.next();
313                if (event == XMLStreamConstants.START_ELEMENT) {
314                    if ("tag".equals(parser.getLocalName())) {
315                        parseTag(uploadChangeset);
316                    } else {
317                        parseUnknown();
318                    }
319                } else if (event == XMLStreamConstants.END_ELEMENT)
320                    return;
321            }
322        } else {
323            jumpToEnd(false);
324        }
325    }
326
327    private void parseTag(Tagged t) throws XMLStreamException {
328        String key = parser.getAttributeValue(null, "k");
329        String value = parser.getAttributeValue(null, "v");
330        try {
331            parseTag(t, key, value);
332        } catch (IllegalDataException e) {
333            throwException(e);
334        }
335        jumpToEnd();
336    }
337
338    protected void parseUnknown(boolean printWarning) throws XMLStreamException {
339        final String element = parser.getLocalName();
340        if (printWarning && ("note".equals(element) || "meta".equals(element))) {
341            // we know that Overpass API returns those elements
342            Logging.debug(tr("Undefined element ''{0}'' found in input stream. Skipping.", element));
343        } else if (printWarning) {
344            Logging.info(tr("Undefined element ''{0}'' found in input stream. Skipping.", element));
345        }
346        while (true) {
347            int event = parser.next();
348            if (event == XMLStreamConstants.START_ELEMENT) {
349                parseUnknown(false); /* no more warning for inner elements */
350            } else if (event == XMLStreamConstants.END_ELEMENT)
351                return;
352        }
353    }
354
355    protected void parseUnknown() throws XMLStreamException {
356        parseUnknown(true);
357    }
358
359    /**
360     * When cursor is at the start of an element, moves it to the end tag of that element.
361     * Nested content is skipped.
362     *
363     * This is basically the same code as parseUnknown(), except for the warnings, which
364     * are displayed for inner elements and not at top level.
365     * @param printWarning if {@code true}, a warning message will be printed if an unknown element is met
366     * @throws XMLStreamException if there is an error processing the underlying XML source
367     */
368    protected final void jumpToEnd(boolean printWarning) throws XMLStreamException {
369        while (true) {
370            int event = parser.next();
371            if (event == XMLStreamConstants.START_ELEMENT) {
372                parseUnknown(printWarning);
373            } else if (event == XMLStreamConstants.END_ELEMENT)
374                return;
375        }
376    }
377
378    protected final void jumpToEnd() throws XMLStreamException {
379        jumpToEnd(true);
380    }
381
382    /**
383     * Read out the common attributes and put them into current OsmPrimitive.
384     * @param current primitive to update
385     * @throws IllegalDataException if there is an error processing the underlying XML source
386     */
387    private void readCommon(PrimitiveData current) throws IllegalDataException {
388        try {
389            parseId(current, getLong("id"));
390            parseTimestamp(current, parser.getAttributeValue(null, "timestamp"));
391            parseUser(current, parser.getAttributeValue(null, "user"), parser.getAttributeValue(null, "uid"));
392            parseVisible(current, parser.getAttributeValue(null, "visible"));
393            parseVersion(current, parser.getAttributeValue(null, "version"));
394            parseAction(current, parser.getAttributeValue(null, "action"));
395            parseChangeset(current, parser.getAttributeValue(null, "changeset"));
396        } catch (UncheckedParseException | XMLStreamException e) {
397            throw new IllegalDataException(e);
398        }
399    }
400
401    private long getLong(String name) throws XMLStreamException {
402        String value = parser.getAttributeValue(null, name);
403        try {
404            return getLong(name, value);
405        } catch (IllegalDataException e) {
406            throwException(e);
407        }
408        return 0; // should not happen
409    }
410
411    /**
412     * Exception thrown after user cancelation.
413     */
414    private static final class OsmParsingCanceledException extends XmlStreamParsingException implements ImportCancelException {
415        /**
416         * Constructs a new {@code OsmParsingCanceledException}.
417         * @param msg The error message
418         * @param location The parser location
419         */
420        OsmParsingCanceledException(String msg, Location location) {
421            super(msg, location);
422        }
423    }
424
425    @Override
426    protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
427        return doParseDataSet(source, progressMonitor, ir -> {
428            try {
429                setParser(XmlUtils.newSafeXMLInputFactory().createXMLStreamReader(ir));
430                parse();
431            } catch (XmlStreamParsingException | UncheckedParseException e) {
432                throw new IllegalDataException(e.getMessage(), e);
433            } catch (XMLStreamException e) {
434                String msg = e.getMessage();
435                Pattern p = Pattern.compile("Message: (.+)");
436                Matcher m = p.matcher(msg);
437                if (m.find()) {
438                    msg = m.group(1);
439                }
440                if (e.getLocation() != null)
441                    throw new IllegalDataException(tr("Line {0} column {1}: ",
442                            e.getLocation().getLineNumber(), e.getLocation().getColumnNumber()) + msg, e);
443                else
444                    throw new IllegalDataException(msg, e);
445            }
446        });
447    }
448
449    /**
450     * Parse the given input source and return the dataset.
451     *
452     * @param source the source input stream. Must not be null.
453     * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed
454     *
455     * @return the dataset with the parsed data
456     * @throws IllegalDataException if an error was found while parsing the data from the source
457     * @throws IllegalArgumentException if source is null
458     */
459    public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
460        return new OsmReader().doParseDataSet(source, progressMonitor);
461    }
462}