001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.io.nmea;
003
004import java.io.BufferedReader;
005import java.io.IOException;
006import java.io.InputStream;
007import java.io.InputStreamReader;
008import java.nio.charset.StandardCharsets;
009import java.text.ParsePosition;
010import java.text.SimpleDateFormat;
011import java.util.ArrayList;
012import java.util.Collection;
013import java.util.Collections;
014import java.util.Date;
015import java.util.Locale;
016import java.util.Objects;
017import java.util.regex.Matcher;
018import java.util.regex.Pattern;
019
020import org.openstreetmap.josm.data.coor.LatLon;
021import org.openstreetmap.josm.data.gpx.GpxConstants;
022import org.openstreetmap.josm.data.gpx.GpxData;
023import org.openstreetmap.josm.data.gpx.ImmutableGpxTrack;
024import org.openstreetmap.josm.data.gpx.WayPoint;
025import org.openstreetmap.josm.io.IGpxReader;
026import org.openstreetmap.josm.io.IllegalDataException;
027import org.openstreetmap.josm.tools.Logging;
028import org.openstreetmap.josm.tools.date.DateUtils;
029import org.xml.sax.SAXException;
030
031/**
032 * Reads a NMEA 0183 file. Based on information from
033 * <a href="http://www.catb.org/gpsd/NMEA.html">http://www.catb.org/gpsd</a>.
034 *
035 * NMEA files are in printable ASCII form and may include information such as position,
036 * speed, depth, frequency allocation, etc.
037 * Typical messages might be 11 to a maximum of 79 characters in length.
038 *
039 * NMEA standard aims to support one-way serial data transmission from a single "talker"
040 * to one or more "listeners". The type of talker is identified by a 2-character mnemonic.
041 *
042 * NMEA information is encoded through a list of "sentences".
043 *
044 * @author cbrill
045 */
046public class NmeaReader implements IGpxReader {
047
048    /**
049     * Course Over Ground and Ground Speed.
050     * <p>
051     * The actual course and speed relative to the ground
052     */
053    enum VTG {
054        COURSE(1), COURSE_REF(2), // true course
055        COURSE_M(3), COURSE_M_REF(4), // magnetic course
056        SPEED_KN(5), SPEED_KN_UNIT(6), // speed in knots
057        SPEED_KMH(7), SPEED_KMH_UNIT(8), // speed in km/h
058        REST(9); // version-specific rest
059
060        final int position;
061
062        VTG(int position) {
063            this.position = position;
064        }
065    }
066
067    /**
068     * Recommended Minimum Specific GNSS Data.
069     * <p>
070     * Time, date, position, course and speed data provided by a GNSS navigation receiver.
071     * This sentence is transmitted at intervals not exceeding 2-seconds.
072     * RMC is the recommended minimum data to be provided by a GNSS receiver.
073     * All data fields must be provided, null fields used only when data is temporarily unavailable.
074     */
075    enum RMC {
076        TIME(1),
077        /** Warning from the receiver (A = data ok, V = warning) */
078        RECEIVER_WARNING(2),
079        WIDTH_NORTH(3), WIDTH_NORTH_NAME(4), // Latitude, NS
080        LENGTH_EAST(5), LENGTH_EAST_NAME(6), // Longitude, EW
081        SPEED(7), COURSE(8), DATE(9),           // Speed in knots
082        MAGNETIC_DECLINATION(10), UNKNOWN(11),  // magnetic declination
083        /**
084         * Mode (A = autonom; D = differential; E = estimated; N = not valid; S = simulated)
085         *
086         * @since NMEA 2.3
087         */
088        MODE(12);
089
090        final int position;
091
092        RMC(int position) {
093            this.position = position;
094        }
095    }
096
097    /**
098     * Global Positioning System Fix Data.
099     * <p>
100     * Time, position and fix related data for a GPS receiver.
101     */
102    enum GGA {
103        TIME(1), LATITUDE(2), LATITUDE_NAME(3), LONGITUDE(4), LONGITUDE_NAME(5),
104        /**
105         * Quality (0 = invalid, 1 = GPS, 2 = DGPS, 6 = estimanted (@since NMEA 2.3))
106         */
107        QUALITY(6), SATELLITE_COUNT(7),
108        HDOP(8), // HDOP (horizontal dilution of precision)
109        HEIGHT(9), HEIGHT_UNTIS(10), // height above NN (above geoid)
110        HEIGHT_2(11), HEIGHT_2_UNTIS(12), // height geoid - height ellipsoid (WGS84)
111        GPS_AGE(13), // Age of differential GPS data
112        REF(14); // REF station
113
114        final int position;
115        GGA(int position) {
116            this.position = position;
117        }
118    }
119
120    /**
121     * GNSS DOP and Active Satellites.
122     * <p>
123     * GNSS receiver operating mode, satellites used in the navigation solution reported by the GGA or GNS sentence,
124     * and DOP values.
125     * If only GPS, GLONASS, etc. is used for the reported position solution the talker ID is GP, GL, etc.
126     * and the DOP values pertain to the individual system. If GPS, GLONASS, etc. are combined to obtain the
127     * reported position solution multiple GSA sentences are produced, one with the GPS satellites, another with
128     * the GLONASS satellites, etc. Each of these GSA sentences shall have talker ID GN, to indicate that the
129     * satellites are used in a combined solution and each shall have the PDOP, HDOP and VDOP for the
130     * combined satellites used in the position.
131     */
132    enum GSA {
133        AUTOMATIC(1),
134        FIX_TYPE(2), // 1 = not fixed, 2 = 2D fixed, 3 = 3D fixed)
135        // PRN numbers for max 12 satellites
136        PRN_1(3), PRN_2(4), PRN_3(5), PRN_4(6), PRN_5(7), PRN_6(8),
137        PRN_7(9), PRN_8(10), PRN_9(11), PRN_10(12), PRN_11(13), PRN_12(14),
138        PDOP(15),   // PDOP (precision)
139        HDOP(16),   // HDOP (horizontal precision)
140        VDOP(17);   // VDOP (vertical precision)
141
142        final int position;
143        GSA(int position) {
144            this.position = position;
145        }
146    }
147
148    /**
149     * Geographic Position - Latitude/Longitude.
150     * <p>
151     * Latitude and Longitude of vessel position, time of position fix and status.
152     */
153    enum GLL {
154        LATITUDE(1), LATITUDE_NS(2), // Latitude, NS
155        LONGITUDE(3), LONGITUDE_EW(4), // Latitude, EW
156        UTC(5), // Universal Time Coordinated
157        STATUS(6), // Status: A = Data valid, V = Data not valid
158        /**
159         * Mode (A = autonom; D = differential; E = estimated; N = not valid; S = simulated)
160         * @since NMEA 2.3
161         */
162        MODE(7);
163
164        final int position;
165        GLL(int position) {
166            this.position = position;
167        }
168    }
169
170    private final InputStream source;
171    GpxData data;
172
173    private static final Pattern DATE_TIME_PATTERN = Pattern.compile("(\\d{12})(\\.\\d+)?");
174
175    private final SimpleDateFormat rmcTimeFmt = new SimpleDateFormat("ddMMyyHHmmss.SSS", Locale.ENGLISH);
176
177    private Date readTime(String p) throws IllegalDataException {
178        // NMEA defines time with "a variable number of digits for decimal-fraction of seconds"
179        // This variable decimal fraction cannot be parsed by SimpleDateFormat
180        Matcher m = DATE_TIME_PATTERN.matcher(p);
181        if (m.matches()) {
182            String date = m.group(1);
183            double milliseconds = 0d;
184            if (m.groupCount() > 1 && m.group(2) != null) {
185                milliseconds = 1000d * Double.parseDouble("0" + m.group(2));
186            }
187            // Add milliseconds on three digits to match SimpleDateFormat pattern
188            date += String.format(".%03d", (int) milliseconds);
189            Date d = rmcTimeFmt.parse(date, new ParsePosition(0));
190            if (d != null)
191                return d;
192        }
193        throw new IllegalDataException("Date is malformed: '" + p + "'");
194    }
195
196    // functons for reading the error stats
197    public NMEAParserState ps;
198
199    public int getParserUnknown() {
200        return ps.unknown;
201    }
202
203    public int getParserZeroCoordinates() {
204        return ps.zeroCoord;
205    }
206
207    public int getParserChecksumErrors() {
208        return ps.checksumErrors+ps.noChecksum;
209    }
210
211    public int getParserMalformed() {
212        return ps.malformed;
213    }
214
215    /**
216     * Returns the number of coordinates that have been successfuly read.
217     * @return the number of coordinates that have been successfuly read
218     */
219    public int getNumberOfCoordinates() {
220        return ps.success;
221    }
222
223    /**
224     * Constructs a new {@code NmeaReader}
225     * @param source NMEA file input stream
226     * @throws IOException if an I/O error occurs
227     */
228    public NmeaReader(InputStream source) throws IOException {
229        this.source = Objects.requireNonNull(source);
230        rmcTimeFmt.setTimeZone(DateUtils.UTC);
231    }
232
233    @Override
234    public boolean parse(boolean tryToFinish) throws SAXException, IOException {
235        // create the data tree
236        data = new GpxData();
237        Collection<Collection<WayPoint>> currentTrack = new ArrayList<>();
238
239        try (BufferedReader rd = new BufferedReader(new InputStreamReader(source, StandardCharsets.UTF_8))) {
240            StringBuilder sb = new StringBuilder(1024);
241            int loopstartChar = rd.read();
242            ps = new NMEAParserState();
243            if (loopstartChar == -1)
244                //TODO tell user about the problem?
245                return false;
246            sb.append((char) loopstartChar);
247            ps.pDate = "010100"; // TODO date problem
248            while (true) {
249                // don't load unparsable files completely to memory
250                if (sb.length() >= 1020) {
251                    sb.delete(0, sb.length()-1);
252                }
253                int c = rd.read();
254                if (c == '$') {
255                    parseNMEASentence(sb.toString(), ps);
256                    sb.delete(0, sb.length());
257                    sb.append('$');
258                } else if (c == -1) {
259                    // EOF: add last WayPoint if it works out
260                    parseNMEASentence(sb.toString(), ps);
261                    break;
262                } else {
263                    sb.append((char) c);
264                }
265            }
266            currentTrack.add(ps.waypoints);
267            data.tracks.add(new ImmutableGpxTrack(currentTrack, Collections.<String, Object>emptyMap()));
268
269        } catch (IllegalDataException e) {
270            Logging.warn(e);
271            return false;
272        }
273        return true;
274    }
275
276    private static class NMEAParserState {
277        protected Collection<WayPoint> waypoints = new ArrayList<>();
278        protected String pTime;
279        protected String pDate;
280        protected WayPoint pWp;
281
282        protected int success; // number of successfully parsed sentences
283        protected int malformed;
284        protected int checksumErrors;
285        protected int noChecksum;
286        protected int unknown;
287        protected int zeroCoord;
288    }
289
290    /**
291     * Determines if the given address denotes the given NMEA sentence formatter of a known talker.
292     * @param address first tag of an NMEA sentence
293     * @param formatter sentence formatter mnemonic code
294     * @return {@code true} if the {@code address} denotes the given NMEA sentence formatter of a known talker
295     */
296    static boolean isSentence(String address, Sentence formatter) {
297        for (TalkerId talker : TalkerId.values()) {
298            if (address.equals('$' + talker.name() + formatter.name())) {
299                return true;
300            }
301        }
302        return false;
303    }
304
305    // Parses split up sentences into WayPoints which are stored
306    // in the collection in the NMEAParserState object.
307    // Returns true if the input made sense, false otherwise.
308    private boolean parseNMEASentence(String s, NMEAParserState ps) throws IllegalDataException {
309        try {
310            if (s.isEmpty()) {
311                throw new IllegalArgumentException("s is empty");
312            }
313
314            // checksum check:
315            // the bytes between the $ and the * are xored
316            // if there is no * or other meanities it will throw
317            // and result in a malformed packet.
318            String[] chkstrings = s.split("\\*");
319            if (chkstrings.length > 1) {
320                byte[] chb = chkstrings[0].getBytes(StandardCharsets.UTF_8);
321                int chk = 0;
322                for (int i = 1; i < chb.length; i++) {
323                    chk ^= chb[i];
324                }
325                if (Integer.parseInt(chkstrings[1].substring(0, 2), 16) != chk) {
326                    ps.checksumErrors++;
327                    ps.pWp = null;
328                    return false;
329                }
330            } else {
331                ps.noChecksum++;
332            }
333            // now for the content
334            String[] e = chkstrings[0].split(",");
335            String accu;
336
337            WayPoint currentwp = ps.pWp;
338            String currentDate = ps.pDate;
339
340            // handle the packet content
341            if (isSentence(e[0], Sentence.GGA)) {
342                // Position
343                LatLon latLon = parseLatLon(
344                        e[GGA.LATITUDE_NAME.position],
345                        e[GGA.LONGITUDE_NAME.position],
346                        e[GGA.LATITUDE.position],
347                        e[GGA.LONGITUDE.position]
348                );
349                if (latLon == null) {
350                    throw new IllegalDataException("Malformed lat/lon");
351                }
352
353                if (LatLon.ZERO.equals(latLon)) {
354                    ps.zeroCoord++;
355                    return false;
356                }
357
358                // time
359                accu = e[GGA.TIME.position];
360                Date d = readTime(currentDate+accu);
361
362                if ((ps.pTime == null) || (currentwp == null) || !ps.pTime.equals(accu)) {
363                    // this node is newer than the previous, create a new waypoint.
364                    // no matter if previous WayPoint was null, we got something better now.
365                    ps.pTime = accu;
366                    currentwp = new WayPoint(latLon);
367                }
368                if (!currentwp.attr.containsKey("time")) {
369                    // As this sentence has no complete time only use it
370                    // if there is no time so far
371                    currentwp.setTime(d);
372                }
373                // elevation
374                accu = e[GGA.HEIGHT_UNTIS.position];
375                if ("M".equals(accu)) {
376                    // Ignore heights that are not in meters for now
377                    accu = e[GGA.HEIGHT.position];
378                    if (!accu.isEmpty()) {
379                        Double.parseDouble(accu);
380                        // if it throws it's malformed; this should only happen if the
381                        // device sends nonstandard data.
382                        if (!accu.isEmpty()) { // FIX ? same check
383                            currentwp.put(GpxConstants.PT_ELE, accu);
384                        }
385                    }
386                }
387                // number of satellites
388                accu = e[GGA.SATELLITE_COUNT.position];
389                int sat = 0;
390                if (!accu.isEmpty()) {
391                    sat = Integer.parseInt(accu);
392                    currentwp.put(GpxConstants.PT_SAT, accu);
393                }
394                // h-dilution
395                accu = e[GGA.HDOP.position];
396                if (!accu.isEmpty()) {
397                    currentwp.put(GpxConstants.PT_HDOP, Float.valueOf(accu));
398                }
399                // fix
400                accu = e[GGA.QUALITY.position];
401                if (!accu.isEmpty()) {
402                    int fixtype = Integer.parseInt(accu);
403                    switch(fixtype) {
404                    case 0:
405                        currentwp.put(GpxConstants.PT_FIX, "none");
406                        break;
407                    case 1:
408                        if (sat < 4) {
409                            currentwp.put(GpxConstants.PT_FIX, "2d");
410                        } else {
411                            currentwp.put(GpxConstants.PT_FIX, "3d");
412                        }
413                        break;
414                    case 2:
415                        currentwp.put(GpxConstants.PT_FIX, "dgps");
416                        break;
417                    case 3:
418                        currentwp.put(GpxConstants.PT_FIX, "pps");
419                        break;
420                    case 4:
421                        currentwp.put(GpxConstants.PT_FIX, "rtk");
422                        break;
423                    case 5:
424                        currentwp.put(GpxConstants.PT_FIX, "float rtk");
425                        break;
426                    case 6:
427                        currentwp.put(GpxConstants.PT_FIX, "estimated");
428                        break;
429                    case 7:
430                        currentwp.put(GpxConstants.PT_FIX, "manual");
431                        break;
432                    case 8:
433                        currentwp.put(GpxConstants.PT_FIX, "simulated");
434                        break;
435                    default:
436                        break;
437                    }
438                }
439            } else if (isSentence(e[0], Sentence.VTG)) {
440                // COURSE
441                accu = e[VTG.COURSE_REF.position];
442                if ("T".equals(accu)) {
443                    // other values than (T)rue are ignored
444                    accu = e[VTG.COURSE.position];
445                    if (!accu.isEmpty() && currentwp != null) {
446                        Double.parseDouble(accu);
447                        currentwp.put("course", accu);
448                    }
449                }
450                // SPEED
451                accu = e[VTG.SPEED_KMH_UNIT.position];
452                if (accu.startsWith("K")) {
453                    accu = e[VTG.SPEED_KMH.position];
454                    if (!accu.isEmpty() && currentwp != null) {
455                        double speed = Double.parseDouble(accu);
456                        currentwp.put("speed", Double.toString(speed)); // speed in km/h
457                    }
458                }
459            } else if (isSentence(e[0], Sentence.GSA)) {
460                // vdop
461                accu = e[GSA.VDOP.position];
462                if (!accu.isEmpty() && currentwp != null) {
463                    currentwp.put(GpxConstants.PT_VDOP, Float.valueOf(accu));
464                }
465                // hdop
466                accu = e[GSA.HDOP.position];
467                if (!accu.isEmpty() && currentwp != null) {
468                    currentwp.put(GpxConstants.PT_HDOP, Float.valueOf(accu));
469                }
470                // pdop
471                accu = e[GSA.PDOP.position];
472                if (!accu.isEmpty() && currentwp != null) {
473                    currentwp.put(GpxConstants.PT_PDOP, Float.valueOf(accu));
474                }
475            } else if (isSentence(e[0], Sentence.RMC)) {
476                // coordinates
477                LatLon latLon = parseLatLon(
478                        e[RMC.WIDTH_NORTH_NAME.position],
479                        e[RMC.LENGTH_EAST_NAME.position],
480                        e[RMC.WIDTH_NORTH.position],
481                        e[RMC.LENGTH_EAST.position]
482                );
483                if (LatLon.ZERO.equals(latLon)) {
484                    ps.zeroCoord++;
485                    return false;
486                }
487                // time
488                currentDate = e[RMC.DATE.position];
489                String time = e[RMC.TIME.position];
490
491                Date d = readTime(currentDate+time);
492
493                if (ps.pTime == null || currentwp == null || !ps.pTime.equals(time)) {
494                    // this node is newer than the previous, create a new waypoint.
495                    ps.pTime = time;
496                    currentwp = new WayPoint(latLon);
497                }
498                // time: this sentence has complete time so always use it.
499                currentwp.setTime(d);
500                // speed
501                accu = e[RMC.SPEED.position];
502                if (!accu.isEmpty() && !currentwp.attr.containsKey("speed")) {
503                    double speed = Double.parseDouble(accu);
504                    speed *= 0.514444444 * 3.6; // to km/h
505                    currentwp.put("speed", Double.toString(speed));
506                }
507                // course
508                accu = e[RMC.COURSE.position];
509                if (!accu.isEmpty() && !currentwp.attr.containsKey("course")) {
510                    Double.parseDouble(accu);
511                    currentwp.put("course", accu);
512                }
513
514                // TODO fix?
515                // * Mode (A = autonom; D = differential; E = estimated; N = not valid; S = simulated)
516                // *
517                // * @since NMEA 2.3
518                //
519                //MODE(12);
520            } else if (isSentence(e[0], Sentence.GLL)) {
521                // coordinates
522                LatLon latLon = parseLatLon(
523                        e[GLL.LATITUDE_NS.position],
524                        e[GLL.LONGITUDE_EW.position],
525                        e[GLL.LATITUDE.position],
526                        e[GLL.LONGITUDE.position]
527                );
528                if (LatLon.ZERO.equals(latLon)) {
529                    ps.zeroCoord++;
530                    return false;
531                }
532                // only consider valid data
533                if (!"A".equals(e[GLL.STATUS.position])) {
534                    return false;
535                }
536
537                // RMC sentences contain a full date while GLL sentences contain only time,
538                // so create new waypoints only of the NMEA file does not contain RMC sentences
539                if (ps.pTime == null || currentwp == null) {
540                    currentwp = new WayPoint(latLon);
541                }
542            } else {
543                ps.unknown++;
544                return false;
545            }
546            ps.pDate = currentDate;
547            if (ps.pWp != currentwp) {
548                if (ps.pWp != null) {
549                    ps.pWp.getDate();
550                }
551                ps.pWp = currentwp;
552                ps.waypoints.add(currentwp);
553                ps.success++;
554                return true;
555            }
556            return true;
557
558        } catch (IllegalArgumentException | IndexOutOfBoundsException | IllegalDataException ex) {
559            if (ps.malformed < 5) {
560                Logging.warn(ex);
561            } else {
562                Logging.debug(ex);
563            }
564            ps.malformed++;
565            ps.pWp = null;
566            return false;
567        }
568    }
569
570    private static LatLon parseLatLon(String ns, String ew, String dlat, String dlon) {
571        String widthNorth = dlat.trim();
572        String lengthEast = dlon.trim();
573
574        // return a zero latlon instead of null so it is logged as zero coordinate
575        // instead of malformed sentence
576        if (widthNorth.isEmpty() && lengthEast.isEmpty()) return LatLon.ZERO;
577
578        // The format is xxDDLL.LLLL
579        // xx optional whitespace
580        // DD (int) degres
581        // LL.LLLL (double) latidude
582        int latdegsep = widthNorth.indexOf('.') - 2;
583        if (latdegsep < 0) return null;
584
585        int latdeg = Integer.parseInt(widthNorth.substring(0, latdegsep));
586        double latmin = Double.parseDouble(widthNorth.substring(latdegsep));
587        if (latdeg < 0) {
588            latmin *= -1.0;
589        }
590        double lat = latdeg + latmin / 60;
591        if ("S".equals(ns)) {
592            lat = -lat;
593        }
594
595        int londegsep = lengthEast.indexOf('.') - 2;
596        if (londegsep < 0) return null;
597
598        int londeg = Integer.parseInt(lengthEast.substring(0, londegsep));
599        double lonmin = Double.parseDouble(lengthEast.substring(londegsep));
600        if (londeg < 0) {
601            lonmin *= -1.0;
602        }
603        double lon = londeg + lonmin / 60;
604        if ("W".equals(ew)) {
605            lon = -lon;
606        }
607        return new LatLon(lat, lon);
608    }
609
610    @Override
611    public GpxData getGpxData() {
612        return data;
613    }
614}