openrvdas.logger.utils.nmea_parser

  1#!/usr/bin/env python3
  2
  3"""Tools for parsing.
  4"""
  5
  6import glob
  7import logging
  8import re
  9
 10from logger.utils import read_config  # noqa: E402
 11from logger.utils.das_record import DASRecord  # noqa: E402
 12from logger.utils.timestamp import timestamp, TIME_FORMAT  # noqa: E402
 13
 14DEFAULT_MESSAGE_PATH = 'local/message/*.yaml'
 15DEFAULT_SENSOR_PATH = 'local/sensor/*.yaml'
 16DEFAULT_SENSOR_MODEL_PATH = 'local/sensor_model/*.yaml'
 17
 18RAW_FIELDS_RE = '(?P<raw_fields>[^*]+)'
 19CHECKSUM_RE = r'(?:\*(?P<checksum>[0-9A-F]{2}))?'
 20NMEA_RE = re.compile(RAW_FIELDS_RE + CHECKSUM_RE)
 21
 22
 23class NMEAParser:
 24    ############################
 25    def __init__(self, message_path=DEFAULT_MESSAGE_PATH,
 26                 sensor_path=DEFAULT_SENSOR_PATH,
 27                 sensor_model_path=DEFAULT_SENSOR_MODEL_PATH,
 28                 time_format=None):
 29        self.messages = self._read_definitions(message_path)
 30        self.sensor_models = self._read_definitions(sensor_model_path)
 31        self.sensors = self._read_definitions(sensor_path)
 32        self.time_format = time_format or TIME_FORMAT
 33
 34    ############################
 35    def parse_record(self, nmea_record):
 36        """Receive an id-prefixed, timestamped NMEA record."""
 37        if not nmea_record:
 38            return None
 39        if not isinstance(nmea_record, str):
 40            logging.info('Record is not NMEA string: "%s"', nmea_record)
 41            return None
 42        try:
 43            (data_id, raw_ts, message) = nmea_record.strip().split(maxsplit=2)
 44            ts = timestamp(raw_ts, time_format=self.time_format)
 45        except ValueError:
 46            logging.info('Record not in <data_id> <timestamp> <NMEA> format: "%s"',
 47                         nmea_record)
 48            return None
 49
 50        # Figure out what kind of message we're expecting, based on data_id
 51        sensor = self.sensors.get(data_id)
 52        if not sensor:
 53            logging.error('Unrecognized data_id ("%s") in record: %s',
 54                          data_id, nmea_record)
 55            return None
 56
 57        model_name = sensor.get('model')
 58        if not model_name:
 59            logging.error('No "model" for sensor %s', sensor)
 60            return None
 61
 62        # If something goes wrong during parsing, we'll get a ValueError
 63        try:
 64            (fields, message_type) = self.parse_nmea(sensor_model_name=model_name,
 65                                                     message=message)
 66        except ValueError as e:
 67            logging.error(str(e))
 68            return None
 69
 70        # Finally, convert field values to variable names specific to sensor
 71        sensor_fields = sensor.get('fields')
 72        if not sensor_fields:
 73            logging.error('No "fields" definition found for sensor %s', data_id)
 74            return None
 75
 76        named_fields = {}
 77        for field_name in fields:
 78            var_name = sensor_fields.get(field_name)
 79            if var_name:
 80                named_fields[var_name] = fields[field_name]
 81
 82        record = DASRecord(data_id=data_id, message_type=message_type,
 83                           timestamp=ts, fields=named_fields)
 84        logging.debug('created DASRecord: %s', str(record))
 85        return record
 86
 87    ############################
 88    def parse_nmea(self, sensor_model_name, message):
 89        """Parse a raw NMEA message; raise ValueError if there are problems."""
 90        # Break the message into an optional message_type, the aggregated
 91        # raw fields and an optional checksum.
 92        match = NMEA_RE.match(message)
 93        if not match:
 94            raise ValueError('Can\'t parse NMEA record: "%s"' % message)
 95
 96        # Parse out the optional checksum from the raw fields. The first
 97        # field may be a message type, but we'll deal with that later.
 98        raw_fields = match.group('raw_fields')
 99        checksum = match.group('checksum')
100
101        logging.debug('Parsed "%s"', message)
102        logging.debug('raw_fields "%s"', raw_fields)
103        logging.debug('checksum "%s"', checksum)
104
105        # Proper NMEA uses commas to delimit fields, but some serial
106        # instruments use spaces or other characters (Gravimeter, for
107        # example, uses both spaces and ':'). Look up sensor model and see
108        # if we have a non-default delimiter defined.
109        sensor_model = self.sensor_models.get(sensor_model_name)
110        if not sensor_model:
111            raise ValueError('No sensor_model  matching "%s"' % sensor_model_name)
112
113        field_delimiter = sensor_model.get('field_delimiter', ',')
114        fields = re.split(field_delimiter, raw_fields)
115        logging.debug('fields "%s"', fields)
116
117        # We need to find what fields are defined by this sensor model. If
118        # the top-level sensor_model definition has 'fields', then it's
119        # easy, and we're done.
120        message_type = ''
121        definition_base = sensor_model
122
123        # If we don't have 'fields' defined at the top level, it means
124        # that this sensor can emit multiple types of messages, which we
125        # expect to find under a 'messages' key. We will count on the
126        # first element of our field list to tell us which of those
127        # messages we've got.
128        while 'fields' not in definition_base:
129            logging.debug('Iterating with message_type "%s", looking for messages '
130                          'in definition_base: %s', message_type, definition_base)
131
132            # If there's no 'fields', there ought to be a 'messages'
133            sensor_messages = definition_base.get('messages')
134            if not sensor_messages:
135                raise ValueError('Sensor model %s must have either "fields" or '
136                                 '"messages" definition.' % sensor_model_name)
137
138            # We count on the first field in the field list to tell us which
139            # message out of our dictionary of messages we actually have.
140            element = fields.pop(0)
141            message_type = message_type + '-' + element if message_type else element
142
143            definition = sensor_messages.get(element)
144            if not definition:
145                raise ValueError('Message "%s" is not one defined by model %s (%s)'
146                                 % (message_type, sensor_model_name, sensor_messages))
147
148            # The value of the 'definition' can be one of two things: 1) a
149            # string referring us to a message definition in self.messages,
150            # or 2) a dictionary that contains the message definition. Note
151            # that the message definition may either directly contain a
152            # 'fields' key or may have a 'messages' key defining
153            # sub-message_types, requiring us to iterate.
154
155            # If it's a str, it's a reference into the definitions we've
156            # previously loaded into self.messages. Look it up, make that
157            # our new definition_base and loop.
158            if isinstance(definition, str):
159                definition_base = self.messages.get(definition)
160                logging.debug('Definition is reference to message "%s"; loaded: %s',
161                              definition, definition_base)
162                if not definition_base:
163                    raise ValueError('Message definition "%s" (%s) not found for %s'
164                                     % (definition, message_type, sensor_model_name))
165
166            # If 'definition' is a dict, make that our new definition_base,
167            # tack the element onto our message_type and iterate.
168            elif isinstance(definition, dict):
169                definition_base = definition
170
171            # If definition is neither dict nor str, something's wrong
172            else:
173                raise ValueError('Bad definition for %s (%s)'
174                                 % (message_type, sensor_model_name))
175
176        # End of while loop. If we're here, we darned well ought to have
177        # field_definitions in our definition_base. Get them and make sure
178        # they line up with the number of fields we have left.
179        field_definitions = definition_base.get('fields')
180        if len(fields) != len(field_definitions):
181            raise ValueError('Sensor model "%s": %s # of fields (%s) != '
182                             '# field definitions (%s): "%s" != "%s"' % (
183                                 sensor_model_name, message_type,
184                                 len(fields), len(field_definitions),
185                                 fields, [f[0] for f in field_definitions]))
186
187        # If still okay, map field values to their definitions
188        field_values = {}
189        for i in range(len(fields)):
190            (name, data_type) = field_definitions[i]
191            field_values[name] = self._convert(fields[i], data_type)
192        return (field_values, message_type)
193
194    ############################
195    def _convert(self, value, data_type):
196        if value == '':
197            return None
198        if not data_type:
199            return value
200        elif data_type == 'int':
201            return int(value)
202        elif data_type == 'float':
203            return float(value)
204        elif data_type == 'str':
205            return str(value)
206        else:
207            raise ValueError('Unknown data type in field definition: "%s"' % data_type)
208
209    ############################
210    def _read_definitions(self, filespec_paths):
211        definitions = {}
212        for filespec in filespec_paths.split(','):
213            logging.debug('reading definitions from %s', filespec)
214            for filename in glob.glob(filespec):
215                new_defs = read_config.read_config(filename)
216                for key in new_defs:
217                    if key in definitions:
218                        logging.warning('Duplicate definition for key "%s" found in %s',
219                                        key, filename)
220                    definitions[key] = new_defs[key]
221        return definitions