openrvdas.logger.writers.regex_logfile_writer

  1#!/usr/bin/env python3
  2
  3import logging
  4import re
  5
  6from logger.utils import timestamp  # noqa: E402
  7from logger.writers.writer import Writer  # noqa: E402
  8from logger.writers.file_writer import FileWriter  # noqa: E402
  9
 10
 11class RegexLogfileWriter(Writer):
 12    """Write to the specified filebase, with datestamp appended. If filebase
 13    is a <regex>:<filebase> dict, write records to every filebase whose
 14    regex appears in the record.
 15    """
 16    def __init__(self, filebase=None, flush=True,
 17                 time_format=timestamp.TIME_FORMAT,
 18                 date_format=timestamp.DATE_FORMAT,
 19                 split_char=' ', suffix='', header=None,
 20                 header_file=None, rollover_hourly=False,
 21                 **kwargs):
 22        """Write timestamped text records to a filebase. The filebase will
 23        have the current date appended, in keeping with R2R format
 24        recommendations (http://www.rvdata.us/operators/directory). When the
 25        timestamped date on records rolls over to next day, create a new file
 26        with the new date suffix.
 27
 28        If filebase is a dict of <string>:<filebase> pairs, The writer will
 29        attempt to match a <string> in the dict to each record it receives.
 30        It will write the record to the filebase corresponding to the first
 31        string it matches (Note that the order of comparison is not
 32        guaranteed!). If no strings match, the record will be written to the
 33        standalone filebase provided.
 34        ```
 35        filebase        A filebase string to write to or a dict mapping
 36                        <string>:<filebase>.
 37
 38        flush           If True (default), flush after every write() call
 39
 40        date_fomat      A strftime-compatible string, such as '%Y-%m-%d';
 41                        defaults to whatever's defined in
 42                        utils.timestamps.DATE_FORMAT.
 43
 44        split_char      Delimiter between timestamp and rest of message
 45
 46        suffix          string to apply to the end of the log filename
 47
 48        header          Add the specified header string to each file.
 49
 50        header_file     Add the content of the specified file to each file.
 51
 52        rollover_hourly Set files to truncate by hour.  By default files will
 53                        truncate by day
 54
 55        quiet           If True, don't complain if a record doesn't match
 56                        any mapped prefix
 57        ```
 58        """
 59        super().__init__(**kwargs)  # processes 'quiet' and type hints
 60
 61        self.filebase = filebase
 62        self.flush = flush
 63        self.time_format = time_format
 64        self.date_format = date_format
 65        self.split_char = split_char
 66        self.suffix = suffix
 67        self.header = header
 68        self.header_file = header_file
 69        self.rollover_hourly = rollover_hourly
 70
 71        # If our filebase is a dict, we're going to be doing our
 72        # fancy pattern->filebase mapping.
 73        self.do_filebase_mapping = isinstance(self.filebase, dict)
 74
 75        if self.do_filebase_mapping:
 76            # Do our matches faster by precompiling
 77            self.compiled_filebase_map = {
 78                pattern: re.compile(pattern) for pattern in self.filebase
 79            }
 80        self.current_filename = {}
 81        self.writer = {}
 82
 83    ############################
 84    def write(self, record: str):
 85        """Note: Assume record begins with a timestamp string."""
 86
 87        # See if it's something we can process, and if not, try digesting
 88        if not self.can_process_record(record):  # inherited from BaseModule()
 89            self.digest_record(record)  # inherited from BaseModule()
 90            return
 91
 92        # Get the timestamp we'll be using
 93        try:  # Try to extract timestamp from record
 94            time_str = record.split(self.split_char)[0]
 95            ts = timestamp.timestamp(time_str, time_format=self.time_format)
 96        except ValueError:
 97            if not self.quiet:
 98                logging.error('LogfileWriter.write() - bad timestamp: "%s"', record)
 99                return
100
101        # Now parse ts into hour and date strings
102        hr_str = self.rollover_hourly and \
103            timestamp.date_str(ts, date_format='_%H00') or ""
104        date_str = timestamp.date_str(ts, date_format=self.date_format)
105        time_str = date_str + hr_str + self.suffix
106        logging.debug('LogfileWriter time_str: %s', time_str)
107
108        # Figure out where we're going to write
109        if self.do_filebase_mapping:
110            matched_patterns = [self.write_if_match(record, pattern, time_str)
111                                for pattern in self.filebase]
112            if True not in matched_patterns:
113                if not self.quiet:
114                    logging.warning(f'No patterns matched in PatternLogfileWriter '
115                                    f'for record "{record}"')
116        else:
117            pattern = 'fixed'  # just an arbitrary fixed pattern
118            filename = self.filebase + '-' + time_str
119            self.write_filename(record, pattern, filename)
120
121    ############################
122    def write_filename(self, record, pattern, filename):
123        """Write record to filename. If it's the first time we're writing to
124        this filename, create the appropriate FileWriter and insert it into
125        the map for the relevant pattern."""
126
127        # Are we currently writing to this file? If not, open/create it.
128        if not filename == self.current_filename.get(pattern):
129            logging.info('LogfileWriter opening new file: %s', filename)
130            self.current_filename[pattern] = filename
131            self.writer[pattern] = FileWriter(filename=filename,
132                                              header=self.header,
133                                              header_file=self.header_file,
134                                              flush=self.flush)
135        # Now, if our logic is correct, should *always* have a matching_writer
136        matching_writer = self.writer.get(pattern)
137        matching_writer.write(record)
138
139    ############################
140    def write_if_match(self, record, pattern, time_str):
141        """If the record matches the pattern, write to the matching filebase."""
142        # Find the compiled regex matching the pattern
143        regex = self.compiled_filebase_map.get(pattern)
144        if not regex:
145            logging.error(f'System error: found no regex pattern matching "{pattern}"!')
146            return None
147
148        # If the pattern isn't in this record, go home quietly
149        if regex.search(record) is None:
150            return None
151
152        # Otherwise, we write.
153        filebase = self.filebase.get(pattern)
154        if filebase is None:
155            logging.error(f'System error: found no filebase matching pattern "{pattern}"!')
156            return None
157
158        filename = filebase + '-' + time_str
159        self.write_filename(record, pattern, filename)
160        return True