openrvdas.logger.writers.file_writer

  1#!/usr/bin/env python3
  2
  3import logging
  4import os.path
  5import sys
  6import re
  7import math
  8
  9from datetime import datetime, timedelta, timezone
 10from typing import Union
 11
 12
 13from logger.utils.timestamp import time_str, DATE_FORMAT  # noqa: E402
 14from logger.writers.writer import Writer  # noqa: E402
 15
 16DEFAULT_DATETIME_STR = '-' + DATE_FORMAT
 17
 18
 19class FileWriter(Writer):
 20    """Write to the specified file. If filename is empty, write to stdout."""
 21
 22    def __init__(self,
 23                 filebase=None,
 24                 filename=None,          # deprecated
 25                 mode='a',
 26                 delimiter='\n',
 27                 flush=True,
 28                 split_by_time=False,    # deprecated
 29                 split_interval=None,
 30                 header=None,
 31                 header_file=None,
 32                 time_format=None,       # deprecated
 33                 date_format=None,
 34                 suffix=None,
 35                 time_zone=timezone.utc,
 36                 create_path=True,
 37                 **kwargs):
 38        """Write text records to a file. If no filename is specified, write to
 39        stdout.
 40        ```
 41
 42        filebase    A filebase string that will be used as for the output
 43                    filename.
 44
 45        filename    DEPRECATED Name of file to write to. If None, write to
 46                    stdout.
 47
 48        mode        Mode with which to open file. 'a' by default to append, but
 49                    can also be 'w' to truncate, 'ab' to append in binary mode,
 50                    or any other valid Python write file mode.
 51
 52        delimiter   By default, append a newline after each record written. Set
 53                    to None to disable appending any record delimiter. Ignored
 54                    if mode is for binary.
 55
 56        flush       If True (default), flush after every write() call
 57
 58        split_by_time   DEPRECATED Create a separate text file for each (by
 59                        default) day, appending a -YYYY-MM-DD string to the
 60                        specified filename. By overridding time_format, other
 61                        split intervals, such as hourly or monthly, may be
 62                        imposed.
 63
 64        split_interval  If set the file will trucate at the specified interval.
 65                        The value must be a string containing an integer
 66                        followed by a 'H' (hours) or 'M' (minutes). Default
 67                        is to not split.
 68
 69        header          A string to add to the beginning of a new file.
 70
 71        header_file     A string containing the path to file containing a
 72                        header string to add to the beginning of a new file.
 73
 74        time_format     DEPRECATED By default ISO 8601-compliant '-%Y-%m-%d'.
 75                        If, e.g. '-%Y-%m' is used, files will be split by
 76                        month; if -%y-%m-%d:%H' is specified, splits will be
 77                        hourly. If '%y+%j' is specified, splits will be daily,
 78                        but named via Julian date.  Putting '-' or '.' on the
 79                        left indicates timestamp suffix, putting it on the
 80                        right indicates timestamp prefix.  If you put '-' or
 81                        '.' on both sides, it's handled as a suffix.
 82
 83        date_fomat      A strftime-compatible string, such as '%Y-%m-%d';
 84                        defaults to '-' plus whatever's defined in
 85                        utils.timestamps.DATE_FORMAT.  If the value starts with
 86                        a '^' character, the string will prepend the file
 87                        name portion of the filebase
 88
 89        suffix          A suffix string to add to the log filename.
 90
 91        time_zone       Timezone to use when constructing the date_format
 92                        portion of the filenames.
 93
 94        create_path     Create directory path to file if it doesn't exist.
 95
 96        quiet           If True, don't complain if a record doesn't match
 97                        any mapped prefix.
 98
 99        encoding        'utf-8' by default. If empty or None, do not attempt
100                        any decoding and return raw bytes. Other possible
101                        encodings are listed in online documentation here:
102                        [https://docs.python.org/3/library/codecs.html]
103                        (https://docs.python.org/3/library/codecs.html)
104
105        encoding_errors 'ignore' by default. Other error strategies are
106                        'strict', 'replace', and 'backslashreplace', described
107                        here: [https://docs.python.org/3/howto/unicode.html]
108                        (https://docs.python.org/3/howto/unicode.html)
109
110        Sample invocations (original vs proposed)
111        - Write to stdout:
112            Original: FileWriter(None)
113            Proposed: FileWriter(None)
114
115        - Write to file, no split:
116            Original: FileWriter(/data/sample_file)
117            Proposed: FileWriter(/data/sample_file)
118
119        - Write to file, no split, with header:
120            Original: FileWriter(/data/sample_file, header='This is a header')
121            Proposed: FileWriter(/data/sample_file, header='This is a header')
122
123        - Write to file, daily split: (filename = /data/sample_file-%Y-%m-%d)
124            Original: FileWriter(/data/sample_file, split_by_time=True)
125            Proposed: FileWriter(/data/sample_file, split_interval='24H')
126
127        - Write to file, daily split, with suffix:
128          (filename = /data/sample_file-%Y-%m-%d.txt)
129            Original: FileWriter(/data/sample_file, split_by_time=True,
130                                 time_format='-%Y-%m-%d.txt')
131            Proposed: FileWriter(/data/sample_file, split_interval='24H',
132                                 suffix='.txt')
133
134        - Write to file, daily split: (filename = /data/sample_file-%Y-%j)
135            Original: FileWriter(/data/sample_file, split_by_time=True,
136                                 time_format='-%Y-%j')
137            Proposed: FileWriter(/data/sample_file, split_interval='24H',
138                                 date_format='-%Y-%j')
139
140        - Write to file, hourly split:
141          (filename = /data/sample_file-%Y-%m-%dT%H00)
142            Original: FileWriter(/data/sample_file, split_by_time=True,
143                                 time_format='-%Y-%m-%d:%H00')
144            Proposed: FileWriter(/data/sample_file, split_interval='1H')
145
146        - Write to file, 15-minute split:
147          (filename = /data/sample_file-%Y-%m-%dT%H%M)
148            Original: FileWriter(/data/sample_file, split_interval='15M',
149                                 time_format='-%Y-%m-%d:%H%M')
150            Proposed: FileWriter(/data/sample_file, split_interval='15M')
151
152        - Write to file, 15-minute split:
153          (filename = /data/%Y-%m-%dT%H%M-sample_file)
154            Original: FileWriter(/data/sample_file, split_interval='15M',
155                                 time_format='%Y-%m-%d:%H%M-')
156            Proposed: FileWriter(/data/sample_file, split_interval='15M',
157                                 date_format='^%Y-%m-%d:%H%M-')
158
159        - Write to file, 15-minute split, with suffix:
160          (filename = /data/%Y-%m-%dT%H%M-sample_file.txt)
161            Original: FileWriter(/data/sample_file.txt, split_interval='15M',
162                                 time_format='%Y-%m-%d:%H%M-')
163            Proposed: FileWriter(/data/sample_file, split_interval='15M',
164                                 date_format='^%Y-%m-%d:%H%M-', suffix='.txt')
165
166        ```
167        """
168        super().__init__(**kwargs)  # processes 'quiet', encoding and hints
169
170        if 'b' in mode and (self.encoding or self.encoding_errors) is not None:
171            logging.warning("Ignoring encoding and encoding_errors because"
172                            " file mode is binary")
173            self.encoding = self.encoding_errors = None
174
175        # --- Deprecated args ---
176        if filename is not None:
177            filebase = filename
178            # warnings.warn(
179            #     "`filename` is deprecated, use `filebase` instead",
180            #     DeprecationWarning,
181            #     stacklevel=2,
182            # )
183        if split_by_time:
184            split_interval = split_interval or '24H'
185            # warnings.warn(
186            #     "`split_by_time` is deprecated, use `split_interval` instead",
187            #     DeprecationWarning,
188            #     stacklevel=2,
189            # )
190        if time_format is not None:
191            # warnings.warn(
192            #     "`time_format` is deprecated, use `date_format` instead",
193            #     DeprecationWarning,
194            #     stacklevel=2,
195            # )
196            date_format = ('^' + time_format[:-1]
197                           if time_format.endswith('-') else time_format)
198
199        # Can't use split_interval or split_by_time if filebase and filename
200        # are None
201        if split_interval and filebase is None:
202            raise ValueError("filebase must be specified")
203
204        # --- Base file name ---
205        self.filebase = filebase
206
207        # --- File settings ---
208        self.mode = mode
209        self.flush = flush
210        self.suffix = suffix or ''
211        self.time_zone = time_zone
212        self.next_file_split = datetime.now(self.time_zone)
213
214        # --- Delimiter ---
215        self.delimiter = self._resolve_delimiter(delimiter)
216
217        # --- Header handling ---
218        self.header = self._load_header(header, header_file)
219
220        # --- Split interval ---
221        self.split_interval = self._validate_split_interval(split_interval)
222        self.split_interval_in_seconds = self._get_split_interval_in_seconds()
223
224        # --- Date/time format ---
225        self.date_format = self._validate_date_format(date_format)
226
227        # --- Ensure path exists ---
228        if self.filebase and create_path:
229            os.makedirs(os.path.dirname(self.filebase), exist_ok=True)
230
231        # --- File state ---
232        self.file = None
233        self.file_date_format = None
234
235        # A hook to aid in debugging; should be None to use system time.
236        self.timestamp = None
237
238    # -----------------------
239    # Validation helpers
240    # -----------------------
241    def _load_header(self, header, header_file):
242        if header and header_file:
243            raise ValueError("Cannot specify both `header` and `header_file`")
244
245        if 'b' in self.mode and header is not None:
246            logging.warning("Ignoring header because file mode is binary")
247            return None
248
249        # Case 1: simple string header
250        if header:
251            return (header.rstrip(self.delimiter) + self.delimiter
252                    if self.delimiter else header)
253
254        # Case 2: header_file is a single path
255        if header_file:
256            try:
257                with open(header_file, "r", encoding="utf-8") as hf:
258                    return (
259                        hf.read().strip().rstrip(self.delimiter)
260                        + self.delimiter
261                        if self.delimiter
262                        else hf.read().strip()
263                    )
264            except OSError as e:
265                raise ValueError(
266                    f"Error reading header_file {header_file}: {e}")
267
268        return None
269
270    def _validate_split_interval(self, split_interval):
271        if split_interval is None:
272            return None
273        if not isinstance(split_interval, str):
274            raise ValueError("split_interval must be a string like '1H' "
275                             "or '30M'")
276        if not split_interval.endswith(("H", "M")):
277            raise ValueError("must be an integer followed by 'H' or 'M'")
278        try:
279            return (int(split_interval[:-1]), split_interval[-1])
280        except ValueError:
281            raise ValueError("must be an integer followed by 'H' or 'M'")
282        return None
283
284    def _resolve_delimiter(self, delimiter):
285        if 'b' in self.mode and delimiter is not None:
286            logging.warning("Ignoring delimiter because file mode is binary")
287            return None
288
289        if delimiter:
290            delimiter = delimiter.encode("utf-8").decode("unicode_escape")
291
292        return delimiter
293
294    def _validate_date_format(self, date_format):
295        if not self.split_interval:
296            return date_format or ""
297
298        unit = self.split_interval[1]
299        value = self.split_interval[0]
300
301        # --- Decide requirements based on interval ---
302        if unit == "H":
303            even_days = value % 24 == 0
304            needs_hour = not even_days
305            needs_minute = False
306        elif unit == "M":
307            even_hours = value % 60 == 0
308            needs_hour = True
309            needs_minute = not even_hours
310        else:
311            return DEFAULT_DATETIME_STR  # fallback
312
313        # --- Default formats if user didn’t supply one ---
314        if not date_format:
315            if unit == "H":
316                return (DEFAULT_DATETIME_STR if even_days
317                        else DEFAULT_DATETIME_STR + "T%H00")
318            if unit == "M":
319                # Concise f-string format
320                return (f"{DEFAULT_DATETIME_STR}T%H"
321                        f"{'%M' if needs_minute else '00'}")
322
323        # --- Extract directives ---
324        found = set(re.findall(r"%[a-zA-Z]", date_format))
325
326        # Must always have year
327        if "%Y" not in found:
328            raise ValueError("date_format must include %Y (year).")
329
330        # Must have either month+day or julian day
331        if not ({"%m", "%d"} <= found or "%j" in found):
332            raise ValueError("date_format must include %m, %d (month, day) "
333                             "or %j (day-of-year).")
334
335        # Hours?
336        if needs_hour and "%H" not in found:
337            raise ValueError("date_format must include %H (hour).")
338
339        # Minutes?
340        if needs_minute and "%M" not in found:
341            raise ValueError("date_format must include %M (minute).")
342
343        return date_format
344
345    ############################
346    def __del__(self):
347        if hasattr(self, 'file') and self.file:
348            self.file.close()
349
350    ############################
351    def _get_split_interval_in_seconds(self):
352
353        if not self.split_interval:
354            return 0
355
356        if self.split_interval[1] == 'H':
357            return self.split_interval[0] * 3600
358
359        if self.split_interval[1] == 'M':
360            return self.split_interval[0] * 60
361
362        return 0
363
364    ############################
365    def _get_file_date_format(self):
366        """Return a string to be used for the file suffix."""
367
368        # Note: the self.timestamp variable exists for debugging, and
369        # should be left as None in actual use, which tells the time_str
370        # method to use current system time.
371
372        # if there is no split interval
373        if self.timestamp:
374            return time_str(timestamp=self.timestamp,
375                            time_zone=self.time_zone,
376                            time_format=self.date_format)
377
378        # if the data is being split by N hours
379        elif self.split_interval[1] == 'H':  # hour
380            timestamp_raw = datetime.now(self.time_zone)
381            # Round down to nearest interval
382            timestamp_hour = (self.split_interval[0] *
383                              math.floor(timestamp_raw.hour /
384                                         self.split_interval[0]))
385            timestamp_proc = timestamp_raw.replace(hour=timestamp_hour,
386                                                   minute=0, second=0)
387            self.next_file_split = (timestamp_proc +
388                                    timedelta(seconds=self
389                                              .split_interval_in_seconds))
390
391            return time_str(timestamp=timestamp_proc.timestamp(),
392                            time_zone=self.time_zone,
393                            time_format=self.date_format)
394
395        # if the data is being split by N minutes
396        elif self.split_interval[1] == 'M':  # minute
397            timestamp_raw = datetime.now(self.time_zone)
398            # Round down to nearest interval
399            timestamp_minute = (self.split_interval[0] *
400                                math.floor(timestamp_raw.minute /
401                                           self.split_interval[0]))
402            timestamp_proc = timestamp_raw.replace(minute=timestamp_minute,
403                                                   second=0)
404            self.next_file_split = (timestamp_proc +
405                                    timedelta(seconds=self
406                                              .split_interval_in_seconds))
407
408            return time_str(timestamp=timestamp_proc.timestamp(),
409                            time_zone=self.time_zone,
410                            time_format=self.date_format)
411
412        return ""
413
414    ############################
415    def _set_file(self, filename):
416        """Set the current file to the specified filename."""
417
418        # If they haven't given us a filename, we'll write to stdout
419        if filename is None:
420            self.file = sys.stdout
421
422            if self.header is not None:
423                self.file.write(self.header)
424
425            return
426
427        # If here, we have a filename. If we already have a file open,
428        # close it, then open the new one.
429        if self.file:
430            self.file.close()
431
432        # Check to see if file already exists
433        file_is_new = not os.path.isfile(filename)
434
435        # Finally, open the specified file with the specified mode and encoding
436        logging.info("opening %s with mode=%s and encoding=%s",
437                     filename, self.mode, self.encoding)
438        self.file = open(filename, self.mode, encoding=self.encoding)
439
440        # Add header record to file if a header was specified and the file was
441        # just created.
442        if file_is_new and self.header:
443            self.file.write(self.header)
444
445    ############################
446    def write(self, record: Union[str, bytes]):
447        """ Write out record, appending a newline at end."""
448
449        # See if it's something we can process, and if not, try digesting
450        if not self.can_process_record(record):  # inherited from BaseModule()
451            self.digest_record(record)  # inherited from BaseModule()
452            return
453
454        if not self.filebase:
455            self._set_file(None)
456
457        # If we're splitting by some time interval, see if it's time to
458        # roll over to a new file.
459        # if self.split_by_time or self.split_interval is not None:
460        elif (self.split_interval and
461              datetime.now(self.time_zone) > self.next_file_split):
462            new_file_date_format = self._get_file_date_format()
463            if new_file_date_format != self.file_date_format:
464                self.file_date_format = new_file_date_format
465                if new_file_date_format.startswith('^'):
466                    self._set_file(
467                        os.path.dirname(self.filebase)
468                        + new_file_date_format[1:]
469                        + os.path.basename(self.filebase)
470                        + self.suffix
471                    )
472                else:
473                    self._set_file(self.filebase + new_file_date_format +
474                                   self.suffix)
475
476        # If we're not splitting by intervals, still check that we've got
477        # a file open we can write to. If not, open it.
478        else:
479            if not self.file:
480                self._set_file(self.filebase + self.suffix)
481
482        # Write the record and flush if requested
483        self.file.write(record)
484        if self.delimiter is not None:
485            self.file.write(self.delimiter)
486        if self.flush:
487            self.file.flush()