openrvdas.logger.transforms.select_fields_transform

  1#!/usr/bin/env python3
  2"""Compute subsamples of input data.
  3"""
  4
  5import copy
  6import logging
  7from typing import Union
  8
  9from logger.utils.das_record import DASRecord  # noqa: E402
 10from logger.transforms.transform import Transform  # noqa: E402
 11
 12
 13################################################################################
 14class SelectFieldsTransform(Transform):
 15    """Cull key:value pairs from a record's field dict. Can accept a
 16    top-level dict, a field dict or a DASRecord, and will return a
 17    record in the same format as it received.
 18    """
 19
 20    def __init__(self, keep=None, delete=None, **kwargs):
 21        """
 22        ```
 23        keep - an optional list of field names to keep
 24
 25        delete - an optional list of field names to delete
 26
 27        One, but not both of these should be present. If both are present,
 28        the delete values will be ignored.
 29
 30        Can accept a top-level dict, a field dict or a DASRecord, and will
 31        return a record in the same format as it received.
 32        ```
 33        """
 34        super().__init__(**kwargs)  # processes 'quiet' and type hints
 35
 36        self.keep = keep or []
 37        self.delete = delete or []
 38
 39        if not keep and not delete:
 40            logging.warning('SelectFieldsTransform has empty "keep" and "delete" arguments; '
 41                            'no modifications will be made to passed records.')
 42        if keep and delete:
 43            logging.warning('SelectFieldsTransform has both "keep" and "delete" arguments; '
 44                            '"delete" arguments will be ignored.')
 45
 46    ############################
 47    def transform(self, record: Union[dict, DASRecord]):
 48        """
 49        Return a copy of the passed record with the relevant fields kept/deleted.
 50        """
 51        # See if it's something we can process, and if not, try digesting
 52        if not self.can_process_record(record):  # inherited from BaseModule()
 53            return self.digest_record(record)  # inherited from BaseModule()
 54
 55        # We need to make a deep copy of the record, because we're going
 56        # to modify it as we go, and the same record may be getting passed
 57        # to multiple transforms at the same time (e.g., if the transform
 58        # is part of a ComposedWriter, and we have multiple
 59        # ComposedWriters).
 60        new_record = copy.deepcopy(record)
 61
 62        # Below, we're counting on Python copying the relevant dict by
 63        # reference so that if we modify the 'fields' dict, it is also
 64        # modified in the record we were passed.
 65
 66        # As warned in the constructor, if we have both keep and delete,
 67        # we're going to pass records through unchanged.
 68        if self.keep and self.delete:
 69            return new_record
 70
 71        # If it's a dict, hope it's a single record.
 72        elif type(new_record) is DASRecord:
 73            fields = new_record.fields
 74
 75        elif type(new_record) is dict:
 76            # If we have a 'fields' dict inside the dict, use that
 77            if 'fields' in new_record and type(new_record['fields']) is dict:
 78                fields = new_record['fields']
 79
 80            # Otherwise treat the entire dict as a field dict
 81            else:
 82                fields = new_record
 83
 84        else:
 85            logging.warning('SelectFieldsTransform Got non-list/dict/DASRecord '
 86                            'record to interpolate: %s', new_record)
 87            return None
 88
 89        if self.delete:
 90            for key in self.delete:
 91                if key in fields:
 92                    del fields[key]
 93        else:
 94            field_list = list(fields.keys())
 95            for key in field_list:
 96                if key not in self.keep:
 97                    del fields[key]
 98
 99        # If no fields left, scrap the record
100        if not fields:
101            return None
102
103        return new_record