openrvdas.logger.utils.read_config

   1#!/usr/bin/env python3
   2"""Utilities for reading/processing JSON data.
   3"""
   4import copy
   5import os
   6import glob
   7import logging
   8import re
   9import json
  10from typing import Dict, List, Any, Union
  11
  12try:
  13    import yaml
  14except ModuleNotFoundError:
  15    pass
  16
  17
  18###############################################################################
  19def read_config(file_path: str) -> Dict[str, Any]:
  20    """
  21    Read a YAML configuration file.
  22
  23    Args:
  24        file_path: Path to the YAML configuration file
  25
  26    Returns:
  27        Dictionary containing the YAML content or empty dict on error
  28    """
  29    try:
  30        # Load the YAML file
  31        with open(file_path, 'r') as file:
  32            file_content = file.read()
  33        return parse(file_content, file_path)
  34
  35    except FileNotFoundError:
  36        logging.error(f'YAML file not found: "{file_path}"')
  37        return {}
  38    except Exception as e:
  39        logging.error(f'Error reading file "{file_path}": {str(e)}')
  40        return {}
  41
  42
  43###################
  44def parse(content: str, file_path: str = None) -> Dict[str, Any]:
  45    """
  46    Parse YAML content and process includes.
  47
  48    Args:
  49        content: The YAML content as a string
  50        file_path: The original file path (for error reporting)
  51
  52    Returns:
  53        Dictionary containing the merged YAML content or empty dict on error
  54    """
  55    try:
  56        # Parse the YAML content
  57        try:
  58            data = yaml.load(content, Loader=yaml.FullLoader)
  59        except AttributeError:
  60            # If they've got an older yaml, it may not have FullLoader)
  61            data = yaml.load(content)
  62        # Handle empty file
  63        if data is None:
  64            return {}
  65        return data
  66
  67    except yaml.YAMLError as e:
  68        logging.error(f'Invalid YAML syntax in "{file_path}": {str(e)}')
  69        return {}
  70    except Exception as e:
  71        logging.error(f'Error parsing YAML in "{file_path}": {str(e)}')
  72        return {}
  73
  74
  75###################
  76def expand_cruise_definition(input_dict):
  77    """
  78    Expand a configuration dictionary with loggers and configs structure.
  79    """
  80    # First handle includes to get all templates loaded
  81    result = expand_includes(input_dict)
  82
  83    # Process both logger and config templates
  84    result = expand_templates(result)
  85
  86    # Now do the global variable substitution
  87    global_variables = result.get('variables', {})
  88    result = substitute_variables(result, global_variables)
  89
  90    # Continue with the rest of the process
  91    result = expand_logger_definitions(result)
  92    result = expand_modes(result)
  93
  94    unmatched_vars = find_unmatched_variables(result)
  95    if unmatched_vars:
  96        logging.error(f'Unexpanded variables found: '
  97                      f'{", ".join(unmatched_vars)}')
  98
  99    return result
 100
 101
 102###################
 103def expand_wildcards(include_pattern: str, base_dir: str) -> List[str]:
 104    """
 105    Expand a potentially wildcard-containing path to a list of matching files.
 106
 107    Args:
 108        include_pattern: Pattern that may contain wildcards (e.g., "*.yaml")
 109        base_dir: Base directory to resolve the pattern from
 110
 111    Returns:
 112        List of matching file paths
 113    """
 114    # Resolve relative paths
 115    if not os.path.isabs(include_pattern):
 116        full_pattern = os.path.normpath(os.path.join(base_dir,
 117                                                     include_pattern))
 118    else:
 119        full_pattern = include_pattern
 120
 121    # Use glob to expand the pattern
 122    matching_files = glob.glob(full_pattern)
 123
 124    if not matching_files:
 125        logging.warning(f'No files found matching pattern: "{include_pattern}"')
 126
 127    return matching_files
 128
 129
 130###################
 131def expand_includes(input_dict: dict) -> Dict[str, Any]:
 132    """
 133    Recursively process any included YAML files and merge them into the top
 134    level.
 135
 136    Args:
 137        input_dict (dict): The input dictionary optionally containing
 138                           'includes' and 'includes_base_dir' keys.
 139
 140    Returns:
 141        Dictionary containing the merged YAML content or empty dict on error
 142
 143    Raises:
 144        ValueError: If any included files are not found.
 145    """
 146
 147    # Default base_dir is top level project directory
 148    base_dir = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
 149
 150    # If they've got an 'includes_base_dir' key in the file, use that.
 151    includes_base_dir = input_dict.get('includes_base_dir')
 152    if includes_base_dir is not None:
 153        if not isinstance(includes_base_dir, str):
 154            logging.error(f'Key "includes_base_dir" must be a dir path str; '
 155                          f'found: {includes_base_dir}. Ignoring.')
 156        else:
 157            base_dir = includes_base_dir
 158
 159    # Handle includes if present
 160    if 'includes' in input_dict and isinstance(input_dict['includes'], list):
 161        included_data = {}
 162
 163        # Process each included file or pattern
 164        for include_pattern in input_dict['includes']:
 165            # Expand wildcards to get list of matching files
 166            if isinstance(include_pattern, str):
 167                include_pattern = include_pattern.strip()
 168            matching_files = expand_wildcards(include_pattern, base_dir)
 169
 170            # Process each matching file
 171            for include_path in matching_files:
 172                # Use the directory of the include_path as the base_dir for
 173                # nested includes
 174                file_path = os.path.join(base_dir, include_path)
 175
 176                # Load the included file
 177                included_content = read_config(file_path)
 178
 179                # Merge with current data
 180                included_data = deep_merge(included_data, included_content)
 181
 182        # Remove the includes key before merging
 183        includes_value = input_dict.pop('includes')
 184
 185        # Merge the original data on top of the included data
 186        result = deep_merge(included_data, input_dict)
 187
 188        # Restore the includes key if needed
 189        input_dict['includes'] = includes_value
 190
 191        return result
 192
 193    return input_dict
 194
 195
 196def deep_merge(base: Dict[str, Any],
 197               overlay: Dict[str, Any]) -> Dict[str, Any]:
 198    """
 199    Deeply merge two dictionaries with special handling for different value
 200    types:
 201    - Scalar values: overwrite
 202    - Lists: append
 203    - Dictionaries: recursively merge
 204
 205    Args:
 206        base: Base dictionary to merge into
 207        overlay: Dictionary to merge on top of base
 208
 209    Returns:
 210        Merged dictionary
 211    """
 212    result = base.copy()
 213
 214    for key, value in overlay.items():
 215        if key in result:
 216            # If both values are dictionaries, merge them recursively
 217            if isinstance(result[key], dict) and isinstance(value, dict):
 218                result[key] = deep_merge(result[key], value)
 219
 220            # If both values are lists, append them
 221            elif isinstance(result[key], list) and isinstance(value, list):
 222                result[key] = result[key] + value
 223
 224            # Otherwise overwrite (handles scalar case)
 225            else:
 226                result[key] = value
 227        else:
 228            # Key doesn't exist in result, just add it
 229            result[key] = value
 230
 231    return result
 232
 233
 234###################
 235def expand_templates(
 236    cruise_definition: Dict[str, Dict[str, Any]]
 237) -> Dict[str, Dict[str, Any]]:
 238    """
 239    Process a complete configuration dictionary with templates.
 240    Handles both logger_templates and config_templates.
 241
 242    Args:
 243        cruise_definition: Dictionary containing 'logger_templates',
 244                           'config_templates', 'loggers', and optionally
 245                           'variables' as top-level keys
 246
 247    Returns:
 248        Dictionary with fully processed logger and config configurations
 249    """
 250    # Extract components from the configuration dictionary
 251    logger_templates = cruise_definition.get('logger_templates', {})
 252    config_templates = cruise_definition.get('config_templates', {})
 253    global_variables = cruise_definition.get('variables', {})
 254
 255    # FIRST PHASE: Process logger templates
 256    for logger_name, logger_def in cruise_definition.get('loggers', {}).items():  # noqa E501
 257        if not isinstance(logger_def, dict):
 258            raise ValueError(f'Malformed logger definition for {logger_name}; '
 259                             f'should be dict.')
 260
 261        # Skip loggers that don't use logger_templates
 262        template_name = logger_def.get('logger_template')
 263        if not template_name:
 264            continue
 265
 266        # Copy global variables so we can modify them
 267        effective_variables = copy.deepcopy(global_variables)
 268        effective_variables['logger'] = logger_name
 269
 270        # Override with logger-specific variables
 271        logger_variables = logger_def.get('variables', {})
 272        effective_variables.update(logger_variables)
 273
 274        # Get the template
 275        template = logger_templates.get(template_name)
 276        if not template:
 277            raise ValueError(f"Template '{template_name}' not found in "
 278                             f"logger_templates")
 279
 280        # Overlay the template on the existing logger definition,
 281        # overwriting configs, etc.
 282        merged_def = copy.deepcopy(template)
 283        for key, value in logger_def.items():
 284            if key not in ['logger_template', 'variables']:
 285                merged_def[key] = value
 286
 287        # Substitute variables
 288        try:
 289            processed_definition = substitute_variables(merged_def,
 290                                                        effective_variables)
 291        except ValueError as e:
 292            logging.error(f"Error processing logger '{logger_name}': {e}")
 293            raise
 294
 295        # Clean up things that aren't needed
 296        if 'variables' in processed_definition:
 297            del processed_definition['variables']
 298        if 'logger_template' in processed_definition:
 299            del processed_definition['logger_template']
 300
 301        # Store the processed definition
 302        cruise_definition['loggers'][logger_name] = processed_definition
 303
 304    # SECOND PHASE: Process config templates
 305    for logger_name, logger_def in cruise_definition.get('loggers', {}).items():  # noqa E501
 306        if not isinstance(logger_def, dict):
 307            continue
 308
 309        # Process configs within this logger
 310        if 'configs' not in logger_def or not isinstance(logger_def['configs'], dict):  # noqa E501
 311            continue
 312
 313        for config_name, config_def in logger_def['configs'].items():
 314            if not isinstance(config_def, dict):
 315                continue
 316
 317            # Skip if no config_template is specified
 318            if 'config_template' not in config_def:
 319                continue
 320
 321            # Get template name and the template itself
 322            template_name = config_def.get('config_template')
 323            template = config_templates.get(template_name)
 324            if not template:
 325                raise ValueError(f"Template '{template_name}' not found in "
 326                                 f"config_templates")
 327
 328            # Setup effective variables by merging global, logger, and config
 329            # variables
 330            effective_variables = copy.deepcopy(global_variables)
 331            effective_variables['logger'] = logger_name
 332
 333            # Add logger-specific variables
 334            if 'variables' in logger_def and isinstance(logger_def['variables'], dict):  # noqa E501
 335                effective_variables.update(logger_def['variables'])
 336
 337            # Add config-specific variables
 338            if 'variables' in config_def and isinstance(config_def['variables'], dict):  # noqa E501
 339                effective_variables.update(config_def['variables'])
 340
 341            # Create a new config by copying the template
 342            merged_config = copy.deepcopy(template)
 343
 344            # Check for missing variables before substitution and use global
 345            # values
 346            unmatched = find_unmatched_variables(merged_config)
 347            for var in unmatched:
 348                # Extract the variable name from the <<var>> format
 349                var_name = var.strip('<>')
 350                # If it's missing from the effective variables but exists
 351                # in globals, use the global value
 352                if var_name not in effective_variables and var_name in global_variables:  # noqa E501
 353                    logging.info(
 354                        f"Using global value for '{var_name}' in config "
 355                        f"'{config_name}' for logger '{logger_name}'")
 356                    effective_variables[var_name] = global_variables[var_name]
 357
 358            # Apply variable substitution to the config
 359            try:
 360                processed_config = substitute_variables(merged_config,
 361                                                        effective_variables)
 362            except ValueError as e:
 363                missing_var = str(e).split("'")[1] if "Variable '" in str(e) else "unknown"  # noqa E501
 364                logging.error(f"Missing variable '{missing_var}' "
 365                              f"in config '{config_name}' for "
 366                              f"logger '{logger_name}'")
 367                logging.error(f"Available variables: "
 368                              f"{', '.join(sorted(effective_variables.keys()))}")  # noqa E501
 369                raise
 370
 371            # Merge any extra non-template keys from the original config
 372            for key, value in config_def.items():
 373                if key not in ['config_template', 'variables']:
 374                    processed_config[key] = value
 375
 376            # Update the config definition with the processed config
 377            logger_def['configs'][config_name] = processed_config
 378
 379    # Clean up template definitions
 380    if 'variables' in cruise_definition:
 381        del cruise_definition['variables']
 382    if 'logger_templates' in cruise_definition:
 383        del cruise_definition['logger_templates']
 384    if 'config_templates' in cruise_definition:
 385        del cruise_definition['config_templates']
 386
 387    # Apply global variables substitution
 388    cruise_definition = substitute_variables(cruise_definition,
 389                                             global_variables)
 390
 391    return cruise_definition
 392
 393
 394# Define recursive ConfigValue type
 395ConfigValue = Union[Dict[str, Any], List[Any], str, int, float, bool, None]
 396
 397
 398def substitute_variables(config: ConfigValue,
 399                         variables: Dict[str, Any]) -> ConfigValue:
 400    """
 401    Recursively substitute template variables in a configuration dictionary.
 402
 403    Args:
 404        config: Dictionary or list containing template variables
 405        variables: Dictionary of variable names and their values
 406
 407    Returns:
 408        Configuration with all variables substituted
 409
 410    Recursively substitute template variables in a configuration structure.
 411
 412    Supports:
 413      - <<var>>                → replaces with variable value
 414      - <<var|default>>        → uses default if var missing
 415      - nested defaults        → <<var|<<fallback|default>>>>
 416      - type conversion        → <<timeout|10>> → int(10)
 417      - pass-through unresolved placeholders
 418    """
 419
 420    def _convert_type(value: str) -> Any:
 421        """
 422        Convert a string default value to a native Python type when possible.
 423        """
 424        value = value.strip()
 425
 426        # Try to parse JSON literals (true/false/null/numbers)
 427        try:
 428            return json.loads(value)
 429        except json.JSONDecodeError:
 430            pass
 431
 432        try:
 433            return int(value)
 434        except ValueError:
 435            try:
 436                return float(value)
 437            except ValueError:
 438                return value
 439
 440    def _split_placeholder(expr: str):
 441        """
 442        Split a placeholder expression into its variable name and optional
 443        default.
 444
 445        Example expr values:
 446            "var"
 447            "var|default"
 448            "var|<<fallback|default>>"
 449
 450        Returns a tuple containing the variable name and default value (or
 451        None):
 452            (var_name, default_expr_or_None)
 453
 454        The split on the '|' character only occurs at the top level (depth == 0).
 455        Any '|' characters inside nested placeholders delimited by '<<' and '>>'
 456        are ignored. Nesting depth is tracked by counting occurrences of '<<' and
 457        '>>' while scanning the string from left to right.
 458
 459        Examples:
 460            _split_placeholder("timeout|10")
 461                -> ("timeout", "10")
 462
 463            _split_placeholder("a|<<b|c>>")
 464                -> ("a", "<<b|c>>")
 465
 466            _split_placeholder("a")
 467                -> ("a", None)
 468        """
 469        depth, i = 0, 0
 470        while i < len(expr):
 471            if expr[i:i+2] == '<<':
 472                depth += 1
 473                i += 2
 474                continue
 475            if expr[i:i+2] == '>>':
 476                depth -= 1
 477                i += 2
 478                continue
 479            if expr[i] == '|' and depth == 0:
 480                return expr[:i], expr[i+1:]
 481            i += 1
 482
 483        return expr, None
 484
 485    def _resolve_variable(expr: str):
 486        """
 487        Resolve a single placeholder expression to its final value.
 488
 489        Given the inner contents of a placeholder (the text between << and >>),
 490        this function:
 491
 492          1. Splits the expression into a variable name and optional default
 493             using `_split_placeholder`, supporting nested defaults.
 494          2. If the variable name exists in `variables`, returns its value.
 495          3. If the variable does not exists and a default expression is
 496             present, recursively resolves the default expr via
 497             `substitute_variables`, allowing chains such as: <<a|<<b|10>>>>
 498          4. Applies type conversion to string defaults
 499             (e.g. "10" → 10, "true" → True).
 500          5. If variable does not exist and no a default can be resolved,
 501             returns the original placeholder in pass-through form
 502             (e.g. "<<a>>").
 503
 504        Args:
 505            expr: The inner contents of a placeholder (without the outer << >>).
 506
 507        Returns:
 508            The resolved value in its native Python type (int, float, bool,
 509            None, str, etc.) or the original placeholder string.
 510        """
 511
 512        name, default_expr = _split_placeholder(expr)
 513        if name in variables:
 514            return variables[name]
 515
 516        if default_expr is not None:
 517            resolved = substitute_variables(default_expr, variables)
 518            return _convert_type(resolved) if isinstance(resolved, str) else resolved
 519
 520        return f"<<{name}>>"
 521
 522    def _walk_string(expr: str) -> str:
 523        """
 524        Scan the expression and replace variable syntax with actual variable
 525        values.
 526
 527        For each complete variable syntax found, the inner expression is
 528        resolved via `_resolve_variable`, and the resolved value is inserted
 529        into the output.
 530
 531        Args:
 532            expr: Input string possibly containing one or more <<...>> placeholders.
 533
 534        Returns:
 535            A new string with all placeholders expanded and substituted.
 536        """
 537        out, i = [], 0
 538        while i < len(expr):
 539            if expr[i:i+2] == '<<':
 540                depth, j = 1, i + 2
 541                while j < len(expr) and depth:
 542                    if expr[j:j+2] == '<<':
 543                        depth += 1
 544                        j += 2
 545                    elif expr[j:j+2] == '>>':
 546                        depth -= 1
 547                        j += 2
 548                    else:
 549                        j += 1
 550                if depth != 0:  # verify there was a closing >>
 551                    raise ValueError(f"Malformed variable syntax '{expr[i:]}'")
 552                out.append(str(_resolve_variable(expr[i+2:j-2])))
 553                i = j
 554            else:
 555                out.append(expr[i])
 556                i += 1
 557        return ''.join(out)
 558
 559    if isinstance(config, dict):
 560        return {substitute_variables(k, variables): substitute_variables(v, variables)
 561                for k, v in config.items()}
 562
 563    if isinstance(config, list):
 564        return [substitute_variables(v, variables) for v in config]
 565
 566    if isinstance(config, str):
 567        # Check whether the entire string is a single top-level placeholder.
 568        # We track nesting depth so that a string like
 569        # "<<file_root>>/<<logger>>/raw/<<cruise>>_<<logger>>" is NOT treated
 570        # as a single placeholder (the first ">>" closes at position 12, not
 571        # at the end of the string).
 572        if config.startswith("<<"):
 573            depth, i = 1, 2
 574            while i < len(config) and depth:
 575                if config[i:i+2] == '<<':
 576                    depth += 1
 577                    i += 2
 578                elif config[i:i+2] == '>>':
 579                    depth -= 1
 580                    i += 2
 581                else:
 582                    i += 1
 583            if depth == 0 and i == len(config):
 584                # The whole string is one placeholder — resolve with type
 585                # preservation (so e.g. <<baud_rate>> can return an int).
 586                return _resolve_variable(config[2:i-2])
 587        return _walk_string(config)
 588
 589    return config
 590
 591
 592###################
 593def expand_logger_definitions(input_dict):
 594    """
 595    Expand a configuration dictionary with loggers and configs structure.
 596
 597    This function processes a dictionary with a 'loggers' key (required) and
 598    an optional 'configs' key. It extracts config dictionaries from each logger
 599    and moves them to the top level 'configs' section, replacing them with a
 600    list of references.
 601
 602    Args:
 603        input_dict (dict): The input dictionary containing 'loggers' and
 604                           optionally 'configs' keys.
 605
 606    Returns:
 607        dict: A new dictionary with expanded configuration structure.
 608
 609    Raises:
 610        ValueError: If the 'loggers' key is missing or if referenced configs
 611                    are missing.
 612
 613    ###This code is to support flexibility in defining cruise configurations.###
 614
 615    In the past, the "loggers" section of a cruise definition only allowed
 616    declaring the names of each configuration associated with a logger. The
 617    actual definition of each configuration had to be placed in a following
 618    top-level "configs" section.
 619
 620    For example:
 621
 622    loggers:
 623     PCOD:
 624       configs:
 625       - PCOD-off
 626       - PCOD-net
 627       - PCOD-net+file
 628     cwnc:
 629       ...
 630
 631    configs:
 632      PCOD-off: {}
 633      PCOD-net:
 634        readers:
 635          key1: value1
 636        writers:
 637          key2: value2
 638      PCOD-net+file:
 639        readers:
 640          key1: value1
 641        writers:
 642          key2: value2
 643
 644    The old declaration-followed-by-definition method still works, but now, if
 645    desired, the relevant configs may instead be defined within the logger
 646    definition itself.
 647
 648    For example:
 649
 650    loggers:
 651     PCOD:
 652       configs:
 653         'off': {}
 654         net:
 655           readers:
 656             key1: value1
 657           writers:
 658             key2: value2
 659         net+file:
 660           readers:
 661             key1: value1
 662           writers:
 663             key2: value2
 664
 665    In this case, the config names will have the logger name prepended
 666    (e.g. 'off' becomes PCOD-off, net becomes PCOD-net, etc.)
 667
 668    Note that both methods may be used in a single cruise definition, though
 669    for clarity, this is not advised.
 670    """
 671    # Validate input
 672    if 'loggers' not in input_dict:
 673        raise ValueError("Input dictionary must have a 'loggers' key")
 674
 675    # Create a new dictionary to avoid modifying the input
 676    result = copy.deepcopy(input_dict)
 677
 678    # Ensure configs key exists in the result
 679    if 'configs' not in result:
 680        result['configs'] = {}
 681
 682    # Process each logger
 683    for logger_name, logger_data in input_dict['loggers'].items():
 684        # Skip if no configs key in this logger
 685        if 'configs' not in logger_data:
 686            continue
 687
 688        # Get the configs for this logger
 689        logger_configs = logger_data['configs']
 690
 691        # Handle the case where configs is a list of strings
 692        if isinstance(logger_configs, list):
 693            # Verify each referenced config exists in top-level configs
 694            for config_name in logger_configs:
 695                if config_name not in result['configs']:
 696                    raise ValueError(f"Referenced config '{config_name}' "
 697                                     "not found in top-level configs")
 698
 699        # Handle the case where configs is a dictionary of dictionaries
 700        elif isinstance(logger_configs, dict):
 701            # Create a new config list for this logger
 702            new_config_list = []
 703
 704            # Process each config in this logger
 705            for config_key, config_value in logger_configs.items():
 706                # Generate the new config name
 707                config_name = f"{logger_name}-{config_key}"
 708
 709                # Add to the config list
 710                new_config_list.append(config_name)
 711
 712                # Check for potential overwrites in the top-level configs
 713                if config_name in result['configs']:
 714                    print(f"Warning: Overwriting existing config "
 715                          f"'{config_name}' in top-level configs")
 716
 717                # Add the config to the top-level configs
 718                result['configs'][config_name] = config_value
 719
 720            # Replace the logger's configs dict with the list of config names
 721            result['loggers'][logger_name]['configs'] = new_config_list
 722
 723    return result
 724
 725
 726###################
 727def expand_modes(input_dict):
 728    """
 729    Expand or infer the modes section of a cruise definition dict.
 730
 731    This function processes a dictionary with a 'loggers' key (required) and
 732    an optional 'configs' key. It extracts config dictionaries from each
 733    logger and moves them to the top level 'configs' section, replacing them
 734    with a list of references.
 735
 736    Args:
 737        input_dict (dict): The input dictionary (possibly) containing 'modes'
 738                           and 'configs' keys.
 739
 740    Returns:
 741        dict: A new dictionary with expanded configuration structure.
 742
 743    Raises:
 744        ValueError: If the 'configs' key is missing or if referenced configs
 745                    are missing.
 746
 747    ### This code is to support flexibility in defining cruise configurations.
 748
 749    In the past, cruise modes were required to be dicts mapping a logger name
 750    to config. We can infer that dict from a simple list of configs.
 751    """
 752    # Validate input
 753    if 'loggers' not in input_dict:
 754        raise ValueError("Cruise definition missing loggers?!?")
 755    if 'configs' not in input_dict:
 756        raise ValueError("Cruise definition missing configs?!?")
 757
 758    # No modes defined (or an empty 'modes' declaration)? Create a default one
 759    modes = input_dict.get('modes')
 760    if not modes:
 761        logging.warning('No "modes" section found. Generating default mode.')
 762        return generate_default_mode(input_dict)
 763
 764    # 'modes' is there. Is it a dict?
 765    if not isinstance(modes, dict):
 766        raise ValueError(f"'modes' definition must be a dict of modes. "
 767                         f"Found {type(modes)}")
 768
 769    # This is the copy we're going to modify and return
 770    result = copy.deepcopy(input_dict)
 771
 772    loggers = input_dict.get('loggers')
 773    for mode_name, mode_configs in input_dict.get('modes').items():
 774        # Mode is already in expanded form - nothing to do
 775        if isinstance(mode_configs, dict):
 776            continue
 777
 778        if not isinstance(mode_configs, list):
 779            raise ValueError(f"Mode {mode_name} must be either dict "
 780                             f"or list; found {type(mode_configs)}")
 781
 782        # If here, we've got a list of configs that should be run in this mode.
 783        # Figure out which loggers they belong to and expand into the normal
 784        # dict form.
 785        mode_dict = {}
 786        for config_name in mode_configs:
 787            # Look through loggers for this config_name
 788            found = False
 789            for logger_name, logger_def in loggers.items():
 790                logger_configs = logger_def.get('configs')
 791                if not logger_configs:
 792                    raise ValueError(f"Logger {logger_name} has no configs!")
 793                if config_name in logger_configs:
 794                    mode_dict[logger_name] = config_name
 795                    found = True
 796                    break
 797            if not found:
 798                raise ValueError(f"No logger found for {config_name} in "
 799                                 f"mode {mode_name}")
 800
 801        # Now confirm that each logger has had a config defined
 802        for logger_name in loggers:
 803            if logger_name not in mode_dict:
 804                raise ValueError(f"No config defined for {logger_name} "
 805                                 f"in mode {mode_name}")
 806
 807        # Replace the config list with newly-created config dict
 808        result['modes'][mode_name] = mode_dict
 809
 810    # Is there a default mode defined? If not, pick the first one in the
 811    # dict and define it as the default.
 812    if 'default_mode' not in result:
 813        first_mode = next(iter(result.get('modes')))
 814        result['default_mode'] = first_mode
 815
 816    return result
 817
 818
 819###################
 820def generate_default_mode(input_dict):
 821    """
 822    If no 'modes' section is present in input_dict, create one that has a
 823    single mode, named 'default', using the first config defined for each
 824    logger.
 825
 826    Args:
 827        input_dict (dict): The input dictionary containing 'loggers' and
 828                           optionally 'configs' keys.
 829
 830    Returns:
 831        dict: A new dictionary with modes and default_mode keys.
 832
 833    Raises:
 834        ValueError: If the 'loggers' key is missing or if referenced configs
 835                    are missing.
 836    """
 837
 838    # Now it's time to check up on modes - do we actually have a modes key?
 839    if 'modes' in input_dict:
 840        return input_dict
 841
 842    # If not, create one.
 843    # Create a new dictionary to avoid modifying the input
 844    result = copy.deepcopy(input_dict)
 845    default_mode = {}
 846
 847    for logger_name, logger_data in input_dict['loggers'].items():
 848        # Skip if no configs key in this logger
 849        if 'configs' not in logger_data:
 850            raise ValueError(f"Logger {logger_name} has no configs")
 851        # Get the configs for this logger
 852        logger_configs = logger_data['configs']
 853
 854        # Handle the case where configs is a list of strings
 855        if not isinstance(logger_configs, list) or not len(logger_configs):
 856            raise ValueError(f"Logger {logger_name} config list is not a "
 857                             f"list? Found type {type(logger_configs)}: "
 858                             f"{logger_configs}")
 859        default_mode[logger_name] = logger_configs[0]
 860    result['modes'] = {'default': default_mode}
 861    result['default_mode'] = 'default'
 862
 863    return result
 864
 865
 866##############################################################################
 867def find_unmatched_variables(data: Union[Dict, List, str, Any]) -> List[str]:
 868    """
 869    Recursively searches through a nested data structure (dicts, lists,
 870    strings) and finds all variables that begin with "<<" and end with ">>",
 871    returning them with the brackets intact. These typically represent template
 872    variables.
 873
 874    Args:
 875        data: A dict, list, string, or other value to search through
 876
 877    Returns:
 878        List of extracted strings with "<<" and ">>" included
 879    """
 880    results = []
 881
 882    if isinstance(data, dict):
 883        # Search through dictionary keys and values
 884        for key, value in data.items():
 885            # Check if key is a string that might contain bracketed strings
 886            if isinstance(key, str):
 887                results.extend(_extract_from_string(key))
 888
 889            # Recursively check the value
 890            results.extend(find_unmatched_variables(value))
 891
 892    elif isinstance(data, list):
 893        # Search through list elements
 894        for item in data:
 895            results.extend(find_unmatched_variables(item))
 896
 897    elif isinstance(data, str):
 898        # Search within the string
 899        results.extend(_extract_from_string(data))
 900
 901    # Return unique results (no duplicates)
 902    return list(set(results))
 903
 904
 905def _extract_from_string(text: str) -> List[str]:
 906    """
 907    Helper function to extract all "<<...>>" patterns from a string
 908
 909    Args:
 910        text: String to search within
 911
 912    Returns:
 913        List of extracted strings with the brackets included
 914    """
 915    pattern = r"(<<[^<]*>>)"
 916    matches = re.findall(pattern, text)
 917    return matches
 918
 919
 920##############################################################################
 921def load_definitions(definition_path):
 922    """
 923    Load and merge device definitions from YAML files.
 924
 925    This is a shared utility used by RegexParser and RecordParser to load
 926    device and device_type definitions from YAML configuration files.
 927
 928    Supports both the new structured format and the legacy format:
 929
 930    New format:
 931        devices:
 932          device_name:
 933            device_type: SomeType
 934        device_types:
 935          SomeType:
 936            format: ...
 937
 938    Legacy format (deprecated):
 939        device_name:
 940          category: device
 941          device_type: SomeType
 942        SomeType:
 943          category: device_type
 944          format: ...
 945
 946    Args:
 947        definition_path: Comma-separated glob patterns for definition files.
 948                        Example: 'local/devices/*.yaml,contrib/devices/*.yaml'
 949
 950    Returns:
 951        Dict with structure:
 952        {
 953            'devices': {device_name: device_def, ...},
 954            'device_types': {type_name: type_def, ...}
 955        }
 956
 957        Returns empty structure if definition_path is None or no files found.
 958    """
 959    definitions = {'devices': {}, 'device_types': {}}
 960
 961    if not definition_path:
 962        return definitions
 963
 964    def_files = []
 965    for path_glob in definition_path.split(','):
 966        matched = glob.glob(path_glob.strip())
 967        if not matched:
 968            logging.debug('No files match definition path "%s"', path_glob.strip())
 969        def_files.extend(matched)
 970
 971    if not def_files:
 972        return definitions
 973
 974    for filename in def_files:
 975        file_defs = read_config(filename)
 976        file_defs = expand_includes(file_defs)
 977
 978        for key, val in file_defs.items():
 979            # New format: 'devices' key contains dict of device definitions
 980            if key == 'devices':
 981                if not isinstance(val, dict):
 982                    logging.error('"devices" value in file %s must be dict. '
 983                                  'Found type "%s"', filename, type(val))
 984                    continue
 985                for name, defn in val.items():
 986                    if name in definitions['devices']:
 987                        logging.warning('Duplicate device definition "%s" in %s',
 988                                        name, filename)
 989                    definitions['devices'][name] = defn
 990
 991            # New format: 'device_types' key contains dict of device type definitions
 992            elif key == 'device_types':
 993                if not isinstance(val, dict):
 994                    logging.error('"device_types" value in file %s must be dict. '
 995                                  'Found type "%s"', filename, type(val))
 996                    continue
 997                for name, defn in val.items():
 998                    if name in definitions['device_types']:
 999                        logging.warning('Duplicate device_type definition "%s" in %s',
1000                                        name, filename)
1001                    definitions['device_types'][name] = defn
1002
1003            # Skip 'includes' - already handled by expand_includes
1004            elif key == 'includes':
1005                pass
1006
1007            # Legacy format: top-level key with 'category' field
1008            elif isinstance(val, dict) and 'category' in val:
1009                category = val.get('category')
1010                if category == 'device':
1011                    if key in definitions['devices']:
1012                        logging.warning('Duplicate device definition "%s" in %s',
1013                                        key, filename)
1014                    definitions['devices'][key] = val
1015                elif category == 'device_type':
1016                    if key in definitions['device_types']:
1017                        logging.warning('Duplicate device_type definition "%s" in %s',
1018                                        key, filename)
1019                    definitions['device_types'][key] = val
1020                else:
1021                    logging.warning('Top-level definition "%s" in file %s has '
1022                                    'unrecognized category "%s" - ignoring',
1023                                    key, filename, category)
1024
1025            # Unknown top-level key
1026            else:
1027                logging.debug('Ignoring unknown top-level key "%s" in %s', key, filename)
1028
1029    return definitions
1030
1031
1032##############################################################################
1033##############################################################################
1034if __name__ == "__main__":
1035    import argparse
1036
1037    parser = argparse.ArgumentParser()
1038    parser.add_argument('-v', '--verbosity', dest='verbosity', default=0,
1039                        action='count', help='Increase output verbosity')
1040    parser.add_argument('filename', type=str, help='Input file to process')
1041    args = parser.parse_args()
1042
1043    LOG_LEVELS = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG}
1044    args.verbosity = min(args.verbosity, max(LOG_LEVELS))
1045    logging.getLogger().setLevel(LOG_LEVELS[args.verbosity])
1046
1047    config = read_config(args.filename)
1048    config = expand_cruise_definition(config)
1049
1050    print(yaml.dump(config, sort_keys=False))