Source code for hispecdrp.driver.config

from __future__ import annotations
import importlib.resources
import os
import yaml
from typing import Any, Callable
from .. import datamodels

from koa_middleware import CalibrationSelector

from packaging.version import Version

_PKG_NAME = __name__.split('.')[0]

import logging
logger = logging.getLogger(__name__)

__all__ = [
    'load_config_file',
    'get_config_filepath',
    'get_config_dir',
    'ConfigFile',
    'ConfigStore',
    'ConfigSelector'
]

class FlowSeqDumper(yaml.SafeDumper):
    pass

def _flow_seq(dumper, data):
    return dumper.represent_sequence('tag:yaml.org,2002:seq', data, flow_style=True)

FlowSeqDumper.add_representer(list, _flow_seq)


[docs] class ConfigFile: """ A class representing a configuration file. Attributes ---------- class_name : str The name of the primitive or pipeline class (with or without the module path). filepath : str Full filepath to the config file. context : dict A dictionary of metadata for this config file. """ _DEFAULT_CONTEXT = { 'quicklook': None, 'instrument_era': ['0.0.1', None], 'instrument_mode': None, 'processing_mode': None, } def __init__( self, name: str, class_name: str, description: str | None = None, filepath: str | None = None, context: dict | None = None, parameters : dict | None = None, primitives : dict | None = None, logging : dict | None = None, ): """ Initialize a ConfigFile object. Parameters ---------- name : str Unique name for all config files = filename without extension. class_name : str The name of the primitive or pipeline class (with or without the module path). description : str, optional A description of the config file. filepath : str, optional Full filepath to the config file. context : dict, optional A dictionary of metadata for this config file. parameters : dict, optional A dictionary of parameters for this config file. primitives : dict, optional A dictionary of primitives for this config file. logging : dict, optional A dictionary of logging configuration for this config file. """ if filepath is not None: self.filepath = os.path.abspath(filepath) else: self.filepath = None self._name = name self.class_name = class_name self.description = description self.context = context or {} for key, default_value in self._DEFAULT_CONTEXT.items(): self.context.setdefault(key, default_value) self.parameters = parameters or {} self.primitives = primitives or {} self.logging = logging or {} @property def name(self): """Unique name for a config file.""" return self._name or os.path.splitext(self.filename)[0] or self.class_name @property def class_(self): if not hasattr(self, '_class') or self._class is not None: try: class_ = get_primitive_class(self.class_name) except KeyError: class_ = None if class_ is None: try: class_ = get_pipeline_class(self.class_name) except KeyError: class_ = None if class_ is None: raise ValueError(f"Could not find class for {self.class_name}.") self._class = class_ return self._class @property def filename(self): if self.filepath is not None: return os.path.basename(self.filepath) return None
[docs] @classmethod def from_file(cls, filepath: str, **kwargs): config_data = load_config_file(filepath) name = config_data.get('name', os.path.splitext(os.path.basename(filepath))[0]) return cls( name=name, class_name=config_data['class'], # Required field description=config_data.get('description'), # Optional field filepath=filepath, context=config_data.get('context', {}), # Optional section parameters=config_data.get('parameters', {}), # Optional section primitives=config_data.get('primitives', {}), # Optional section **kwargs )
[docs] @classmethod def from_dict(cls, config: dict): return cls( name=config.get('name'), filepath=None, class_name=config['class'], description=config.get('description'), context=config.get('context'), parameters=config.get('parameters'), primitives=config.get('primitives'), )
[docs] def to_dict(self) -> dict: return { 'name': self.name, 'class': self.class_name, 'description': self.description, 'context': self.context, 'parameters': self.parameters, 'primitives': self.primitives, }
[docs] def save(self, output_path : str, output_dir : str | None = None) -> str: if output_path is None: if output_dir is None: raise ValueError("Either output_path or output_dir must be specified.") output_path = os.path.join(output_dir, f"{self.name}.yaml") config_dict = self.to_dict() with open(output_path, 'w') as f: yaml.dump(config_dict, f, Dumper=FlowSeqDumper, sort_keys=False) return output_path
@property def is_pipeline(self): from ..pipelines.base_pipeline import BasePipeline class_ = self.class_ if class_ is None: return False return issubclass(class_, BasePipeline) def __repr__(self): return ( f"ConfigFile(name={self.name!r}, description={self.description!r}, class_name={self.class_name!r}, " f"filepath={self.filepath!r}, context={self.context!r}, " f"parameters={self.parameters!r}, primitives={self.primitives!r})" )
[docs] class ConfigStore: """ A store for managing configuration files. Examples -------- >>> with ConfigStore() as store: config_file : ConfigFile = store.select_and_get_config( dataset, selector ) filepath = config_file.filepath """ _cache: list[ConfigFile] | None = None def __init__(self, keep_in_memory: bool = True, config_dir: str | None = None): self.config_dir = config_dir # Instances pointed at a custom config_dir never touch the shared # cache -- it exists only for the real, installed configs directory. self.keep_in_memory = keep_in_memory and config_dir is None if self.keep_in_memory and ConfigStore._cache is not None: self.configs = ConfigStore._cache else: self.configs = self.populate() if self.keep_in_memory: ConfigStore._cache = self.configs
[docs] @classmethod def clear_cache(cls): cls._cache = None
[docs] def populate(self): configs = [] config_dir = self.config_dir or get_config_dir() for filename in os.listdir(config_dir): if filename.endswith(('.yaml', '.yml')): filepath = os.path.join(config_dir, filename) try: config_file = ConfigFile.from_file(filepath) except KeyError: #logger.warning(f"Skipping non-config YAML file in {config_dir}: {filename}") continue configs.append(config_file) return configs
def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): if not self.keep_in_memory: self.configs = None # Main method called to retrieve a config file for a dataset and primitives class
[docs] def select_and_get_config( self, dataset: Any, selector : ConfigSelector, ) -> ConfigFile: """ Given an input dataset and a primitive class to run, select a config file. Parameters ---------- dataset : Any The dataset for which to select a config. selector : ConfigSelector The config selector to use for selecting the config. Returns ------- config_file : ConfigFile The selected config file object. """ model = datamodels.utils.get_first_model(dataset, meta_only=True) meta = model.meta # Select the config file config_file = selector.select(meta, self.configs) return config_file
# Dict interface def __iter__(self): return iter(self.configs) def __len__(self): return len(self.configs) def __getitem__(self, key): return self.configs[key] def __setitem__(self, key, value): self.configs[key] = value def __delitem__(self, key): del self.configs[key] def __contains__(self, config_name : str): return config_name in self.configs
class AmbiguousConfigError(ValueError): pass class NoConfigFoundError(ValueError): pass def _get_nested(meta : Any, path : str) -> Any: """Resolve a dotted path against dict-like or attribute-like metadata.""" value = meta for part in path.split('.'): if value is None: return None if isinstance(value, dict): value = value.get(part) else: value = getattr(value, part, None) return value
[docs] class ConfigSelector(CalibrationSelector): _CUSTOM_FILTERS: dict[str, Callable[[Any, dict], bool]] = {} def __init__( self, class_name : str | type, context : dict | None = None, skip_filters : list[str] = None, ): self.class_name = class_name self.context = context or {} self.skip_filters = skip_filters or [] #### API ####
[docs] def select(self, meta, candidates : list[ConfigFile]) -> ConfigFile | None: """ Select the most appropriate config file for the given metadata. Parameters ---------- meta : dict Data model metadata. candidates : list[ConfigFile] List of candidate config files to consider. Returns ------- result : ConfigFile | None The selected config file if one is found, otherwise None. """ result = self._select(meta, candidates) if result is None: raise NoConfigFoundError(f"No config file could be selected for class {self.class_name} and the given metadata.") return result
[docs] def _select(self, meta, candidates : list[ConfigFile]) -> ConfigFile | None: """ Internal method to perform the core config file selection logic. This method first retrieves a set of candidate config files using ``get_candidates`` and then selects the best one (or ones) from these candidates using ``select_best``. Subclasses can override this method to implement custom selection workflows, but it's often more appropriate to override ``get_candidates`` or ``select_best``. Parameters ---------- meta : dict The metadata for which a config file is to be selected. candidates : list[ConfigFile] The list of candidate config files to consider. Returns ------- result : ConfigFile | None The selected config file if one is found, otherwise None. """ candidates = self.get_candidates(meta, candidates) result = self.select_best(meta, candidates) return result
[docs] def get_candidates(self, meta, candidates : list[ConfigFile]) -> ConfigFile: """ Select the most appropriate config file for the given metadata. Parameters ---------- meta : dict Data metadata to match against the ``meta`` blocks of the candidate config files. candidates : list[ConfigFile] List of candidate config files to consider. Returns ------- config_file : ConfigFile """ if not candidates: raise NoConfigFoundError(f"No config files available for class {self.class_name}.") # Run through the standard filters candidates = self._filter_class(candidates) if "quicklook" not in self.skip_filters: candidates = self._filter_quicklook(candidates) if "instrument_era" not in self.skip_filters: candidates = self._filter_instrument_era(meta, candidates) if "instrument_mode" not in self.skip_filters: candidates = self._filter_instrument_mode(meta, candidates) if "processing_mode" not in self.skip_filters: candidates = self._filter_processing_mode(candidates) # Run additional filters for filter_name, filter_func in self._CUSTOM_FILTERS.items(): if filter_name not in self.skip_filters: candidates = filter_func(meta, candidates) return candidates
[docs] def select_best(self, meta, candidates : list[ConfigFile]) -> ConfigFile: if len(candidates) == 0: raise NoConfigFoundError(f"No config files available for class {self.class_name}.") if len(candidates) > 1: raise AmbiguousConfigError(f"Multiple config files for class {self.class_name} match the given dataset and context. Must specify a more specific config.") # NOTE: Config files are versioned by the DRP, # so only one should remain candidate = candidates[0] return candidate
#### Filters ####
[docs] def _filter_class(self, candidates : list[ConfigFile]) -> list[ConfigFile]: """ Filter config files by the primitive or pipeline class they are meant to configure. Parameters ---------- candidates : list[ConfigFile] The list of candidate config files to filter. Returns ------- filtered_candidates : list[ConfigFile] The filtered list of config files that match the class. """ if isinstance(self.class_name, type): class_name = self.class_name.__name__ else: class_name = self.class_name.rsplit('.', 1)[-1] candidates_out = [ cf for cf in candidates if cf.class_name.rsplit('.', 1)[-1] == class_name ] if len(candidates_out) == 0: raise ValueError(f"No config files match the class '{class_name}'.") return candidates_out
[docs] def _filter_instrument_era(self, meta, candidates : list[ConfigFile]) -> list[ConfigFile]: """ Filter candidates by the valid instrument_era range. Parameters ---------- meta : dict Datamodel metadata candidates : list[ConfigFile] List of candidate config files to filter. Returns ------- filtered_candidates : list[ConfigFile] The filtered list of config files that match the instrument era. """ era = _get_nested(meta, 'instrument_era') if era is None: return candidates era = Version(era) candidates_out = [] for cf in candidates: # The valid instrument era for this config file valid_era = cf.context.get('instrument_era') # No era specified if ( valid_era is None or ( isinstance(valid_era, str) and valid_era.lower() == 'any' ) ): candidates_out.append(cf) continue # era bounds bounds = valid_era if isinstance(valid_era, list) else [valid_era, valid_era] assert len(bounds) == 2, f"Invalid instrument_era in config {cf.name}: {valid_era}" def _to_version(bound): """None or 'any' means unbounded; otherwise parse as a Version.""" if bound is None or (isinstance(bound, str) and bound.lower() == 'any'): return None return Version(bound) era_min, era_max = (_to_version(b) for b in bounds) # Check if the dataset's era falls within the bounds if (era_min is None or era >= era_min) and (era_max is None or era <= era_max): candidates_out.append(cf) return candidates_out
[docs] def _filter_instrument_mode(self, meta, candidates : list[ConfigFile]) -> list[ConfigFile]: """ Filter candidates by instrument mode. Parameters ---------- meta : dict Datamodel metadata candidates : list[ConfigFile] List of candidate config files to filter. Returns ------- filtered_candidates : list[ConfigFile] The filtered list of config files that match the instrument mode. """ inst_mode = _get_nested(meta, 'instrument_mode') if inst_mode is None: return candidates inst_mode = inst_mode.lower() candidates_out = [] for cf in candidates: # Valid instrument modes for this config file inst_mode_valid = cf.context.get('instrument_mode') # Applies to all modes if inst_mode_valid is None: candidates_out.append(cf) continue if isinstance(inst_mode_valid, str): inst_mode_valid = [inst_mode_valid.lower()] elif isinstance(inst_mode_valid, list): inst_mode_valid = [m.lower() for m in inst_mode_valid] else: raise ValueError(f"Invalid instrument_mode type in config {cf.name}: {inst_mode_valid}") # Check if the dataset's instrument mode matches any of the valid modes if ( inst_mode in inst_mode_valid or 'any' in inst_mode_valid ): candidates_out.append(cf) return candidates_out
[docs] def _filter_ifs_mode(self, meta, candidates : list[ConfigFile]): """ Filter candidates by IFS mode. Parameters ---------- meta : dict Datamodel metadata candidates : list[ConfigFile] List of candidate config files to filter. Returns ------- filtered_candidates : list[ConfigFile] The filtered list of config files that match the IFS mode. """ ifs_mode = getattr(meta.instrument, 'ifs_mode', None) if ifs_mode is None: return candidates ifs_mode = ifs_mode.lower() if ifs_mode != 'ifs': return candidates candidates_out = [] for cf in candidates: # Valid instrument modes for this config file inst_mode_valid = cf.context.get('ifs_mode') # Applies to all modes if inst_mode_valid is None: candidates_out.append(cf) continue if isinstance(inst_mode_valid, str): inst_mode_valid = [inst_mode_valid.lower()] elif isinstance(inst_mode_valid, list): inst_mode_valid = [m.lower() for m in inst_mode_valid] else: raise ValueError(f"Invalid instrument_mode type in config {cf.name}: {inst_mode_valid}") # Check if the dataset's instrument mode matches any of the valid modes if ( ifs_mode in inst_mode_valid or 'any' in inst_mode_valid ): candidates_out.append(cf) return candidates_out
[docs] def _filter_quicklook(self, candidates : list[ConfigFile]): """ Filter candidates by the quicklook flag (``self.quicklook``). A config file that omits ``quicklook`` (context value ``None``) is a wildcard: it matches any requested value. It's only used as a fallback when no candidate explicitly matches the requested value, so an explicit ``True``/``False`` config always takes precedence over a wildcard one for the same class. Cases ----- 1. Input context = None ConfigFile contexts = (None, ...) -> All ConfigFile contexts = (True, False) -> False 2. Input context = False ConfigFile contexts = (True, False) -> False ConfigFile contexts = (None, ...) -> All 3. Input context = True ConfigFile contexts = (True, None) -> True Parameters ---------- candidates : list[ConfigFile] List of candidate config files to filter. Returns ------- filtered_candidates : list[ConfigFile] The filtered list of config files that match the quicklook criteria. """ ql_input = self.context.get('quicklook') if ql_input is None: ql_input = False matched = [cf for cf in candidates if cf.context['quicklook'] is ql_input] if matched: return matched return [cf for cf in candidates if cf.context['quicklook'] is None]
[docs] def _filter_processing_mode(self, candidates : list[ConfigFile]) -> list[ConfigFile]: """ Filter candidates by the processing mode (``self.context.processing_mode``). Unlike ``quicklook``, ``processing_mode`` has no natural default value, so an unspecified input (``None``) leaves the candidates unconstrained. When a value *is* specified, a config file that explicitly matches it always wins; a config file that omits ``processing_mode`` (``None``, a wildcard) is only used as a fallback when nothing explicitly matches. Parameters ---------- candidates : list[ConfigFile] List of candidate config files to filter. Returns ------- filtered_candidates : list[ConfigFile] The filtered list of config files that match the processing mode criteria. """ pm_input = self.context.get('processing_mode') if pm_input is None: return candidates matched = [cf for cf in candidates if cf.context['processing_mode'] == pm_input] if matched: return matched return [cf for cf in candidates if cf.context['processing_mode'] is None]
[docs] @classmethod def register_filter(cls, key: str): """Register custom match logic for a selection key with non-equality semantics.""" def _register(fn): cls._CUSTOM_FILTERS[key] = fn return fn return _register
@classmethod def __init_subclass__(cls, **kwargs): super().__init_subclass__(**kwargs) cls._CUSTOM_FILTERS = dict(cls._CUSTOM_FILTERS) # copy-on-inherit
#### Utility methods for config file handling ####
[docs] def get_config_dir(): config_dir = importlib.resources.files(_PKG_NAME).joinpath("configs") if config_dir.is_dir(): return str(config_dir) else: raise FileNotFoundError("Config directory not found in hispecdrp.configs")
[docs] def get_config_filepath(config_file : str) -> str: """ Resolve the path to a configuration file. Parameters ---------- config_file : str The base filename of the configuration file or a valid file path. Returns ------- config_filepath : str The resolved path to the configuration file. """ # If already a ConfigFile object, get its filepath if isinstance(config_file, ConfigFile): config_file = config_file.filepath # If already valid file or filepath, return it. if os.path.isfile(config_file): return os.path.abspath(config_file) # Get full path config_filepath = os.path.join(get_config_dir(), config_file) # Check if valid filepath. if os.path.isfile(config_filepath): return config_filepath raise FileNotFoundError(f"No valid config file found for: {config_file}")
def _load_yaml_config_file(config_filepath : str) -> dict: with open(config_filepath, 'r') as f: return yaml.safe_load(f)
[docs] def load_config_file(config_filepath : str) -> dict: """ Read a configuration file and return its contents as a dictionary.""" ext = os.path.splitext(config_filepath)[1].lower() is_file = os.path.isfile(config_filepath) if not is_file: raise FileNotFoundError(f"Config file not found: {config_filepath}") if config_filepath.endswith(('.yaml', '.yml')): return _load_yaml_config_file(config_filepath) raise ValueError(f"Unsupported config file extension: {ext}. Only .yaml and .yml are supported.")
def get_primitive_class(name: str) -> type | None: """ Lazily resolve and return a primitive or pipeline class. Accepts: - Short class name: "AssignWCSStep" - Full dot path: "some_external_pkg.module.CustomStep" """ from ..primitives import PRIMITIVE_REGISTER return PRIMITIVE_REGISTER[name] def get_pipeline_class(name: str) -> type | None: """ Lazily resolve and return a primitive or pipeline class. Accepts: - Short class name: "AssignWCSStep" - Full dot path: "some_external_pkg.module.CustomStep" """ from ..pipelines import PIPELINE_REGISTER return PIPELINE_REGISTER[name]