import os
import logging
from copy import deepcopy
from typing import Any
from ..driver.logging import logging_context
from ..driver import ConfigFile, ConfigStore, ConfigSelector, get_config_filepath
from ..processing_component import ProcessingComponent
logger = logging.getLogger(__name__)
[docs]
class BasePipeline(ProcessingComponent):
"""
Base pipeline class for managing a sequence of primitives.
PARAMETERS
---------
output_dir (str | None): Directory to save results.
suffix (str | None): Suffix to append to the output filename for the pipeline only.
"""
PARAMETERS = {
'save_result': True,
}
PRIMITIVES = {}
########################
#### INITIALIZATION ####
########################
def __init__(
self,
dataset : Any = None,
config : dict | ConfigFile | str | None = None,
context : dict | None = None,
log_config : dict | str | None = None,
primitives : dict | None = None,
**kwargs
):
"""
Parameters
----------
config : dict | ConfigFile | str | None
The configuration file or Config to further specify parameters.
log_config : dict | str | None
Logging configuration override applied to this pipeline and all its
primitives. Can be a dict of logging keys, a path to a logging YAML
file, or None to use package defaults.
primitives : dict | None
A dictionary of primitive instances to use for this pipeline.
**kwargs
Any additional keyword arguments to set as direct attributes for this pipeline.
"""
self.dataset = dataset
# Load the configuration file or Config object
self.init_config(dataset, config, context)
# Resolve logging config (merges defaults -> config file -> kwarg)
self._init_log_config(log_config)
# Set any kwargs passed to init
self.init_parameters(**kwargs)
# Initialize the primitives
self.init_primitives(context, primitives)
# Init the output, status, and calibrations used
self._output = None
self._status = None
self._calibrations_used = {}
[docs]
def init_primitives(
self,
context : dict | None = None,
primitives : dict | None = None,
):
"""
Initialize all primitives in the pipeline.
Parameters
----------
context : dict, optional
A dictionary of context information to use for selecting the configuration file for each primitive.
primitives : dict, optional
A dictionary of primitive aliases and user-specified parameters.
"""
# Default steps dict from user input
primitives = primitives or {}
context = context or {}
# Alias pipeline context
pipe_context = self.config.context
valid_primitives = set(self.PRIMITIVES.keys())
for key in primitives.keys():
if key not in valid_primitives:
raise ValueError(f"Invalid primitive alias '{key}' in pipeline {self.__class__.__name__}. Valid primitive aliases are: {valid_primitives}")
# Loop over and instantiate all primitives in the pipeline
for prim_alias, prim_class in self.PRIMITIVES.items():
# TODO: Consider resolving step config file here to more robustly handle step parameters if needed. Currently, step config files are not resolved until the step is instantiated. If we want to override step parameters like output_dir, etc, we may want to refactor this loop.
# Config for this step from pipeline config file (if any)
prim_config_from_pipe_config = self.config.primitives.get(prim_alias, {})
prim_config_from_steps_dict = primitives.get(prim_alias, {})
# Explicit config file for this step (if any). The steps dict (user-provided at
# construction time) takes precedence over the pipeline's own config file. If set,
# this fully determines the step's config; any resolved `context` for this step is
# ignored (mirrors BasePrimitive/BasePipeline.init_config, where an explicit `config`
# always wins over `context`).
prim_config_file = prim_config_from_steps_dict.get(
'config', prim_config_from_pipe_config.get('config')
)
if prim_config_file is not None:
prim_config = ConfigFile.from_file(get_config_filepath(prim_config_file))
else:
prim_config = None
# Context for this step
prim_context_from_pipe_config = prim_config_from_pipe_config.get('context', {})
prim_context_from_steps_dict = prim_config_from_steps_dict.get('context', {})
prim_context = pipe_context | context | prim_context_from_pipe_config | prim_context_from_steps_dict
# Parameters for this step
if 'parameters' in prim_config_from_pipe_config:
prim_pars_from_pipe_config = prim_config_from_pipe_config['parameters']
else:
prim_pars_from_pipe_config = prim_config_from_pipe_config
if 'parameters' in prim_config_from_steps_dict:
prim_pars_from_steps_dict = prim_config_from_steps_dict['parameters']
else:
prim_pars_from_steps_dict = prim_config_from_steps_dict
#prim_pars_from_pipe_config = prim_config_from_pipe_config
#prim_pars_from_steps_dict = prim_config_from_steps_dict
prim_pars = prim_pars_from_pipe_config | prim_pars_from_steps_dict
prim_pars.pop('context', None)
prim_pars.pop('config', None)
# Initialize the step
primitive = prim_class(
dataset=self.dataset,
parent=self,
context=prim_context,
config=prim_config,
log_config=self.log_config,
**prim_pars
)
# Set as attribute of the pipeline
setattr(self, prim_alias, primitive)
[docs]
def init_config(
self,
dataset : Any = None,
config : dict | None = None,
context : dict | None = None
):
if config is not None:
if isinstance(config, str):
self.config = ConfigFile.from_file(config)
elif isinstance(config, dict):
self.config = ConfigFile.from_dict(config)
elif isinstance(config, ConfigFile):
self.config = config
else:
raise TypeError("config must be a str (filename or filepath), dict, or ConfigFile instance.")
else:
if dataset is None:
raise ValueError(
"A dataset must be passed to the Pipeline constructor if no config file is provided."
)
self.config = self.get_config(
dataset, context=context
)
[docs]
def init_parameters(self, **kwargs):
"""
Initialize parameter values as top-level attributes.
Parameters
----------
**kwargs
Any additional parameter values to override.
"""
# Get all step parameter values from config file and input kwargs
pars = self.PARAMETERS | self.config.parameters | kwargs
# Set attributes
allowed = self.allowed_parameters
for k, v in pars.items():
if k not in allowed:
msg = f"Invalid parameter name '{k}' for {self.__class__.__name__}. Valid parameters are: {allowed}"
logger.error(msg)
raise NameError(msg)
setattr(self, k, v)
[docs]
@classmethod
def from_config_file(
cls,
config : ConfigFile | str,
dataset : Any = None,
log_config : dict | str | None = None,
**kwargs
):
"""
Initialize a pipeline from a configuration file.
Parameters
----------
dataset : Any, optional
The dataset to process.
config : str, dict, or ConfigFile, optional.
The configuration for the pipeline. Can be one of:
- str: A full path to a configuration file or a filename in the ConfigStore.
- dict: A dictionary of configuration parameters.
- ConfigFile: A ConfigFile instance.
If not provided, the pipeline will attempt to auto-configure based on the input dataset and context.
If the dataset is required for auto-configuration, it must be provided.
log_config : dict, str, or None, optional
The logging configuration to use for the pipeline.
kwargs : dict
Additional keyword arguments to pass to the pipeline constructor.
Returns
-------
BasePipeline
An instance of the pipeline with the configuration loaded.
"""
# If ConfigFile object is passed, use it directly.
# Otherwise, load the config file.
if isinstance(config, str):
config_filepath = get_config_filepath(config)
config_file = ConfigFile.from_file(config_filepath)
elif isinstance(config, dict):
config_file = ConfigFile.from_dict(config)
elif isinstance(config, ConfigFile):
config_file = config
else:
raise ValueError("Input config must be a filepath (str) or a ConfigFile object")
# If class is not provided, then use the current cls context
pipe_class = config_file.class_
if pipe_class is None:
pipe_class = cls
# Check the class is a valid Step or Pipeline class
if not issubclass(pipe_class, cls):
raise ValueError(f"Config file {config_file} specifies class {pipe_class}, which is not a subclass of {cls}")
from ..primitives.base_primitive import BasePrimitive
if pipe_class in (BasePrimitive, BasePipeline):
raise ValueError(f"Config file {config_file} does not specify a class, and the current context is {cls}, which is not a valid class to initialize.")
if issubclass(pipe_class, BasePrimitive):
raise ValueError(f"Config file {config_file} specifies class {pipe_class}, which is a Primitive. Use BasePrimitive.from_config_file to initialize a Primitive.")
return pipe_class(
dataset=dataset,
config=config_file,
log_config=log_config,
**kwargs
)
##################
#### MAIN API ####
##################
[docs]
def apply(
self,
dataset : Any | None = None,
primitives: dict | None = None,
**kwargs
):
"""
The main method a user calls to run a pipeline.
Parameters
----------
dataset
The dataset to process and is up to the subclass to decide what this represents, but is typically a datamodel, or simple containers of datamodels (or filepaths).
primitives : dict | None
A dictionary of primitive aliases and user-specified parameters to override the default primitive parameters for this pipeline.
**kwargs
Additional keyword arguments to set or override for this pipeline. These will override any arguments set during initialization.
Returns
-------
dict
The main output of the pipeline (self._output['output']), which is typically a datamodel or a container of datamodels. This is typically the last primitive's output, but is up to the pipeline implementation.
"""
self._output = None
with logging_context(self.log_config):
try:
if dataset is not None:
self.dataset = dataset
self.set_pars(**kwargs)
# Set additional parameters from the primitives dict.
if primitives is not None:
for prim_alias, prim_config in primitives.items():
if prim_alias not in self.PRIMITIVES:
raise ValueError(f"Invalid primitive alias '{prim_alias}' for pipeline {self.__class__.__name__}. Valid primitive aliases are: {list(self.PRIMITIVES.keys())}")
if 'parameters' in prim_config:
prim_pars = prim_config['parameters']
else:
prim_pars = prim_config
prim_pars.pop('context', None)
prim_instance = getattr(self, prim_alias)
prim_instance.set_pars(**prim_pars)
for prim_alias in self.PRIMITIVES:
prim_instance = getattr(self, prim_alias)
# Forward output_dir to primitives.
if (
prim_instance.output_dir is None
and (prim_instance.save_result or prim_instance.output_file is not None)
):
prim_instance.output_dir = self.output_dir
# Forward max_cores to primitives
if prim_instance.max_cores is None:
prim_instance.max_cores = self.max_cores
#input_dir = self.get_input_dir()
#if self.save_result in (None, False) and self.output_dir is not None:
#if self.save_result is None and self.output_dir is not None:
# self.save_result = True
#if self.save_result and self.output_dir is None:
# self.output_dir = input_dir
logger.info(f"Applying {self.__class__.__name__} with parameters:\n{self.pars_str}")
self._output = self._perform()
if self.save_result:
self.save_output() # Already finalized in the primitive's perform.
except Exception as e:
self._output = dict(output=input)
self._status = "FAILED"
logger.exception(
f"Failed executing pipeline {self.__class__.__name__}"
)
raise
# Get main output
output = self._output.get('output', None) if self._output is not None else None
# Reset input
self.dataset = None
# Return main output
return output
[docs]
@classmethod
def call(
cls,
dataset : Any,
config : dict | str | ConfigFile | None = None,
context : dict | None = None,
primitives : dict | None = None,
return_instance : bool = False,
**kwargs
):
"""
Top level method to instantiate a Step or Pipeline and run it in one call.
Parameters
----------
dataset : object
The dataset to process.
context : object, optional
The context in which the step is being called.
config : object, optional
The configuration to use for the step. Can be a dict, a config file path, or a ConfigFile object.
parent : LigerIRISStep, optional
The parent step or pipeline of this step. Used to determine a fully-qualified name for this step, and to determine the mode in which to run this step.
return_instance : bool, optional
If True, return a tuple of (result, step_instance).
If False, return only the result.
kwargs : dict
Override step parameters.
Returns
-------
results: Any
The result of processing the input with the step.
"""
pipe = cls(
dataset,
config=config,
context=context,
primitives=primitives,
**kwargs
)
result = pipe.apply()
if return_instance:
return result, pipe
else:
return result
###################
#### UTILITIES ####
###################
[docs]
def get_effective_config(self):
config_file = deepcopy(self.config)
config_file.parameters |= self.get_pars()
for prim_alias, prim_class in self.PRIMITIVES.items():
prim = getattr(self, prim_alias)
prim_config = prim.get_effective_config()
config_file.primitives[prim_alias] = {
'class': prim_class.__name__,
'name' : prim_config.name,
'context': prim_config.context,
'parameters': prim_config.parameters,
'description': prim_config.description,
}
return config_file
def __repr__(self):
s = f"{self.__class__.__name__}:\n"
s += " Primitives:\n"
for prim_alias, prim_class in self.PRIMITIVES.items():
s += f" {prim_alias}: {prim_class.__name__}\n"
return s