Source code for hispecdrp.calibrations.hispec_store

from __future__ import annotations
import glob
from koa_middleware import CalibrationStore, CalibrationSelector
import os
from .. import datamodels
from koa_middleware.utils import generate_koa_filehandle, is_valid_uuid, get_env_var_bool

from typing import TYPE_CHECKING, Sequence
if TYPE_CHECKING:
    from datamodels.base_model import CalibrationModel

import logging
logger = logging.getLogger(__name__)

from .download_verify import (
    ensure_calibration_on_disk,
    normalize_downloaded_calibration_path,
)

__all__ = ['HISPECCalibrationStore']


def _default_connect_remote() -> bool:
    """Remote KOA unless ``HISPEC_KOA_CONNECT_REMOTE`` / ``KOA_CONNECT_REMOTE`` is false."""
    if "HISPEC_KOA_CONNECT_REMOTE" in os.environ:
        return get_env_var_bool("HISPEC_KOA_CONNECT_REMOTE", default=True)
    if "KOA_CONNECT_REMOTE" in os.environ:
        return get_env_var_bool("KOA_CONNECT_REMOTE", default=True)
    return True


def _is_recoverable_remote_init_error(exc: BaseException) -> bool:
    msg = str(exc).lower()
    markers = (
        "failed login",
        "incorrect password",
        "incorrect password or email",
        "must be set as environment variables",
        "keck_observer",
        "failed to query metadata",
        "404",
        "connection",
        "timeout",
        "timed out",
        "name or service not known",
        "nodename nor servname",
        "temporary failure",
    )
    return any(m in msg for m in markers)


[docs] class HISPECCalibrationStore(CalibrationStore): def __init__( self, *args, instrument_name : str | None = None, connect_remote : bool | None = None, **kwargs ): if instrument_name is None: instrument_name = os.getenv( 'HISPECDRP_INSTRUMENT_NAME', datamodels.DataModel._DEFAULT_INSTRUMENT ) if connect_remote is None: connect_remote = _default_connect_remote() if connect_remote: try: super().__init__( *args, instrument_name=instrument_name, connect_remote=True, **kwargs ) return except (ValueError, RuntimeError, OSError) as exc: if not _is_recoverable_remote_init_error(exc): raise logger.warning( "Remote KOA unavailable (%s: %s); using local caldb only.", type(exc).__name__, str(exc).splitlines()[0] if str(exc) else exc, ) super().__init__( *args, instrument_name=instrument_name, connect_remote=False, **kwargs )
[docs] def register_calibration( self, cal : CalibrationModel, origin : str | None = None, new_version : bool = False, ) -> tuple[str, dict]: """ Registers a calibration to the local cache and metadata database. Parameters ---------- cal : CalibrationModel | str The filepath or Calibration Datamodel object to register. origin : str, optional The origin to register the calibration under. new_version : bool, optional Whether to generate a new version for this calibration. If False, the method will check if a calibration with the same version family already exists in the cache and skip registration if so. Defaults to False. Returns ------- tuple[str, dict] A tuple containing: - ``str``: The local file path where the calibration was saved. - ``dict``: The calibration metadata dictionary as added to the database. Notes ----- Upstream ``koa_middleware.CalibrationStore.register_calibration`` snapshots ``cal.to_record()`` *before* ``cal.save()`` and discards the return value of ``generate_calibration_version``, so ``filename`` / ``koa_id`` / ``cal_version`` can be stored as ``None`` even though the FITS file is written. This override assigns those fields on the model and record before insert, and returns any existing version-family match instead of ``(None, None)``. """ #from koa_middleware.utils import generate_md5_file cal = datamodels.open(cal) return super().register_calibration(cal, origin=origin, new_version=new_version)
[docs] def _existing_calibration_path_and_record( self, existing: dict | list | None ) -> tuple[str | None, dict | None]: """Return ``(path, record)`` for a pre-existing cache row, or ``(None, None)``. Upstream ``calibration_record_in_cache(..., mode="version-family")`` returns a ``list[dict]`` when multiple version-family matches exist (default ``include_version=False``). Normalize to a single record before path lookup. """ if not existing: return None, None # rows_where / some cache lookups can yield a generator or other iterable. if not isinstance(existing, (dict, list)): try: existing = list(existing) except TypeError: return None, None if isinstance(existing, list): chosen = None for row in existing: if isinstance(row, dict) and row.get("filename"): chosen = row break existing = chosen or (existing[0] if existing else None) if not isinstance(existing, dict): return None, None if not isinstance(existing, dict): return None, None try: path = self.calibration_file_in_cache(existing) except Exception: # noqa: BLE001 - missing/null filename should not crash callers logger.warning( "Existing calibration record is not usable (filename=%r).", existing.get("filename") if isinstance(existing, dict) else None, ) return None, existing if isinstance(existing, dict) else None if path and existing.get("filename"): return path, existing return None, existing
# Select and get calibration
[docs] def select_and_get_calibration( self, input, selector : CalibrationSelector, ) -> tuple[str, dict]: """ Selects the best calibration based on input data and a selection rule, then retrieves it. This method uses a `CalibrationSelector` to identify the most appropriate calibration for the given input data. Once selected, it retrieves the calibration file, downloading it if it's not already cached locally. Parameters ---------- input The input data product for which a calibration is needed. selector : CalibrationSelector An instance of a ``CalibrationSelector`` class. Returns ------- tuple[str, dict] - ``str``: The local file path of the retrieved calibration file. - ``dict``: The record of the selected calibration from the local database. Example: >>> # Assuming ``my_input_data`` and ``my_selector`` are defined >>> local_filepath, calibration_record = store.select_and_get_calibration(my_input_data, my_selector) >>> print(f"Calibration file: {local_filepath}") >>> print(f"Calibration ID: {calibration_record['id']}") """ if isinstance(input, datamodels.DataModel): input = input.meta elif isinstance(input, str): if os.path.isfile(input): model = datamodels.open(input, meta_only=True) input = model.meta else: raise ValueError(f"Input string is not a valid file path: {input}") # koa_middleware logs result['filename'] without checking for None when # select_best finds no candidates. Guard here and raise a clear error. logger.info( "Selecting calibration for input=%s using %s", input.get("filename") if hasattr(input, "get") else None, selector, ) result = selector.select(input, self.local_db) if result is None: raise ValueError( f"No matching calibration found for selector {type(selector).__name__} " f"(instrument_era={input.get('instrument_era')!r}, " f"spectrograph={input.get('spectrograph')!r}, " f"mjd_start={input.get('mjd_start')!r})." ) logger.info( "Selected calibration filename=%s ID=%s", result.get("filename"), result.get("id"), ) return self.get_calibration(result)
[docs] def download_calibration_file( self, calibration: dict | str, ) -> str: """ Download a remote calibration into ``data_dir`` under the DB filename. Wraps ``koa_middleware`` so a zip that uses a different member name is moved to the expected cache path, then verifies the file exists and is non-trivial before returning. """ # Initialize the remote DB if file is if self.remote_db is None: self._init_remote_db() if isinstance(calibration, dict): cal_record = calibration elif isinstance(calibration, str): cal_record = self.calibration_record_in_cache(calibration, mode="id") or { "id": calibration } else: raise TypeError("Calibration must be a dict or str.") expected = self._get_local_filepath(cal_record) if expected is None: # Fall back to middleware behavior when filename is absent. path = super().download_calibration_file(calibration) return ensure_calibration_on_disk(path, cal_record=cal_record) path = super().download_calibration_file(calibration) path = normalize_downloaded_calibration_path(path, expected) return ensure_calibration_on_disk( path, expected_filename=os.path.basename(expected), cal_record=cal_record if isinstance(cal_record, dict) else None, )
[docs] def get_calibration(self, cal: dict | str) -> tuple[str, dict]: """Retrieve a calibration file and verify it is present on disk.""" path, record = super().get_calibration(cal) expected = self._get_local_filepath(record) if expected is not None: path = normalize_downloaded_calibration_path(path, expected) try: path = ensure_calibration_on_disk( path, expected_filename=(record or {}).get("filename"), cal_record=record if isinstance(record, dict) else None, ) return path, record except (FileNotFoundError, RuntimeError) as first_err: # Stale empty/truncated cache hits: remove and force a fresh download. if path and os.path.isfile(path): try: os.remove(path) logger.warning( "Removed invalid cached calibration %s (%s); re-downloading.", path, first_err, ) except OSError: raise first_err from None if self.remote_db is None: raise path = self.download_calibration_file(record) path = ensure_calibration_on_disk( path, expected_filename=(record or {}).get("filename"), cal_record=record if isinstance(record, dict) else None, ) return path, record
[docs] def save_calibration_file(self, cal: CalibrationModel, cal_record : dict | None = None) -> str: """ Saves a calibration file to the local cache directory. Parameters ---------- cal : CalibrationModel The calibration data model instance to save. cal_record : dict | None The corresponding record. Returns ------- str The absolute local file path where the calibration file was saved. """ # Add cal_version and origin to cal file if cal_record is not None: cal.meta.cal_version = cal_record.get('cal_version') cal.meta.origin = cal_record.get('origin') # Generate filename cal.generate_filename() # Set the KOA metadtata fields in the model cal.meta.koa_id = cal_record.get('koa_id', None) cal.meta.koa_filepath = generate_koa_filehandle( cal.meta.instrument_name, cal.meta.datetime_obs, cal.meta.koa_id, ) # Call super to save file and return filename return super().save_calibration_file(cal, cal_record)
[docs] def sync_records_from_cached_files( self, cals : str | CalibrationModel | Sequence[str | CalibrationModel] | None = None, ) -> None: """ Populates the local database from existing cached calibration files. Parameters ---------- cals : str | CalibrationModel | Sequence[str | CalibrationModel] A single calibration metadata dictionary or a data model instance, or a list of these. Notes ----- This method may be removed in the future if not found useful. """ if cals is None: cals = [ f for f in glob.glob(os.path.join(self.data_dir, "*")) if os.path.isfile(f) and not os.path.basename(f).startswith(".") ] if isinstance(cals, (str, datamodels.CalibrationModel)): cals = [cals] cal_records_added = [] for cal in cals: # Add new record (one at a time in case memory issues for many files) if isinstance(cal, str): cal_model = datamodels.open(cal, meta_only=True) else: cal_model = cal cal_record = self._prepare_cal_record(cal_model, origin='LOCAL') cal_record_added = self.local_db.add(cal_record) cal_records_added.append(cal_record_added) # Return new new records return cal_records_added
[docs] def sync_records_from_remote( self, cals=None, mode: str = "id", ) -> list[dict]: """ Synchronize the local DB with the remote KOA catalog. ``cals`` is accepted for compatibility with callers that pass an explicit remote query result; when omitted, records are discovered via ``get_missing_records`` (same as ``koa_middleware``). Remote metadata failures (e.g. KOA API 404) are logged and treated as a no-op so pipelines can continue with the local caldb/cache. Callers that need a hard failure should inspect logs or sync explicitly with their own error handling. """ if cals is not None: mode = str(mode).lower() if mode == "id": ids_local = {cal["id"] for cal in self.local_db.query()} missing = [cal for cal in cals if cal["id"] not in ids_local] elif mode == "last_updated": last_updated_local = self.get_last_updated(source="local") missing = [ cal for cal in cals if last_updated_local is None or cal.get("last_updated", "") > last_updated_local ] else: raise ValueError( f"Invalid mode {mode!r} for sync_records_from_remote()." ) if missing: logger.info( "Adding %d explicit remote record(s) to local DB (mode=%r).", len(missing), mode, ) return self.local_db.add(missing) logger.info( "Local DB already contains explicit remote record(s) (mode=%r).", mode, ) return [] if self.remote_db is None: logger.info("Skipping remote sync (remote_db is None).") return [] try: return super().sync_records_from_remote(mode=mode) except RuntimeError as exc: msg = str(exc) if "Failed to query metadata" in msg or "404" in msg: logger.warning( "Remote KOA metadata sync failed (%s); continuing with local caldb only.", msg.splitlines()[0] if msg else exc, ) return [] raise
[docs] def calibration_record_in_cache( self, cal: dict | str | 'CalibrationModel', mode: str = 'id' ) -> dict | None: """ Checks if a calibration is already present in the local cache. Parameters ---------- calibration : dict | str | DataModel Can be one of: - `str` : A calibration ID string or filepath. - `dict` : A calibration metadata dict. - `DataModel` : A calibration data model instance. mode : str The mode to check the cache. Can be one of: - 'id' : Check by calibration ID (cal_id), the primary key in the database. - 'version-family' : Check by the version family (cal_type, datetime_obs, master_cal, spectrograph) + version (cal_version). - 'md5' : Check by the MD5 checksum of the calibration file (*Not Implemented yet!*). Returns ------- dict | None The calibration metadata record if found, otherwise None. """ if isinstance(cal, str) and os.path.isfile(cal): model = datamodels.open(cal, meta_only=True) return super().calibration_record_in_cache(model, mode=mode) return super().calibration_record_in_cache(cal, mode=mode)
[docs] def generate_calibration_version( self, calibration, origin: str | None = None, ) -> str: """Generate the next version string; accept a model or a metadata dict. Upstream ``koa_middleware`` assigns ``calibration['origin']`` and therefore only works with dicts. Datamodel callers (and :meth:`~hispecdrp.datamodels.base_model.CalibrationModel.generate_calibration_version`) pass the model itself, so convert to a record first and write the version back onto the model when applicable. """ model = None if isinstance(calibration, datamodels.CalibrationModel): model = calibration if origin is not None: model.meta.origin = origin elif getattr(model.meta, "origin", None) is None: model.meta.origin = os.getenv("KOA_CALIBRATION_ORIGIN", "LOCAL") calibration = model.to_record() version = super().generate_calibration_version(calibration, origin=origin) # Upstream sets origin on the dict but does not assign the returned version. if isinstance(calibration, dict): calibration["cal_version"] = version if origin is not None: calibration["origin"] = origin.upper() if model is not None: model.meta.cal_version = version if calibration.get("origin"): model.meta.origin = calibration["origin"] return version
[docs] def get_version_family_column_names(self, cal_type : str): """ Retrieves the column names for the version family attributes. This takes in cal_type in case we want to have different version family fields for different calibration types in the future. Parameters ---------- cal_type : str The type of calibration (e.g. 'dark', 'flat', etc.) to get the version family fields for. """ return ['cal_type', 'datetime_obs', 'master_cal', 'spectrograph']
[docs] def _convert_origin(self, cal : str | 'CalibrationModel', origin : str) -> str: """ Convert an existing calibration file's origin (record and file) to a new origin. This creates a new file and record. This does not modify the input file or record. TODO: Need to generate new claibration ID. Parameters ---------- cal : str | CalibrationModel The calibration file or model to copy the origin from. origin : str The new origin string to set. Returns ------- str The standardized origin string. """ if isinstance(cal, str): if os.path.isfile(cal): model = datamodels.open(cal) elif is_valid_uuid(cal): record = self.query(cal_id=cal) if record is None: raise ValueError(f"No calibration found with ID: {cal}") local_filepath = self._get_local_filepath(record) model = datamodels.open(local_filepath) elif isinstance(cal, datamodels.CalibrationModel): model = cal # New model with updated origin origin = origin.upper() model_out = model.copy() model_out.meta.origin = origin # Register new_local_filepath, new_record = self.register_calibration(model_out) return new_local_filepath, new_record
# def _prepare_cal_record( # self, # cal : dict | CalibrationModel, # origin: str | None = None, # version : bool = True, # last_updated : bool = True # ) -> dict: # """ # Extract and version a calibration's metadata record before saving to disk. # """ # cal_record = self.record_from(cal) # if version: # if origin is None: # origin = self.origin or cal_record.get('origin') # if origin is None: # raise ValueError(f"Origin must be provided or already set in record for {cal}.") # cal_record['cal_version'] = self.generate_calibration_version(cal_record, origin=origin) # cal_record['origin'] = origin # if last_updated: # cal_record['last_updated'] = datetime_to_isot_ms(datetime.now(timezone.utc)) # return cal_record
[docs] def _finalize_cal_record( self, cal, cal_record: dict | str | None = None, local_filepath: str | None = None, ) -> dict: """ Finalize a calibration record after the file has been saved to disk. Adds filename/MD5 via the base store, then fills KOA identifiers that are set on the saved file during :meth:`~datamodels.CalibrationModel.generate_filename` but are not yet present on the pre-save metadata record. Compatible with both koa_middleware call signatures: * ``_finalize_cal_record(cal, cal_record, local_filepath)`` (current git) * ``_finalize_cal_record(cal_record, local_filepath)`` (older releases) """ cal_record = super()._finalize_cal_record(cal, cal_record, local_filepath) cal_record["koa_id"] = cal.meta.filename cal_record["koa_filepath"] = generate_koa_filehandle( cal.meta.instrument_name, cal.meta.datetime_obs, cal_record["koa_id"], ) return cal_record