"""Helpers to sync KOA calibration metadata and download missing cache files."""
from __future__ import annotations
import logging
import os
from typing import Any, Sequence
from .download_verify import ensure_calibration_on_disk
from .hispec_store import HISPECCalibrationStore
logger = logging.getLogger(__name__)
__all__ = [
"verify_koa_calibrations",
"seed_koa_calibration_cache",
"ensure_koa_calibrations",
]
[docs]
def verify_koa_calibrations(
spectrograph: str = "BSPEC",
*,
connect_remote: bool = True,
sync_remote: bool = True,
download_missing: bool = True,
require_complete: bool = True,
ignore_missing_cal_types: list[str] | tuple[str, ...] | None = None,
fallback_local: bool = True,
) -> dict[str, Any]:
"""
Sync KOA catalog metadata and download missing calibration files for one spectrograph.
Uses ``KOA_CALIBRATION_CACHE`` for the local cache. When ``connect_remote`` /
``sync_remote`` are True, also requires ``KECK_OBSERVER_EMAIL`` and
``KECK_OBSERVER_PASSWORD`` for remote KOA access.
Each download is verified on disk (exists, non-trivial size, expected basename)
before it is counted as successful. Metadata sync alone is not treated as a
successful file download.
Parameters
----------
spectrograph : str, optional
Spectrograph to retain / download (``'BSPEC'`` or ``'RSPEC'``). Default ``'BSPEC'``.
connect_remote : bool, optional
Open a remote KOA connection. Default True.
sync_remote : bool, optional
Sync remote catalog IDs into the local SQLite DB. Default True.
download_missing : bool, optional
Download calibrations listed in the local DB but missing from the cache,
filtered to ``spectrograph``. Default True.
require_complete : bool, optional
If True (default), raise ``RuntimeError`` when any download fails or files
remain missing after the pass. Set False to only return the summary.
ignore_missing_cal_types : list of str or None, optional
Calibration types that may be absent from the cache without failing
``require_complete`` (and are skipped for download attempts). Useful for
simulation notebooks that register a packaged ``lfc_wavesol`` instead of
the remote geometric product. Example: ``["lfc_wavesol"]``.
fallback_local : bool, optional
If True (default) and a remote connect/sync fails with a recoverable
auth/network/API error, retry once with ``connect_remote=False`` and
``sync_remote=False`` so notebooks need no try/except boilerplate.
Returns
-------
dict
Summary with keys ``spectrograph``, ``n_records``, ``cal_types``,
``n_missing``, ``n_downloaded``, ``n_failed``, ``downloaded``, ``failed``,
``verified_paths``, ``still_missing``, ``ignored_missing``.
On local fallback, also ``remote_mode='local_fallback'`` and ``remote_error``.
"""
try:
summary = _verify_koa_calibrations_once(
spectrograph,
connect_remote=connect_remote,
sync_remote=sync_remote,
download_missing=download_missing,
require_complete=require_complete,
ignore_missing_cal_types=ignore_missing_cal_types,
)
summary.setdefault("remote_mode", "remote" if connect_remote else "local")
return summary
except (RuntimeError, ValueError, OSError) as exc:
if not (
fallback_local
and connect_remote
and _is_recoverable_remote_error(exc)
):
raise
logger.warning(
"Remote KOA sync/login failed (%s: %s); "
"retrying with local cache only.",
type(exc).__name__,
str(exc).splitlines()[0] if str(exc) else exc,
)
summary = _verify_koa_calibrations_once(
spectrograph,
connect_remote=False,
sync_remote=False,
download_missing=download_missing,
require_complete=require_complete,
ignore_missing_cal_types=ignore_missing_cal_types,
)
summary["remote_mode"] = "local_fallback"
summary["remote_error"] = f"{type(exc).__name__}: {exc}"
return summary
def _verify_koa_calibrations_once(
spectrograph: str = "BSPEC",
*,
connect_remote: bool = True,
sync_remote: bool = True,
download_missing: bool = True,
require_complete: bool = True,
ignore_missing_cal_types: list[str] | tuple[str, ...] | None = None,
) -> dict[str, Any]:
"""Single-pass KOA verify (no local fallback). See :func:`verify_koa_calibrations`."""
spec = str(spectrograph).strip().upper()
if spec not in ("BSPEC", "RSPEC"):
raise ValueError(f"spectrograph must be BSPEC or RSPEC, got {spectrograph!r}")
ignore_types = {
str(t).strip().lower()
for t in (ignore_missing_cal_types or ())
if str(t).strip()
}
def _ignored(rec: dict) -> bool:
return str(rec.get("cal_type", "")).strip().lower() in ignore_types
downloaded: list[str] = []
verified_paths: list[str] = []
failed: list[dict[str, str]] = []
still_missing: list[str] = []
ignored_missing: list[str] = []
with HISPECCalibrationStore(connect_remote=connect_remote, sync_on_init=False) as store:
if sync_remote:
if not connect_remote or store.remote_db is None:
logger.warning(
"Skipping remote sync (connect_remote=%s, remote_db=%s).",
connect_remote,
getattr(store, "remote_db", None),
)
else:
try:
added = store.sync_records_from_remote(mode="id")
except RuntimeError as exc:
# Belt-and-suspenders: store soft-fails most remote 404s, but
# keep verify usable when the Keck catalog API is down.
msg = str(exc)
if "Failed to query metadata" not in msg and "404" not in msg:
raise
logger.warning(
"Remote KOA metadata sync failed (%s); "
"continuing with local caldb only.",
msg.splitlines()[0] if msg else exc,
)
added = []
logger.info(
"Synced %d KOA calibration record(s) into the local caldb.",
len(added or []),
)
missing: list[dict] = []
if download_missing:
missing_all = [
r
for r in (store.get_missing_local_files() or [])
if str(r.get("spectrograph", "")).upper() == spec
]
ignored_missing = [
str(r.get("filename") or r.get("id") or "?")
for r in missing_all
if _ignored(r)
]
missing = [r for r in missing_all if not _ignored(r)]
if ignored_missing:
logger.info(
"Ignoring %d missing %s calibration(s) of type(s) %s: %s",
len(ignored_missing),
spec,
sorted(ignore_types),
ignored_missing[:5],
)
if missing:
logger.info(
"Found %d %s calibration file(s) in the local DB but missing from "
"KOA_CALIBRATION_CACHE=%s",
len(missing),
spec,
os.environ.get("KOA_CALIBRATION_CACHE", "<unset>"),
)
for i, rec in enumerate(missing, 1):
filename = str(rec.get("filename") or "?")
try:
path, _ = store.get_calibration(cal=rec)
path = ensure_calibration_on_disk(
path,
expected_filename=filename if filename != "?" else None,
cal_record=rec,
)
downloaded.append(filename)
verified_paths.append(path)
size = os.path.getsize(path)
logger.info(
"[%d/%d] Verified download %s -> %s (%d bytes)",
i,
len(missing),
filename,
path,
size,
)
except Exception as e:
failed.append({"filename": filename, "error": str(e)})
logger.warning(
"[%d/%d] Failed to download/verify %s: %s",
i,
len(missing),
filename,
e,
)
# Re-scan so we catch zip-basename mismatches and silent no-ops.
still_missing_recs = [
r
for r in (store.get_missing_local_files() or [])
if str(r.get("spectrograph", "")).upper() == spec and not _ignored(r)
]
still_missing = [
str(r.get("filename") or r.get("id") or "?")
for r in still_missing_recs
]
# Refresh ignored list from the full re-scan.
ignored_missing = [
str(r.get("filename") or r.get("id") or "?")
for r in (store.get_missing_local_files() or [])
if str(r.get("spectrograph", "")).upper() == spec and _ignored(r)
]
rows = store.local_db.query() or []
if not isinstance(rows, list):
rows = [rows]
spec_rows = [r for r in rows if str(r.get("spectrograph", "")).upper() == spec]
cal_types = sorted({r.get("cal_type") for r in spec_rows if r.get("cal_type")})
summary = {
"spectrograph": spec,
"n_records": len(spec_rows),
"cal_types": cal_types,
"n_missing": len(missing),
"n_downloaded": len(downloaded),
"n_failed": len(failed),
"downloaded": downloaded,
"failed": failed,
"verified_paths": verified_paths,
"still_missing": still_missing,
"ignored_missing": ignored_missing,
}
logger.info(
"%s caldb: %d record(s); downloaded %d/%d missing file(s) (%d failed); "
"still_missing=%d; ignored_missing=%d; types=%s",
spec,
summary["n_records"],
summary["n_downloaded"],
summary["n_missing"],
summary["n_failed"],
len(still_missing),
len(ignored_missing),
summary["cal_types"],
)
if require_complete and (failed or still_missing):
raise RuntimeError(
f"KOA calibration download incomplete for {spec}: "
f"{len(failed)} failed, {len(still_missing)} still missing from cache "
f"({still_missing[:5]}{'...' if len(still_missing) > 5 else ''}). "
"Check KECK_OBSERVER_* credentials, network access to the Keck calibrations "
"API, and that KOA_CALIBRATION_CACHE is writable."
)
return summary
def _koa_cache_fit_count(cache_dir: str | os.PathLike[str]) -> int:
root = os.path.join(str(cache_dir), "calibrations")
if not os.path.isdir(root):
return 0
n = 0
for _dirpath, _dirnames, filenames in os.walk(root):
n += sum(1 for name in filenames if name.lower().endswith(".fits"))
return n
def _koa_cache_has_db(cache_dir: str | os.PathLike[str]) -> bool:
cache_dir = str(cache_dir)
for name in (
os.path.join(cache_dir, "database", "hispec_calibrations.db"),
os.path.join(cache_dir, "hispec_calibrations.db"),
):
if os.path.isfile(name):
return True
return False
def _discover_koa_cache_sources(
dest: str,
*,
extra: Sequence[str | os.PathLike[str]] | None = None,
) -> list[str]:
"""Candidate seed caches, richest first (by FITS count), excluding ``dest``."""
dest_abs = os.path.abspath(dest)
cands: list[str] = []
def _add(path: str | os.PathLike[str] | None) -> None:
if not path:
return
p = os.path.abspath(os.path.expanduser(str(path)))
if p == dest_abs or not os.path.isdir(p):
return
if p not in cands:
cands.append(p)
for path in extra or ():
_add(path)
env_seed = (os.environ.get("HISPEC_KOA_CACHE_SEED") or "").strip()
if env_seed:
_add(env_seed)
home = os.path.expanduser("~")
try:
for name in os.listdir(home):
if not name.startswith("hispec"):
continue
_add(os.path.join(home, name, "cals", "KOA_CALIBRATION_CACHE"))
except OSError:
pass
# Rank by FITS count then DB presence.
ranked = sorted(
cands,
key=lambda p: (_koa_cache_fit_count(p), _koa_cache_has_db(p)),
reverse=True,
)
return [p for p in ranked if _koa_cache_fit_count(p) > 0 or _koa_cache_has_db(p)]
[docs]
def seed_koa_calibration_cache(
dest: str | os.PathLike[str] | None = None,
*,
sources: Sequence[str | os.PathLike[str]] | None = None,
min_fits: int = 1,
force: bool = False,
) -> dict[str, Any]:
"""
Copy a populated KOA cache into ``dest`` when the destination is empty/thin.
Parameters
----------
dest
Target ``KOA_CALIBRATION_CACHE`` directory. Defaults to the current
``KOA_CALIBRATION_CACHE`` environment variable.
sources
Optional explicit seed directories (tried first). Otherwise discovers
``~/hispec*/cals/KOA_CALIBRATION_CACHE`` and ``HISPEC_KOA_CACHE_SEED``.
min_fits
Destination is considered populated once it has at least this many
FITS files under ``calibrations/``.
force
If True, replace the destination from the richest source even when
``dest`` already meets ``min_fits``.
Returns
-------
dict
``dest``, ``seeded``, ``source``, ``n_fits_before``, ``n_fits_after``,
``candidates``.
"""
import shutil
dest_path = (str(dest).strip() if dest is not None else "") or (
os.environ.get("KOA_CALIBRATION_CACHE") or ""
).strip()
if not dest_path:
raise ValueError(
"dest or KOA_CALIBRATION_CACHE must be set to seed a KOA cache."
)
dest_path = os.path.abspath(os.path.expanduser(dest_path))
os.makedirs(dest_path, exist_ok=True)
os.environ["KOA_CALIBRATION_CACHE"] = dest_path
n_before = _koa_cache_fit_count(dest_path)
candidates = _discover_koa_cache_sources(dest_path, extra=sources)
result: dict[str, Any] = {
"dest": dest_path,
"seeded": False,
"source": None,
"n_fits_before": n_before,
"n_fits_after": n_before,
"candidates": candidates,
}
if not force and n_before >= int(min_fits) and _koa_cache_has_db(dest_path):
logger.info(
"KOA cache already populated (%d FITS) at %s; skip seed.",
n_before,
dest_path,
)
return result
if not candidates:
logger.warning(
"No KOA cache seed found for dest=%s (set HISPEC_KOA_CACHE_SEED "
"or pass sources=...).",
dest_path,
)
return result
source = candidates[0]
if os.path.abspath(source) == dest_path:
return result
logger.info(
"Seeding KOA cache %s <- %s (%d FITS in source).",
dest_path,
source,
_koa_cache_fit_count(source),
)
# Copy tree contents into dest (preserve existing unrelated files).
for name in os.listdir(source):
src_item = os.path.join(source, name)
dst_item = os.path.join(dest_path, name)
if os.path.isdir(src_item):
shutil.copytree(src_item, dst_item, dirs_exist_ok=True)
elif os.path.isfile(src_item):
os.makedirs(os.path.dirname(dst_item) or dest_path, exist_ok=True)
shutil.copy2(src_item, dst_item)
n_after = _koa_cache_fit_count(dest_path)
result.update(
seeded=True,
source=source,
n_fits_after=n_after,
)
logger.info(
"Seeded KOA cache: %d -> %d FITS under %s",
n_before,
n_after,
dest_path,
)
return result
def _is_recoverable_remote_error(exc: BaseException) -> bool:
msg = str(exc).lower()
markers = (
"failed to query metadata",
"404",
"failed login",
"incorrect password",
"incorrect password or email",
"must be set as environment variables",
"keck_observer",
"connection",
"timeout",
"timed out",
"name or service not known",
"nodename nor servname",
"temporary failure",
"remote",
)
return any(m in msg for m in markers)
[docs]
def ensure_koa_calibrations(
spectrograph: str = "BSPEC",
*,
cache_dir: str | os.PathLike[str] | None = None,
seed_sources: Sequence[str | os.PathLike[str]] | None = None,
ignore_missing_cal_types: list[str] | tuple[str, ...] | None = None,
require_complete: bool = True,
connect_remote: bool = True,
) -> dict[str, Any]:
"""
Seed the local KOA cache if thin, then verify/download calibrations.
Tries a remote sync first. On auth / network / API failures, falls back to
the local cache only (after seeding from another cache when needed).
"""
if ignore_missing_cal_types is None:
ignore_missing_cal_types = ["lfc_wavesol"]
if cache_dir is not None:
cache_path = os.path.abspath(os.path.expanduser(str(cache_dir)))
os.makedirs(cache_path, exist_ok=True)
os.environ["KOA_CALIBRATION_CACHE"] = cache_path
elif not (os.environ.get("KOA_CALIBRATION_CACHE") or "").strip():
raise ValueError(
"KOA_CALIBRATION_CACHE is unset; pass cache_dir= or set the env var."
)
seed_info = seed_koa_calibration_cache(
os.environ["KOA_CALIBRATION_CACHE"],
sources=seed_sources,
)
summary = verify_koa_calibrations(
spectrograph,
connect_remote=bool(connect_remote),
sync_remote=bool(connect_remote),
ignore_missing_cal_types=ignore_missing_cal_types,
require_complete=require_complete,
fallback_local=bool(connect_remote),
)
summary["seed"] = seed_info
return summary