Source code for hispecdrp.primitives.spectral_extraction.extract_spectroperfectionism

"""Full spectroperfectionism: sparse banded LS with HDF PSF deposit operator.

The forward operator matches :func:`_deposit_oversampled_psf_numba` (native
detector) followed by the sim orientation map used by Stage-1
(``orders_are_vertical=True`` / :func:`_orient_hdf_pixel_xy`). Extraction
solves ``(Aᵀ N⁻¹ A) f = Aᵀ N⁻¹ d`` with SciPy sparse factorization.

Memory notes
------------
HDF PSF stamps are large (``~256×256`` oversampled). Full-order assembly of
every stamp weight for all ``4096`` spectral samples can exceed 16 GB. Prefer:

- ``pix_range=(lo, hi)`` for windowed extracts (readiness / fidelity),
- ``psf_rel_threshold`` to crop near-zero stamp wings,
- a single process-wide HDF cache (see ``e2e_pipeline.clear_hdf_model_cache``).
"""

from __future__ import annotations

import numpy as np
from numba import njit
from scipy import sparse
from scipy.interpolate import PchipInterpolator
from scipy.sparse.linalg import factorized


[docs] @njit(fastmath=True) def _emit_oriented_deposit_weights( out_row: np.ndarray, out_col: np.ndarray, out_val: np.ndarray, psf: np.ndarray, xc: float, yc: float, oversample: float, nx_native: int, abs_threshold: float, ) -> int: """ Write oriented ``(row, col, weight)`` samples for one normalized PSF deposit. Skips ``|psf| < abs_threshold``. Returns the number of written entries. Capacity must be ``>=`` number of above-threshold samples × 4. """ H, W = psf.shape y0 = yc * oversample - (H * 0.5 + 1.0) x0 = xc * oversample - (W * 0.5 + 1.0) n = 0 n_max = out_row.size for j in range(H): y_s = (j + y0) / oversample i0 = int(np.floor(y_s)) dy = y_s - i0 for i in range(W): val = psf[j, i] if val < abs_threshold and val > -abs_threshold: continue x_s = (i + x0) / oversample j0 = int(np.floor(x_s)) dx = x_s - j0 for di in range(2): iy = i0 + di wy = (1.0 - dy) if di == 0 else dy if wy == 0.0 or iy < 0 or iy >= nx_native: continue for dj in range(2): ix = j0 + dj wx = (1.0 - dx) if dj == 0 else dx if wx == 0.0 or ix < 0 or ix >= nx_native: continue w = val * wy * wx if w == 0.0: continue if n >= n_max: return n out_row[n] = (nx_native - 1) - ix out_col[n] = (nx_native - 1) - iy out_val[n] = w n += 1 return n
[docs] def crop_psf_stamp( stamp: np.ndarray, *, rel_threshold: float = 1.0e-2, ) -> tuple[np.ndarray, float, float]: """ Crop a PSF stamp to the bounding box of significant flux. Returns ``(cropped, dx_det, dy_det)`` where the detector-pixel offsets restore the original stamp-center → ``(xc, yc)`` deposit convention after the array shrinks (add to ``xc`` / ``yc`` before deposit). """ psf = np.asarray(stamp, dtype=np.float64) if psf.ndim != 2: raise ValueError(f"stamp must be 2-D; got {psf.shape}") h, w = psf.shape peak = float(np.max(np.abs(psf))) if not np.isfinite(peak) or peak <= 0.0: return psf, 0.0, 0.0 msk = np.abs(psf) >= float(rel_threshold) * peak if not np.any(msk): return psf, 0.0, 0.0 ys, xs = np.where(msk) y0, y1 = int(ys.min()), int(ys.max()) + 1 x0, x1 = int(xs.min()), int(xs.max()) + 1 cropped = np.ascontiguousarray(psf[y0:y1, x0:x1]) # Original geometric center in full-stamp index space. cy0 = 0.5 * (h - 1) cx0 = 0.5 * (w - 1) hc, wc = cropped.shape cy1 = cy0 - y0 cx1 = cx0 - x0 # Deposit centers on 0.5*(H_new-1); shift (xc, yc) to compensate. dy_os = cy1 - 0.5 * (hc - 1) dx_os = cx1 - 0.5 * (wc - 1) return cropped, float(dx_os), float(dy_os)
[docs] @njit(fastmath=True) def deposit_psf_to_oriented( det_oriented: np.ndarray, psf: np.ndarray, xc: float, yc: float, oversample: float, scale: float, nx_native: int, ) -> None: """Deposit one scaled PSF into an oriented rate map (in-place).""" H, W = psf.shape ny_o, nx_o = det_oriented.shape y0 = yc * oversample - (H * 0.5 + 1.0) x0 = xc * oversample - (W * 0.5 + 1.0) for j in range(H): y_s = (j + y0) / oversample i0 = int(np.floor(y_s)) dy = y_s - i0 for i in range(W): x_s = (i + x0) / oversample j0 = int(np.floor(x_s)) dx = x_s - j0 val = psf[j, i] * scale if val == 0.0: continue for di in range(2): iy = i0 + di wy = (1.0 - dy) if di == 0 else dy if wy == 0.0 or iy < 0 or iy >= nx_native: continue for dj in range(2): ix = j0 + dj wx = (1.0 - dx) if dj == 0 else dx if wx == 0.0 or ix < 0 or ix >= nx_native: continue orow = (nx_native - 1) - ix ocol = (nx_native - 1) - iy if 0 <= orow < ny_o and 0 <= ocol < nx_o: det_oriented[orow, ocol] += val * wy * wx
[docs] def _psf_oversample(block: dict, pixelsize: float) -> float: spacing = float(block.get("psf_data_spacing", pixelsize)) if spacing <= 0.0: return 1.0 return float(pixelsize) / spacing
[docs] def _nearest_psf_index(psfw: np.ndarray, lam: float) -> int: return int(np.argmin(np.abs(psfw - float(lam))))
[docs] def forward_psf_deposit_oriented( spec: np.ndarray, wave: np.ndarray, *, fx_native, fy_native, psfw: np.ndarray, psf_stamps, oversample: float, nx_native: int, det_shape: tuple[int, int] | None = None, psf_rel_threshold: float = 1.0e-2, ) -> np.ndarray: """Forward model ``M = A f`` via direct oriented deposits.""" spec = np.asarray(spec, dtype=np.float64) wave = np.asarray(wave, dtype=np.float64) if det_shape is None: det = np.zeros((int(nx_native), int(nx_native)), dtype=np.float64) else: det = np.zeros(det_shape, dtype=np.float64) psfw = np.asarray(psfw, dtype=np.float64) cropped: dict[int, tuple[np.ndarray, float, float]] = {} for s in range(wave.size): fx = float(spec[s]) if not np.isfinite(fx) or fx == 0.0: continue lam = float(wave[s]) if not np.isfinite(lam): continue xc = float(fx_native(lam)) yc = float(fy_native(lam)) if not (np.isfinite(xc) and np.isfinite(yc)): continue idx = _nearest_psf_index(psfw, lam) if idx not in cropped: psf = np.asarray(psf_stamps[idx], dtype=np.float64) psf_sum = float(np.sum(psf)) if psf_sum <= 0.0: continue c, dx, dy = crop_psf_stamp( psf / psf_sum, rel_threshold=psf_rel_threshold ) cropped[idx] = (c, dx / float(oversample), dy / float(oversample)) psf_n, dx_det, dy_det = cropped[idx] deposit_psf_to_oriented( det, psf_n, xc + dx_det, yc + dy_det, float(oversample), fx, int(nx_native), ) return det
[docs] def extract_spectrum_spectroperfectionism( rate: np.ndarray, rate_var: np.ndarray, wave: np.ndarray, *, fx_native, fy_native, psfw: np.ndarray, psf_stamps, oversample: float, nx_native: int, spectral_bandwidth: int | None = None, pix_range: tuple[int, int] | None = None, psf_rel_threshold: float = 1.0e-2, rate_dq: np.ndarray | None = None, do_not_use: int = 1, ) -> dict[str, np.ndarray]: """ Banded weighted LS extract with HDF PSF deposit operator. ``rate`` / ``rate_var`` are oriented (spectral rows × cross-disp cols). Parameters ---------- pix_range Optional ``(i0, i1)`` half-open spectral-row window to solve. Use this on 16 GB machines; full ``4096``-pixel orders with ``256×256`` stamps can exhaust RAM. psf_rel_threshold Crop each stamp to flux ≥ this fraction of the stamp peak before depositing (default ``1e-2``). """ rate = np.asarray(rate, dtype=np.float64) rate_var = np.asarray(rate_var, dtype=np.float64) wave = np.asarray(wave, dtype=np.float64) psfw = np.asarray(psfw, dtype=np.float64) ny, nx = rate.shape n_spec = int(wave.size) nx_native = int(nx_native) if len(psf_stamps) == 0: raise ValueError("No PSF stamps provided") if pix_range is None: i0, i1 = 0, n_spec else: i0 = max(0, int(pix_range[0])) i1 = min(n_spec, int(pix_range[1])) if i1 <= i0: raise ValueError(f"empty pix_range={pix_range}") # Crop only stamps needed for this spectral window (full-order pre-crop of # every 256×256 HDF stamp can exhaust RAM on 16 GB machines). cropped: dict[int, tuple[np.ndarray, float, float]] = {} def _crop_stamp(idx: int) -> tuple[np.ndarray, float, float] | None: hit = cropped.get(idx) if hit is not None: return hit psf = np.asarray(psf_stamps[idx], dtype=np.float64) psf_sum = float(np.sum(psf)) if psf_sum <= 0.0 or not np.isfinite(psf_sum): return None c, dx, dy = crop_psf_stamp( psf / psf_sum, rel_threshold=float(psf_rel_threshold) ) item = (c, dx / float(oversample), dy / float(oversample)) cropped[idx] = item return item needed: set[int] = set() for s in range(i0, i1): lam = float(wave[s]) if not np.isfinite(lam): continue needed.add(_nearest_psf_index(psfw, lam)) max_h = max_w = 1 for idx in sorted(needed): item = _crop_stamp(idx) if item is None: continue max_h = max(max_h, item[0].shape[0]) max_w = max(max_w, item[0].shape[1]) max_per_col = int(max_h * max_w * 4) if spectral_bandwidth is None: spectral_bandwidth = max( 8, int(np.ceil(0.5 * max_w / float(oversample))) + 4 ) half_b = int(spectral_bandwidth) n_win = i1 - i0 est_nnz = int(n_win) * max_per_col # ~40M float64 COO triples ≈ 1 GB; keep headroom for AtA + HDF cache. if est_nnz > 20_000_000: raise MemoryError( f"SP extract window too large for safe memory use: " f"pix_range=[{i0},{i1}) ≈ {est_nnz:.2e} COO weights " f"({len(cropped)} stamps). Pass a shorter pix_range " f"(e.g. 64–128 pixels) or raise psf_rel_threshold." ) pix_key_to_i: dict[tuple[int, int], int] = {} obs_data: list[float] = [] obs_ivar: list[float] = [] def _pix_index(r: int, c: int) -> int | None: key = (r, c) if key in pix_key_to_i: return pix_key_to_i[key] if r < 0 or c < 0 or r >= ny or c >= nx: return None val = float(rate[r, c]) var = float(rate_var[r, c]) if rate_dq is not None and (int(rate_dq[r, c]) & int(do_not_use)) != 0: return None if not np.isfinite(val) or not np.isfinite(var) or var <= 0.0: return None idx = len(obs_data) pix_key_to_i[key] = idx obs_data.append(val) obs_ivar.append(1.0 / var) return idx buf_r = np.empty(max_per_col, dtype=np.int64) buf_c = np.empty(max_per_col, dtype=np.int64) buf_v = np.empty(max_per_col, dtype=np.float64) cap = min(est_nnz, max(65536, n_win * 4096)) coo_r = np.empty(cap, dtype=np.int32) coo_c = np.empty(cap, dtype=np.int32) coo_v = np.empty(cap, dtype=np.float64) n_coo = 0 def _append_coo(iobs: int, scol: int, w: float) -> None: nonlocal n_coo, coo_r, coo_c, coo_v, cap if n_coo >= cap: cap = int(cap * 1.5) + 1 coo_r = np.resize(coo_r, cap) coo_c = np.resize(coo_c, cap) coo_v = np.resize(coo_v, cap) coo_r[n_coo] = iobs coo_c[n_coo] = scol coo_v[n_coo] = w n_coo += 1 abs_thr = 0.0 for s in range(i0, i1): lam = float(wave[s]) if not np.isfinite(lam): continue xc = float(fx_native(lam)) yc = float(fy_native(lam)) if not (np.isfinite(xc) and np.isfinite(yc)): continue idx = _nearest_psf_index(psfw, lam) item = _crop_stamp(idx) if item is None: continue psf_n, dx_det, dy_det = item n_w = _emit_oriented_deposit_weights( buf_r, buf_c, buf_v, psf_n, xc + dx_det, yc + dy_det, float(oversample), nx_native, abs_thr, ) local_col = s - i0 for k in range(n_w): orow = int(buf_r[k]) ocol = int(buf_c[k]) if abs(orow - s) > half_b: continue iobs = _pix_index(orow, ocol) if iobs is None: continue _append_coo(iobs, local_col, float(buf_v[k])) # Drop cropped stamps before allocating AtA (can be tens–hundreds of MB). del cropped, buf_r, buf_c, buf_v spec = np.full(n_spec, np.nan, dtype=np.float64) spec_err = np.full(n_spec, np.nan, dtype=np.float64) if n_coo == 0: return { "spec": spec, "err": spec_err, "pix_range": (i0, i1), "n_stamps": len(needed), } A = sparse.coo_matrix( (coo_v[:n_coo], (coo_r[:n_coo], coo_c[:n_coo])), shape=(len(obs_data), n_win), ).tocsr() del coo_r, coo_c, coo_v, pix_key_to_i ivar = np.asarray(obs_ivar, dtype=np.float64) d = np.asarray(obs_data, dtype=np.float64) del obs_data, obs_ivar sqrt_ivar = np.sqrt(ivar) A_w = A.multiply(sqrt_ivar[:, None]) AtA = (A_w.T @ A_w).tocsc() Atb = A.T @ (ivar * d) try: solve = factorized(AtA) f = solve(Atb) except Exception: f = sparse.linalg.lsqr(A_w, sqrt_ivar * d, atol=1e-12, btol=1e-12)[0] spec[i0:i1] = f diag = np.asarray(AtA.diagonal(), dtype=np.float64) with np.errstate(divide="ignore", invalid="ignore"): err_win = np.where(diag > 0.0, 1.0 / np.sqrt(diag), np.nan) spec_err[i0:i1] = err_win return { "spec": spec, "err": spec_err, "pix_range": (i0, i1), "n_stamps": int(len(needed)), "n_coo": int(n_coo), }
[docs] def extract_order_spectroperfectionism_from_hdf( rate: np.ndarray, rate_var: np.ndarray, *, spectrograph: str, fiber: int, order: int, wave: np.ndarray | None = None, spectral_bandwidth: int | None = None, pix_range: tuple[int, int] | None = None, psf_rel_threshold: float = 1.0e-2, rate_dq: np.ndarray | None = None, ) -> dict[str, np.ndarray]: """SP extract one echelle order using HDF PSF stamps + deposit-inverse WAVE.""" from ...simulations.e2e_pipeline import ( _load_hdf_model_data, get_sim_hdf_trace_grids, ) spectrograph = str(spectrograph).upper() fiber = int(fiber) order = int(order) _, model_data = _load_hdf_model_data(spectrograph) pixelsize = float(model_data.get("pixelsize", 1.0)) nx = int(model_data.get("nx", rate.shape[0])) block = model_data[f"fiber_{fiber}"][str(order)] ow = np.asarray(block["xy_wavelengths"], dtype=np.float64) ox = np.asarray(block["x"], dtype=np.float64) oy = np.asarray(block["y"], dtype=np.float64) psfw = np.asarray(block["psf_wavelengths"], dtype=np.float64) psf_stamps = block["psf_stamps"] oversample = _psf_oversample(block, pixelsize) fx = PchipInterpolator(ow, ox, extrapolate=True) fy = PchipInterpolator(ow, oy, extrapolate=True) if wave is None: grids = get_sim_hdf_trace_grids(spectrograph) wave = np.asarray(grids[fiber][order]["wavelength"], dtype=np.float64) return extract_spectrum_spectroperfectionism( rate, rate_var, wave, fx_native=fx, fy_native=fy, psfw=psfw, psf_stamps=psf_stamps, oversample=oversample, nx_native=nx, spectral_bandwidth=spectral_bandwidth, pix_range=pix_range, psf_rel_threshold=psf_rel_threshold, rate_dq=rate_dq, )