Source code for hispecdrp.primitives.ramp_fitting.ramp_fit_primitive

from ..base_primitive import BasePrimitive
from ... import datamodels
from ...calibrations import GainSelector,ReadNoiseSelector
from ...datamodels.dqflags import DQ_FLAGS
import logging

from .fit_ramp_numba import fit_ramps_ols, fit_ramps_mcds
from .fit_ramp_jwst import fit_ramps_jwst

import numpy as np

logger = logging.getLogger(__name__)

__all__ = [
    'FitRampPrimitive',
    'sanitize_ramp_fit_rate_err',
    'clip_noise_driven_poisson_variance',
]

_DO_NOT_USE = int(DQ_FLAGS["DO_NOT_USE"])

# Optional detection gate for mode='detect_nsigma' (legacy empty-sky policy).
_POISSON_DETECT_NSIGMA = 3.0

# smooth_detect: gate on a spatially smoothed rate so empty sky stays RN-only
# while faint illuminated structure (orders) keeps pixel Poisson.
_POISSON_SMOOTH_SIGMA = 1.0
_POISSON_SMOOTH_NSIGMA = 1.0

# Default balances empty-sky pulls with faint illuminated emp/mean(err²).
_POISSON_VARIANCE_MODE_DEFAULT = "smooth_detect"
_POISSON_VARIANCE_MODES = frozenset({
    "positive_only",
    "detect_nsigma",
    "smooth_detect",
    "stcal",
})


def _nan_safe_gaussian_filter(rate: np.ndarray, sigma: float) -> np.ndarray:
    """Gaussian smooth that ignores non-finite pixels (normalized by weight)."""
    from scipy.ndimage import gaussian_filter

    sig = float(sigma)
    if sig <= 0.0:
        return np.asarray(rate, dtype=np.float64)

    rate_a = np.asarray(rate, dtype=np.float64)
    finite = np.isfinite(rate_a)
    fill = np.where(finite, rate_a, 0.0)
    weight = finite.astype(np.float64)
    # Support 1-D unit tests and 2-D detector images.
    num = gaussian_filter(fill, sigma=sig, mode="nearest")
    den = gaussian_filter(weight, sigma=sig, mode="nearest")
    out = np.zeros_like(num)
    np.divide(num, den, out=out, where=den > 1e-6)
    return out


[docs] def clip_noise_driven_poisson_variance( rate, rate_var_poisson, rate_var_rnoise, *, mode: str = _POISSON_VARIANCE_MODE_DEFAULT, n_sigma: float = _POISSON_DETECT_NSIGMA, smooth_sigma: float = _POISSON_SMOOTH_SIGMA, smooth_nsigma: float = _POISSON_SMOOTH_NSIGMA, rate_err=None, ): """ Adjust reported Poisson variance after the ramp fit. Ramp fits (including ``stcal`` likely) evaluate photon-noise variance at a non-negative function of the fitted rate (Wald plug-in). Modes: - ``'smooth_detect'`` (default): keep the fit Poisson term where ``rate > 0`` **and** a Gaussian-smoothed rate exceeds ``smooth_nsigma * sqrt(rate_var_rnoise)``. Empty sky stays RN-limited (isolated positive noise spikes are dropped); faint illuminated structure keeps Poisson because neighbors hold the smooth rate up. - ``'positive_only'``: zero Poisson only where ``rate <= 0``. - ``'detect_nsigma'``: also zero Poisson unless ``rate > n_sigma * sqrt(rate_var_rnoise)`` (legacy; underestimates faint-source ``rate_err``). - ``'stcal'``: leave ``rate_var_poisson`` unchanged; only rebuild ``rate_err = sqrt(var_poisson + var_rnoise)``. If ``rate_err`` is provided, it is preserved wherever ``var_poisson`` / ``var_rnoise`` cannot form a finite rebuilt uncertainty (e.g. legacy fitters that omit the variance split). Returns ------- rate_var_poisson_out, rate_err_out, n_clipped """ rate_a = np.asarray(rate, dtype=np.float64) vp = np.asarray(rate_var_poisson, dtype=np.float64).copy() vr = np.asarray(rate_var_rnoise, dtype=np.float64) if rate_a.shape != vp.shape or rate_a.shape != vr.shape: raise ValueError( f"shape mismatch: rate={rate_a.shape} var_poisson={vp.shape} " f"var_rnoise={vr.shape}" ) mode_key = str(mode).strip().lower() if mode_key not in _POISSON_VARIANCE_MODES: raise ValueError( f"poisson variance mode must be one of {sorted(_POISSON_VARIANCE_MODES)}, " f"got {mode!r}" ) # Only gate / rebuild where variance planes are usable. Fitters that leave # NaN var_* (historical cython_likely) must not wipe a finite rate_err. var_ok = np.isfinite(vp) & np.isfinite(vr) & (vr >= 0.0) & (vp >= 0.0) n_clipped = 0 if mode_key != "stcal": nondetect = ~np.isfinite(rate_a) | (rate_a <= 0.0) if mode_key == "detect_nsigma": sig_rn = np.sqrt(np.maximum(vr, 0.0)) thresh = float(n_sigma) * sig_rn nondetect = nondetect | ~np.isfinite(sig_rn) | (rate_a < thresh) elif mode_key == "smooth_detect": sig_rn = np.sqrt(np.maximum(vr, 0.0)) # Smooth only where rates are finite; invalid vr already in ~var_ok. rate_s = _nan_safe_gaussian_filter(rate_a, smooth_sigma) thresh = float(smooth_nsigma) * sig_rn nondetect = ( nondetect | ~np.isfinite(sig_rn) | ~np.isfinite(rate_s) | (rate_s < thresh) ) clip_mask = nondetect & var_ok & (vp > 0.0) n_clipped = int(np.count_nonzero(clip_mask)) vp = vp.copy() vp[clip_mask] = 0.0 rebuilt = np.sqrt(np.maximum(vp, 0.0) + np.maximum(vr, 0.0)) rebuild_ok = var_ok & np.isfinite(rebuilt) & (rebuilt >= 0.0) if rate_err is not None: rate_err_out = np.asarray(rate_err, dtype=np.float64).copy() if rate_err_out.shape != rate_a.shape: raise ValueError( f"shape mismatch: rate={rate_a.shape} rate_err={rate_err_out.shape}" ) rate_err_out[rebuild_ok] = rebuilt[rebuild_ok] else: rate_err_out = rebuilt rate_err_out = np.where(rebuild_ok, rate_err_out, np.nan) bad = ~(np.isfinite(rate_err_out) & (rate_err_out >= 0.0)) rate_err_out = rate_err_out.astype( np.result_type(rate_var_poisson, np.float32), copy=False ) vp_out = vp.astype(np.asarray(rate_var_poisson).dtype, copy=False) if np.any(bad): rate_err_out = np.array(rate_err_out, copy=True) rate_err_out[bad] = np.nan return vp_out, rate_err_out, n_clipped
[docs] def sanitize_ramp_fit_rate_err( rate, rate_err, dq, *, rate_var_rnoise=None, do_not_use: int = _DO_NOT_USE, ): """ Make ramp-fit ``rate_err`` safe for downstream extraction. Ramp fits often leave empty / inter-order pixels as ``rate==0`` with NaN ``rate_err``. Flagging those as ``DO_NOT_USE`` and letting box extract renormalize the aperture caused large spectral spikes for undersampled PSFs. Instead: - Where ``rate`` is finite and ``rate_err`` is not: fill ``rate_err`` with ``sqrt(rate_var_rnoise)`` when that variance is finite and positive, otherwise ``0``. Do **not** mark ``DO_NOT_USE``. - Where ``rate`` is non-finite: OR ``do_not_use`` into ``dq`` and set ``rate_err`` to ``0`` for array hygiene. Returns ------- rate_err_out, dq_out, n_filled_err, n_flagged_rate ``rate_err`` / ``dq`` are copied only when modified. """ rate_a = np.asarray(rate) rate_err_a = np.asarray(rate_err) dq_a = np.asarray(dq) rate_finite = np.isfinite(rate_a) err_bad = ~np.isfinite(rate_err_a) fill = rate_finite & err_bad flag = ~rate_finite n_filled = int(np.count_nonzero(fill)) n_flagged = int(np.count_nonzero(flag)) if n_filled == 0 and n_flagged == 0: return rate_err_a, dq_a, 0, 0 rate_err_out = rate_err_a.copy() dq_out = dq_a if n_filled: fill_val = np.zeros(n_filled, dtype=rate_err_out.dtype) if rate_var_rnoise is not None: vr = np.asarray(rate_var_rnoise) if vr.shape == rate_a.shape: vr_fill = vr[fill] ok = np.isfinite(vr_fill) & (vr_fill > 0) fill_val = fill_val.astype(np.result_type(fill_val, np.float64), copy=False) fill_val[ok] = np.sqrt(vr_fill[ok]).astype(fill_val.dtype, copy=False) rate_err_out[fill] = fill_val if n_flagged: dq_out = dq_a.copy() dq_out[flag] = dq_out[flag] | int(do_not_use) rate_err_out[flag] = np.asarray(0, dtype=rate_err_out.dtype) return rate_err_out, dq_out, n_filled, n_flagged
def flag_nonfinite_rate_pixels(rate, rate_err, dq, *, do_not_use: int = _DO_NOT_USE): """ Backward-compatible wrapper around :func:`sanitize_ramp_fit_rate_err`. Historically this ORed ``DO_NOT_USE`` for non-finite ``rate`` **or** ``rate_err``. That mistreated empty pixels (``rate==0``, NaN err) and caused extraction spikes. The sanitizer now fills finite ``rate_err`` for those pixels and only flags non-finite ``rate``. Returns ``(dq, n_flagged_rate)`` for compatibility with existing callers. """ _rate_err, dq_out, _n_filled, n_flagged = sanitize_ramp_fit_rate_err( rate, rate_err, dq, do_not_use=do_not_use ) return dq_out, n_flagged
[docs] class FitRampPrimitive(BasePrimitive): """ This step fits the slope (DN/s) from the UTR for each pixel. While fitting the ramps, this step will also calculate the wavelength dependent mid-exposure times, which is then used to calculate the wavelength dependent barycentric correction. HISPEC leverages the UTR instead of having an exposure meter for this purpose. The entire ramp does not need to be discarded for a single jump, which is a significant advantage of UTR. Each ramp might be divided into separate chunks after the jump detection step. The final slope will be the combined slope estimate from every valid chunk in that pixel. Descriptions of the different ramp fitting algorithms: - **Ordinary Least Squares ('ols'):** This method fits a straight line to the ramp data using the ordinary least squares approach. This method is simple and computationally efficient but is not optimal with significant photon noise. - **Correlated Double Sampling ('cds'):** This method uses the difference between last read and the first read to estimate the slope. It is very fast but not optimal for readnoise dominated ramps. - **Multiple Correlated Double Sampling ('mcds'):** This method generalizes CDS by using multiple reads (see argument mcds_num_coadd) at the beginning and end of the ramp to reduce the readnoise. - **JWST Fixsen ('jwst_fixsen'):** This method assumes that the covariance is piece-wise constant by dividing 9 different regimes of various ratios of readnoise and photon noise. This is referenced as OLS in the JWST pipeline. Fixsen+2000 https://ui.adsabs.harvard.edu/abs/2000PASP..112.1350F/abstract - **JWST Likely ('jwst_likely'):** Uses the maximum-likelihood from Brandt (2024) to fit the ramps, which is the optimal approach (but assuming constant illumination). Reference: Brandt, T. D., PASP, 136, 045005 (2024) https://ui.adsabs.harvard.edu/abs/2024PASP..136d5005B/abstract - **Cython Likely ('cython_likely'):** Same as JWST Likely but implemented in Cython for better performance (author: Brandt, Tim). todo: As a future research avenue, although not a DRP requirement, we plan to explore using the UTR and the ATC images to optimize the slope fitting to account for the variable slopes due to cloud or seeing variability. For example, a simple strategy could be that reads acquired during spikes in seeing (as monitored in the ATC data) could simply be masked in the same manner as cosmic rays. This could be particularly beneficial for high contrast spectroscopy of companions because spikes in seeing means that there is more speckles and less planet signal getting through the fiber. Parameters ---------- dataset : str or RampModel Input file path or data model to process. This must be a raw up-the-ramp (UTR) ramp data product in which ramps are to be fitted. suffix : str or None, optional Suffix to append to the output filename. Default is ``'rampfit'``. method : {'ols', 'cds', 'mcds', 'jwst_likely', 'jwst_fixsen', 'cython_likely'}, optional Ramp fitting algorithm to use. See descriptions above for details. Default is ``'cython_likely'``. mcds_num_coadd : int or None, optional Number of reads to coadd for MCDS ramp fitting. If None, it will be set to n_reads/4. Only used if method is set to 'mcds'. Default is None. max_cores : int, optional Maximum number of CPU cores to use for parallel processing. Default is 1 (no parallel execution). jwst_jump_detection : bool, optional If True, use shared memory optimization for jump detection in combination with JWST ramp fitting methods. Default is False. jwst_jump_rejection_threshold : float, optional Rejection threshold (approximately in signal-to-noise) above which discontinuities are flagged as jumps when using JWST ramp fitting methods. Default is 4.0. poisson_variance_mode : {'smooth_detect', 'positive_only', 'detect_nsigma', 'stcal'}, optional How to adjust reported ``rate_var_poisson`` / ``rate_err`` after the fit. ``'smooth_detect'`` (default) gates Poisson on a smoothed rate so empty sky stays RN-limited while faint illuminated structure keeps Poisson; ``'positive_only'`` clears Poisson only for ``rate <= 0``; ``'detect_nsigma'`` uses a per-pixel ``poisson_detect_nsigma`` gate (legacy); ``'stcal'`` leaves the fit Poisson term unchanged. poisson_detect_nsigma : float, optional Per-pixel detection threshold for ``poisson_variance_mode='detect_nsigma'``. Default is 3.0. poisson_smooth_sigma : float, optional Gaussian smoothing scale (pixels) for ``smooth_detect``. Default is 1.0. poisson_smooth_nsigma : float, optional Smoothed-rate detection threshold in RN σ for ``smooth_detect``. Default is 1.0. Calibrations ------------ gain : GainModel - selector: :class:`~hispecdrp.calibrations.selectors.GainSelector` rn : ReadNoiseModel - selector: :class:`~hispecdrp.calibrations.selectors.ReadNoiseSelector` Returns ------- output : Level0Model Level 0 data model with fitted slope (rate) and associated arrays. Notes ----- Roman calibration pipeline reference for ramp fitting algorithms: https://github.com/spacetelescope/romancal Ramp fitting roman reference: https://www.stsci.edu/files/live/sites/www/files/home/roman/_documents/Roman-STScI-000394_DeterminingTheBestFittingSlope.pdf See JWST primitive: https://jwst-pipeline.readthedocs.io/en/latest/jwst/ramp_fitting/description.html CHARIS reference: https://ui.adsabs.harvard.edu/abs/2017JATIS...3d8002B/abstract HPF DRP: https://github.com/indiajoe/HxRGproc """ PARAMETERS = { 'suffix' : 'rampfit', 'method' : 'cython_likely', 'mcds_num_coadd' : None, 'output_model' : 'L0Model', 'jwst_jump_detection': False, 'jwst_jump_rejection_threshold': 4.0, 'poisson_variance_mode': _POISSON_VARIANCE_MODE_DEFAULT, 'poisson_detect_nsigma': _POISSON_DETECT_NSIGMA, 'poisson_smooth_sigma': _POISSON_SMOOTH_SIGMA, 'poisson_smooth_nsigma': _POISSON_SMOOTH_NSIGMA, } CALIBRATIONS = { 'gain': { 'selector': GainSelector, 'selector_kwargs': {} }, # 'dark': { # would need to be defined if we implement dark subtraction before ramp fitting. # 'selector': DarkSelector, # 'selector_kwargs': {} # }, 'rn': { 'selector': ReadNoiseSelector, 'selector_kwargs': {} } } def _perform(self): if self.jwst_jump_detection and self.method.lower() not in ('jwst_likely', 'jwst_fixsen'): raise ValueError("JWST jump detection is only for shared memory optimization in combination with JWST ramp fitting methods. " "Please set jwst_jump_detection to False or change ramp fitting algorithm.") input_model = datamodels.open(self.dataset, copy=False) logger.info(f"Fitting ramp to {input_model} with method={self.method.lower()}") gain_cal, _ = self.get_calibration(input_model, 'gain') gain_model = datamodels.open(gain_cal) # Dark would need to be defined if we implement dark subtraction before ramp fitting. # dark_cal, _ = self.get_calibration(input_model, 'dark') # dark_model = datamodels.open(dark_cal) rn_cal, _ = self.get_calibration(input_model, 'rn') rn_model = datamodels.open(rn_cal) _mcds_num_coadd = self.mcds_num_coadd delta_readtimes = input_model.get_delta_readtimes() first_read_mjd = input_model.get_first_read_mjd() max_cores = self.max_cores or 1 if self.method.lower() == 'ols': result = fit_ramps_ols( delta_readtimes, input_model.data, dq_raw=input_model.dq_raw, dq=input_model.dq, rdnoise=rn_model.rn, gain=gain_model.gain, start_times=first_read_mjd, dark_current=None, # Not used because we assume dark subtraction is done in rate maps. max_cores=max_cores ) elif self.method.lower() in ('cds'): result = fit_ramps_mcds( delta_readtimes, input_model.data, dq_raw=input_model.dq_raw, dq=input_model.dq, rdnoise=rn_model.rn, gain=gain_model.gain, start_times=first_read_mjd, dark_current=None, # Not used because we assume dark subtraction is done in rate maps. num_coadd=1, max_cores=max_cores ) elif self.method.lower() in ('mcds'): if _mcds_num_coadd is None: n_read = input_model.data.shape[0] _mcds_num_coadd = np.max([n_read // 4, 1]) # Ensure at least 1 read is used. result = fit_ramps_mcds( delta_readtimes, input_model.data, dq_raw=input_model.dq_raw, dq=input_model.dq, rdnoise=rn_model.rn, gain=gain_model.gain, start_times=first_read_mjd, dark_current=None, # Not used because we assume dark subtraction is done in rate maps. num_coadd=_mcds_num_coadd, max_cores=max_cores ) elif self.method.lower() in ('jwst_likely', 'jwst_fixsen', 'cython_likely'): result = fit_ramps_jwst( delta_readtimes, input_model.data, dq_raw=input_model.dq_raw, dq=input_model.dq, rdnoise=rn_model.rn, gain=gain_model.gain, start_times=first_read_mjd, dark_current=None, # Not used because we assume dark subtraction is done in rate maps. max_cores=max_cores, algorithm=self.method.lower(), jump_detection = self.jwst_jump_detection, jump_rejection_threshold = self.jwst_jump_rejection_threshold ) else: logger.error(f"Invalid ramp fit method {self.method.lower()}") raise ValueError(f"Invalid ramp fit method {self.method.lower()}") output_model = self.get_output_model(input_model) output_model.rate = result['rate'] output_model.rate_err = result['rate_err'] output_model.rate_var_poisson = result['rate_var_poisson'] output_model.rate_var_rnoise = result['rate_var_rnoise'] output_model.dq = result['dq'] output_model.mid_time = result['mid_time'] # Adjust reported Poisson variance (see clip_noise_driven_poisson_variance). # Default 'smooth_detect' keeps empty-sky RN pulls while preserving # faint illuminated Poisson; see poisson_variance_mode for alternatives. ( output_model.rate_var_poisson, output_model.rate_err, n_pois_clipped, ) = clip_noise_driven_poisson_variance( output_model.rate, output_model.rate_var_poisson, output_model.rate_var_rnoise, mode=self.poisson_variance_mode, n_sigma=self.poisson_detect_nsigma, smooth_sigma=self.poisson_smooth_sigma, smooth_nsigma=self.poisson_smooth_nsigma, rate_err=output_model.rate_err, ) if n_pois_clipped: logger.info( "Ramp fit: cleared Poisson variance on %d pixel(s) " "(mode=%s) for reported rate_err.", n_pois_clipped, self.poisson_variance_mode, ) # Empty / unfitted pixels often return rate=0 with NaN rate_err. Fill a # finite rate_err (0 or RN floor) so box extract does not drop aperture # edge pixels and renormalize; only flag DO_NOT_USE when rate itself is # non-finite. ( output_model.rate_err, output_model.dq, n_filled, n_flagged, ) = sanitize_ramp_fit_rate_err( output_model.rate, output_model.rate_err, output_model.dq, rate_var_rnoise=output_model.rate_var_rnoise, ) if n_filled or n_flagged: logger.info( "Ramp fit: filled non-finite rate_err on %d finite-rate pixel(s); " "flagged %d non-finite-rate pixel(s) as DO_NOT_USE.", n_filled, n_flagged, ) return dict(output=output_model, method=self.method, mcds_num_coadd=_mcds_num_coadd)