Source code for hispecdrp.primitives.coadd.coadd_spectra_primitive

from __future__ import annotations
from ..base_primitive import BasePrimitive
from ...datamodels import DataModel
from ... import datamodels
import numpy as np
from numba import njit, prange
from ..utils.array_utils import bitwise_or_reduce_uint32
from .coadd_frames_primitive import sigma_clip
import logging

logger = logging.getLogger(__name__)

__all__ = ['CoaddSpectraPrimitive', 'coadd_spectra']

from ...datamodels.dqflags import DQ_FLAGS
DO_NOT_USE = DQ_FLAGS['DO_NOT_USE']


[docs] class CoaddSpectraPrimitive(BasePrimitive): """ Coadd multiple spectral-like models (e.g. Level 2 spectra) of the same kind to increase signal-to-noise. An ancestor FITS extension is created to include the list of filenames that were combined. This primitive combines input data products using a weighted average (if mean_calc = 'wmean'), where the weights are given by the inverse variances (1 / model.err^2). The output error and variance arrays are computed using the corresponding weighted standard error. Data quality (DQ) flags from all inputs are propagated to the output using a bitwise OR. Parameters ---------- input : list[str or DataModel] Input files or data models to coadd. All inputs are combined into a single output model (each fiber/order trace is coadded independently). suffix : str or None, optional Suffix to append to the output filename. Default is ``'coadded'``. mean_calc : str, optional Method for calculating the output data values. Currently, only ``'wmean'`` (weighted mean) is implemented. err_calc : str, optional Method for calculating the output error and variance. Currently, only ``'wse'`` (weighted standard error) is implemented. do_fibers : list of int, optional Fibers to coadd. If None, all fibers on the input model are used. do_orders : list of int, optional Orders to coadd. If None, all orders on the input model are used. Returns ------- output : DataModel Coadded data model with propagated errors and data quality flags. """ PARAMETERS = { 'suffix' : 'coadded', 'output_model' : None, 'do_sigma_clip' : False, 'sigma_thresh_low' : 5.0, 'sigma_thresh_high' : 5.0, 'min_keep' : 3, 'mean_calc' : 'wmean', 'err_calc' : 'wse', 'do_fibers' : None, 'do_orders' : None, } def _perform(self): input_type = type(self.dataset) # Make sure input is a list of models if input_type in (str, datamodels.DataModel): logger.warning("Input is a single model or path, returning without coadding.") self.run = False return dict(output=self.dataset) # Determine which fibers/orders to coadd input_models = [datamodels.open(f) for f in self.dataset] do_fibers, do_orders = self.get_do_traces(input_models[0], self.do_fibers, self.do_orders) logger.info(f"Coadding {len(self.dataset)} spectra:\n" + "\n".join([str(m) for m in input_models])) # Coadd output_model = coadd_spectra( input_models, do_fibers=do_fibers, do_orders=do_orders, do_sigma_clip=self.do_sigma_clip, sigma_thresh_low=self.sigma_thresh_low, sigma_thresh_high=self.sigma_thresh_high, min_keep=self.min_keep, mean_calc=self.mean_calc, err_calc=self.err_calc, primitive=self, ) return dict(output=output_model)
[docs] def coadd_spectra( input : list[str | 'DataModel'], do_fibers : list[str | int], do_orders : list[int], do_sigma_clip : bool = False, sigma_thresh_low : float | None = None, sigma_thresh_high : float | None = None, min_keep : int | float = 3, mean_calc : str = 'wmean', err_calc : str = 'wse', primitive : BasePrimitive | None = None, ): # If only one input is provided, return it as is. if len(input) == 1: only = input[0] return datamodels.open(only) if isinstance(only, str) else only # Open input models input_models = [datamodels.open(f) for f in input] # Init output model model_class_out = input_models[0].__class__ model_out = model_class_out.from_datamodels(input_models, primitive=primitive) orders = model_out.make_order_vec().tolist() # Loop over fibers/orders, coadding each spectral trace separately for fiber in do_fibers: attrs = _get_spec_data_attrs(input_models[0], fiber=fiber) for order in do_orders: # Make the batch for this fiber/order spec_batch = _make_spec_batch(input_models, attrs, order=order) # Do coadding result = _coadd_spectra( spec_batch, do_sigma_clip=do_sigma_clip, sigma_thresh_low=sigma_thresh_low, sigma_thresh_high=sigma_thresh_high, min_keep=min_keep, mean_calc=mean_calc, err_calc=err_calc, ) # Store results in the output model idx_order = orders.index(order) for attr_common, attr_unique in attrs.items(): if not model_out.has_extension_value(attr_unique): setattr( model_out, attr_unique, np.zeros(input_models[0].shape, dtype=result[attr_common].dtype) ) getattr(model_out, attr_unique)[idx_order] = result[attr_common] return model_out
def _make_spec_batch( input_models : list[str | DataModel], attrs : dict[str, str], order : int ) -> dict[str, np.ndarray]: """Build (n_frames, n_pix) arrays for one order from a list of spectral models.""" out = {} n_frames = len(input_models) for i in range(n_frames): model = datamodels.open(input_models[i]) n_pix = model.shape[1] orders = model.make_order_vec().tolist() idx_order = orders.index(order) for attr_common, attr_unique in attrs.items(): arr = getattr(model, attr_unique) if i == 0: out[attr_common] = np.zeros((n_frames, n_pix), dtype=arr.dtype) out[attr_common][i, :] = arr[idx_order] return out def _get_spec_data_attrs(input_model : datamodels.DataModel, fiber : str | int) -> dict[str, str]: attrs = {} # Data and error attributes; fall back to the uncalibrated spectrum if the # calibrated one hasn't been populated yet. if input_model.has_extension_value(f'spec_fib{fiber}'): attrs['data'] = f'spec_fib{fiber}' attrs['err'] = f'spec_err_fib{fiber}' attrs['var_poisson'] = f'spec_var_poisson_fib{fiber}' attrs['var_rnoise'] = f'spec_var_rnoise_fib{fiber}' else: attrs['data'] = f'spec_uncal_fib{fiber}' attrs['err'] = f'spec_uncal_err_fib{fiber}' attrs['var_poisson'] = f'spec_uncal_var_poisson_fib{fiber}' attrs['var_rnoise'] = f'spec_uncal_var_rnoise_fib{fiber}' # DQ attrs['dq'] = f'spec_dq_fib{fiber}' # Mid-time attrs['mid_time'] = f'mid_time_fib{fiber}' return attrs def _coadd_spectra( input_frames : dict[str, np.ndarray], do_sigma_clip : bool = False, sigma_thresh_low : float | None = None, sigma_thresh_high : float | None = None, min_keep : int | float = 3, mean_calc : str = 'wmean', err_calc : str = 'wse', ) -> dict: """ Coadd a (n_frames, n_pix) stack of 1D spectra using a specified method. Parameters ---------- input_frames : dict[str, np.ndarray] Dictionary containing the data array and associated error, variance, DQ, and mid-time arrays. do_sigma_clip : bool, optional Whether to perform sigma clipping to reject outliers before coadding. Default is False. sigma_thresh_low : float | None, optional Lower threshold for sigma clipping. Default is None (no sigma clipping). sigma_thresh_high : float | None, optional Upper threshold for sigma clipping. Default is None (no sigma clipping). min_keep : int or float, optional Minimum number of good frames to keep after sigma clipping. If a float between 0 and 1, it is interpreted as a fraction of the total number of frames. If an integer greater than or equal to 1, it is interpreted as the absolute number of frames. Default is 3 (int). mean_calc : str, optional Method for calculating the mean. Currently only 'wmean' (weighted mean) is implemented. Default is 'wmean'. err_calc : str, optional Method for calculating the error. Currently only 'wse' (weighted standard error) is implemented. Default is 'wse'. """ # Data data_2d = input_frames['data'] dq_2d = input_frames['dq'] err_2d = input_frames.get('err') # Optional err input, but always output # Shape n_frames, n_pix = data_2d.shape # Mask mask = np.isfinite(data_2d) if 'dq' in input_frames: mask &= dq_2d == 0 # Weights from error if 'err' in input_frames: weights = 1 / err_2d**2 else: weights = np.ones_like(data_2d, dtype=np.float32) # Set weights to 0 where mask is False weights[~mask] = 0 # Sigma clipping if requested if do_sigma_clip and n_frames > 3: if min_keep < 1: min_keep = int(np.ceil(min_keep * n_frames)) min_keep = int(min_keep) sigma_clip_spectra( data_2d, mask, sigma_thresh_low=sigma_thresh_low, sigma_thresh_high=sigma_thresh_high, min_keep=min_keep, ) bad = ~mask weights[bad] = 0 dq_2d[bad] |= DO_NOT_USE # Create output arrays data_out = np.full(n_pix, np.nan, dtype=data_2d.dtype) err_out = np.full(n_pix, np.nan, dtype=data_2d.dtype) dq_out = np.zeros(n_pix, dtype=dq_2d.dtype) var_poisson_2d = input_frames.get('var_poisson') var_rnoise_2d = input_frames.get('var_rnoise') if var_poisson_2d is not None: var_poisson_out = np.full(n_pix, np.nan, dtype=data_2d.dtype) else: var_poisson_out = None if var_rnoise_2d is not None: var_rnoise_out = np.full(n_pix, np.nan, dtype=data_2d.dtype) else: var_rnoise_out = None mid_time_2d = input_frames.get('mid_time') if mid_time_2d is not None: mid_times_out = np.full(n_pix, np.nan, dtype=mid_time_2d.dtype) else: mid_times_out = None _coadd_spectra_numba( spec_2d=data_2d, err_2d=err_2d, var_poisson_2d=var_poisson_2d, var_rnoise_2d=var_rnoise_2d, dq_2d=dq_2d, mid_time_2d=mid_time_2d, spec_out=data_out, err_out=err_out, var_poisson_out=var_poisson_out, var_rnoise_out=var_rnoise_out, dq_out=dq_out, mid_times_out=mid_times_out, ) return dict( data=data_out, err=err_out, var_poisson=var_poisson_out, var_rnoise=var_rnoise_out, dq=dq_out, mid_time=mid_times_out ) @njit(nogil=True, parallel=False, cache=True) def _coadd_spectra_numba( spec_2d : np.ndarray, spec_out : np.ndarray, dq_2d : np.ndarray, err_2d : np.ndarray | None = None, var_poisson_2d : np.ndarray | None = None, var_rnoise_2d : np.ndarray | None = None, mid_time_2d : np.ndarray | None = None, err_out : np.ndarray | None = None, var_poisson_out : np.ndarray | None = None, var_rnoise_out : np.ndarray | None = None, dq_out : np.ndarray | None = None, mid_times_out : np.ndarray | None = None, ): n_frames, n_pix = spec_2d.shape for i in range(n_pix): # Good indices where dq == 0 good = np.where(dq_2d[:, i] == 0)[0] n_good = len(good) # Case 1: No good pixels, nans if n_good == 0: spec_out[i] = np.nan if err_out is not None: err_out[i] = np.nan if var_poisson_out is not None: var_poisson_out[i] = np.nan if var_rnoise_out is not None: var_rnoise_out[i] = np.nan if dq_out is not None: dq_out[i] = DO_NOT_USE if mid_times_out is not None: mid_times_out[i] = np.nan # Case 2: One good pixel, just take that value elif n_good == 1: k = good[0] spec_out[i] = spec_2d[k, i] if err_out is not None: err_out[i] = err_2d[k, i] if var_poisson_out is not None: var_poisson_out[i] = var_poisson_2d[k, i] if var_rnoise_out is not None: var_rnoise_out[i] = var_rnoise_2d[k, i] if dq_out is not None: dq_out[i] = dq_2d[k, i] if mid_times_out is not None: mid_times_out[i] = mid_time_2d[k, i] # Case 3: Multiple good pixels, weighted mean else: if err_2d is None: w = np.ones(n_good, dtype=np.float32) else: w = 1 / err_2d[good, i]**2 spec_out[i] = np.sum(w * spec_2d[good, i]) / np.sum(w) if err_out is not None: err_out[i] = 1 / np.sqrt(np.sum(w)) if var_poisson_out is not None: var_poisson_out[i] = 1 / np.sum(1 / var_poisson_2d[good, i]) if var_rnoise_out is not None: var_rnoise_out[i] = 1 / np.sum(1 / var_rnoise_2d[good, i]) if dq_out is not None: dq_out[i] = bitwise_or_reduce_uint32(dq_2d[good, i]) if mid_times_out is not None: mid_times_out[i] = np.mean(mid_time_2d[good, i]) @njit(nogil=True, parallel=True, cache=True) def sigma_clip_spectra( data_2d : np.ndarray, mask_2d : np.ndarray, sigma_thresh_low : float | None = None, sigma_thresh_high : float | None = None, min_keep : int = 3, ): _, n_pix = data_2d.shape for j in prange(n_pix): sigma_clip( data_2d[:, j], mask_2d[:, j], sigma_thresh_low=sigma_thresh_low, sigma_thresh_high=sigma_thresh_high, min_keep=min_keep, )