Source code for hispecdrp.primitives.bias_subtraction.make_bias_primitive

from ..base_primitive import BasePrimitive
from ...datamodels.dqflags import DQ_FLAGS
import numpy as np
from copy import copy
from typing import Sequence
from ... import datamodels
from ..jump_detection.jump_detection_difference import median_axis0_3d
import logging

from numba import njit, prange
from ...utils.parallelization_utils import numba_thread_scope

logger = logging.getLogger(__name__)

__all__ = ['MakeBiasPrimitive']

UNRELIABLE_BIAS_FLAG =  DQ_FLAGS['UNRELIABLE_BIAS']
DO_NOT_USE_FLAG = DQ_FLAGS["DO_NOT_USE"]

[docs] class MakeBiasPrimitive(BasePrimitive): """ Derive the detector bias and kTC noise from a set of dark exposures. 1) Median combine the first read to create the bias frame. 2) Compute the kTC noise as the standard deviation across the first reads, using sigma-clipping to reject outliers. 3) Flag pixels with high bias level or high kTC noise in the DQ array. Parameters ---------- input : list of str or list of RampModel List of input file paths or ramp models to process. max_cores : int, optional Maximum number of CPU cores to use for parallel processing. Default is 1. kTC_threshold : float, optional Threshold kTC above which pixels are flagged as bad in the DQ array. Default is 100 DN. bias_threshold : float, optional Threshold bias above which pixels are flagged as bad in the DQ array. Default is 1000 DN. Returns ------- output : BiasModel The derived bias calibration model containing the bias frame, kTC noise map, and updated DQ array. """ PARAMETERS = { 'output_model' : 'BiasModel', 'kTC_threshold' : 100, # DN, Threshold kTC above which pixels are bad 'bias_threshold' : 1000, # DN, Threshold bias above which pixels are bad } def _perform(self): if isinstance(self.dataset, (datamodels.DataModel, str)): _input_list_tmp = [self.dataset] if isinstance(self.dataset, Sequence): _input_list_tmp = self.dataset nfiles = len(_input_list_tmp) logger.info(f"Deriving the detector bias and kTC noise from a list of {nfiles} inputs.") input_models = [datamodels.open(f) for f in _input_list_tmp] # Define output model from inputs output_bias = self.get_output_model(input_models) bias_arr = [] for _input in input_models: bias_arr.append(_input.data[0,:,:]) bias_arr = np.array(bias_arr) max_cores = self.max_cores or 1 with numba_thread_scope(max_cores): output_bias.bias = median_axis0_3d(bias_arr) output_bias.ktc = std_axis0_3d_sigclip(bias_arr, nsig=5, min_keep=10) output_bias.dq = copy(input_models[0].dq) _find_bad_pixels_bias( output_bias.bias, output_bias.ktc, output_bias.dq, self.kTC_threshold, self.bias_threshold ) median_bias_level = np.nanmedian(output_bias.bias) output_bias.meta.median_bias_level = median_bias_level median_kTC_level = np.nanmedian(output_bias.ktc) output_bias.meta.median_kTC_level = median_kTC_level return dict(output=output_bias)
@njit(nogil=True, parallel=True, cache=True) def _find_bad_pixels_bias(bias: np.ndarray, kTC: np.ndarray, dq: np.ndarray, kTC_threshold: float = 100, bias_threshold: float = 1000): ny, nx = bias.shape for j in prange(ny): for i in range(nx): if kTC[j, i] > kTC_threshold: dq[j, i] |= UNRELIABLE_BIAS_FLAG if bias[j, i] > bias_threshold: dq[j, i] |= UNRELIABLE_BIAS_FLAG @njit(nogil=True, parallel=True, cache=True) def std_axis0_3d_sigclip(bias_arr, nsig=5.0, min_keep=10): """ Sigma-clipped std over axis=0 for bias_arr with shape (n_files, ny, nx) Parameters ---------- bias_arr : float array (n_files, ny, nx) nsig : sigma threshold min_keep : minimum number of samples required after clipping Returns ------- std : (ny, nx) array """ n_files, ny, nx = bias_arr.shape std = np.zeros((ny, nx), dtype=bias_arr.dtype) for y in prange(ny): for x in range(nx): # --------- # Pass 1: initial mean/std (Welford) mean = 0.0 M2 = 0.0 count = 0 for f in range(n_files): v = bias_arr[f, y, x] count += 1 delta = v - mean mean += delta / count M2 += delta * (v - mean) if count < 2: std[y, x] = 0.0 continue sigma = np.sqrt(M2 / count) # --------- # Pass 2: sigma clipping + recompute stats mean2 = 0.0 M2_2 = 0.0 count2 = 0 thresh = nsig * sigma for f in range(n_files): v = bias_arr[f, y, x] if abs(v - mean) <= thresh: count2 += 1 delta = v - mean2 mean2 += delta / count2 M2_2 += delta * (v - mean2) if count2 < min_keep: # fallback: unclipped std std[y, x] = sigma else: std[y, x] = np.sqrt(M2_2 / count2) return std