from . import fit_gain_Brandt
from ..ramp_fitting import fitramp_Brandt
# from ..coadd.coadd_frames_primitive import CoaddFramesPrimitive
from ..base_primitive import BasePrimitive
from ... import datamodels
from ...calibrations import GainSelector,ReadNoiseSelector
import numpy as np
import matplotlib.pyplot as plt
from typing import Sequence
from ..utils.stats import get_err_from_posterior
import logging
logger = logging.getLogger(__name__)
from ...datamodels.dqflags import DQ_FLAGS
import multiprocessing as mp
import ctypes
from ...utils.parallelization_utils import _arraytonumpy, numba_thread_scope
from multiprocessing.sharedctypes import RawArray
from progressbar import progressbar
import numbers
__all__ = ['MakeGainPrimitive', 'fit_gain_and_readnoise','plot_2d_posterior']
SATURATED_FLAG_uint8 = np.uint8(DQ_FLAGS["SATURATED"])
DO_NOT_USE_FLAG_uint8 = np.uint8(DQ_FLAGS["DO_NOT_USE"])
JUMP_DET_FLAG_uint8 = np.uint8(DQ_FLAGS["JUMP_DET"])
[docs]
class MakeGainPrimitive(BasePrimitive):
"""
Estimate the detector electronic gain (and optionally read noise) from up-the-ramp data.
This primitive fits detector gain and, optionally, read noise parameters
from raw up-the-ramp (UTR) ramps using the maximum-likelihood framework
described in Brandt (2025; PASP, 137, 125006). Three fitting strategies are
supported: a grid-based maximum-likelihood search, a local quadratic
approximation, or a hybrid approach that uses the grid search to locate
the optimum and then refines it with a quadratic approximation.
Initial gain and read-noise values may be provided explicitly (e.g., as
arrays defining a search grid). If not provided, the primitive is
expected to obtain appropriate defaults from the calibration database.
Optional arguments allow fixing the read noise while fitting only the
gain, and treating all ramps as having identical count rates.
The fitting may be performed on a per-pixel basis or using a single
model for the full detector. Although the full detector mode is only for
testing.
Parameters
----------
input : list of str or list of RampModel
List of input file paths or ramp models to process.
max_cores : int, optional
Maximum number of CPU cores to use for parallel processing. Default
is 1.
init_gain : None or array-like, optional
Initial gain value(s) used to define the maximum-likelihood grid
search. If None, obtain an initial value from the calibration
database. If provided, this may be a scalar or an array defining the
gain sampling (e.g., ``np.linspace(0.1, 10, 101)``).
init_RN : None or array-like, optional
Initial read noise value(s) used to define the maximum-likelihood
grid search. If None, obtain an initial value from the calibration
database. If provided, this may be a scalar or an array defining the
read-noise sampling (e.g., ``np.linspace(1, 20, 101)``).
mode : {'per_pixel', 'channel', 'full_detector'}, optional
Mode used to fit the parameters. ``'per_pixel'`` fits independent
parameters for each pixel. ``'full_detector'`` fits a single set of
parameters for the detector. ``'channel'`` is reserved for future
implementation.
method : {'quad', 'grid', 'both'}, optional
Fitting strategy. ``'grid'`` performs a maximum-likelihood grid
search. ``'quad'`` uses a quadratic approximation around the initial
parameters. ``'both'`` runs a grid search first and then refines the
solution with a quadratic approximation.
init_delta_gains : array-like or None, optional
Gain grid offsets used for the maximum-likelihood grid search. The
gain sampling is defined as ``init_gain + init_delta_gains``.
init_delta_RNs : array-like or None, optional
Read-noise grid offsets used for the maximum-likelihood grid search.
The read-noise sampling is defined as ``init_RN + init_delta_RNs``.
quad_delta_gain : float, optional
Only used when ``method='quad'``. Defines the initial gain step size
around ``init_gain`` for the quadratic approximation. Default is 0.2.
quad_delta_RN : float, optional
Only used when ``method='quad'``. Defines the initial read-noise step
size around ``init_RN`` for the quadratic approximation. Default is
0.5.
fix_RN : bool, optional
If True, fix the read noise to ``init_RN`` and fit only the gain.
Default is False.
countratesidentical : bool, optional
If True, assume all ramps have identical count rates. Default is
False.
Calibrations
------------
gain_init : GainModel
- selector: :class:`~hispecdrp.calibrations.selectors.GainSelector`
rn_init : ReadNoiseModel
- selector: :class:`~hispecdrp.calibrations.selectors.ReadNoiseSelector`
Returns
-------
output : GainModel
Data model containing the fitted gain (and, if applicable, read
noise) parameters and associated fit diagnostics.
References
----------
Brandt, T. D. (2025), *Computing the Electronic Gain for Detectors Read
Out Up-the-ramp*, PASP, 137, 125006.
https://ui.adsabs.harvard.edu/abs/2025PASP..137l5006B/abstract
Roman calibration pipeline reference:
https://github.com/spacetelescope/romancal
"""
PARAMETERS = {
'init_gain' : None, # None | int, Define the initial gain value used for the maximum likelihood grid search.
# If None, use calibration database, otherwise specify a np.array (eg, np.linspace(0.1, 10, 101)).
'init_RN' : None, # None | in, Define the initial readnoise value used for the maximum likelihood grid search.
# If None, use calibration database, otherwise specify a np.array (eg, np.linspace(1, 20, 101)).
'mode': 'per_pixel', # str, 'per_pixel' | 'channel' | 'full_detector', mode to fit the parameters.
# 'per_pixel' compute gain for each pixel independently,
# 'channel' (Not implemented yet) compute gain for each readout channel independently,
# 'full_detector' compute gain for the full detector at once. This can be used when a single exposure is available.
'method': 'quad',# str, 'quad' | 'grid' | 'both', method to fit the parameters.
# 'quad' uses quadratic approximation,
# 'grid' uses maximum likelihood grid search.
# 'both' start with grid search and then apply quadratic approximation.
'init_delta_gains' : None, # np.array, the grid sampling is defined as init_gain + init_delta_gains for the maximum likelihood grid search.
# eg, np.linspace(0.1, 10, 101))
'init_delta_RNs' : None, # np.array, the grid sampling is defined as init_RN + init_delta_RNs for the maximum likelihood grid search.
# specify a np.array (eg, np.linspace(1, 20, 101)).
'quad_delta_gain' : 0.2, # float, used only in 'quad' mode, defines the initial delta gain around init_gain for the quadratic approximation.
'quad_delta_RN' : 0.5, # float, used only in 'quad' mode, defines the initial delta read noise around init_RN for the quadratic approximation.
'fix_RN' : False, # bool, if True, fix the read noise to init_RN value and only fit for gain.
'countratesidentical': False, # bool, if True, assume all ramps have identical count rates.
}
CALIBRATIONS = {
'gain_init': {
'selector': GainSelector,
'selector_kwargs': {}
},
'rn_init': {
'selector': ReadNoiseSelector,
'selector_kwargs': {}
}
}
def _perform(self):
if self.mode.lower() != 'full_detector' and self.mode.lower() != 'per_pixel':
raise NotImplementedError(f"Only 'full_detector' and 'per_pixel' modes are implemented currently. Not {self.mode}")
if isinstance(self.dataset, (datamodels.DataModel, str)):
_input_list_tmp = [self.dataset]
if isinstance(self.dataset, Sequence):
_input_list_tmp = self.dataset
nfiles = len(_input_list_tmp)
logger.info(f"Deriving the detector gain from a list of {nfiles} inputs.")
input_models = [datamodels.open(f) for f in _input_list_tmp]
datamodel0 = input_models[0]
nreads,ny,nx = datamodel0.data.shape
if self.init_gain is None:
gain_init, _ = self.get_calibration(datamodel0, 'gain_init')
gain_init_model = datamodels.open(gain_init)
_init_center_gain = gain_init_model.gain
logger.info(f"Using {gain_init} as initial detector gain values with median value {np.nanmedian(_init_center_gain)}.")
elif isinstance(self.init_gain, numbers.Number):
_init_center_gain = self.init_gain
logger.info(f"Using user-defined initial detector gain values: {self.init_gain}.")
if self.init_RN is None:
rn_cal_init, _ = self.get_calibration(datamodel0, 'rn_init')
rn_model_init = datamodels.open(rn_cal_init)
_init_center_RN = rn_model_init.RN
logger.info(f"Using {rn_cal_init} as initial read noise values with median value {np.nanmedian(_init_center_RN)} DN.")
elif isinstance(self.init_RN, numbers.Number):
_init_center_RN = self.init_RN
logger.info(f"Using user-defined initial read noise values: {self.init_RN}.")
delta_readtimes = datamodel0.get_delta_readtimes()
if self.method == 'quad':
_init_delta_gains = np.array([-self.quad_delta_gain,0,+self.quad_delta_gain])
if self.fix_RN:
_init_delta_RNs = None
else:
_init_delta_RNs = np.array([-self.quad_delta_RN,0,+self.quad_delta_RN])
elif self.method == 'grid' or self.method == 'both':
if self.init_delta_gains is None:
_init_delta_gains = np.linspace(-1, 1, 51)
else:
_init_delta_gains = self.init_delta_gains
if self.fix_RN:
_init_delta_RNs = None
else:
if self.init_delta_RNs is None:
_init_delta_RNs = np.linspace(-10,10, 51)
else:
_init_delta_RNs = self.init_delta_RNs
if self.mode.lower() == 'full_detector':
logger.info(f"Running in full_detector mode. All detector pixels will be fitted jointly and a single gain and/or readnoise value will be returned for the entire chip.")
elif self.mode.lower() == 'per_pixel':
logger.info(f"Running in single_pixel mode. Each detector pixel will be fitted independently and a map of gain and/or readnoise will be returned.")
# Define output model from inputs
output_gain = self.get_output_model(input_models,"GainModel")
# Make all ramps from all files into a big cube
ramp_arr = []
dq_raw_arr = []
for opened_input in input_models:
if nreads != opened_input.data.shape[0]:
raise ValueError("All input ramps must have the same number of reads.")
ramp_arr.append(opened_input.data)
dq_raw_arr.append(opened_input.dq_raw)
ramp_arr = np.array(ramp_arr)
dq_raw_arr = np.array(dq_raw_arr)
max_cores = self.max_cores or 1
_out = fit_gain_and_readnoise(
delta_readtimes, ramp_arr, dq_raw_arr, _init_center_gain, _init_center_RN,_init_delta_gains,_init_delta_RNs,
countratesidentical=self.countratesidentical,
mode=self.mode,
method=self.method,
max_cores=max_cores
)
bestfit_gain, bestfit_rn, cov = _out
# Output
ny, nx = input_models[0].data.shape[1:]
if bestfit_gain.ndim == 1:
output_gain.gain = np.full((ny, nx), bestfit_gain, dtype=np.float32)
output_gain.rn = np.full((ny, nx), bestfit_rn, dtype=np.float32)
output_gain.cov = np.broadcast_to(cov[:, :, None, None], (2, 2, ny, nx))
else:
output_gain.gain = bestfit_gain
output_gain.rn = bestfit_rn
output_gain.cov = cov
output_gain.dq = np.zeros(bestfit_gain.shape, dtype=np.uint32)
return dict(output=output_gain)
[docs]
def fit_gain_and_readnoise(
readtimes : np.ndarray, ramps : np.ndarray, dq_raw : np.ndarray,
center_gains : np.ndarray, center_rns : np.ndarray,
delta_gains: np.ndarray = None, delta_RNs : np.ndarray = None,
countratesidentical: bool = False, mode : str = "per_pixel", method: str = 'quad',
max_cores : int = 1,
):
"""
Fit for the detector gain and/or read noise.
Parameters
----------
readtimes : np.ndarray
1D array of read times.
ramps : np.ndarray
4D array of input rates. Size n_files x nreads x ny x nx.
dq_raw : np.ndarray
4D array of input UTR data quality flags. Size n_files x nreads x ny x nx.
center_gains : np.ndarray | float
Initial center guess for gain (e-/ADU).
center_rns : np.ndarray | float
Initial center guess for read noise (in DN).
delta_gains : np.ndarray
The grid sampling is defined as center_gains[i,j] + delta_gains for the maximum likelihood grid search.
If None, only fit for read noise while fixing gain to this value.
If method is 'quad', should be three elements [left, 0, right].
delta_RNs : np.ndarray
The grid sampling is defined as center_rns[i,j] + delta_RNs for the maximum likelihood grid search.
If None, only fit for gain while fixing read noise to this value.
If method is 'quad', should be three elements [left, 0, right].
countratesidentical: bool
If True, assume all ramps have identical count rates.
mode : str
'full_detector' to fit a single gain and read noise for the entire detector,
'per_pixel' to fit gain and read noise for each pixel independently.
method : str
Method to use for fitting. Options are 'quad' for quadratic approximation, 'grid' for maximum likelihood grid search, 'both' for grid search followed by quadratic approximation.
max_cores : int, optional
Number of cores for parallelization. Default is 1 (no parallelization).
Not used for mode == 'full_detector'.
Returns
-------
gain : np.ndarray
Best fit Gain (e-/ADU).
Singleton if mode is 'full_detector', 2D array if mode is 'per_pixel'.
rn : np.ndarray
Best fit read noise (in DN).
Singleton if mode is 'full_detector', 2D array if mode is 'per_pixel'.
cov : np.ndarray
Covariance matrix of the fit.
2D matrix if mode is 'full_detector', 4D array if mode is 'per_pixel'.
"""
if delta_gains is not None:
if method == 'quad' and len(delta_gains) != 3:
raise ValueError("In 'quad' method, delta_gains must have three elements: [left, 0, right]. Or be set to None.")
elif (method == 'grid' or method == 'both') and len(delta_gains) < 4:
raise ValueError("In 'grid' or 'both' method, delta_gains must have more than 3 elements.")
if delta_RNs is not None:
if method == 'quad' and len(delta_RNs) != 3:
raise ValueError("In 'quad' method, delta_RNs must have three elements: [left, 0, right]. Or be set to None.")
elif (method == 'grid' or method == 'both') and len(delta_RNs) < 4:
raise ValueError("In 'grid' or 'both' method, delta_RNs must have more than 3 elements.")
if mode.lower() == 'full_detector':
C = fitramp_Brandt.Covar(readtimes)
if isinstance(center_gains, np.ndarray):
_center_gain = np.nanmedian(center_gains)
else:
_center_gain = center_gains
if isinstance(center_rns, np.ndarray):
_center_rn = np.nanmedian(center_rns)
else:
_center_rn = center_rns
if delta_gains is not None:
gains_init = _center_gain + delta_gains
else:
gains_init = np.array([_center_gain])
if delta_RNs is not None:
rns_init = _center_rn + delta_RNs
else:
rns_init = np.array([_center_rn])
if method == 'quad':
_out = _fit_single_gain_readnoise_quad_approx(readtimes, ramps, dq_raw, C,
gains_init, rns_init,
do_once=False, countratesidentical=countratesidentical)
elif method == 'grid' or method == 'both':
if method == 'grid':
error_computation = 'posterior'
elif method == 'both':
error_computation = 'quad'
_out = _fit_single_gain_readnoise(readtimes, ramps, dq_raw, C,
gains_init, rns_init,
error_computation=error_computation, countratesidentical=countratesidentical)
bestfit_gain, bestfit_rn, cov = np.array([_out[0]]), np.array([_out[1]]), _out[2]
elif mode.lower() == 'per_pixel':
n_files, n_reads, ny,nx = ramps.shape
if isinstance(center_gains, np.ndarray):
_center_gains = center_gains
elif isinstance(center_gains, float):
_center_gains = np.full((ny,nx), center_gains)
if isinstance(center_rns, np.ndarray):
_center_rns = center_rns
elif isinstance(center_rns, float):
_center_rns = np.full((ny,nx), center_rns)
_out = _fit_gain_and_readnoise_per_pixel(readtimes, ramps, dq_raw,
_center_gains, _center_rns,
delta_gains, delta_RNs,
countratesidentical=countratesidentical,
method= method,max_cores =max_cores)
bestfit_gain, bestfit_rn, cov = _out
return bestfit_gain, bestfit_rn, cov
def _fit_gain_and_readnoise_per_pixel(
readtimes : np.ndarray, ramps : np.ndarray, dq_raw : np.ndarray,
center_gains : np.ndarray, center_rns : np.ndarray,
delta_gains: np.ndarray = None, delta_RNs : np.ndarray = None,
countratesidentical: bool = False, method: str = 'quad',
max_cores : int = 1,
):
"""
Fit the gain and read noise for each pixel on the detector using either quadratic approximation or maximum likelihood grid search.
Parameters
----------
readtimes : np.ndarray
1D array of read times.
ramps : np.ndarray
2D array of input rates. Size n_files x nreads x ny x nx.
dq_raw : np.ndarray
2D array of input UTR data quality flags. Size n_files x nreads x ny x nx.
center_gains : np.ndarray
Initial center guess for gain (e-/ADU).
center_rns : np.ndarray
Initial center guess for read noise (in DN).
delta_gains : np.ndarray
The grid sampling is defined as center_gains[i,j] + delta_gains for the maximum likelihood grid search.
If None, only fit for read noise while fixing gain to this value.
If method is 'quad', should be three elements [left, 0, right].
delta_RNs : np.ndarray
The grid sampling is defined as center_rns[i,j] + delta_RNs for the maximum likelihood grid search.
If None, only fit for gain while fixing read noise to this value.
If method is 'quad', should be three elements [left, 0, right].
countratesidentical: bool
If True, assume all ramps have identical count rates.
method : str
Method to use for fitting. Options are 'quad' for quadratic approximation, 'grid' for maximum likelihood grid search, 'both' for grid search followed by quadratic approximation.
max_cores : int, optional
Number of cores for parallelization. Default is 1 (no parallelization).
Returns
-------
gain : float
Best fit Gain (e-/ADU)
rn : float
Best fit read noise (in DN)
cov : np.ndarray
Covariance matrix of the fit.
"""
n_files, n_reads, ny, nx = ramps.shape
if ramps.dtype is np.uint16:
mp_data_type = ctypes.c_ushort
else:
mp_data_type = ctypes.c_float
mp_float_type = ctypes.c_float
mp_dq_raw_type = ctypes.c_uint8
mp_dq_type = ctypes.c_uint
types_tuple = (mp_data_type, mp_float_type, mp_dq_raw_type, mp_dq_type)
gain_mp = RawArray(mp_float_type, nx * ny)
gain_shape = (ny, nx)
gain_np = _arraytonumpy(gain_mp, gain_shape, dtype=mp_float_type)
gain_np[:] = np.zeros(gain_shape, dtype=np.float32)
readnoise_mp = RawArray(mp_float_type, nx * ny)
readnoise_shape = (ny, nx)
readnoise_np = _arraytonumpy(readnoise_mp, readnoise_shape, dtype=mp_float_type)
readnoise_np[:] = np.zeros(readnoise_shape, dtype=np.float32)
cov_mp = RawArray(mp_float_type, 2 * 2 * nx * ny)
cov_shape = (2,2,ny, nx)
cov_np = _arraytonumpy(cov_mp, cov_shape, dtype=mp_float_type)
cov_np[:] = np.zeros(cov_shape, dtype=np.float32)
center_gains_mp = RawArray(mp_float_type, nx * ny)
center_gains_shape = (ny, nx)
center_gains_np = _arraytonumpy(center_gains_mp, center_gains_shape, dtype=mp_float_type)
center_gains_np[:] = center_gains.astype(np.float32)
center_rns_mp = RawArray(mp_float_type, nx * ny)
center_rns_shape = (ny, nx)
center_rns_np = _arraytonumpy(center_rns_mp, center_rns_shape, dtype=mp_float_type)
center_rns_np[:] = center_rns.astype(np.float32)
if max_cores > 1:
ramps_mp = RawArray(mp_data_type, nx * ny * n_reads * n_files)
ramps_shape = (n_files,n_reads, ny, nx)
ramps_np = _arraytonumpy(ramps_mp, ramps_shape, dtype=mp_data_type)
ramps_np[:] = ramps # copy original array into shared array
ramps = ramps_np # free the original array from memory to avoid duplicates
dq_raw_mp = RawArray(mp_dq_raw_type, nx * ny * n_reads * n_files)
dq_raw_shape = (n_files,n_reads, ny, nx)
dq_raw_np = _arraytonumpy(dq_raw_mp, dq_raw_shape, dtype=mp_dq_raw_type)
if dq_raw is None:
dq_raw_np[:] = np.zeros(dq_raw_np, dtype=np.uint8)
else:
dq_raw_np[:] = dq_raw # copy original array into shared array
dq_raw = dq_raw_np # free the original array from memory to avoid duplicates
# Prepare blocks
block_size = max(1, ny // (4*max_cores)) if max_cores > 1 else ny
_init_args = (
ramps_mp, ramps_shape,
dq_raw_mp, dq_raw_shape,
gain_mp, gain_shape,
readnoise_mp, readnoise_shape,
cov_mp, cov_shape,
center_gains_mp, center_gains_shape,
center_rns_mp, center_rns_shape
)
ctx = mp.get_context("spawn") # avoids unsafe fork after OpenMP init
tpool = ctx.Pool(
processes=max_cores,
initializer=_tpool_init_fit_gain_readnoise,
initargs=_init_args,
maxtasksperchild=50
)
args_list = []
for block_start in range(0, ny, block_size):
block_end = min(block_start + block_size, ny)
args = (block_start, block_end, readtimes,
delta_gains,delta_RNs,
countratesidentical, method, types_tuple)
args_list.append(args)
try:
with numba_thread_scope(max_cores):
tasks = [
tpool.apply_async(_fit_gain_and_readnoise_worker, args=(args_tuple,))
for args_tuple in args_list
]
for t in progressbar(tasks, desc="Fitting gain and read noise", fast=False):
t.wait()
finally:
tpool.close()
tpool.join()
else:
# If not parallelizing, ie max_cores=1, just run directly without shared memory
results = []
for row_id in np.arange(ny):
print(f"Fitting detector gain and/or readnoise: Processing row {row_id + 1} of {ny}.")
args = (
row_id,
readtimes,
ramps[:, :, [row_id], :],
dq_raw[:, :, [row_id], :],
center_gains_np[[row_id], :],
center_rns_np[[row_id], :],
delta_gains,delta_RNs,
countratesidentical,
method
)
results.append(_fit_gain_and_readnoise_row(args))
for row_element in results:
row_id, out_bestfit_gain, out_bestfit_rn, out_cov = row_element
gain_np[row_id, :] = out_bestfit_gain
readnoise_np[row_id, :] = out_bestfit_rn
cov_np[:,:,row_id, :] = out_cov
return gain_np, readnoise_np, cov_np
def _tpool_init_fit_gain_readnoise(
data : np.ndarray, data_shape : tuple,
dq_raw : np.ndarray, dq_raw_shape : tuple,
gain : np.ndarray, gain_shape : tuple,
readnoise : np.ndarray, readnoise_shape : tuple,
cov : np.ndarray, cov_shape : tuple,
center_gains : np.ndarray, center_gains_shape : tuple,
center_rns : np.ndarray, center_rns_shape : tuple,
):
"""
Initialize shared global variables for the multiprocessing pool.
"""
global shared_data, shared_data_shape, \
shared_dq_raw, shared_dq_raw_shape, \
shared_gain, shared_gain_shape, \
shared_readnoise, shared_readnoise_shape, \
shared_cov, shared_cov_shape, \
shared_center_gains, shared_center_gains_shape, \
shared_center_rns, shared_center_rns_shape
shared_data = data
shared_data_shape = data_shape
shared_dq_raw = dq_raw
shared_dq_raw_shape = dq_raw_shape
shared_gain = gain
shared_gain_shape = gain_shape
shared_readnoise = readnoise
shared_readnoise_shape = readnoise_shape
shared_cov = cov
shared_cov_shape = cov_shape
shared_center_gains = center_gains
shared_center_gains_shape = center_gains_shape
shared_center_rns = center_rns
shared_center_rns_shape = center_rns_shape
def _fit_gain_and_readnoise_row(indata : tuple) -> tuple:
"""
Helper function for parallelization fitting the gain and readnoise per detector row.
"""
row_id, delta_readtimes, ramp_row, dq_raw_row, center_gains_row, center_rns_row, delta_gains,delta_RNs, countratesidentical,method = indata
n_files, n_reads, _, nx = ramp_row.shape
# print(row_id, ramp_row.shape, ramp_row[0,:,0,0])
C = fitramp_Brandt.Covar(delta_readtimes)
if method == 'grid':
error_computation = 'posterior'
elif method == 'both':
error_computation = 'quad'
out_bestfit_gain = np.zeros((nx,), dtype=np.float32)
out_bestfit_rn = np.zeros((nx,), dtype=np.float32)
out_cov = np.zeros((2,2,nx), dtype=np.float32)
for i in range(nx):
if delta_gains is not None:
gains_init = center_gains_row[0,i]+ delta_gains
else:
gains_init = np.array([center_gains_row[0,i]])
if delta_RNs is not None:
rns_init = center_rns_row[0,i]+delta_RNs
else:
rns_init = np.array([center_rns_row[0,i]])
if method == 'quad':
_out = _fit_single_gain_readnoise_quad_approx(delta_readtimes, ramp_row[:,:,0,i], dq_raw_row[:,:,0,i], C,
gains_init, rns_init,
do_once=False,countratesidentical=countratesidentical)
elif method == 'grid' or method == 'both':
_out = _fit_single_gain_readnoise(delta_readtimes, ramp_row[:,:,0,i], dq_raw_row[:,:,0,i], C,
gains_init, rns_init,
error_computation=error_computation,countratesidentical=countratesidentical)
bestfit_gain, bestfit_rn, cov = _out
out_bestfit_gain[i] = bestfit_gain
out_bestfit_rn[i] = bestfit_rn
out_cov[:,:,i] = cov
return (row_id, out_bestfit_gain, out_bestfit_rn, out_cov)
def _fit_gain_and_readnoise_worker(args : tuple) -> tuple:
"""
Worker function for multiprocessing. Runs _fit_gain_and_readnoise_row on a block of rows using shared arrays.
Parameters
----------
args : tuple
Tuple containing: (block_start, block_end, delta_readtimes,delta_gains,delta_RNs, countratesidentical, method, types_tuple)
Returns
-------
block_start : int
Starting index of the block of rows processed.
block_end : int
Ending index of the block of rows processed.
"""
block_start, block_end, delta_readtimes,delta_gains,delta_RNs, countratesidentical, method, types_tuple = args
mp_data_type, mp_float_type, mp_dq_raw_type, mp_dq_type = types_tuple
ramps_np = _arraytonumpy(shared_data, shared_data_shape, dtype=mp_data_type)
dq_raw_np = _arraytonumpy(shared_dq_raw, shared_dq_raw_shape, dtype=mp_dq_raw_type)
gain_np = _arraytonumpy(shared_gain, shared_gain_shape, dtype=mp_float_type)
readnoise_np = _arraytonumpy(shared_readnoise, shared_readnoise_shape, dtype=mp_float_type)
cov_np = _arraytonumpy(shared_cov, shared_cov_shape, dtype=mp_float_type)
center_gains_np = _arraytonumpy(shared_center_gains, shared_center_gains_shape, dtype=mp_float_type)
center_rns_np = _arraytonumpy(shared_center_rns, shared_center_rns_shape, dtype=mp_float_type)
results = []
for row_id in range(block_start, block_end):
args_row = (
row_id,
delta_readtimes,
ramps_np[:,:, [row_id], :],
dq_raw_np[:,:, [row_id], :],
center_gains_np[[row_id], :],
center_rns_np[[row_id], :],
delta_gains,delta_RNs,
countratesidentical,
method
)
results.append(_fit_gain_and_readnoise_row(args_row))
for row_element in results:
row_id, out_bestfit_gain, out_bestfit_rn, out_cov = row_element
gain_np[row_id, :] = out_bestfit_gain
readnoise_np[row_id, :] = out_bestfit_rn
cov_np[:,:,row_id, :] = out_cov
return (block_start, block_end)
def _fit_single_gain_readnoise_quad_approx(
readtimes : np.ndarray, ramps : np.ndarray, dq_raw : np.ndarray, covar : fitramp_Brandt.Covar,
gains_init : np.ndarray, rns_init : np.ndarray, do_once : bool = False,
countratesidentical: bool = False
):
"""
Fit the gain and read noise for each pixel on the detector using a quadratic approximation.
The fit is actually performed twice to refine the best guess.
The first iteration uses gains_init and rns_init.
The second iteration recenters gains_init and rns_init around the best fit from the first iteration and use the covariance for scaling.
Parameters
----------
readtimes : np.ndarray
1D array of read times.
ramps : np.ndarray
2D array of input rates. Size nramps x nreads (x ny x nx).
Extra y and x dimensions are flattened with nramps if they exist.
dq_raw : np.ndarray
2D array of input UTR data quality flags. Size nramps x nreads (x ny x nx).
Extra y and x dimensions are flattened with nramps if they exist.
covar : fitramp_Brandt.Covar class
Should be appropriate for diffs_lin
gains_init : np.ndarray
Initial guess for gain (e-/ADU).
Should be three elements (left, middle, right) with all gains_init > 0.
Or if singleton, only fit for read noise while fixing gain to this value.
rns_init : np.ndarray
Initial guess for read noise (in DN).
Should be three elements (left, middle, right) with all rns_init > 0.
Or if singleton, only fit for gain while fixing read noise to this value.
do_once : bool
If True, perform the fit only once (no refinement).
countratesidentical: bool
If True, assume all ramps have identical count rates.
Returns
-------
gain : float
Best fit Gain (e-/ADU)
rn : float
Best fit read noise (in DN)
cov : np.ndarray
Covariance matrix of the fit.
"""
nreads = readtimes.shape[0]
_ramps = np.array(ramps).swapaxes(0, 1) # n_reads, n_files, ny, nx
_dq_raw = np.array(dq_raw).swapaxes(0, 1) # n_reads, n_files, ny, nx
_ramps = _ramps.reshape(nreads, -1) # shape (n_reads, nfiles*ny*nx)
_dq_raw = _dq_raw.reshape(nreads, -1) # shape (n_reads, nfiles*ny*nx)
y0 = 0
diff_times = np.diff(readtimes)[:, None]
diffs_lin = np.diff(_ramps, axis=0).astype(np.float64)/diff_times
diffs_quad = np.diff((_ramps - y0) ** 2, axis=0).astype(np.float64)/diff_times
n_reads,n_ramps = _dq_raw.shape
mask = ((_dq_raw & DO_NOT_USE_FLAG_uint8) | (_dq_raw & SATURATED_FLAG_uint8)).astype(np.bool)
mask2 = (_dq_raw & JUMP_DET_FLAG_uint8).astype(np.bool)
diffs2use = ~(mask[1:n_reads] | mask[0:n_reads - 1] | mask2[1:n_reads])
# diffs2use = np.full((n_reads-1,n_ramps),1,dtype=np.uint8)
# diffs2use[5,:] = 0
# diffs2use[10,:] = 0
# diffs2use[20,:] = 0
diffs2use = diffs2use.astype(np.uint8)
# The quadratic form fit, followed by the maximum value that the quadratic
# form takes and the maximum value seen on the dense grid. Run the
# quadratric form fit twice to make sure we get very close to the maximum
# of the likelihood.
if do_once:
n_iter = 1
else:
n_iter = 2
for i in range(n_iter):
# three points each in read noise and gain to fit a quadratic form
if i == 0:
gains_short = gains_init.astype(np.float64)
sigs_short = rns_init.astype(np.float64)
else:
if len(gains_short) > 1:
bestfit_gain_err = np.sqrt(cov[0, 0])
gains_short = [bestfit_gain- bestfit_gain_err,bestfit_gain, bestfit_gain + bestfit_gain_err]
if len(sigs_short) > 1:
bestfit_rn_err = np.sqrt(cov[1,1])
sigs_short = [bestfit_rn- bestfit_rn_err,bestfit_rn, bestfit_rn + bestfit_rn_err]
bestfit_gain, bestfit_rn, cov = _fitquad_gain_rn(diffs_lin, diffs_quad, covar,sigs_short, gains_short,
d2use=diffs2use,countratesidentical=countratesidentical)
return bestfit_gain, bestfit_rn, cov
def _fitquad_gain_rn(
diffs_lin,
diffs_quad,
covar,
sigs_short,
gains_short,
d2use=None,
countratesidentical=False,
):
"""
Estimate best-fit (gain, read-noise) and an approximate covariance matrix.
Logic:
- If both grids have >1 point: call fit_gain_Brandt.fitquad(...)
- If only sigs_short has 1 point: compute logL vs gain, fit a quadratic,
take maximum at -b/(2a), and estimate sigma_gain ~ sqrt(-1/(2a)).
- If only gains_short has 1 point: compute logL vs rn, fit a quadratic,
take maximum at -b/(2a), and estimate sigma_rn ~ sqrt(-1/(2a)).
Returns
-------
bestfit_gain : float
bestfit_rn : float
cov : (2,2) ndarray
cov[0,0] ~ var_gain (as in your snippet), cov[1,1] ~ var_rn;
off-diagonals are NaN unless fitquad returns something else.
"""
if len(sigs_short) > 1 and len(gains_short) > 1:
(bestfit_gain, bestfit_rn), cov, coefs = fit_gain_Brandt.fitquad(diffs_lin, diffs_quad, covar,
sigs_short, gains_short, log=False,d2use=d2use,
countratesidentical=countratesidentical)
elif len(sigs_short) == 1:
logL = fit_gain_Brandt.gen_likelihood_map(diffs_lin, diffs_quad, covar,
sigs_short, gains_short, [0],
countratesidentical=countratesidentical,d2use=d2use)
a,b,c = _quad_1d_through_3pts(gains_short, logL[0,:,0])
bestfit_gain = -b/(2*a)
bestfit_rn = sigs_short[0]
cov = np.full((2,2),np.nan)
cov[0,0] = -1/(2*a)
elif len(gains_short) == 1:
logL = fit_gain_Brandt.gen_likelihood_map(diffs_lin, diffs_quad, covar,
sigs_short, gains_short, [0],
countratesidentical=countratesidentical,d2use=d2use)
a,b,c = _quad_1d_through_3pts(sigs_short, logL[:,0,0])
bestfit_gain = gains_short[0]
bestfit_rn = -b/(2*a)
cov = np.full((2,2),np.nan)
cov[1,1] = -1/(2*a)
return bestfit_gain, bestfit_rn, cov
def _fit_single_gain_readnoise(
readtimes: np.ndarray, ramps: np.ndarray, dq_raw: np.ndarray, covar: fitramp_Brandt.Covar,
gains_init: np.ndarray, rns_init: np.ndarray, error_computation: str = 'quad',
countratesidentical: bool = False, return_likelihood: bool = False
):
"""
Fit the gain and read noise using a grid search maximum likelihood.
The grid search is defined by gains_init and rns_init.
Parameters
----------
readtimes : np.ndarray
1D array of read times.
ramps : np.ndarray
2D array of input rates. Size nramps x nreads.
dq_raw : np.ndarray
2D array of input UTR data quality flags. Size nramps x nreads.
covar : fitramp_Brandt.Covar class
Should be appropriate for diffs_lin
gains_init : np.ndarray
Grid sampling for gain (e-/ADU). Elements should be all positive.
rns_init : float
Grid sampling for read noise (in DN). Elements should be all positive.
error_computation : str
Mode to compute the errors. Options are 'quad' for quadratic approximation, 'posterior' for posterior marginalization.
countratesidentical: bool
If True, assume all ramps have identical count rates.
Returns
-------
gain : float
Best fit Gain (e-/ADU)
gain_err : float
Best fit Gain error (e-/ADU)
rn : float
Best fit read noise (in DN)
rn_err : float
Best fit read noise error (in DN)
"""
nreads = readtimes.shape[0]
_ramps = np.array(ramps).swapaxes(0, 1) # n_reads, n_files, ny, nx
_dq_raw = np.array(dq_raw).swapaxes(0, 1) # n_reads, n_files, ny, nx
_ramps = _ramps.reshape(nreads, -1) # shape (n_reads, nfiles *ny*nx)
_dq_raw = _dq_raw.reshape(nreads, -1) # shape (n_reads, nfiles*ny*nx)
y0 = 0
diff_times = np.diff(readtimes)[:, None]
diffs_lin = np.diff(_ramps, axis=0).astype(np.float64)/diff_times
diffs_quad = np.diff((_ramps - y0) ** 2, axis=0).astype(np.float64)/diff_times
n_reads,n_ramps = _dq_raw.shape
mask = ((_dq_raw & DO_NOT_USE_FLAG_uint8) | (_dq_raw & SATURATED_FLAG_uint8)).astype(np.bool)
mask2 = (_dq_raw & JUMP_DET_FLAG_uint8).astype(np.bool)
diffs2use = ~(mask[1:n_reads] | mask[0:n_reads - 1] | mask2[1:n_reads])
# diffs2use = np.full((n_reads-1,n_ramps),1,dtype=np.uint8)
# diffs2use[5,:] = 0
# diffs2use[10,:] = 0
# diffs2use[20,:] = 0
diffs2use = diffs2use.astype(np.uint8)
_rns_init = rns_init.astype(np.float64)
_rns_init = _rns_init[_rns_init > 0]
_gains_init = gains_init.astype(np.float64)
_gains_init = _gains_init[_gains_init > 0]
likelihoods = fit_gain_Brandt.gen_likelihood_map(diffs_lin, diffs_quad, covar, _rns_init, _gains_init, [0],
d2use=diffs2use,countratesidentical=countratesidentical)[..., 0]
maxL = np.amax(likelihoods)
k, l = np.unravel_index(np.argmax(likelihoods), likelihoods.shape)
# bestfit_gain, bestfit_rn = _gains_init[l], rns_init[k]
if error_computation == 'posterior':
gain_post = np.sum(np.exp(likelihoods - np.amax(likelihoods)), axis=0)
gain_post /= np.sum(gain_post)
gains_init_diffs = np.diff(_gains_init, prepend=_gains_init[0] - (_gains_init[1] - _gains_init[0]))
gain_post /= gains_init_diffs
bestfit_gain, gain_left_error, gain_right_error = get_err_from_posterior(_gains_init, gain_post)
bestfit_gain_err = np.max([gain_left_error, gain_right_error])
if np.size(_rns_init) > 3:
rn_post = np.sum(np.exp(likelihoods - np.amax(likelihoods)), axis=1)
rn_post /= np.sum(rn_post)
rns_init_diffs = np.diff(_rns_init, prepend=_rns_init[0] - (_rns_init[1] - _rns_init[0]))
rn_post /= rns_init_diffs
bestfit_rn, rn_left_error, rn_right_error = get_err_from_posterior(_rns_init, rn_post)
bestfit_rn_err = np.max([rn_left_error, rn_right_error])
else:
bestfit_rn, bestfit_rn_err = np.nan, np.nan
cov = np.full((2,2),np.nan)
cov[0,0] = bestfit_gain_err
cov[1,1] = bestfit_rn_err
if return_likelihood:
other_outputs = {}
other_outputs["rns_init"] = _rns_init
other_outputs["gains_init"] = _gains_init
other_outputs["likelihoods"] = likelihoods
return bestfit_gain, bestfit_rn, cov, other_outputs
else:
return bestfit_gain, bestfit_rn, cov
elif error_computation == 'quad':
# three points each in read noise and gain to fit a quadratic form
n_iter = 2
for i in range(n_iter):
# three points each in read noise and gain to fit a quadratic form
if i == 0:
if len(_gains_init) > 1:
delta_gain = (_gains_init[1] - _gains_init[0])/2
gains_short = [_gains_init[l]-delta_gain,_gains_init[l], _gains_init[l]+delta_gain]
else:
gains_short = _gains_init
if len(_rns_init) > 1:
delta_rn = (_rns_init[1] - _rns_init[0])/2
sigs_short = [_rns_init[k]-delta_rn,_rns_init[k], _rns_init[k]+delta_rn]
else:
sigs_short = _rns_init
else:
if len(gains_short) > 1:
bestfit_gain_err = np.sqrt(cov[0, 0])
gains_short = [bestfit_gain- bestfit_gain_err,bestfit_gain, bestfit_gain + bestfit_gain_err]
if len(sigs_short) > 1:
bestfit_rn_err = np.sqrt(cov[1,1])
sigs_short = [bestfit_rn- bestfit_rn_err,bestfit_rn, bestfit_rn + bestfit_rn_err]
bestfit_gain, bestfit_rn, cov = _fitquad_gain_rn(diffs_lin, diffs_quad, covar,sigs_short, gains_short,
d2use=diffs2use,countratesidentical=countratesidentical)
return bestfit_gain, bestfit_rn, cov
def _quad_1d_through_3pts(x, y):
"""
Fit y = a*x^2 + b*x + c through exactly three points (xi, yi),
without explicit matrix inversion.
x, y: array-like of length 3
Returns: (a, b, c)
"""
x = np.asarray(x, dtype=float)
y = np.asarray(y, dtype=float)
if x.shape != (3,) or y.shape != (3,):
raise ValueError("x and y must be length-3 arrays")
if np.unique(x).size != 3:
raise ValueError("x values must be distinct")
x1, x2, x3 = x
y1, y2, y3 = y
d1 = (x1 - x2) * (x1 - x3)
d2 = (x2 - x1) * (x2 - x3)
d3 = (x3 - x1) * (x3 - x2)
a = y1 / d1 + y2 / d2 + y3 / d3
b = -(y1 * (x2 + x3) / d1 + y2 * (x1 + x3) / d2 + y3 * (x1 + x2) / d3)
c = y1 * x2 * x3 / d1 + y2 * x1 * x3 / d2 + y3 * x1 * x2 / d3
return a, b, c
[docs]
def plot_2d_posterior(
likelihoods,
gains_init,
rns_init,
*,
aspect=0.4,
vmin=-20,
vmax=0,
cmap="bone_r",
contour_deltas=(11.8/2, 6.2/2, 2.3/2),
contour_color="r",
contour_lw=2,
cbar_label=r"$\log {\cal L}/{\cal L}_{\rm max}$",
xlabel=r"Gain ($e^-$/DN)",
ylabel=r"Read Noise (DN)",
title_2d=None,
title_1d=None,
show=False,
):
"""
Plot a 2D log-likelihood map (shifted by max) with confidence contours,
and optionally a 1D marginalized gain posterior.
Parameters
----------
likelihoods : (Nr, Ng) array
Log-likelihood values on the (rns_init, gains_init) grid.
Assumes axis-0 corresponds to rns_init and axis-1 corresponds to gains_init.
gains_init : (Ng,) array
Gain grid (x-axis).
rns_init : (Nr,) array
Read-noise grid (y-axis).
show : bool
If True, calls plt.show() at the end.
Returns
-------
figs : dict
{"2d": fig2d, "1d": fig1d_or_None}
axes : dict
{"2d": ax2d, "1d": ax1d_or_None}
"""
likelihoods = np.asarray(likelihoods)
gains_init = np.asarray(gains_init)
rns_init = np.asarray(rns_init)
gain_post = np.sum(np.exp(likelihoods - np.amax(likelihoods)), axis=0)
gain_post /= np.sum(gain_post)
gains_init_diffs = np.diff(gains_init, prepend=gains_init[0] - (gains_init[1] - gains_init[0]))
gain_post /= gains_init_diffs
if likelihoods.ndim != 2:
raise ValueError("likelihoods must be 2D (Nr, Ng).")
if likelihoods.shape != (rns_init.size, gains_init.size):
raise ValueError(
f"Shape mismatch: likelihoods has shape {likelihoods.shape} but "
f"expected ({rns_init.size}, {gains_init.size}) = (len(rns_init), len(gains_init))."
)
maxL = np.nanmax(likelihoods)
extent = [gains_init[0], gains_init[-1], rns_init[0], rns_init[-1]]
# --- 2D figure ---
fig2d, ax2d = plt.subplots(figsize=(8, 6))
im = ax2d.imshow(
likelihoods - maxL,
origin="lower",
aspect=aspect,
extent=extent,
vmin=vmin,
vmax=vmax,
cmap=cmap,
)
cbar = fig2d.colorbar(im, ax=ax2d)
cbar.set_label(cbar_label)
ax2d.set_xlabel(xlabel)
ax2d.set_ylabel(ylabel)
if title_2d is not None:
ax2d.set_title(title_2d)
# Contours at maxL - delta
levels = [maxL - d for d in contour_deltas]
ax2d.contour(
likelihoods,
extent=extent,
levels=levels,
colors=[contour_color] * len(levels),
linewidths=[contour_lw] * len(levels),
)
gain_post = np.asarray(gain_post)
if gain_post.shape != (gains_init.size,):
raise ValueError(
f"gain_post has shape {gain_post.shape} but expected ({gains_init.size},)."
)
fig1d, ax1d = plt.subplots(figsize=(8, 6))
ax1d.plot(gains_init, gain_post, label="Full Numerical Marginalization")
ax1d.set_xlabel(xlabel)
ax1d.set_ylabel("Posterior (arb.)")
if title_1d is not None:
ax1d.set_title(title_1d)
ax1d.legend()
if show:
plt.show()
return {"2d": fig2d, "1d": fig1d}, {"2d": ax2d, "1d": ax1d}