Source code for hispecdrp.primitives.gain.fit_gain_Brandt

#!/usr/bin/env python

# Author: Tim Brandt
# Shared with HISPEC team in Aug 2025
#
# Reference:
# Brandt, T. D., "Computing the Electronic Gain for Detectors Read Out Up-the-ramp", PASP, 137, 125006 (2025)
# https://ui.adsabs.harvard.edu/abs/2025PASP..137l5006B/abstract

import numpy as np
import hispecdrp.primitives.ramp_fitting.fitramp_Brandt as fitramp
import hispecdrp.primitives.ramp_fitting.fitramp_cython_Brandt as fitramp_cython

import os
os.environ['OPENBLAS_NUM_THREADS'] = '1'
os.environ['MKL_NUM_THREADS'] = '1'
os.environ['NUMEXPR_NUM_THREADS'] = '1'


[docs] def gen_likelihood_map(diffs_lin, diffs_quad, covar, sigs, gains, alphas, countratesidentical=False, asig=0, cguess=None, d2use=None,readtime=None): """ Generate a 3D array of the log likelihood of the ramp fit marginalized over the count rates ramp-by-ramp (or, if the countratesidentical variable is set to True, assume that all ramps have the same count rate). Returns a 3D array of the size of len(sigs) x len(gains) x len(alphas), where sig is the read noise, gain is the gain in electrons/DN, and alpha is the nonlinearity coefficient. Chi squared is equal to this returned array times -2. Parameters ---------- diffs_lin : 2D array of floats Size nreads-1 x nramps. Differences between consecutive reads. diffs_quad : 2D array of floats Size nreads-1 x nramps. Differences between the squares of the values (from a designated pedestal) in consecutive reads covar : fitramp.Covar class Should be appropriate for diffs_lin sigs : list of floats Trial read noise values gains : list of floats Trial gain values alphas : list of floats Trial nonlinearity coefficient values asig : float, optional Standard deviation of the read noise from other constraints. If the best-fit read noise from this other constraint is sigs[1], asig will incorporate this constraint to the likelihood. countratesidentical : boolean, optional Assume that the count rate is the same for all ramps? Default False. cguess : 1D array of floats or None, optional Count rates for the purposes of computing a covariance matrix. Default None. If None, compute cguess from the nonlinearity-corrected ramps. d2use : 2D array of bools or None, optional Boolean mask of differences between reads to use for computing a likelihood. If None, use all read differences. Returns ------- logL : 3D array of floats log likelihood, shape (len(sigs), len(gains), len(alphas)) """ nsig = len(sigs) ngain = len(gains) nalph = len(alphas) logL = np.zeros((nsig, ngain, nalph)) # Normally, the lines below will not be used; these arrays # should be supplied as input to the routine. n0, n1 = diffs_lin.shape if d2use is None: diffs2use = np.ones((n0, n1), dtype=np.uint8) else: diffs2use = d2use # Estimate the covariance matrix inside this routine? This will # be done using the linearity-corrected ramps. if cguess is None: dynamic_cguess = True else: dynamic_cguess = False for k in range(nalph): diffs_nonlinscaled = diffs_lin + alphas[k]*diffs_quad if dynamic_cguess: cguess = np.sum(diffs_nonlinscaled*diffs2use, axis=0) cguess /= np.sum(diffs2use, axis=0) cguess *= cguess > 0 for j in range(ngain): for i in range(nsig): r = fitramp_cython.fit_ramps( diffs_nonlinscaled, diffs2use, covar.alpha_phnoise/gains[j], covar.beta_phnoise/gains[j], covar.alpha_readnoise, covar.beta_readnoise, sigs[i], cguess, n1, n0) logdetC, A, B, C, countrate, chisq, uncert = r # Can only use the lines below if we are *sure* that the count # rates are all the same. if countratesidentical: c = np.sum(countrate*C)/np.sum(C) chisqval = A - 2*c*B + c**2*C logsigval = -0.5*np.log(np.sum(C)) else: chisqval = chisq logsigval = fitramp_cython.logsum(uncert, n1) logL[i, j, k] = np.sum(-chisqval/2 - 0.5*logdetC) + logsigval # Incorporate other information about the read noise. if asig != 0: if i == 0 or i == 2: logL[i, j, k] += asig return logL
[docs] def fitquad(diffs_lin, diffs_quad, covar, sigs, gains, alphas=[0], asig=0, countratesidentical=False, cguess=None, d2use=None, log=True): """ Fit a quadratic form to the probability distributions from marginalizing the ramps over the count rates. This function will return the vertex of the fit, the covariance matrix implied by the coefficients, and the coefficients themselves. Parameters ---------- diffs_lin : 2D array of float Size (nreads-1, nramps). Differences between consecutive reads. diffs_quad : 2D array of float Size (nreads-1, nramps). Differences between the squares of the values (from a designated pedestal) in consecutive reads. covar : fitramp.Covar Should be appropriate for diffs_lin. sigs : list of float Trial read noise values. gains : list of float Trial gain values. alphas : list of float Trial nonlinearity coefficient values. asig : float, optional Standard deviation of the read noise from other constraints. If the best-fit read noise from this other constraint is sigs[1], asig will incorporate this constraint into the likelihood. countratesidentical : bool, optional Assume that the count rate is the same for all ramps. Default is False. cguess : 1D array of float or None, optional Count rates for computing a covariance matrix. Default is None. If None, compute cguess from the nonlinearity-corrected ramps. d2use : 2D array of bool or None, optional Boolean mask of differences between reads to use for computing a likelihood. If None, use all read differences. log : bool, optional Use logarithmic coordinates for the quadratic form. Default is True. Returns ------- center : tuple of float Center of the quadratic form in gain, read noise, nonlinearity. The coordinates are given in log(gain) and log(readnoise) if log is True. cov : 2D array of float Covariance matrix of the log likelihood as estimated by the quadratic form fit. c_2d : list of float Coefficients describing the quadratic form. """ logL = gen_likelihood_map(diffs_lin, diffs_quad, covar, sigs, gains, alphas, asig=asig, countratesidentical=countratesidentical, cguess=cguess, d2use=d2use) arr = np.ones((len(sigs)*len(gains)*len(alphas), 10)) if log: x, y, z = np.meshgrid(np.log(gains), np.log(sigs), alphas) else: x, y, z = np.meshgrid(gains, sigs, alphas) x = x.flatten() y = y.flatten() z = z.flatten() arr[:, 1] = x arr[:, 2] = y arr[:, 3] = z arr[:, 4] = x*y arr[:, 5] = x*z arr[:, 6] = y*z arr[:, 7] = x**2 arr[:, 8] = y**2 arr[:, 9] = z**2 c = np.linalg.lstsq(arr, logL.flatten(), rcond=None)[0] # the covariance matrix interpreting the quadratic form as a log likelihood Cinv = -np.array([[2*c[7], c[4], c[5]], [c[4], 2*c[8], c[6]], [c[5], c[6], 2*c[9]]]) # We might be fitting a quadratic form in 3d; we might be fitting one # in 2d. if len(alphas) > 1: cov = np.linalg.inv(Cinv) # the vertex of the quadratic form xc, yc, zc = np.linalg.solve(Cinv, np.array([c[1], c[2], c[3]])) return (xc, yc, zc), cov, c else: cov = np.linalg.inv(Cinv[:2, :2]) # the vertex of the quadratic form xc, yc = np.linalg.solve(Cinv[:2, :2], np.array([c[1], c[2]])) c_2d = np.array([c[0], c[1], c[2], c[4], c[7], c[8]]) return (xc, yc), cov, c_2d
[docs] def get_best_nonlinearity(diffs_lin, diffs_quad, covar, sigs, gains, alpha_offset, dalpha, asig, cguess=None, d2use=None): """ Fit a quadratic form for three values of the nonlinearity coefficient, and then use a parzabolic fit to find the highest gain and the nonlinearity coefficient corresponding to this highest gain. Parameters ---------- diffs_lin : 2D array of float Size (nreads-1, nramps). Differences between consecutive reads. diffs_quad : 2D array of float Size (nreads-1, nramps). Differences between the squares of the values (from a designated pedestal) in consecutive reads. covar : fitramp.Covar Should be appropriate for diffs_lin. sigs : list of float Trial read noise values. gains : list of float Trial gain values. alpha_offset : float Trial nonlinearity coefficient value. dalpha : float Offset in nonlinearity coefficient to try to fit a parabola to maximize the gain. asig : float, optional Standard deviation of the read noise from other constraints. If the best-fit read noise from this other constraint is sigs[1], asig will incorporate this constraint into the likelihood. cguess : 1D array of float or None, optional Count rates for computing a covariance matrix. Default is None. If None, compute cguess from the nonlinearity-corrected ramps. d2use : 2D array of bool or None, optional Boolean mask of differences between reads to use for computing a likelihood. If None, use all read differences. Returns ------- bestloggain : float Best-fit log gain. alphabest : float Best nonlinearity coefficient (the one that yields the highest best-fit gain). alpha_inc : float Offset between alphabest and the input alpha_offset. Can be used to assess convergence. """ alphalist = [alpha_offset + i*dalpha for i in [-1, 0, 1]] xclist = [] for alpha in alphalist: (xc, yc), cov, _ = fitquad( diffs_lin, diffs_quad, covar, sigs, gains, alphas=[alpha], asig=asig, cguess=cguess, d2use=d2use) xclist += [xc*1.] a = 0.5*(xclist[2] + xclist[0] - 2*xclist[1]) b = 0.5*(xclist[2] - xclist[0]) c = xclist[1] alphabest = alphalist[1] - dalpha*b/(2*a) alpha_inc = -dalpha*b/(2*a) bestloggain = c - b**2/(4*a) return bestloggain, alphabest, alpha_inc
[docs] def fit_gain_linearity(ramps_dark, ramps_illum, fit_nonlin=True, alpha_offset=2e-6, dalpha=2e-6, dalpha2=5e-7, gainguess=1.85, maxiter_nonlin=10, snrscalefac=1): """ Main function: fit gain, read noise, nonlinearity. Parameters ---------- diffs_lin : 2D array of floats Size nreads-1 x nramps. Differences between consecutive reads. diffs_quad : 2D array of floats Size nreads-1 x nramps. Differences between the squares of the values (from a designated pedestal) in consecutive reads covar : fitramp.Covar class Should be appropriate for diffs_lin sigs : list of floats Trial read noise values gains : list of floats Trial gain values alphas : list of floats Trial nonlinearity coefficient values asig : float, optional Standard deviation of the read noise from other constraints. If the best-fit read noise from this other constraint is sigs[1], asig will incorporate this constraint to the likelihood. countratesidentical : boolean, optional Assume that the count rate is the same for all ramps? Default False. cguess : 1D array of floats or None, optional Count rates for the purposes of computing a covariance matrix. Default None. If None, compute cguess from the nonlinearity-corrected ramps. d2use : 2D array of bools or None, optional Boolean mask of differences between reads to use for computing a likelihood. If None, use all read differences. log : bool, optional Use logarithmic coordinates for the quadratic form? Default True. Returns ------- center : tuple of two or three floats Center of the quadratic form in gain, read noise, nonlinearity. The coordinates will be given in log(gain) and log(readnoise) if the argument log is True. cov : 2D array of floats The covariance matrix of the log likelihood as estimated by the quadratic form fit c_2d : list of floats The coefficients describing the quadratic form """ Nres = ramps_dark.shape[0] Nramps = ramps_dark.shape[1] + ramps_illum.shape[1] readtimes_dark = np.arange(ramps_dark.shape[0]) + 1 readtimes_illum = np.arange(ramps_illum.shape[0]) + 1 covar_dark = fitramp.Covar(readtimes_dark) covar_illum = fitramp.Covar(readtimes_illum) # Anchor the nonlinearity correction to the average pixel value # of the dark ramps. y0 = np.median(ramps_dark) darkdiffs_lin = np.diff(ramps_dark, axis=0) darkdiffs_quad = np.diff((ramps_dark - y0)**2, axis=0) # Initial estimate of the read noise sig_rn = np.median(np.std(darkdiffs_lin/np.sqrt(2), axis=0, ddof=1)) d2u_dark = np.abs(darkdiffs_lin) < 6*sig_rn d2u_dark = d2u_dark.astype(np.uint8) # Should raise a ValueError here. if sig_rn == 0: raise ValueError("Zero dispersion in dark ramps implies no read noise.") # This parameter controls the spacing of the samples when fitting # a quadratic form. snrscale = snrscalefac*np.sqrt(75/(Nramps*Nres)) # Run this twice to get the best read noise: first with a larger # interval, then with a smaller interval around the best value # from the first iteration. cen = 0 for fac in [1, 0.2]: sigs = sig_rn*np.exp((np.arange(-1, 2)*fac + cen)*snrscale) # Unit gain, zero nonlinearity for the darks. gains = [1] alphas = [0] logL = gen_likelihood_map(darkdiffs_lin, darkdiffs_quad, covar_dark, sigs, gains, alphas, d2use=d2u_dark) logL = logL.flatten() b = 0.5*(logL[2] - logL[0]) # asig will combine the likelihood from the read noise derived # from the dark frames with the likelihood derived from the # illuminated frames. asig = 0.5*(logL[2] + logL[0] - 2*logL[1]) # Displacement (in units of dsig) of the new best-fit read noise cen = -b/(2*asig) # The rest of the calculations will use this array of read noise # values. The central value is the best-fit read noise from the # darks. sigs *= np.exp(cen*fac*snrscale) gains = gainguess*np.exp(np.arange(-1, 2)*snrscale) illumdiffs_lin = np.diff(ramps_illum, axis=0) illumdiffs_quad = np.diff((ramps_illum - y0)**2, axis=0) # Mask illum read differences with more than 10sigma discrepancies # The 10 sigma threshold accounts for a poorly estimated gain, # where the actual gain is significantly lower than estimated. # Need to estimate count rates first. ctrates = np.median(illumdiffs_lin, axis=0) ctrates *= ctrates > 0 totalnoise = np.sqrt(sig_rn**2 + ctrates/gainguess) d2u_illum = np.abs(illumdiffs_lin - ctrates) < 10*totalnoise d2u_illum = d2u_illum.astype(np.uint8) # If we are not fitting for any nonlinearity, we can take a shortcut. if not fit_nonlin: (xc, yc), cov, _ = fitquad( illumdiffs_lin, illumdiffs_quad, covar_illum, sigs, gains, d2use=d2u_illum) return xc, yc, 0, cov # If we are fitting for nonlinearity, we have a bit more work to do. xcbest, alphabest, alpha_inc = get_best_nonlinearity( illumdiffs_lin, illumdiffs_quad, covar_illum, sigs, gains, alpha_offset, dalpha, asig, d2use=d2u_illum) # Update the array of trial gains. The 0.6 is hardcoded below-- # perhaps make this tunable by the user? gains = np.exp(np.arange(-1, 2)*snrscale + xcbest) # Iterate to find the nonlinearity correction that gives # the highest best-fit gain. nonlin_ok = False for k in range(maxiter_nonlin): xcbest, alphabest, alpha_inc = get_best_nonlinearity( illumdiffs_lin, illumdiffs_quad, covar_illum, sigs, gains, alphabest, dalpha2, asig, d2use=d2u_illum) # Continue for maxiter_nonlin iterations, or until our # new best-fit value is within the searched interval. if np.abs(alpha_inc/dalpha2) < 1: nonlin_ok = True break # Update the array of trial gains one more time. gains = np.exp(np.arange(-1, 2)*0.6*snrscale + xcbest) # One last quadratic form fit at the best nonlinearity correction (xc, yc), cov, _ = fitquad( illumdiffs_lin, illumdiffs_quad, covar_illum, sigs, gains, alphas=[alphabest], asig=asig, d2use=d2u_illum) # Apply a very small statistical correction to the gain xc -= np.sqrt(1/(Nramps*(Nres - 1))) zc = alphabest # Unconverged nonlinearity if not nonlin_ok: raise RuntimeWarning("Nonlinearity calculation did not converge.") if cov[0, 0] <= 0: raise RuntimeWarning("Invalid derived covariance for gain, nonlinearity.") return xc, yc, zc, cov