Source code for hispecdrp.primitives.nonlinear_correction.make_nonlin_primitive

import hispecdrp.primitives.nonlinear_correction.calc_nonlin_coefs_Brandt as nonlin_Brandt
from ..base_primitive import BasePrimitive
from ... import datamodels
from ...calibrations import GainSelector, ReadNoiseSelector, SaturationSelector, BiasSelector
import numpy as np
import matplotlib.pyplot as plt
from typing import Sequence
from ..utils.stats import get_err_from_posterior
from ...datamodels.dqflags import DQ_FLAGS
import logging

import multiprocessing as mp
import ctypes

logger = logging.getLogger(__name__)
from ...utils.parallelization_utils import _arraytonumpy, numba_thread_scope
from multiprocessing.sharedctypes import RawArray
from progressbar import progressbar


__all__ = ['MakeNonLinPrimitive','fit_nonlin_coefs']

SATURATED_FLAG_uint8 =  np.uint8(DQ_FLAGS["SATURATED"])
DO_NOT_USE_FLAG_uint8 = np.uint8(DQ_FLAGS["DO_NOT_USE"])
JUMP_DET_FLAG_uint8 = np.uint8(DQ_FLAGS["JUMP_DET"])

[docs] class MakeNonLinPrimitive(BasePrimitive): """ This primitive computes the coefficients of a polynomial nonlinearity correction model from raw up-the-ramp (UTR) detector data Thresholds are also computed to characterize the onset and maximum extent of correctable nonlinearity (eg, 1% and 10% deviation from linearity). The primitive uses the algorithm described in Brandt (2025; PASP, 137, 125005). The fitted model characterizes the deviation of the measured signal from an ideal linear ramp and is intended to be applied later by the nonlinearity correction step. No science data are modified by this primitive; instead, a nonlinearity calibration model is produced. Use saturated flat fields to derive the nonlinearity correction model. The non-linearity correction is assumed to follow: P(DN) = a0 + a1*DN + a2*DN^2 + ... + an*DN^n where P(DN) is the corrected signal, DN is the measured signal, and a0, a1, ..., an are the polynomial coefficients derived in this primitive. Note that the ordering of the coefficients in the calibration file is from lowest order to highest order, which is the reverse of the usual numpy.polyval ordering. Polynomial fits may be carried out either in the standard power basis or in a Legendre polynomial basis. Reads that exceed a specified fraction of the saturation level are excluded from the fit. The fitting may be performed on a per-pixel basis or using a single model for the full detector. Although the full detector mode is only for testing since there are too many pixels on a normal detector and the matrix becomes too large with so many ramps. Parameters ---------- input : list of str or list of RampModel List of input file paths or ramp models to process. max_cores : int, optional Maximum number of CPU cores to use for the fitting. Default is 1. mode : {'per_pixel', 'channel', 'full_detector'}, optional Mode used to fit the nonlinearity parameters. ``'per_pixel'`` fits an independent model for each pixel. ``'full_detector'`` fits a single model to the entire detector. ``'channel'`` is reserved for future implementation. use_legendre : bool, optional If True, perform the polynomial fit using a Legendre polynomial basis. If False, use the standard power basis. Default is True. order : int, optional Order of the polynomial used to model the detector nonlinearity. Default is 5. readstart : int, optional Index (zero-based) of the first read to include in the fit. Default is 0. satval_frac : float, optional Fraction of the saturation level beyond which reads are excluded from the fit. Default is 0.95. criterion_for_nonlin_thresh : float, optional Fractional deviation from a linear ramp used to define the nonlinearity threshold, above which data are considered non-linear but still correctable. Default is 0.01. criterion_for_nonlin_max : float, optional Fractional deviation from a linear ramp used to define the maximum nonlinearity, above which data are considered too non-linear to be corrected accurately. Default is 0.05. Calibrations ------------ gain : GainModel - selector: :class:`~hispecdrp.calibrations.selectors.GainSelector` rn : ReadNoiseModel - selector: :class:`~hispecdrp.calibrations.selectors.ReadNoiseSelector` saturation : SaturationModel - selector: :class:`~hispecdrp.calibrations.selectors.SaturationSelector` bias : BiasModel - selector: :class:`~hispecdrp.calibrations.selectors.BiasSelector` Returns ------- output : DataModel Nonlinearity correction calibration model containing the fitted polynomial coefficients and associated nonlinearity thresholds. References ---------- Brandt, T. D. (2025), *A Classic Nonlinearity Correction Algorithm for Detectors Read Out Up-the-ramp*, PASP, 137, 125005. https://ui.adsabs.harvard.edu/abs/2025PASP..137l5005B/abstract Roman calibration pipeline reference: https://github.com/spacetelescope/romancal Notes ----- Relevant HISPEC DRP requirement: REQ-3-DRP-183: The DRP shall correct for the detector non-linearity to better than 1% when deriving pixel fluxes. """ PARAMETERS = { 'output_model': 'NonlinearCorrectionModel', 'mode': 'per_pixel', # str, 'per_pixel' | 'channel' | 'full_detector', mode to fit the parameters. # 'per_pixel' compute gain for each pixel independently, # 'channel' (Not implemented yet) compute gain for each readout channel independently, # 'full_detector' compute gain for the full detector at once. This can be used when a single exposure is available. 'use_legendre': True, # fit in the Legendre polynomial basis? 'order' : 5, # int, order of polynomial to fit for the nonlinearity correction 'readstart': 0, # int, first read (indexed from zero) that we use 'satval_frac': 0.95, # float, fractional value of saturation beyond which reads are ignored. 'criterion_for_nonlin_thresh': 0.01, # float, criterion to define the nonlinearity threshold. # ie, if the data DN deviates from a linear ramp by more than this fraction, it is considered non-linear, but it can be corrected reasonably accurately. 'criterion_for_nonlin_max': 0.05, # float, criterion to define the nonlinearity maximum. # ie, if the data DN deviates from a linear ramp by more than this fraction, it is considered too non-linear and cannot be corrected accurately. } CALIBRATIONS = { 'gain': { 'selector': GainSelector, 'selector_kwargs': {} }, 'rn': { 'selector': ReadNoiseSelector, 'selector_kwargs': {} }, 'saturation': { 'selector': SaturationSelector, 'selector_kwargs': {} }, 'bias': { 'selector': BiasSelector, 'selector_kwargs': {} } } def _perform(self): if self.mode.lower() != 'full_detector' and self.mode.lower() != 'per_pixel': raise NotImplementedError(f"Only 'full_detector' and 'per_pixel' modes are implemented currently. Not {self.mode}") if self.mode.lower() == 'full_detector': logger.info(f"Running in full_detector mode. All detector pixels will be fitted jointly and a single gain and/or readnoise value will be returned for the entire chip.") elif self.mode.lower() == 'per_pixel': logger.info(f"Running in single_pixel mode. Each detector pixel will be fitted independently and a map of gain and/or readnoise will be returned.") if isinstance(self.dataset, (datamodels.DataModel, str)): _input_list_tmp = [self.dataset] if isinstance(self.dataset, Sequence): _input_list_tmp = self.dataset nfiles = len(_input_list_tmp) logger.info(f"Deriving the detector non linearity polynomial coefficients from a list of {nfiles} inputs.") input_models = [datamodels.open(f) for f in _input_list_tmp] datamodel0 = input_models[0] nreads,ny,nx = datamodel0.data.shape gain, _ = self.get_calibration(datamodel0, 'gain') gain_model = datamodels.open(gain) rn_cal, _ = self.get_calibration(datamodel0, 'rn') rn_model = datamodels.open(rn_cal) # Get the saturation threshold model sat, _ = self.get_calibration(datamodel0, 'saturation') saturation_model = datamodels.open(sat) # Get the bias model bias, _ = self.get_calibration(datamodel0, 'bias') bias_model = datamodels.open(bias) # Define output model from inputs output_nonlin = self.get_output_model(input_models) # Make all ramps from all files into a big cube ramp_arr = [] dq_raw_arr = [] for opened_input in input_models: if nreads != opened_input.data.shape[0]: raise ValueError("All input ramps must have the same number of reads.") ramp_arr.append(opened_input.data) dq_raw_arr.append(opened_input.dq_raw) ramp_arr = np.array(ramp_arr) dq_raw_arr = np.array(dq_raw_arr) satval = self.satval_frac*(saturation_model.sat_thresh-bias_model.bias) max_cores = self.max_cores or 1 _out = fit_nonlin_coefs( ramp_arr, dq_raw_arr, gain_model.gain, rn_model.RN, order=self.order, satval=satval, mode=self.mode, use_legendre=self.use_legendre, readstart=self.readstart, criterion_for_nonlin_thresh=self.criterion_for_nonlin_thresh, criterion_for_nonlin_max=self.criterion_for_nonlin_max, max_cores=max_cores ) nonlin_coeffs, nonlin_thresh,nonlin_max = _out # Output output_nonlin.coeffs = nonlin_coeffs output_nonlin.nonlin_thresh = nonlin_thresh # in DN output_nonlin.nonlin_max = nonlin_max # in DN output_nonlin.dq = np.zeros(gain_model.gain.shape, dtype=np.uint32) return dict(output=output_nonlin)
[docs] def fit_nonlin_coefs( ramps : np.ndarray, dq_raw : np.ndarray, gain: np.ndarray, rn : np.ndarray, order : int = 5, satval : np.ndarray = None, mode : str = "per_pixel", use_legendre: bool = 'quad', readstart : int = 0, criterion_for_nonlin_thresh : float = 0.01, criterion_for_nonlin_max : float = 0.1, max_cores : int = 1, ): """ Fit for the detector nonlinearity polynomial coefficients. Parameters ---------- ramps : np.ndarray 4D array of input rates. Size n_files x nreads x ny x nx. dq_raw : np.ndarray 4D array of input UTR data quality flags. Size n_files x nreads x ny x nx. gain : np.ndarray gain map (ny,nx) (e-/ADU). rn : np.ndarray read noise map (ny,nx) (in DN). order : int = 5 Order of polynomial to fit for the nonlinearity correction. satval : np.ndarray | None Map of values beyond which reads will be ignored in the fit (ny,nx) (in DN). Default is 64000 mode : str 'full_detector' to fit a single gain and read noise for the entire detector, 'per_pixel' to fit gain and read noise for each pixel independently. use_legendre : bool If True, fit in the Legendre polynomial basis. readstart : int First read (indexed from zero) that we use. criterion_for_nonlin_thresh : float Criterion to define the nonlinearity threshold. ie, if the data DN deviates from a linear ramp by more than this fraction, it is considered non-linear, but it can be corrected reasonably accurately. criterion_for_nonlin_max : float Criterion to define the nonlinearity maximum. ie, if the data DN deviates from a linear ramp by more than this fraction, it is considered too non-linear and cannot be corrected accurately. max_cores : int, optional Number of cores for parallelization. Default is 1 (no parallelization). Not used for mode == 'full_detector'. Returns ------- nonlin_coefs : np.ndarray Nonlinearity coefficients. Size (order+1, ny, nx) if mode is 'per_pixel', else (order+1,). nonlin_thresh : np.ndarray Nonlinearity threshold map (ny,nx) (in DN). nonlin_max : np.ndarray Nonlinearity maximum map (ny,nx) (in DN). """ n_files, n_reads, ny, nx = ramps.shape if satval is None: satval = np.full((ny,nx), 64000) if mode.lower() == 'full_detector': row_id = 0 ramp_row = np.array(ramps).swapaxes(1, 3) # n_files, ny, nx, n_reads dq_raw_row = np.array(dq_raw).swapaxes(1, 3) # n_files, ny, nx, n_reads ramp_row = ramp_row.reshape(n_files*ny*nx,n_reads)[:,:,np.newaxis] # shape (nfiles*ny*nx,n_reads,1) dq_raw_row = dq_raw_row.reshape(n_files*ny*nx,n_reads)[:,:,np.newaxis] # shape (nfiles*ny*nx,n_reads,1) gains_row = np.array([np.nanmedian(gain)]) rns_row = np.array([np.nanmedian(rn)]) satval_row = np.array([np.nanmedian(satval)]) args = row_id, ramp_row, dq_raw_row, gains_row, rns_row, satval_row, order, use_legendre, readstart, criterion_for_nonlin_thresh, criterion_for_nonlin_max _out = _fit_nonlin_coefs_row(args) _,nonlin_coeffs, nonlin_thresh,nonlin_max = _out elif mode.lower() == 'per_pixel': n_files, n_reads, ny,nx = ramps.shape _out = _fit_nonlin_coefs_per_pixel(ramps, dq_raw, gain, rn, order = order, satval = satval, use_legendre = use_legendre, readstart = readstart, criterion_for_nonlin_thresh = criterion_for_nonlin_thresh, criterion_for_nonlin_max = criterion_for_nonlin_max, max_cores =max_cores) nonlin_coeffs, nonlin_thresh,nonlin_max = _out return nonlin_coeffs, nonlin_thresh,nonlin_max
def _fit_nonlin_coefs_per_pixel( ramps : np.ndarray, dq_raw : np.ndarray, gain: np.ndarray, rn : np.ndarray, order : int = 5, satval : np.ndarray = None, use_legendre: bool = 'quad', readstart : int = 0, criterion_for_nonlin_thresh : float = 0.01, criterion_for_nonlin_max : float = 0.1, max_cores : int = 1, ): """ Fit for the nonlinearity polynomial coefficients on a per pixel basis. Parameters ---------- ramps : np.ndarray 4D array of input rates. Size n_files x nreads x ny x nx. dq_raw : np.ndarray 4D array of input UTR data quality flags. Size n_files x nreads x ny x nx. gain : np.ndarray gain map (ny,nx) (e-/ADU). rn : np.ndarray read noise map (ny,nx) (in DN). order : int = 5 Order of polynomial to fit for the nonlinearity correction. satval : np.ndarray | None Map of values beyond which reads will be ignored in the fit (ny,nx) (in DN). Default is 64000 use_legendre : bool If True, fit in the Legendre polynomial basis. readstart : int First read (indexed from zero) that we use. criterion_for_nonlin_thresh : float Criterion to define the nonlinearity threshold. ie, if the data DN deviates from a linear ramp by more than this fraction, it is considered non-linear, but it can be corrected reasonably accurately. criterion_for_nonlin_max : float Criterion to define the nonlinearity maximum. ie, if the data DN deviates from a linear ramp by more than this fraction, it is considered too non-linear and cannot be corrected accurately. max_cores : int, optional Number of cores for parallelization. Default is 1 (no parallelization). Returns ------- nonlin_coefs : np.ndarray Nonlinearity coefficients. Size (order+1, ny, nx). nonlin_thresh : np.ndarray Nonlinearity threshold map (ny,nx) (in DN). nonlin_max : np.ndarray Nonlinearity maximum map (ny,nx) (in DN). """ n_files, n_reads, ny, nx = ramps.shape if ramps.dtype is np.uint16: mp_data_type = ctypes.c_ushort else: mp_data_type = ctypes.c_float mp_float_type = ctypes.c_float mp_dq_raw_type = ctypes.c_uint8 mp_dq_type = ctypes.c_uint types_tuple = (mp_data_type, mp_float_type, mp_dq_raw_type, mp_dq_type) nonlin_coeffs_mp = RawArray(mp_float_type, (order+1)*nx * ny) nonlin_coeffs_shape = ((order+1), ny, nx) nonlin_coeffs_np = _arraytonumpy(nonlin_coeffs_mp, nonlin_coeffs_shape, dtype=mp_float_type) nonlin_coeffs_np[:] = np.zeros(nonlin_coeffs_shape, dtype=np.float32) nonlin_thresh_mp = RawArray(mp_float_type, nx * ny) nonlin_thresh_shape = (ny, nx) nonlin_thresh_np = _arraytonumpy(nonlin_thresh_mp, nonlin_thresh_shape, dtype=mp_float_type) nonlin_thresh_np[:] = np.zeros(nonlin_thresh_shape, dtype=np.float32) nonlin_max_mp = RawArray(mp_float_type, nx * ny) nonlin_max_shape = (ny, nx) nonlin_max_np = _arraytonumpy(nonlin_max_mp, nonlin_max_shape, dtype=mp_float_type) nonlin_max_np[:] = np.zeros(nonlin_max_shape, dtype=np.float32) gains_mp = RawArray(mp_float_type, nx * ny) gains_shape = (ny, nx) gains_np = _arraytonumpy(gains_mp, gains_shape, dtype=mp_float_type) gains_np[:] = gain.astype(np.float32) rns_mp = RawArray(mp_float_type, nx * ny) rns_shape = (ny, nx) rns_np = _arraytonumpy(rns_mp, rns_shape, dtype=mp_float_type) rns_np[:] = rn.astype(np.float32) satval_mp = RawArray(mp_float_type, nx * ny) satval_shape = (ny, nx) satval_np = _arraytonumpy(satval_mp, satval_shape, dtype=mp_float_type) satval_np[:] = satval.astype(np.float32) if max_cores > 1: ramps_mp = RawArray(mp_data_type, nx * ny * n_reads * n_files) ramps_shape = (n_files,n_reads, ny, nx) ramps_np = _arraytonumpy(ramps_mp, ramps_shape, dtype=mp_data_type) ramps_np[:] = ramps # copy original array into shared array ramps = ramps_np # free the original array from memory to avoid duplicates dq_raw_mp = RawArray(mp_dq_raw_type, nx * ny * n_reads * n_files) dq_raw_shape = (n_files,n_reads, ny, nx) dq_raw_np = _arraytonumpy(dq_raw_mp, dq_raw_shape, dtype=mp_dq_raw_type) if dq_raw is None: dq_raw_np[:] = np.zeros(dq_raw_np, dtype=np.uint8) else: dq_raw_np[:] = dq_raw # copy original array into shared array dq_raw = dq_raw_np # free the original array from memory to avoid duplicates # Prepare blocks block_size = max(1, ny // (4*max_cores)) if max_cores > 1 else ny _init_args = ( ramps_mp, ramps_shape, dq_raw_mp, dq_raw_shape, nonlin_coeffs_mp, nonlin_coeffs_shape, nonlin_thresh_mp, nonlin_thresh_shape, nonlin_max_mp, nonlin_max_shape, gains_mp, gains_shape, rns_mp, rns_shape, satval_mp, satval_shape, ) ctx = mp.get_context("spawn") # avoids unsafe fork after OpenMP init tpool = ctx.Pool( processes=max_cores, initializer=_tpool_init_fit_nonlin_coefs, initargs=_init_args, maxtasksperchild=50 ) args_list = [] for block_start in range(0, ny, block_size): block_end = min(block_start + block_size, ny) args = (block_start, block_end, order,use_legendre,readstart,criterion_for_nonlin_thresh,criterion_for_nonlin_max, types_tuple) args_list.append(args) try: with numba_thread_scope(max_cores): tasks = [ tpool.apply_async(_fit_nonlin_coefs_worker, args=(args_tuple,)) for args_tuple in args_list ] for t in progressbar(tasks, desc="Fitting Nonlinear coefficients", fast=False): t.wait() finally: tpool.close() tpool.join() else: # If not parallelizing, ie max_cores=1, just run directly without shared memory results = [] for row_id in np.arange(ny): print(f"Fitting detector gain and/or readnoise: Processing row {row_id + 1} of {ny}.") args = ( row_id, ramps[:, :, row_id, :], dq_raw[:, :, row_id, :], gains_np[row_id, :], rns_np[row_id, :], satval_np[row_id, :], order, use_legendre, readstart, criterion_for_nonlin_thresh, criterion_for_nonlin_max ) results.append(_fit_nonlin_coefs_row(args)) for row_element in results: row_id, nonlin_coeffs, nonlin_thresh,nonlin_max = row_element nonlin_coeffs_np[:,row_id, :] = nonlin_coeffs nonlin_thresh_np[row_id, :] = nonlin_thresh nonlin_max_np[row_id, :] = nonlin_max return nonlin_coeffs_np, nonlin_thresh_np,nonlin_max_np def _tpool_init_fit_nonlin_coefs( data : np.ndarray, data_shape : tuple, dq_raw : np.ndarray, dq_raw_shape : tuple, nonlin_coeffs : np.ndarray, nonlin_coeffs_shape : tuple, nonlin_thresh : np.ndarray, nonlin_thresh_shape : tuple, nonlin_max : np.ndarray, nonlin_max_shape : tuple, gains : np.ndarray, gains_shape : tuple, rns : np.ndarray, rns_shape : tuple, satval : np.ndarray, satval_shape : tuple, ): """ Initialize shared global variables for the multiprocessing pool. """ global shared_data, shared_data_shape, \ shared_dq_raw, shared_dq_raw_shape, \ shared_nonlin_coeffs, shared_nonlin_coeffs_shape, \ shared_nonlin_thresh, shared_nonlin_thresh_shape, \ shared_nonlin_max, shared_nonlin_max_shape, \ shared_gains, shared_gains_shape, \ shared_rns, shared_rns_shape, \ shared_satval, shared_satval_shape shared_data = data shared_data_shape = data_shape shared_dq_raw = dq_raw shared_dq_raw_shape = dq_raw_shape shared_nonlin_coeffs = nonlin_coeffs shared_nonlin_coeffs_shape = nonlin_coeffs_shape shared_nonlin_thresh = nonlin_thresh shared_nonlin_thresh_shape = nonlin_thresh_shape shared_nonlin_max = nonlin_max shared_nonlin_max_shape = nonlin_max_shape shared_gains = gains shared_gains_shape = gains_shape shared_rns = rns shared_rns_shape = rns_shape shared_satval = satval shared_satval_shape = satval_shape def _fit_nonlin_coefs_row(indata : tuple) -> tuple: """ Helper function for parallelization fitting the gain and readnoise per detector row. """ row_id, ramp_row, dq_raw_row, gains_row, rns_row,satval_row, order,use_legendre,readstart,criterion_for_nonlin_thresh,criterion_for_nonlin_max = indata n_files, n_reads, nx = ramp_row.shape # ramp_row should be (n_files, n_reads, nx) _ramps = ramp_row[:, readstart:, :] pedestal = np.nanmedian(_ramps[:,0,:],axis=0) n_files, n_reads,n_ramps = dq_raw_row.shape mask = ((dq_raw_row & DO_NOT_USE_FLAG_uint8) | (dq_raw_row & SATURATED_FLAG_uint8)).astype(np.bool) mask2 = (dq_raw_row & JUMP_DET_FLAG_uint8).astype(np.bool) diffs2use = ~(mask[:, 1:n_reads,:] | mask[:, 0:n_reads - 1,:] | mask2[:, 1:n_reads,:]) diffs2use = diffs2use.astype(np.uint8) res = nonlin_Brandt.get_nonlin_coefs(_ramps, rns_row, gains_row, order, pedestal,satval=satval_row, d2u_list=diffs2use,use_legendre=use_legendre) res = nonlin_Brandt.get_nonlin_coefs(_ramps, rns_row, gains_row, order, pedestal,satval=satval_row, d2u_list=diffs2use,use_legendre=use_legendre, last_pars=res) nonlin_coefs = res.nonlin_coefs.T # N_coeffs x nx nonlin_coefs =np.insert(nonlin_coefs, 0, np.zeros((nx,)),axis=0) # add zero-th order term # figure out nonlin_thresh,nonlin_max nonlin_thresh = _find_first_valid_root_batch(nonlin_coefs[::-1,:], criterion_for_nonlin_thresh,satval=satval_row) nonlin_max = _find_first_valid_root_batch(nonlin_coefs[::-1,:], criterion_for_nonlin_max,satval=satval_row) return (row_id, nonlin_coefs, nonlin_thresh,nonlin_max) def _fit_nonlin_coefs_worker(args : tuple) -> tuple: """ Worker function for multiprocessing. Runs _fit_nonlin_coefs_row on a block of rows using shared arrays. Parameters ---------- args : tuple Tuple containing: (block_start, block_end, order,use_legendre,readstart,criterion_for_nonlin_thresh,criterion_for_nonlin_max, types_tuple) Returns ------- block_start : int Starting index of the block of rows processed. block_end : int Ending index of the block of rows processed. """ block_start, block_end, order,use_legendre,readstart,criterion_for_nonlin_thresh,criterion_for_nonlin_max, types_tuple = args mp_data_type, mp_float_type, mp_dq_raw_type, mp_dq_type = types_tuple ramps_np = _arraytonumpy(shared_data, shared_data_shape, dtype=mp_data_type) dq_raw_np = _arraytonumpy(shared_dq_raw, shared_dq_raw_shape, dtype=mp_dq_raw_type) nonlin_coeffs_np = _arraytonumpy(shared_nonlin_coeffs, shared_nonlin_coeffs_shape, dtype=mp_float_type) nonlin_thresh_np = _arraytonumpy(shared_nonlin_thresh, shared_nonlin_thresh_shape, dtype=mp_float_type) nonlin_max_np = _arraytonumpy(shared_nonlin_max, shared_nonlin_max_shape, dtype=mp_float_type) gains_np = _arraytonumpy(shared_gains, shared_gains_shape, dtype=mp_float_type) rns_np = _arraytonumpy(shared_rns, shared_rns_shape, dtype=mp_float_type) satval_np = _arraytonumpy(shared_satval, shared_satval_shape, dtype=mp_float_type) results = [] for row_id in range(block_start, block_end): args_row = ( row_id, ramps_np[:, :, row_id, :], dq_raw_np[:, :, row_id, :], gains_np[row_id, :], rns_np[row_id, :], satval_np[row_id, :], order, use_legendre, readstart, criterion_for_nonlin_thresh, criterion_for_nonlin_max ) results.append(_fit_nonlin_coefs_row(args_row)) for row_element in results: row_id, nonlin_coeffs, nonlin_thresh,nonlin_max = row_element nonlin_coeffs_np[:,row_id, :] = nonlin_coeffs nonlin_thresh_np[row_id, :] = nonlin_thresh nonlin_max_np[row_id, :] = nonlin_max return (block_start, block_end) def _find_first_valid_root_batch( nonlin_correction_coefs, my_threshold=0.01, satval=np.iinfo(np.uint16).max, min_root=1, atol=1e-10, ): """ Batch-solve for the smallest admissible real root x, independently for each column ix, where the polynomial P_ix(x) is defined by coefficients in ``nonlin_correction_coefs[:, ix]`` (NumPy / ``np.polyval`` convention: highest degree first), and x satisfies: (P_ix(x) - x) / x = my_threshold This is equivalent to solving, for each ix: P_ix(x) - (1 + my_threshold) * x = 0 and selecting the smallest real root in (min_root[ix], satval[ix]]. Parameters ---------- nonlin_correction_coefs : array_like 2D array of shape (n_coeffs, nx). Column ix contains the polynomial coefficients for P_ix(x), ordered from highest degree to constant term. satval : float or array_like, optional Upper bound(s) on acceptable roots. Either a scalar or a 1D array of length nx. Default is max uint16. min_root : float or array_like, optional Strict lower bound(s) on acceptable roots. Either a scalar or a 1D array of length nx. Default is 1. my_threshold : float, optional Threshold T in the equation (P(x) - x)/x = T. Default is 0.01. atol : float, optional Absolute tolerance for deciding whether a root is (numerically) real. Default is 1e-10. Returns ------- root0 : ndarray 1D float array of shape (nx,). Each element is the smallest admissible real root for that column, or NaN if no admissible root exists (or if the polynomial is invalid). Notes ----- - Uses ``np.roots`` per column; there is no true vectorized root-finding in NumPy. - If the polynomial degree is < 1 (i.e., no linear term), that column returns NaN. """ coefs = np.asarray(nonlin_correction_coefs, dtype=float) if coefs.ndim != 2: raise ValueError("nonlin_correction_coefs must be 2D with shape (n_coeffs, nx).") n_coeffs, nx = coefs.shape if n_coeffs < 2: # No linear term exists, cannot subtract (1+T)*x robustly return np.full(nx, np.nan, dtype=float) # Broadcast satval/min_root to per-column arrays sat_arr = np.asarray(satval, dtype=float) min_arr = np.asarray(min_root, dtype=float) if sat_arr.ndim == 0: sat_arr = np.full(nx, float(sat_arr), dtype=float) if min_arr.ndim == 0: min_arr = np.full(nx, float(min_arr), dtype=float) if sat_arr.shape != (nx,): raise ValueError(f"satval must be a scalar or shape (nx,), got {sat_arr.shape}.") if min_arr.shape != (nx,): raise ValueError(f"min_root must be a scalar or shape (nx,), got {min_arr.shape}.") out = np.full(nx, np.nan, dtype=float) for ix in range(nx): # Build Q(x) = P(x) - (1+T)*x tmp = coefs[:, ix].copy() tmp[-2] -= (1.0 + my_threshold) # If leading coefficients are zero, np.roots can behave poorly; trim them # (while preserving at least degree-1 if possible). nz = np.flatnonzero(tmp) if nz.size == 0: continue first_nz = nz[0] tmp_trim = tmp[first_nz:] # Need at least a linear polynomial after trimming if tmp_trim.size < 2: continue roots = np.roots(tmp_trim) # Keep (numerically) real roots real_roots = roots[np.isclose(roots.imag, 0.0, atol=atol)].real if real_roots.size == 0: continue # Enforce bounds: (min_root, satval] valid = real_roots[(real_roots > min_arr[ix]) & (real_roots <= sat_arr[ix])] if valid.size == 0: continue out[ix] = np.min(valid) return out