import logging
import multiprocessing as mp
import ctypes
from contextlib import contextmanager
import numpy as np
from stcal.ramp_fitting import ramp_fit
from stcal.ramp_fitting.ramp_fit_class import RampData
from multiprocessing.sharedctypes import RawArray
from tqdm import tqdm
from ...utils.parallelization_utils import _arraytonumpy, numba_thread_scope
from ...datamodels.dqflags import DQ_FLAGS
from .utils import get_mid_delta_times, SECONDS_PER_DAY
from ..jump_detection.jump_detection_jwst import _jump_detection_jwst_row
import hispecdrp.primitives.ramp_fitting.fitramp_Brandt as fitramp_Brandt
import hispecdrp.primitives.ramp_fitting.fitramp_cython_Brandt as fitramp_cython
from progressbar import progressbar
logger = logging.getLogger(__name__)
__all__ = ['fit_ramps_jwst']
@contextmanager
def _stcal_likely_fit_allow_negative_rates():
"""
Allow fitted / interim count rates to be negative in ``stcal`` likely_fit.
Photon-noise covariance still uses ``max(rate, 0)`` (variance cannot be
negative). The previous ``rate *= (rate > 0)`` pattern zeroed low-SNR /
negative guesses and made empty background look like a hard floor at 0
instead of a Gaussian around the true rate.
"""
import stcal.ramp_fitting.likely_fit as lf
orig_init = lf.initial_count_rate_guess
orig_alphas = lf.compute_alphas_betas
orig_fit_ramps = lf.fit_ramps
def initial_count_rate_guess(diffs, diffs2use):
num = np.sum((diffs * diffs2use), axis=0)
den = np.sum(diffs2use, axis=0)
with np.errstate(invalid="ignore", divide="ignore"):
return num / den
def compute_alphas_betas(count_rate_guess, gain, rnoise, covar, rescale, diffs, dn_scale):
guess = np.maximum(np.asarray(count_rate_guess, dtype=np.float64), 0.0)
return orig_alphas(guess, gain, rnoise, covar, rescale, diffs, dn_scale)
def fit_ramps(*args, count_rate_guess=None, **kwargs):
# Callers (jump search / final fit) may pass clipped guesses; restore a
# non-negative *covariance* rate without forcing the solved countrate ≥ 0.
if count_rate_guess is not None:
count_rate_guess = np.maximum(np.asarray(count_rate_guess, dtype=np.float64), 0.0)
return orig_fit_ramps(*args, count_rate_guess=count_rate_guess, **kwargs)
lf.initial_count_rate_guess = initial_count_rate_guess
lf.compute_alphas_betas = compute_alphas_betas
lf.fit_ramps = fit_ramps
try:
yield
finally:
lf.initial_count_rate_guess = orig_init
lf.compute_alphas_betas = orig_alphas
lf.fit_ramps = orig_fit_ramps
SATURATED_FLAG = DQ_FLAGS["SATURATED"]
DO_NOT_USE_FLAG = DQ_FLAGS["DO_NOT_USE"]
JUMP_DET_FLAG = DQ_FLAGS["JUMP_DET"]
SATURATED_FLAG_uint8 = np.uint8(DQ_FLAGS["SATURATED"])
DO_NOT_USE_FLAG_uint8 = np.uint8(DQ_FLAGS["DO_NOT_USE"])
JUMP_DET_FLAG_uint8 = np.uint8(DQ_FLAGS["JUMP_DET"])
def _fit_ramps_cython_row(indata: tuple) -> tuple:
"""
Passes data for each row of a ramp to Tim Brandt's likelihood-based ramp fitting algorithm in Cython.
Parameters
----------
indata : Tuple[int, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, str]
Tuple containing the following elements:
ramp row index (int)
read times (np.ndarray)
ramp data (np.ndarray)
readnoise (np.ndarray)
gain (np.ndarray)
DQ cube (np.ndarray)
DQ image (np.ndarray)
dark current (np.ndarray)
algorithm (str)
Returns
-------
i : int
Index of input row from the ramps, same as input.
slope : np.ndarray
Slope image (1, nx).
dq_out : np.ndarray
Output data quality image with flags from algorithm (1, nx).
var_poisson : np.ndarray
Output variance from poisson noise (1, nx).
var_rnoise : np.ndarray
Output variance from read noise (1, nx).
err : np.ndarray
Output error in slope image (1, nx).
"""
i, readtimes, data_3d, readnoise_2d, gain_2d, groupdq, pixeldq_2d, dark_current_2d, algorithm = indata
# We are implicitly fitting by row, so columns is only other axis and nrows=1
nints, n_reads, ny, nx = data_3d.shape
_ramps = data_3d[0, :, 0,:]
_dq_raw = groupdq[0, :, 0,:]
y0 = 0
diff_times = np.diff(readtimes)[:, None]
diffs = np.diff(_ramps, axis=0).astype(np.float64) / diff_times
n0, n1 = diffs.shape
n_reads,n_ramps = _dq_raw.shape
mask = ((_dq_raw & DO_NOT_USE_FLAG_uint8) | (_dq_raw & SATURATED_FLAG_uint8)).astype(np.bool)
mask2 = (_dq_raw & JUMP_DET_FLAG_uint8).astype(np.bool)
diffs2use = ~(mask[1:n_reads] | mask[0:n_reads - 1] | mask2[1:n_reads])
diffs2use = diffs2use.astype(np.uint8)
# todo: try to put covar in shared memory?
covar = fitramp_Brandt.Covar(readtimes.astype(np.float64))
# Mean difference as count-rate guess. Do NOT floor negatives to 0 — empty /
# low-SNR pixels must keep a signed guess so the solved rate is Gaussian
# about the true background (including negative rates). Photon noise in the
# Cython kernel uses max(cguess, 0) via the covariance construction below.
with np.errstate(invalid="ignore", divide="ignore"):
cguess = np.sum(diffs * diffs2use, axis=0) / np.sum(diffs2use, axis=0)
cguess_phnoise = np.maximum(cguess, 0.0)
cython_out = fitramp_cython.fit_ramps_row(
diffs, diffs2use,
covar.alpha_phnoise, covar.beta_phnoise,
covar.alpha_readnoise, covar.beta_readnoise,
readnoise_2d[0,:].astype(np.float64),gain_2d[0,:].astype(np.float64), cguess_phnoise, n1, n0)
if len(cython_out) >= 9:
(
logdetC_np,
A_np,
B_np,
C_np,
countrate_np,
chisq_np,
uncert_np,
var_poisson_np,
var_rnoise_np,
) = cython_out[:9]
var_poisson = np.asarray(var_poisson_np, dtype=np.float64)
var_rnoise = np.asarray(var_rnoise_np, dtype=np.float64)
uncert = np.asarray(uncert_np, dtype=np.float64)
# Prefer the GLS total when the split is incomplete / non-finite.
total = np.square(uncert)
split = var_poisson + var_rnoise
bad = ~(np.isfinite(split) & (split >= 0.0)) | (split <= 0.0)
# If split is usable, rebuild err from the split (stcal-like); else keep uncert.
err = np.where(bad, uncert, np.sqrt(np.maximum(split, 0.0)))
var_poisson = np.where(bad, 0.0, var_poisson)
var_rnoise = np.where(bad, total, var_rnoise)
else:
logdetC_np, A_np, B_np, C_np, countrate_np, chisq_np, uncert_np = cython_out[:7]
# Legacy cython: total uncertainty only — park it in var_rnoise.
uncert = np.asarray(uncert_np, dtype=np.float64)
var_rnoise = np.square(uncert)
var_poisson = np.zeros_like(var_rnoise, dtype=np.float64)
err = uncert
dq = np.zeros((1, nx), dtype=np.uint32)
return (i, countrate_np, dq, var_poisson, var_rnoise, err)
def _fit_ramps_jwst_row(indata : tuple) -> tuple:
"""
Passes data for each row of a ramp to stcal.ramp_fitting.likely_fit algorithms.
Parameters
----------
indata : Tuple[int, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, str]
Tuple containing the following elements:
ramp row index (int)
read times (np.ndarray)
ramp data (np.ndarray)
readnoise (np.ndarray)
gain (np.ndarray)
DQ cube (np.ndarray)
DQ image (np.ndarray)
dark current (np.ndarray)
algorithm (str)
Returns
-------
i : int
Index of input row from the ramps, same as input.
slope : np.ndarray
Slope image (1, nx).
dq_out : np.ndarray
Output data quality image with flags from algorithm (1, nx).
var_poisson : np.ndarray
Output variance from poisson noise (1, nx).
var_rnoise : np.ndarray
Output variance from read noise (1, nx).
err : np.ndarray
Output error in slope image (1, nx).
"""
i, times, data_3d, readnoise_2d, gain_2d, groupdq, pixeldq_2d, dark_current_2d, algorithm = indata
# We are implicitly fitting by row, so columns is only other axis and nrows=1
nints, nframes, ny, nx = data_3d.shape
# Set static parameters for stcal.ramp_fitting algorithms
save_opt = False # Save optional output
weighting = "optimal"
suppress_one_group = False # Suppress saturated ramps with good 0th group
# Initialize ramp data model
ramp_data = RampData()
# Explicitly set flags required by JWST pipeline
ramp_data.flags_do_not_use = DQ_FLAGS['DO_NOT_USE']
ramp_data.flags_jump_det = DQ_FLAGS['JUMP_DET']
ramp_data.flags_saturated = DQ_FLAGS['SATURATED']
ramp_data.flags_no_gain_val = DQ_FLAGS['UNRELIABLE_FLAT'] # Check if this is best/optimal flag
ramp_data.flags_unreliable_slope = DQ_FLAGS['UNRELIABLE_SLOPE']
ramp_data.flags_chargeloss = DQ_FLAGS['OUTLIER'] # Check if this is best/optimal flag - only required by FIXSEN_C
# Explicitly set data arrays
ramp_data.data = data_3d
ramp_data.groupdq = groupdq
ramp_data.pixeldq = pixeldq_2d
ramp_data.average_dark_current = dark_current_2d
# Get average exposure time
diffs = [times[i+1] - times[i] for i in range(len(times)-1)]
group_time = np.average(diffs)
# Set other required parameters
ramp_data.algorithm = algorithm
ramp_data.start_row = 0
ramp_data.num_rows = ny
ramp_data.nframes = 1 # this corresponds to nints instead of nframes as above
# Required by FIXSEN_C for frame time
ramp_data.frame_time = 0.0
ramp_data.group_time = group_time
ramp_data.groupgap = 0
ramp_data.drop_frames1 = 0
ramp_data.suppress_one_group_ramps = suppress_one_group
# Times information is set as read_pattern parameter of RampData
ramp_data.read_pattern = list(times)
#todo: we are disabling the jump detection for now, because we cannot get the group DQ flags to return from the function.
# and we might need it to estimate the mid_time correctly.
ramp_data.rejection_threshold = np.inf
# Run ramp fitting
if algorithm == 'LIKELY':
if nframes < ramp_fit.likely_fit.LIKELY_MIN_NGROUPS:
raise Exception(f'Input ramps have {nframes} frames which is less than LIKELY algorithm requirement of {ramp_fit.likely_fit.LIKELY_MIN_NGROUPS} frames')
# stcal.likely_fit divides readnoise by √2 because JWST READNOISE
# reference files store CDS (pairwise-difference) noise
# (σ_CDS = √2 σ_single). HISPEC RN calibrations and create_ramp use
# single-read σ, so convert to CDS units before calling stcal; otherwise
# rate_var_rnoise is under-reported by exactly 2×.
readnoise_for_stcal = np.asarray(readnoise_2d, dtype=np.float64) * np.sqrt(2.0)
# Call directly to likely ramp fit code (allow signed / low-SNR rates)
with _stcal_likely_fit_allow_negative_rates():
image_info, integ_info, opt_info = ramp_fit.likely_fit.likely_ramp_fit(
ramp_data, readnoise_for_stcal, gain_2d
)
elif algorithm == 'FIXSEN_C':
# Call to C extension for FIXSEN slope fitting
image_info, _, _ = ramp_fit.ols_fit.ols_slope_fitter(
ramp_data, gain_2d, readnoise_2d, weighting, save_opt)
# Parse output image info that we need to pass to output data model
outdata, dq, var_poisson, var_rnoise, err = image_info
return (i, outdata, dq, var_poisson, var_rnoise, err)
def _fit_ramps_worker(args : tuple) -> tuple:
"""
Worker function for multiprocessing. Runs _fit_ramps_jwst_row on a block of rows using shared arrays.
Parameters
----------
args : tuple
(block_start, block_end, times, algorithm, types_tuple)
Returns
-------
block_start : int
Starting index of the block of rows processed.
block_end : int
Ending index of the block of rows processed.
results : list
List of results from processing each row in the block.
"""
block_start, block_end, times, algorithm, types_tuple = args
mp_data_type, mp_float_type, mp_dq_raw_type, mp_dq_type = types_tuple
ramps_np = _arraytonumpy(shared_data, shared_data_shape, dtype=mp_data_type)
dq_raw_np = _arraytonumpy(shared_dq_raw, shared_dq_raw_shape, dtype=mp_dq_raw_type)
dq_image_np = _arraytonumpy(shared_dq_image, shared_dq_image_shape, dtype=mp_dq_type)
gain_np = _arraytonumpy(shared_gain, shared_gain_shape, dtype=mp_float_type)
dark_np = _arraytonumpy(shared_dark, shared_dark_shape, dtype=mp_float_type)
rdnoise_np = _arraytonumpy(shared_read_noise, shared_read_noise_shape, dtype=mp_float_type)
start_times_np = _arraytonumpy(shared_start_times, shared_start_times_shape, dtype=mp_float_type)
slope_image_np = _arraytonumpy(shared_slope_image, shared_slope_image_shape, dtype=mp_float_type)
slope_errors_np = _arraytonumpy(shared_slope_errors, shared_slope_errors_shape, dtype=mp_float_type)
var_poisson_image_np = _arraytonumpy(shared_var_poisson_image, shared_var_poisson_image_shape, dtype=mp_float_type)
var_rnoise_image_np = _arraytonumpy(shared_var_rnoise_image, shared_var_rnoise_image_shape, dtype=mp_float_type)
mid_times_np = _arraytonumpy(shared_mid_times, shared_mid_times_shape, dtype=mp_float_type)
results = []
for row in range(block_start, block_end):
args_row = (
row,
times,
ramps_np[np.newaxis,:, [row], :],
rdnoise_np[[row], :],
gain_np[[row], :],
dq_raw_np[np.newaxis,:, [row], :],
dq_image_np[[row], :],
dark_np[[row], :],
algorithm
)
if algorithm == 'CYTHON_LIKELY':
results.append(_fit_ramps_cython_row(args_row))
else:
results.append(_fit_ramps_jwst_row(args_row))
for row_element in results:
row, outdata, dq, var_poisson, var_rnoise, err = row_element
slope_image_np[row, :] = outdata
slope_errors_np[row, :] = err
dq_image_np[row, :] = dq_image_np[row, :] | dq[0]
var_poisson_image_np[row, :] = var_poisson
var_rnoise_image_np[row, :] = var_rnoise
mid_times_np[block_start:block_end, :] = start_times_np[block_start:block_end, :] +\
get_mid_delta_times(times, dq_raw_np[:, block_start:block_end, :]) / SECONDS_PER_DAY
_dq_raw = dq_raw_np[:, block_start:block_end, :] - (dq_raw_np[:, block_start:block_end, :] & DO_NOT_USE_FLAG_uint8)
dq_image_np[block_start:block_end, :] |= np.bitwise_or.reduce(_dq_raw)
return (block_start, block_end)
def _jump_detection_worker(args : tuple) -> tuple:
"""
Worker function for multiprocessing. Runs _jump_detection_worker on a block of rows using shared arrays.
Parameters
----------
args : tuple
Tuple containing: (block_start, block_end, times, algorithm, types_tuple)
Returns
-------
tuple
(block_start, block_end)
"""
block_start, block_end, times, rejection_threshold, types_tuple = args
mp_data_type, mp_float_type, mp_dq_raw_type, mp_dq_type = types_tuple
ramps_np = _arraytonumpy(shared_data, shared_data_shape, dtype=mp_data_type)
dq_raw_np = _arraytonumpy(shared_dq_raw, shared_dq_raw_shape, dtype=mp_dq_raw_type)
dq_image_np = _arraytonumpy(shared_dq_image, shared_dq_image_shape, dtype=mp_dq_type)
gain_np = _arraytonumpy(shared_gain, shared_gain_shape, dtype=mp_float_type)
rdnoise_np = _arraytonumpy(shared_read_noise, shared_read_noise_shape, dtype=mp_float_type)
results = []
for row in range(block_start, block_end):
args_row = (
row,
rejection_threshold,
ramps_np[np.newaxis, :, [row], :],
# Make sure to add a new axis to each array to maintain number of dimensions
rdnoise_np[[row], :],
gain_np[[row], :],
dq_raw_np[np.newaxis, :, [row], :],
dq_image_np[[row], :]
)
results.append(_jump_detection_jwst_row(args_row))
for row_element in results:
_row, _dq = row_element
dq_raw_np[:, [_row], :] = _dq
return (block_start, block_end)
[docs]
def fit_ramps_jwst(
times : np.ndarray,
ramps : np.ndarray,
dq_raw : np.ndarray = None,
dq : np.ndarray = None,
rdnoise : np.ndarray = None,
gain : np.ndarray = None,
dark_current : np.ndarray | None = None,
start_times : np.ndarray | None = None,
jump_detection : bool = False,
jump_rejection_threshold : float = 4.0,
max_cores : int = 1,
algorithm : str = 'JWST_LIKELY',
) -> dict:
"""
Fit ramps using JWST ramp fitting algorithms from stcal.ramp_fitting.
Parameters
----------
times : np.ndarray
Read times.
ramps : np.ndarray
3D array of ramp data with shape (Nreads, Ny, Nx).
dq_raw : np.ndarray, optional
3D array of data quality flags across detector (Nreads, Ny, Nx)
dq : np.ndarray, optional
2D array of data quality flags across detector (Ny, Nx)
rdnoise : np.ndarray, optional
2D array of read noise across detector (Ny, Nx)
gain : np.ndarray, optional
2D array of gain across detector (Ny, Nx)
dark_current : np.ndarray, optional
2D array of dark current across detector (Ny, Nx)
start_times : np.ndarray, optional
MJD time of the start of the first read.
jump_detection : bool, optional
Whether to perform jump detection before ramp fitting. Default is False.
jump_rejection_threshold : float, optional
Sigma threshold for jump detection. Default is 4.0.
max_cores : int, optional
Number of cores for parallelization. Default is 1 (no parallelization).
algorithm : str, optional
Ramp fitting algorithm to use. Options are 'JWST_LIKELY' or 'JWST_FOXSEN'. Default is 'JWST_LIKELY'.
Returns
-------
output : dict[str, np.ndarray]
Dictionary containing the L0 rate arrays.
"""
n_reads, ny, nx = ramps.shape
if algorithm.lower() in ('jwst_likely', 'jwst_like', 'jwst_likelihood'):
algorithm = 'LIKELY'
elif algorithm.lower() in ('jwst_fixsen', 'jwst_fixsen_c'):
algorithm = 'FIXSEN_C'
elif algorithm.lower() in ('cython_likely', 'cython_like', 'cython_likelihood'):
algorithm = 'CYTHON_LIKELY'
else:
raise ValueError(f'Unrecognized JWST ramp fitting algorithm {algorithm}')
if ramps.dtype is np.uint16:
mp_data_type = ctypes.c_ushort
else:
mp_data_type = ctypes.c_float
mp_float_type = ctypes.c_float
mp_dq_raw_type = ctypes.c_uint8
mp_dq_type = ctypes.c_uint
types_tuple = (mp_data_type, mp_float_type, mp_dq_raw_type, mp_dq_type)
slope_image_mp = RawArray(mp_float_type, nx * ny)
slope_image_shape = (ny, nx)
slope_image_np = _arraytonumpy(slope_image_mp, slope_image_shape, dtype=mp_float_type)
slope_image_np[:] = np.zeros((ny, nx), dtype=np.float32)
slope_errors_mp = RawArray(mp_float_type, nx * ny)
slope_errors_shape = (ny, nx)
slope_errors_np = _arraytonumpy(slope_errors_mp, slope_errors_shape, dtype=mp_float_type)
slope_errors_np[:] = np.zeros((ny, nx), dtype=np.float32)
var_poisson_image_mp = RawArray(mp_float_type, nx * ny)
var_poisson_image_shape = (ny, nx)
var_poisson_image_np = _arraytonumpy(var_poisson_image_mp, var_poisson_image_shape, dtype=mp_float_type)
var_poisson_image_np[:] = np.zeros((ny, nx), dtype=np.float32)
var_rnoise_image_mp = RawArray(mp_float_type, nx * ny)
var_rnoise_image_shape = (ny, nx)
var_rnoise_image_np = _arraytonumpy(var_rnoise_image_mp, var_rnoise_image_shape, dtype=mp_float_type)
var_rnoise_image_np[:] = np.zeros((ny, nx), dtype=np.float32)
dq_image_mp = RawArray(mp_dq_type, nx * ny)
dq_image_shape = (ny, nx)
dq_image_np = _arraytonumpy(dq_image_mp, dq_image_shape, dtype=mp_dq_type)
if dq is None:
dq_image_np[:] = np.zeros((ny, nx), dtype=np.uint32)
else:
dq_image_np[:] = dq
start_times_mp = RawArray(mp_float_type, nx * ny)
start_times_shape = (ny, nx)
start_times_np = _arraytonumpy(start_times_mp, start_times_shape, dtype=mp_float_type)
if start_times is None:
start_times_np[:] = np.zeros((ny, nx), dtype=np.float32)
else:
start_times_np[:] = start_times
mid_times_mp = RawArray(mp_float_type, nx * ny)
mid_times_shape = (ny, nx)
mid_times_np = _arraytonumpy(mid_times_mp, mid_times_shape, dtype=mp_float_type)
mid_times_np[:] = np.zeros((ny, nx), dtype=np.float32)
gain_mp = RawArray(mp_float_type, nx * ny)
gain_shape = (ny, nx)
gain_np = _arraytonumpy(gain_mp, gain_shape, dtype=mp_float_type)
if gain is None:
# TODO? - optimize gain when array is not passed
gain_np[:] = np.ones((ny, nx), dtype=np.float32)
else:
gain_np[:] = gain
dark_mp = RawArray(mp_float_type, nx * ny)
dark_shape = (ny, nx)
dark_np = _arraytonumpy(dark_mp, dark_shape, dtype=mp_float_type)
if dark_current is None:
dark_np[:] = np.zeros((ny, nx), dtype=np.float32)
else:
dark_np[:] = dark_current
rdnoise_mp = RawArray(mp_float_type, nx * ny)
rdnoise_shape = (ny, nx)
rdnoise_np = _arraytonumpy(rdnoise_mp, dark_shape, dtype=mp_float_type)
if rdnoise is None:
# TODO? - optimize rdnoise when array is not passed
rdnoise_np[:] = np.ones((ny, nx), dtype=np.float32)
else:
rdnoise_np[:] = rdnoise
if max_cores > 1:
ramps_mp = RawArray(mp_data_type, nx * ny * n_reads)
ramps_shape = (n_reads, ny, nx)
ramps_np = _arraytonumpy(ramps_mp, ramps_shape, dtype=mp_data_type)
ramps_np[:] = ramps # copy original array into shared array
ramps = ramps_np # free the original array from memory to avoid duplicates
dq_raw_mp = RawArray(mp_dq_raw_type, nx * ny * n_reads)
dq_raw_shape = (n_reads, ny, nx)
dq_raw_np = _arraytonumpy(dq_raw_mp, dq_raw_shape, dtype=mp_dq_raw_type)
if dq_raw is None:
dq_raw_np[:] = np.zeros((n_reads, ny, nx), dtype=np.uint8)
else:
dq_raw_np[:] = dq_raw # copy original array into shared array
dq_raw = dq_raw_np # free the original array from memory to avoid duplicates
# Prepare blocks
block_size = max(1, ny // (4*max_cores)) if max_cores > 1 else ny
_init_args = (
ramps_mp, ramps_shape,
dq_raw_mp, dq_raw_shape,
gain_mp, gain_shape,
dark_mp, dark_shape,
rdnoise_mp, rdnoise_shape,
slope_image_mp, slope_image_shape,
slope_errors_mp, slope_errors_shape,
var_poisson_image_mp, var_poisson_image_shape,
var_rnoise_image_mp, var_rnoise_image_shape,
dq_image_mp, dq_image_shape,
start_times_mp, start_times_shape,
mid_times_mp, mid_times_shape
)
ctx = mp.get_context("spawn") # avoids unsafe fork after OpenMP init
tpool = ctx.Pool(
processes=max_cores,
initializer=_tpool_init_jwst_ramp,
initargs=_init_args,
maxtasksperchild=50
)
if jump_detection:
args_list = []
for block_start in range(0, ny, block_size):
block_end = min(block_start + block_size, ny)
args = (block_start, block_end, times, jump_rejection_threshold, types_tuple)
args_list.append(args)
tasks = [
tpool.apply_async(_jump_detection_worker, args=(args_tuple,))
for args_tuple in args_list
]
for t in progressbar(tasks, desc="Fitting Ramps", fast=False):
t.wait()
args_list = []
for block_start in range(0, ny, block_size):
block_end = min(block_start + block_size, ny)
args = (block_start, block_end, times, algorithm, types_tuple)
args_list.append(args)
try:
with numba_thread_scope(max_cores):
tasks = [
tpool.apply_async(_fit_ramps_worker, args=(args_tuple,))
for args_tuple in args_list
]
for t in progressbar(tasks, desc="Fitting Ramps", fast=False):
t.wait()
finally:
tpool.close()
tpool.join()
else:
# If not parallelizing, ie max_cores=1, just run directly without shared memory
results = []
for row in np.arange(ny):
logger.info("JWST Ramp Fitting: Processing row %s of %s.", row + 1, ny)
if jump_detection:
args_row = (
row,
jump_rejection_threshold,
ramps[np.newaxis, :, [row], :],
# Make sure to add a new axis to each array to maintain number of dimensions
rdnoise_np[[row], :],
gain_np[[row], :],
dq_raw[np.newaxis, :, [row], :],
dq_image_np[[row], :]
)
_row, _dq = _jump_detection_jwst_row(args_row)
dq_raw[:, [_row], :] = _dq
args = (
row,
times,
# Reshape to add the nints axis that JWST pipeline requires
ramps[np.newaxis, :, [row], :],
# Make sure to add a new axis to each array to maintain number of dimensions
rdnoise_np[[row], :],
gain_np[[row], :],
dq_raw[np.newaxis, :, [row], :],
dq_image_np[[row], :],
dark_np[[row], :],
algorithm
)
if algorithm == 'CYTHON_LIKELY':
results.append(_fit_ramps_cython_row(args))
else:
results.append(_fit_ramps_jwst_row(args))
for row_element in results:
row, outdata, dq, var_poisson, var_rnoise, err = row_element
slope_image_np[row, :] = outdata
slope_errors_np[row, :] = err
dq_image_np[row, :] = dq_image_np[row, :] | dq[0]
var_poisson_image_np[row, :] = var_poisson
var_rnoise_image_np[row, :] = var_rnoise
with numba_thread_scope(max_cores):
mid_times_np = start_times + get_mid_delta_times(times, dq_raw) / SECONDS_PER_DAY
output = {}
output['rate'] = slope_image_np
output['rate_err'] = slope_errors_np
output['rate_var_rnoise'] = var_rnoise_image_np
output['rate_var_poisson'] = var_poisson_image_np
output['mid_time'] = mid_times_np
output['dq'] = dq_image_np
return output
def _tpool_init_jwst_ramp(
data : np.ndarray, data_shape : tuple,
dq_raw : np.ndarray, dq_raw_shape : tuple,
gain : np.ndarray, gain_shape : tuple,
dark : np.ndarray, dark_shape : tuple,
read_noise : np.ndarray, read_noise_shape : tuple,
slope_image : np.ndarray, slope_image_shape : tuple,
slope_errors : np.ndarray, slope_errors_shape : tuple,
var_poisson_image : np.ndarray, var_poisson_image_shape : tuple,
var_rnoise_image : np.ndarray, var_rnoise_image_shape : tuple,
dq_image : np.ndarray, dq_image_shape : tuple,
start_times : np.ndarray, start_times_shape : tuple,
mid_times : np.ndarray, mid_times_shape : tuple
):
"""
Initialize shared global variables for the multiprocessing pool.
"""
global shared_data, shared_data_shape, \
shared_dq_raw, shared_dq_raw_shape, \
shared_gain, shared_gain_shape, \
shared_dark, shared_dark_shape, \
shared_read_noise, shared_read_noise_shape, \
shared_slope_image, shared_slope_image_shape, \
shared_slope_errors, shared_slope_errors_shape, \
shared_var_poisson_image, shared_var_poisson_image_shape, \
shared_var_rnoise_image, shared_var_rnoise_image_shape, \
shared_dq_image, shared_dq_image_shape, \
shared_start_times, shared_start_times_shape, \
shared_mid_times, shared_mid_times_shape
shared_data = data
shared_data_shape = data_shape
shared_dq_raw = dq_raw
shared_dq_raw_shape = dq_raw_shape
shared_gain = gain
shared_gain_shape = gain_shape
shared_dark = dark
shared_dark_shape = dark_shape
shared_read_noise = read_noise
shared_read_noise_shape = read_noise_shape
shared_slope_image = slope_image
shared_slope_image_shape = slope_image_shape
shared_slope_errors = slope_errors
shared_slope_errors_shape = slope_errors_shape
shared_var_poisson_image = var_poisson_image
shared_var_poisson_image_shape = var_poisson_image_shape
shared_var_rnoise_image = var_rnoise_image
shared_var_rnoise_image_shape = var_rnoise_image_shape
shared_dq_image = dq_image
shared_dq_image_shape = dq_image_shape
shared_start_times = start_times
shared_start_times_shape = start_times_shape
shared_mid_times = mid_times
shared_mid_times_shape = mid_times_shape