Source code for hispecdrp.primitives.spectral_extraction.extract_cross_talk
import numpy as np
from ...datamodels.dqflags import DQ_FLAGS
from .extraction_utils import get_extraction_window
import matplotlib.pyplot as plt
from numba import njit
DO_NOT_USE = DQ_FLAGS['DO_NOT_USE']
[docs]
def extract_spectra_cross_talk(
input_data : np.ndarray,
input_err : np.ndarray,
input_dq : np.ndarray,
input_var_rnoise : np.ndarray,
input_var_poisson : np.ndarray,
input_mid_time : np.ndarray,
trace_positions : list[np.ndarray],
profile : list[np.ndarray],
extraction_aperture : float,
pix_range : tuple | None = None,
spec_output : dict[str, np.ndarray] = None,
err_output : dict[str, np.ndarray] = None,
var_rnoise_output : dict[str, np.ndarray] = None,
var_poisson_output : dict[str, np.ndarray] = None,
dq_output : dict[str, np.ndarray] = None,
mid_time_output : dict[str, np.ndarray] = None,
orders_are_vertical : bool = True,
) -> dict:
"""
Extract spectra for all fibers for a given order accounting for cross-talk between fibers.
Parameters
----------
input_data : np.ndarray
2D array of input data.
input_err : np.ndarray
2D array of input errors.
input_dq : np.ndarray
2D array of input data quality flags.
input_var_rnoise : np.ndarray
2D array of input read noise variance.
input_var_poisson : np.ndarray
2D array of input Poisson noise variance.
trace_positions : np.ndarray
2D array of trace positions for each fiber. Shape = (n_fibers, nx).
profile : np.ndarray
2D array of profiles for each fiber. Shape = (n_fibers, ny, nx).
extraction_aperture : float
Width of the aperture to use for extraction.
pix_range : tuple
Range of pixel-coordinates to extract (min, max) in the spectral direction. Defaults to the full range.
spec_output : np.ndarray
2D array to store extracted spectrum. Shape = (n_fibers, nx).
err_output : np.ndarray
2D array to store extracted errors. Shape = (n_fibers, nx).
var_rnoise_output : np.ndarray
2D array to store extracted read noise variance. Shape = (n_fibers, nx).
var_poisson_output : np.ndarray
2D array to store extracted Poisson noise variance. Shape = (n_fibers, nx).
dq_output : np.ndarray
2D array to store extracted data quality flags. Shape = (n_fibers, nx).
mid_time_output : np.ndarray
2D array to store extracted mid times. Shape = (n_fibers, nx).
orders_are_vertical : bool
Whether the orders are oriented vertically.
Returns
-------
dict
Dictionary containing extracted flux, error, read noise variance for the box extracted spectrum.
"""
# Default X range
ny, nx = input_data.shape
if pix_range is None:
if orders_are_vertical:
pix_range = (0, ny - 1)
npix = ny
else:
pix_range = (0, nx - 1)
npix = nx
# Sanity check
n_fibers = trace_positions.shape[0]
assert profile.shape == (n_fibers, ny, nx), f"profiles must have shape ({n_fibers}, {ny}, {nx}), got {profile.shape}"
# Mock / sim L0s may omit optional companion arrays.
data_dtype = np.asarray(input_data).dtype
if input_err is None:
input_err = np.zeros((ny, nx), dtype=data_dtype)
if input_var_rnoise is None:
input_var_rnoise = np.zeros((ny, nx), dtype=data_dtype)
if input_var_poisson is None:
input_var_poisson = np.zeros((ny, nx), dtype=data_dtype)
if input_dq is None:
input_dq = np.zeros((ny, nx), dtype=np.uint32)
if input_mid_time is None:
input_mid_time = np.zeros((ny, nx), dtype=np.float64)
# Default outputs
if spec_output is None:
spec_output = np.full((n_fibers, npix), np.nan, dtype=data_dtype)
if err_output is None:
err_output = np.full((n_fibers, npix), np.nan, dtype=input_err.dtype)
if var_rnoise_output is None:
var_rnoise_output = np.full((n_fibers, npix), np.nan, dtype=input_var_rnoise.dtype)
if var_poisson_output is None:
var_poisson_output = np.full((n_fibers, npix), np.nan, dtype=input_var_poisson.dtype)
if dq_output is None:
dq_output = np.full((n_fibers, npix), 0, dtype=input_dq.dtype)
if mid_time_output is None:
mid_time_output = np.full((n_fibers, npix), np.nan, dtype=input_mid_time.dtype)
# Call numba extraction routine
_extract_cross_talk_numba(
input_data=input_data,
input_err=input_err,
input_dq=input_dq,
input_var_rnoise=input_var_rnoise,
input_var_poisson=input_var_poisson,
trace_positions=trace_positions,
profile=profile,
extraction_aperture=extraction_aperture,
pix_range=pix_range,
spec_output=spec_output,
err_output=err_output,
var_rnoise_output=var_rnoise_output,
var_poisson_output=var_poisson_output,
dq_output=dq_output,
mid_time_output=mid_time_output,
orders_are_vertical=orders_are_vertical,
)
# Return
return dict(
spec=spec_output,
err=err_output,
var_rnoise=var_rnoise_output,
var_poisson=var_poisson_output,
dq=dq_output,
mid_time=mid_time_output,
)
[docs]
@njit(nogil=True, parallel=False, cache=True)
def _extract_cross_talk_numba(
input_data : np.ndarray,
input_err : np.ndarray,
input_dq : np.ndarray,
input_var_rnoise : np.ndarray,
input_var_poisson : np.ndarray,
input_mid_time : np.ndarray,
trace_positions : np.ndarray,
profile : np.ndarray,
extraction_aperture : float,
pix_range : tuple,
spec_output : np.ndarray,
err_output : np.ndarray,
var_rnoise_output : np.ndarray,
var_poisson_output : np.ndarray,
dq_output : np.ndarray,
mid_time_output : np.ndarray,
orders_are_vertical : bool = True,
):
n_fibers = trace_positions.shape[0]
for x in range(pix_range[0], pix_range[1] + 1):
trace_positions_x = trace_positions[:, x]
profiles_x = profile[:, :, x]
# Build the profile matrix P for this column or row
P, W, window_full = get_design_matrix(
trace_positions_x=trace_positions_x,
profiles_x=profiles_x,
extraction_aperture=extraction_aperture,
orders_are_vertical=orders_are_vertical,
)
if orders_are_vertical:
D = input_data[x, window_full].copy()
D_var = input_err[x, window_full]**2
D_var_poisson = input_var_poisson[x, window_full].copy()
D_var_rnoise = input_var_rnoise[x, window_full].copy()
D_dq = input_dq[x, window_full].copy()
else:
D = input_data[window_full, x].copy()
D_var = input_err[window_full, x]**2
D_var_poisson = input_var_poisson[window_full, x].copy()
D_var_rnoise = input_var_rnoise[window_full, x].copy()
D_dq = input_dq[window_full, x].copy()
# Mask bad pixels
# NOTE: Ignore column if any profile value is bad, keep eye on this behavior
M = (
((D_dq & DO_NOT_USE) == 0)
& np.isfinite(D)
& np.isfinite(D_var)
& (D > 0)
& (D_var > 0)
& (D_var_poisson > 0)
& (D_var_rnoise > 0)
)
for i in range(n_fibers):
if np.any(~np.isfinite(P[:, i])):
M[:] = False
break
bad = np.where(~M)[0]
n_good = len(M) - len(bad)
if n_good == 0:
spec_output[:, x] = np.nan
err_output[:, x] = np.nan
var_rnoise_output[:, x] = np.nan
var_poisson_output[:, x] = np.nan
dq_output[:, x] = DO_NOT_USE
continue
# Zero bad pixels (M already excludes them; avoid M*inf → NaN).
D[bad] = 0
D_var[bad] = 0
D_var_poisson[bad] = 0
D_var_rnoise[bad] = 0
# Extract the spectrum at this column
spec_fibers, err_fibers, var_rnoise_fibers, var_poisson_fibers = solve_cross_talk(
D=D,
D_err=D_var,
D_var_rnoise=D_var_rnoise,
D_var_poisson=D_var_poisson,
P=P,
W=W,
M=M,
)
# Store the results
spec_output[:, x] = spec_fibers
err_output[:, x] = err_fibers
var_rnoise_output[:, x] = var_rnoise_fibers
var_poisson_output[:, x] = var_poisson_fibers
dq_output[:, x] = 0
# TODO: Compute mid times for cross talk extraction.
# mid_time_output
#if x == 1018:
# breakpoint()
[docs]
@njit(nogil=True, parallel=False, cache=True)
def get_design_matrix(
trace_positions_x : np.ndarray,
profiles_x : np.ndarray,
extraction_aperture : float,
orders_are_vertical : bool = True,
) -> np.ndarray:
"""
Build the profile matrix P for cross-talk extraction.
"""
n_fibers = len(trace_positions_x)
window_first, W_first = get_extraction_window(
yc=trace_positions_x[0],
window_size=extraction_aperture
)
window_last, W_last = get_extraction_window(
yc=trace_positions_x[-1],
window_size=extraction_aperture
)
if trace_positions_x[0] < trace_positions_x[-1]:
window_full = np.arange(window_first[0], window_last[-1] + 1, dtype=np.int64)
else:
window_full = np.arange(window_last[0], window_first[-1] + 1, dtype=np.int64)
ny_full = len(window_full)
P = np.zeros((ny_full, n_fibers), dtype=profiles_x.dtype)
W = np.zeros((ny_full, n_fibers), dtype=profiles_x.dtype)
for i in range(n_fibers):
# NOTE: If i == 0 or i == n_fibers - 1, use first or last window above eventually.
window_i, W_i = get_extraction_window(
yc=trace_positions_x[i],
window_size=extraction_aperture
)
window_sub = window_i - window_full[0]
if orders_are_vertical: # NOTE: Confirm this is correct
P[window_sub, i] = profiles_x[window_i, i]
W[window_sub, i] = W_i
else:
P[window_sub, i] = profiles_x[i, window_i]
W[window_sub, i] = W_i
return P, W, window_full
# NOTE: Look closer at numba C vs. Fortran ordering issues and performance implications.
[docs]
@njit(nogil=True, parallel=False, cache=False)
def solve_cross_talk(
D: np.ndarray,
D_err: np.ndarray,
D_var_rnoise: np.ndarray,
D_var_poisson: np.ndarray,
P: np.ndarray,
W: np.ndarray,
M: np.ndarray,
) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
denom = np.sum(P * W, axis=0)
P_norm = P / denom[None, :]
D_var = D_err**2
# Inverse variance for each element, INCLUDING W (W is already 2D!)
Vinv = M[:, None] * P_norm**2 * W / D_var[:, None]
Vinv = np.where(np.isfinite(Vinv), Vinv, np.float32(0))
# Weighted least squares matrix A and vector b
A = np.ascontiguousarray(P_norm.T) @ np.ascontiguousarray(Vinv * P_norm)
b = np.ascontiguousarray((P_norm * Vinv).T) @ D
# Solve for spectrum
spec = np.linalg.pinv(A) @ b
# Covariance and errors
cov = np.linalg.pinv(A)
spec_err = np.sqrt(np.diag(cov))
#spec_err = np.sqrt(np.sum(P_norm * W, axis=0) / np.sum(M[:, None] * P_norm**2 * W / D_var[:, None], axis=0))
# Separate variance contributions
spec_var_rnoise = np.sum(P_norm * W, axis=0) / np.sum(M[:, None] * P_norm**2 * W / D_var_rnoise[:, None], axis=0)
spec_var_poisson = np.sum(P_norm * W, axis=0) / np.sum(M[:, None] * P_norm**2 * W / D_var_poisson[:, None], axis=0)
return spec, spec_err, spec_var_rnoise, spec_var_poisson