from ..base_primitive import BasePrimitive
from ... import datamodels
import numpy as np
from numba import njit
import scipy.signal
from typing import Sequence
from ..utils.array_utils import median_filter_2d, nanmedian_axis0_ignore_empty_columns
import matplotlib.pyplot as plt
from ..spectral_extraction.extraction_utils import get_extraction_window
import plotly.graph_objects as go
from plotly.subplots import make_subplots
import logging
logger = logging.getLogger(__name__)
__all__ = ['TracePositionPrimitive']
[docs]
class TracePositionPrimitive(BasePrimitive):
"""
Calculate the trace positions for each fiber/order.
Parameters
----------
input : DataModel, str, Sequence[DataModel or str], or dict
Input data. Can be a single DataModel or filename, a sequence of DataModels or filenames, or a dict mapping fiber numbers to DataModels or filenames.
deg_pos : int
Degree of polynomial to fit for trace position as a function of pixel.
deg_width : int
Degree of polynomial to fit for trace width as a function of pixel.
window_size : int
Size of window (in pixels) to use when computing centroids and widths.
xdisp_range : tuple or dict
Range in the cross dispersion direction (in pixels) to restrict trace finding for the central slice.
Can be a single tuple for all fibers or a dict mapping fiber numbers to tuples.
disp_range : tuple or dict
Range in the dispersion direction (in pixels) to restrict trace finding.
Can be a single tuple for all fibers or a dict mapping fiber numbers to tuples.
prominence_factor : float
Initial peak-finding threshold as a fraction of the maximum cross-dispersion
profile value (default 0.05). Lower values recover fainter echelle orders on
high dynamic-range fiber flats.
min_flux_frac : float
Per-order signal floor for the trace walker (default 0.05). Rows whose windowed
flux falls below this fraction of the order's peak windowed flux are dropped from
the polynomial fit, so faint blaze ends and partially illuminated orders cannot
pull the trace into unphysical curves.
Returns
-------
output : TracePosModel
Output model containing fitted trace positions and widths for each fiber.
"""
PARAMETERS = {
'deg_pos' : None,
'deg_width' : None,
'window_size' : 8,
'xdisp_range' : None,
'disp_range' : None,
'prominence_factor' : 0.05,
'min_flux_frac' : 0.05,
'output_model' : 'TracePosModel',
}
def _perform(self):
# Associate inputs by fiber number
if isinstance(self.dataset, (datamodels.DataModel, str)):
self._input_by_fiber = datamodels.group_by_fiber_combination([self.dataset])
if isinstance(self.dataset, Sequence):
self._input_by_fiber = datamodels.group_by_fiber_combination(self.dataset)
elif isinstance(self.dataset, dict):
self._input_by_fiber = self.dataset
else:
raise ValueError(f"Unsupported input type: {type(self.dataset)}")
first_key = next(iter(self._input_by_fiber))
# Current keys: ('WHITELIGHT', 'BLOCKED', 'BLOCKED') : file, etc.
# Want: ('1' : file, '2' : file, '3' : file)
if not str(first_key).isdigit():
self._input_by_fiber = {
str(t.index(next(x for x in t if x.upper() != "BLOCKED")) + 1) : v
for t, v in self._input_by_fiber.items()
}
# Prepare output model
first_model = datamodels.open(list(self._input_by_fiber.values())[0])
output_model = datamodels.TracePosModel.from_datamodels(list(self._input_by_fiber.values()), primitive=self)
# Init trace dict
trace_results = {}
# Pixel array pos(xarr), xarr either rows or columns depending on orientation of orders
ny, nx = first_model.rate.shape
if first_model.meta.orders_are_vertical:
pixel_arr = np.arange(ny)
else:
pixel_arr = np.arange(nx)
# Order array
order_vec = first_model.make_order_vec()
n_orders = len(order_vec)
# Loop over fibers and fit traces
for fiber in self._input_by_fiber.keys():
# Open input model for this fiber
input_model = datamodels.open(self._input_by_fiber[fiber])
# Get xdisp_range, disp_range, and prominence for this fiber.
xdisp_range = self.xdisp_range[fiber] if isinstance(self.xdisp_range, dict) else self.xdisp_range
disp_range = self.disp_range[fiber] if isinstance(self.disp_range, dict) else self.disp_range
if isinstance(self.prominence_factor, dict):
prominence_factor = self.prominence_factor[fiber]
else:
prominence_factor = self.prominence_factor
# Transpose if needed
if input_model.meta.orders_are_vertical:
_input_rate, _input_dq = input_model.rate, input_model.dq
else:
_input_rate, _input_dq = input_model.rate.T, input_model.dq.T
logger.info(f"Performing trace finding on {input_model} fiber {fiber}")
# Find trace positions
r = find_trace_positions(
data=_input_rate, dq=_input_dq,
disp_range=disp_range, xdisp_range=xdisp_range,
deg_pos=self.deg_pos, deg_width=self.deg_width,
window_size=self.window_size,
prominence_factor=prominence_factor,
min_flux_frac=self.min_flux_frac,
n_expected_orders=n_orders,
)
# Sanity check
num_orders_found = r['coeffs_pos'].shape[0]
assert num_orders_found == n_orders, f"Found {num_orders_found} traces, expected {n_orders} for fiber {fiber}"
logger.info(f"Found {num_orders_found} traces")
# Store results in output model
# Evaluate polynomials
trace_results[fiber] = r
pos = np.full((n_orders, len(pixel_arr)), np.nan)
fwhm = np.full((n_orders, len(pixel_arr)), np.nan)
for i in range(num_orders_found):
cent_row = r['trace_centroids'][i, : len(pixel_arr)]
width_row = r['trace_widths'][i, : len(pixel_arr)]
pos[i, :] = np.polyval(r['coeffs_pos'][i, :], pixel_arr)
fwhm[i, :] = np.polyval(r['coeffs_width'][i, :], pixel_arr)
# Do not extrapolate trace polynomials into rows where the walker failed.
pos[i, ~np.isfinite(cent_row)] = np.nan
fwhm[i, ~np.isfinite(width_row)] = np.nan
setattr(output_model, f'COEFF_POS_FIB{fiber}', r['coeffs_pos'])
setattr(output_model, f'COEFF_FWHM_FIB{fiber}', r['coeffs_width'])
setattr(output_model, f'POS_FIB{fiber}', pos)
setattr(output_model, f'FWHM_FIB{fiber}', fwhm)
# Return
return dict(output=output_model, trace_results=trace_results)
[docs]
def _create_preview(self):
active_fibers = list(self._input_by_fiber.keys())
if not active_fibers:
logger.warning("No trace position data found for preview.")
return
tracepos_model = self._output['output']
# Create subplots
n_fibers = len(active_fibers)
fig, axes = plt.subplots(
nrows=1, ncols=n_fibers,
figsize=(4 * n_fibers, 4),
squeeze=False,
dpi=150,
)
for i, fiber in enumerate(active_fibers):
ax = axes[0, i]
ax.set_title(f"Fiber {fiber}")
input_model = datamodels.open(self._input_by_fiber[fiber])
im_data = input_model.rate
ny, nx = im_data.shape
zmin = 0
zmax = np.nanquantile(im_data, 0.98)
# Downsample image for preview to save space
# Use slice factor of 4 (16x smaller size)
res = 2
im_data_small = im_data[::res, ::res]
# Adjust x/y coordinates to center the large pixels
x_small = np.arange(0, nx, res) + res/2 - 0.5
y_small = np.arange(0, ny, res) + res/2 - 0.5
# Heatmap
extent = [x_small[0] - res/2, x_small[-1] + res/2,
y_small[-1] + res/2, y_small[0] - res/2]
ax.imshow(
im_data_small,
origin='upper',
extent=extent,
vmin=zmin, vmax=zmax,
cmap='viridis',
interpolation='nearest',
aspect='equal'
)
# Get traces
pos = getattr(tracepos_model, f'POS_FIB{fiber}')
n_orders = pos.shape[0]
orders_are_vertical = tracepos_model.meta.orders_are_vertical
pixarr = np.arange(pos.shape[1])
for ord_idx in range(n_orders):
# Center line
trace_y = pos[ord_idx, :]
# Check orientation for plotting
if orders_are_vertical:
# If vertical, x is the spatial dimension (trace position), y is spectral (pixel index)
# The pos array stores position as function of pixel index.
# Handle nan values for filled region
valid = np.isfinite(trace_y)
if not np.any(valid):
continue
curr_trace_pos = trace_y[valid]
curr_pixels = pixarr[valid]
# Center line coords
x_trace = curr_trace_pos
y_trace = curr_pixels
else:
# Horizontal traces
# x=pixel_index, y=pos
# Handle nan values for filled region
valid = np.isfinite(trace_y)
if not np.any(valid):
continue
curr_trace_pos = trace_y[valid]
curr_pixels = pixarr[valid]
# Center line coords
x_trace = curr_pixels
y_trace = curr_trace_pos
# Add center line
ax.plot(
x_trace,
y_trace,
color='red', linewidth=1,
label=f'Order {ord_idx}'
)
ax.grid(False)
# Update layout
fig.suptitle(f"Trace Positions: {tracepos_model.meta.filename}")
fig.tight_layout()
# File paths (same directory as FITS products from save_output)
output_path = self.generate_output_path(tracepos_model).replace(
".fits", "_preview.png"
)
if self.save_preview:
fig.savefig(output_path, dpi=150, bbox_inches='tight')
if self.show_preview:
logger.info("Showing trace position preview...")
plt.show()
plt.close(fig)
self._output['preview'] = output_path
def find_trace_positions(
data : np.ndarray,
dq : np.ndarray,
deg_pos : int,
deg_width : int,
xdisp_range : tuple = None,
disp_range : tuple = None,
fwhm : float = 2.5,
window_size : int | None = None,
distance_factor : float = 0.5,
prominence_factor : float = 0.05,
min_flux_frac : float = 0.05,
max_step : float | None = None,
n_expected_orders : int | None = None,
) -> dict:
# Initial parameters
ny, nx = data.shape
if xdisp_range is None:
xdisp_range = (16, nx - 16)
if disp_range is None:
disp_range = (16, ny - 16)
if window_size is None:
window_size = int(np.round(fwhm * 2.5))
window_size_half = int(np.round(window_size / 2))
min_distance = max(1, int(fwhm * distance_factor))
# When the expected order count is known, require detected peaks to be separated by at
# least a fraction of the mean inter-order spacing. This suppresses spurious split
# peaks on bright orders (a single order resolving into two adjacent maxima) without
# merging genuinely distinct orders, which are spaced far wider than this floor.
if n_expected_orders is not None and n_expected_orders > 0:
span = max(1, int(xdisp_range[1]) - int(xdisp_range[0]))
spacing_distance = int(0.35 * span / n_expected_orders)
min_distance = max(min_distance, spacing_distance)
# Maximum allowed centroid shift between adjacent rows. Real echelle traces curve by
# well under a pixel per row, so a tight cap rejects order-hopping and runaway drift
# into inter-order gaps while still following genuine trace curvature.
if max_step is None:
max_step = float(window_size_half)
# Fix negative values in data
data = data.copy()
data[data < 0] = 0
# Find peaks for center column
y0 = ny // 2
ki, kf = max(0, y0 - 3), min(ny, y0 + 3)
y = nanmedian_axis0_ignore_empty_columns(
median_filter_2d(data[ki:kf+1, :], dq[ki:kf+1, :], window_size=(3, 3))
)
y = y[xdisp_range[0]:xdisp_range[1] + 1]
# Interpolate over NaNs (masked / flagged pixels) before peak finding: scipy.signal
# find_peaks silently drops nearly all peaks when NaNs are present because comparisons
# against NaN are False, so a handful of scattered bad pixels can collapse the detected
# order count. Linear interpolation across finite samples preserves the order profile.
finite = np.isfinite(y)
if finite.any() and not finite.all():
idx = np.arange(y.size)
y = np.interp(idx, idx[finite], y[finite])
y_max = np.nanmax(y) if np.any(np.isfinite(y)) else 0.0
min_prominence = y_max * prominence_factor
peaks0, _ = scipy.signal.find_peaks(
y,
prominence=min_prominence,
distance=min_distance
)
peaks0 += xdisp_range[0]
num_traces = len(peaks0)
coeffs_pos = np.full((num_traces, deg_pos + 1), np.nan)
coeffs_width = np.full((num_traces, deg_width + 1), np.nan)
trace_centroids = np.full((num_traces, ny), np.nan)
trace_widths = np.full((num_traces, ny), np.nan)
trace_flux = np.full((num_traces, ny), np.nan)
pixel_arr = np.arange(nx)
# Replace NaN / non-finite pixels (e.g. dq-flagged) with zero before walking: a single
# flagged pixel inside the centroid window makes the windowed flux non-finite, which
# aborts the walk for the whole order. Treating bad pixels as zero signal lets the walk
# bridge isolated flags while contributing negligibly to the local centroid.
data_for_walk = np.where(np.isfinite(data), data, 0.0)
# Walk each trace up/down from initial peak position
for i in range(num_traces):
trace_centroids[i, :], trace_widths[i, :], trace_flux[i, :] = walk_trace_up_down(
pixel_arr,
data_for_walk,
dq,
y0,
peaks0[i],
disp_range,
window_size_half,
max_step,
)
# Reject low-signal rows: where the windowed flux falls below a fraction of the
# order's peak windowed flux, the centroid is dominated by noise / inter-order
# background and must not enter the polynomial fit (otherwise faint blaze ends and
# partially illuminated orders pull the fit into wild, unphysical curves).
flux_row = trace_flux[i, :]
if np.any(np.isfinite(flux_row)):
peak_flux = np.nanmax(flux_row)
if np.isfinite(peak_flux) and peak_flux > 0:
low = ~(flux_row > (min_flux_frac * peak_flux))
trace_centroids[i, low] = np.nan
trace_widths[i, low] = np.nan
good = np.where(np.isfinite(trace_centroids[i, :]))[0]
if len(good) >= deg_pos + 1:
coeffs_pos[i, :] = np.polyfit(good, trace_centroids[i, good], deg_pos)
#breakpoint()
#plt.plot(good, trace_centroids[i, good], 'o')
#plt.plot(np.arange(ny), np.polyval(coeffs_pos[i, :], np.arange(ny)))
# plt.plot(good, trace_centroids[i, good] - np.polyval(coeffs_pos[i, :], good), 'o'); plt.show()
#plt.show()
good = np.where(np.isfinite(trace_widths[i, :]))[0]
if len(good) >= deg_width + 1:
coeffs_width[i, :] = np.polyfit(good, trace_widths[i, good], deg_width)
return dict(
coeffs_pos=coeffs_pos,
coeffs_width=coeffs_width,
trace_centroids=trace_centroids,
trace_widths=trace_widths,
trace_flux=trace_flux,
)
def walk_trace_up_down(
pixels : np.ndarray,
data : np.ndarray,
dq : np.ndarray,
disp0 : float,
xdisp0 : float,
disp_range : tuple,
window_size_half : float,
max_step : float,
) -> np.ndarray:
trace_centroids = np.full(data.shape[0], np.nan)
trace_widths = np.full(data.shape[0], np.nan)
trace_flux = np.full(data.shape[0], np.nan)
_walk_trace_up_down(
pixels,
data,
dq,
int(disp0),
float(xdisp0),
disp_range,
window_size_half,
max_step,
trace_centroids,
trace_widths,
trace_flux,
)
return trace_centroids, trace_widths, trace_flux
@njit(nogil=True, cache=True)
def _walk_trace_up_down(
pixels: np.ndarray,
data: np.ndarray,
dq: np.ndarray,
y0: int,
xc0: float,
disp_range: tuple,
window_size_half: float,
max_step: float,
trace_centroids: np.ndarray,
trace_widths: np.ndarray,
trace_flux: np.ndarray,
):
# Central slice
c, w, f = _compute_centroid_and_width_2pass(
pixels,
data[y0, :],
dq[y0, :],
xc0,
window_size_half,
)
trace_centroids[y0], trace_widths[y0], trace_flux[y0] = c, w, f
anchor0 = c if np.isfinite(c) else xc0
# Walk down. The window is anchored on the last accepted centroid (not the previous
# row's raw value) so that a row with no real signal cannot seed a runaway drift; a
# centroid that jumps by more than ``max_step`` is rejected (order hop / noise spike).
anchor = anchor0
for y in range(y0 - 1, disp_range[0] - 1, -1):
c, w, f = _compute_centroid_and_width_2pass(
pixels,
data[y, :],
dq[y, :],
anchor,
window_size_half,
)
trace_flux[y] = f
if np.isfinite(c) and abs(c - anchor) <= max_step:
trace_centroids[y], trace_widths[y] = c, w
anchor = c
else:
trace_centroids[y], trace_widths[y] = np.nan, np.nan
# Walk up
anchor = anchor0
for y in range(y0 + 1, disp_range[1] + 1):
c, w, f = _compute_centroid_and_width_2pass(
pixels,
data[y, :],
dq[y, :],
anchor,
window_size_half,
)
trace_flux[y] = f
if np.isfinite(c) and abs(c - anchor) <= max_step:
trace_centroids[y], trace_widths[y] = c, w
anchor = c
else:
trace_centroids[y], trace_widths[y] = np.nan, np.nan
# @njit(nogil=True, cache=True)
# def _compute_centroid_and_width(
# pixels : np.ndarray,
# trace_slice : np.ndarray,
# trace_dq : np.ndarray,
# center_loc : float,
# window_size_half : float,
# ):
# """
# Compute the centroid, width, and integrated flux of a trace slice.
# Returns
# -------
# centroid : float
# Flux-weighted centroid, or NaN when the window has no usable signal.
# fwhm : float
# Gaussian-equivalent FWHM, or NaN when the window has no usable signal.
# total_flux : float
# Summed flux inside the window (0.0 when the window is out of bounds / empty).
# """
# if not np.isfinite(center_loc):
# return np.nan, np.nan, 0.0
# xi = int(np.round(center_loc - window_size_half))
# xf = int(np.round(center_loc + window_size_half + 1))
# if xi < 0 or xf > trace_slice.size - 1:
# return np.nan, np.nan, 0.0
# slice_x = pixels[xi : xf + 1]
# slice_y = trace_slice[xi : xf + 1]
# slice_dq = trace_dq[xi : xf + 1]
# n_good = np.sum(slice_dq == 0)
# if n_good < slice_dq.size:
# return np.nan, np.nan, 0.0
# total_flux = np.sum(slice_y)
# if total_flux > 0:
# centroid = np.sum(slice_x * slice_y) / total_flux
# variance = np.sum(((slice_x - centroid) ** 2) * slice_y) / total_flux
# if variance < 0:
# return np.nan, np.nan, total_flux
# fwhm = 2 * np.sqrt(2 * np.log(2)) * np.sqrt(variance)
# return centroid, fwhm, total_flux
# else:
# return np.nan, np.nan, 0.0
@njit(nogil=True, cache=True)
def _compute_centroid_and_width(
pixels : np.ndarray,
trace_slice : np.ndarray,
trace_dq : np.ndarray,
center_loc : float,
window_size_half : float,
):
"""
Compute the centroid, width, and integrated flux of a trace slice.
The window is the continuous interval ``[center_loc - window_size_half,
center_loc + window_size_half]``; ``get_extraction_window`` supplies the fractional
overlap of each pixel with that interval, so the pixels straddling the window edges
contribute partially. No interpolation is performed — pixel values are used as-is and
only their weights vary.
Returns
-------
centroid : float
Flux-weighted centroid, or NaN when the window has no usable signal.
fwhm : float
Gaussian-equivalent FWHM, or NaN when the window has no usable signal.
total_flux : float
Weighted flux inside the window (0.0 when the window is out of bounds / empty).
"""
if not np.isfinite(center_loc):
return np.nan, np.nan, 0.0
n = trace_slice.size
# Reject windows that are not fully on the detector. get_extraction_window would
# silently clip them to the edge, which biases the centroid inward; the walker wants
# these rows dropped instead.
if center_loc - window_size_half < -0.5 or center_loc + window_size_half > n - 0.5:
return np.nan, np.nan, 0.0
idx, W = get_extraction_window(center_loc, 2.0 * window_size_half, n)
if idx.size == 0:
return np.nan, np.nan, 0.0
# Any flagged pixel touching the window invalidates the measurement.
for j in range(idx.size):
if W[j] > 0.0 and trace_dq[idx[j]] != 0:
return np.nan, np.nan, 0.0
total_flux = 0.0
moment1 = 0.0
for j in range(idx.size):
i = idx[j]
fi = W[j] * trace_slice[i]
total_flux += fi
moment1 += pixels[i] * fi
if total_flux <= 0:
return np.nan, np.nan, 0.0
centroid = moment1 / total_flux
variance = 0.0
for j in range(idx.size):
i = idx[j]
d = pixels[i] - centroid
variance += d * d * W[j] * trace_slice[i]
variance /= total_flux
if variance < 0:
return np.nan, np.nan, total_flux
fwhm = 2 * np.sqrt(2 * np.log(2)) * np.sqrt(variance)
return centroid, fwhm, total_flux
@njit(nogil=True, cache=True)
def _compute_centroid_and_width_2pass(
pixels : np.ndarray,
trace_slice : np.ndarray,
trace_dq : np.ndarray,
center_loc : float,
window_size_half : float,
):
"""
Two-pass centroid: the first pass centers the window on ``center_loc`` (the anchor
carried from the previous row), the second re-centers it on the first pass's centroid
so the fractional edge weights are symmetric about the measured trace rather than about
the anchor. Falls back to the first-pass result if the refinement fails (e.g. the
re-centered window runs off the edge or picks up a flagged pixel).
"""
c, w, f = _compute_centroid_and_width(
pixels, trace_slice, trace_dq, center_loc, window_size_half,
)
if not np.isfinite(c):
return c, w, f
c2, w2, f2 = _compute_centroid_and_width(
pixels, trace_slice, trace_dq, c, window_size_half,
)
if not np.isfinite(c2):
return c, w, f
return c2, w2, f2