Source code for hispecdrp.primitives.spectral_extraction.spectral_extraction_primitive

from ..base_primitive import BasePrimitive
from ... import datamodels
import numpy as np
from typing import Sequence
import logging

from .extract_box import extract_spectrum_box
from .extract_optimal import extract_spectrum_optimal
from .extract_cross_talk import extract_spectra_cross_talk
from ...calibrations import TracePosSelector, TraceProfSelector

import matplotlib.pyplot as plt

logger = logging.getLogger(__name__)

__all__ = ['SpectralExtractionPrimitive']


[docs] class SpectralExtractionPrimitive(BasePrimitive): """ Extracts the 2D echelleogram into 1D spectra for each fiber and order. Pre-determined trace positions and profiles (for optimal extraction) must be provided. TODO: Fix normalization to extend to effective infinity regardless of aperture size. Aperture size only determines fit window and we can round here. Parameters ---------- suffix : str Suffix to add to the output filename. Default is 'specextract'. method : str Extraction method to use. Options are: - **box**: Sum all pixels within a fixed aperture around the trace position. - **optimal**: Fit a 1D profile to the data. - **cross_talk**: Simultaneously fit 1D profiles for all fibers to account for cross-talk between fibers. Only applicable when multiple fibers are present. extraction_aperture : float Size of the extraction aperture in pixels, centered on the trace position. do_fibers : list[int] or None List of fiber numbers to extract. If None, all fibers will be extracted. do_orders : list[int] or None List of order numbers to extract. If None, all orders will be extracted. pix_range : tuple[int, int] or None Range of x pixels to extract. If None, the full range will be extracted. Returns ------- output : datamodels.L1Model or list[datamodels.L1Model] Extracted spectra for each fiber and order. """ PARAMETERS = { 'suffix' : 'specextract', 'method' : 'box', 'extraction_aperture' : 10, 'do_fibers' : None, 'do_orders' : None, 'pix_range' : None, 'output_model' : 'L1Model', 'reverse_spectral_axis' : True, } CALIBRATIONS = { 'tracepos': { 'selector': TracePosSelector, 'selector_kwargs': {} }, 'traceprof': { 'selector': TraceProfSelector, 'selector_kwargs': {} }, } def _perform(self): # Open the input model input_model = datamodels.open(self.dataset) npix = input_model.shape[0] if input_model.meta.orders_are_vertical else input_model.shape[1] # Get calibrations (get_calibration returns a filepath or model) tracepos_model, _ = self.get_calibration(input_model, 'tracepos') tracepos_model = datamodels.open(tracepos_model) traceprof_model, _ = self.get_calibration(input_model, 'traceprof') traceprof_model = datamodels.open(traceprof_model) # Init output model output_model = self.get_output_model(input_model) # Which fibers and orders to process do_fibers, do_orders = self.get_do_traces(input_model, self.do_fibers, self.do_orders) # Dispatch to extraction method if self.method.lower() == 'box': self.extract_box( input_model=input_model, output_model=output_model, tracepos_model=tracepos_model, do_fibers=do_fibers, do_orders=do_orders ) elif self.method.lower() == 'optimal': self.extract_optimal( input_model=input_model, output_model=output_model, tracepos_model=tracepos_model, traceprof_model=traceprof_model, do_fibers=do_fibers, do_orders=do_orders ) elif self.method.lower() == 'cross_talk': self.extract_cross_talk( input_model=input_model, output_model=output_model, tracepos_model=tracepos_model, traceprof_model=traceprof_model, do_fibers=do_fibers, do_orders=do_orders ) else: raise ValueError(f"Unknown extraction method: {self.method}") if self.reverse_spectral_axis: for fiber in do_fibers: output_model[fiber].spec_uncal = output_model[fiber].spec_uncal[:, ::-1] output_model[fiber].spec_uncal_err = output_model[fiber].spec_uncal_err[:, ::-1] output_model[fiber].spec_uncal_var_rnoise = output_model[fiber].spec_uncal_var_rnoise[:, ::-1] output_model[fiber].spec_uncal_var_poisson = output_model[fiber].spec_uncal_var_poisson[:, ::-1] output_model[fiber].spec_uncal_dq = output_model[fiber].spec_uncal_dq[:, ::-1] output_model[fiber].mid_time = output_model[fiber].mid_time[:, ::-1] # Return return dict(output=output_model)
[docs] def extract_box( self, input_model : datamodels.L0Model, output_model : datamodels.L1Model, tracepos_model : datamodels.TracePosModel, do_fibers : np.ndarray, do_orders : np.ndarray, ): # All orders order_vec = input_model.make_order_vec() for fiber in do_fibers: trace_positions = getattr(tracepos_model, f'pos_fib{fiber}') # Sanity check assert trace_positions.shape[0] == len(order_vec), "Trace position coefficients do not match number of orders" # Alias outputs spec_out = getattr(output_model, f'SPEC_UNCAL_FIB{fiber}') err_out = getattr(output_model, f'SPEC_UNCAL_ERR_FIB{fiber}') var_rnoise_out = getattr(output_model, f'SPEC_UNCAL_VAR_RNOISE_FIB{fiber}') var_poisson_out = getattr(output_model, f'SPEC_UNCAL_VAR_POISSON_FIB{fiber}') dq_out = getattr(output_model, f'SPEC_UNCAL_DQ_FIB{fiber}') mid_time_out = getattr(output_model, f'MID_TIME_FIB{fiber}') for j, order in enumerate(order_vec): if order not in do_orders: continue logger.info(f"Extracting {input_model} with method {self.method} [Fiber {fiber}][Order {order}]") # Get the data attributes based on the primary ext name pext_name = input_model.primary_extension_name input_data = getattr(input_model, pext_name) input_err = getattr(input_model, pext_name + '_err', None) input_dq = getattr(input_model, 'dq', None) input_var_rnoise = getattr(input_model, pext_name + '_var_rnoise', None) input_var_poisson = getattr(input_model, pext_name + '_var_poisson', None) input_mid_time = getattr(input_model, 'mid_time', None) # Sanity check assert trace_positions.shape[1] == input_data.shape[0], f"Trace positions shape[1]={trace_positions.shape[1]} does not match number of pixels={input_data.shape[0]}" # Perform box extraction #breakpoint() # plt.imshow(input_data, vmin=0, vmax=np.nanquantile(input_data, 0.99)); plt.plot(trace_positions[j], np.arange(4096), c='red'); plt.show() result = extract_spectrum_box( input_data=input_data, input_err=input_err, input_dq=input_dq, input_var_rnoise=input_var_rnoise, input_var_poisson=input_var_poisson, input_mid_time=input_mid_time, trace_positions=trace_positions[j, :], extraction_aperture=self.extraction_aperture, pix_range=self.pix_range, orders_are_vertical=input_model.meta.orders_are_vertical, ) spec_out[j, :] = result['spec'] err_out[j, :] = result['err'] var_rnoise_out[j, :] = result['var_rnoise'] var_poisson_out[j, :] = result['var_poisson'] dq_out[j, :] = result['dq'] mid_time_out[j, :] = result['mid_time']
# NOTE: Keep for debugging # breakpoint() # import matplotlib.pyplot as plt; plt.plot(spec_out[j, :]); plt.show()
[docs] def extract_optimal( self, input_model : datamodels.L0Model, output_model : datamodels.L1Model, tracepos_model : datamodels.TracePosModel, traceprof_model : datamodels.TraceProfModel, do_fibers : np.ndarray, do_orders : np.ndarray, ): # All orders order_vec = input_model.make_order_vec() # Sanity check #assert trace_positions.shape[0] == len(order_vec), f"Trace positions shape[0]={trace_positions.shape[0]} does not match number of orders={len(order_vec)}" for fiber in do_fibers: # Cals trace_positions = getattr(tracepos_model, f'pos_fib{fiber}') trace_profile = getattr(traceprof_model, f'prof_fib{fiber}') # Alias outputs spec_out = getattr(output_model, f'SPEC_UNCAL_FIB{fiber}') err_out = getattr(output_model, f'SPEC_UNCAL_ERR_FIB{fiber}') var_rnoise_out = getattr(output_model, f'SPEC_UNCAL_VAR_RNOISE_FIB{fiber}') var_poisson_out = getattr(output_model, f'SPEC_UNCAL_VAR_POISSON_FIB{fiber}') dq_out = getattr(output_model, f'SPEC_UNCAL_DQ_FIB{fiber}') mid_time_out = getattr(output_model, f'MID_TIME_FIB{fiber}') for j, order in enumerate(order_vec): if order not in do_orders: continue logger.info(f"Extracting {input_model} with method {self.method} [Fiber {fiber}][Order {order}]") # Get the data attributes based on the primary ext name pext_name = input_model.primary_extension_name input_data = getattr(input_model, pext_name) input_err = getattr(input_model, pext_name + '_err', None) input_dq = getattr(input_model, 'dq', None) input_var_rnoise = getattr(input_model, pext_name + '_var_rnoise', None) input_var_poisson = getattr(input_model, pext_name + '_var_poisson', None) input_mid_time = getattr(input_model, 'mid_time', None) # Sanity check assert trace_positions.shape[1] == input_data.shape[0], f"Trace positions shape[1]={trace_positions.shape[1]} does not match number of pixels={input_data.shape[0]}" assert trace_profile.shape == input_data.shape, f"Trace profile shape: {trace_profile.shape} does not match input data shape: {input_data.shape}" # # NOTE: Keep for debugging # if order == 94: # breakpoint() # pixarr = np.arange(input_model.rate.shape[0]) # plt.imshow(input_data, vmin=0, vmax=np.nanquantile(input_data, 0.995)); plt.plot(trace_positions[j+1], pixarr, c='red'); plt.plot(trace_positions[j], pixarr, c='red'); plt.show() # Perform optimal extraction result = extract_spectrum_optimal( input_data=input_data, input_err=input_err, input_dq=input_dq, input_var_rnoise=input_var_rnoise, input_var_poisson=input_var_poisson, input_mid_time=input_mid_time, trace_positions=trace_positions[j, :], profile=trace_profile, extraction_aperture=self.extraction_aperture, pix_range=self.pix_range, orders_are_vertical=input_model.meta.orders_are_vertical, ) spec_out[j, :] = result['spec'] err_out[j, :] = result['err'] var_rnoise_out[j, :] = result['var_rnoise'] var_poisson_out[j, :] = result['var_poisson'] dq_out[j, :] = result['dq'] mid_time_out[j, :] = result['mid_time']
# NOTE: Keep for debugging # import matplotlib.pyplot as plt; plt.imshow(input_data, vmin=0, vmax=np.nanquantile(input_data, 0.995)); plt.plot(trace_positions,color='r'); plt.show() # plt.plot(spec_out[j, :]); plt.show()
[docs] def extract_cross_talk( self, input_model : datamodels.L0Model, output_model : datamodels.L1Model, tracepos_model : datamodels.TracePosModel, traceprof_model : datamodels.TraceProfModel, do_fibers : np.ndarray, do_orders : np.ndarray, ): n_fibers = len(do_fibers) input_data = getattr(input_model, pext_name) pext_name = input_model.primary_extension_name input_data = getattr(input_model, pext_name) ny, nx = input_data.shape order_vec = input_model.make_order_vec() npix = ny if input_model.meta.orders_are_vertical else nx # Trace profiles for all fibers for a single order (resuse for all orders) trace_profiles = np.zeros((n_fibers, ny, nx), dtype=input_data.dtype) for j, fiber in enumerate(do_fibers): trace_profiles[j, :, :] = getattr(traceprof_model, f'prof_fib{fiber}') # Loop over orders, extract all fibers simultaneously for i, order in enumerate(order_vec): if order not in do_orders: continue # Trace positions for all fibers but only this order trace_positions = np.zeros((n_fibers, npix), dtype=np.float64) for j, fiber in enumerate(do_fibers): trace_positions[j, :] = getattr(tracepos_model, f'pos-fib{fiber}')[i, :] logger.info(f"Extracting {input_model} with method {self.method} [Fibers {do_fibers}][Order {order}]") # NOTE: Keep for debugging #breakpoint() # plt.imshow(input_model.rate); plt.plot(trace_positions[0], c='red'); plt.plot(trace_positions[1], c='red'); plt.show() # Get the data attributes based on the primary ext name input_err = getattr(input_model, pext_name + '_err', None) input_dq = getattr(input_model, 'dq', None) input_var_rnoise = getattr(input_model, pext_name + '_var_rnoise', None) input_var_poisson = getattr(input_model, pext_name + '_var_poisson', None) input_mid_time = getattr(input_model, 'mid_time', None) # Sanity check assert trace_positions.shape[1] == input_data.shape[0], f"Trace positions shape[1]={trace_positions.shape[1]} does not match number of pixels={input_data.shape[0]}" #assert trace_profile.shape == input_data.shape, f"Trace profile shape: {trace_profile.shape} does not match input data shape: {input_data.shape}" # Extract all pixels, all fibers for this order result = extract_spectra_cross_talk( input_data=input_data, input_err=input_err, input_dq=input_dq, input_var_rnoise=input_var_rnoise, input_var_poisson=input_var_poisson, input_mid_time=input_mid_time, trace_positions=trace_positions, profile=trace_profiles, extraction_aperture=self.extraction_aperture, pix_range=self.pix_range, orders_are_vertical=input_model.meta.orders_are_vertical, ) for j, fiber in enumerate(do_fibers): spec_uncal = getattr(output_model, f'SPEC_UNCAL_FIB{fiber}') spec_uncal[i, :] = result['spec'][j, :] err_output_fiber = getattr(output_model, f'SPEC_UNCAL_ERR_FIB{fiber}') err_output_fiber[i, :] = result['err'][j, :] var_rnoise_output_fiber = getattr(output_model, f'SPEC_UNCAL_VAR_RNOISE_FIB{fiber}') var_rnoise_output_fiber[i, :] = result['var_rnoise'][j, :] var_poisson_output_fiber = getattr(output_model, f'SPEC_UNCAL_VAR_POISSON_FIB{fiber}') var_poisson_output_fiber[i, :] = result['var_poisson'][j, :] dq_output_fiber = getattr(output_model, f'SPEC_UNCAL_DQ_FIB{fiber}') dq_output_fiber[i, :] = result['dq'][j, :] mid_time_output_fiber = getattr(output_model, f'MID_TIME_FIB{fiber}') mid_time_output_fiber[i, :] = result['mid_time'][j, :]
# NOTE: Keep for debugging # breakpoint() # import matplotlib.pyplot as plt; plt.imshow(input_model.rate, vmin=0, vmax=np.nanquantile(input_model.rate, 0.995)); plt.plot(trace_positions,color='r'); plt.show() # plt.plot(spec_uncal[j, :]); plt.show()