Source code for hispecdrp.primitives.sky_subtraction.sky_primitive

from ..base_primitive import BasePrimitive
from ... import datamodels
import numpy as np
from typing import Sequence


from ...calibrations import FiberFlatSelector

import plotly.graph_objects as go
from plotly.subplots import make_subplots

import logging

logger = logging.getLogger(__name__)

__all__ = ['SkySubtractionPrimitive']

[docs] class SkySubtractionPrimitive(BasePrimitive): """ Subtract the sky background from the input data. This primitive applies the appropriate sky calibration to the input data product. The sky frame is subtracted from the science data, the associated uncertainties are propagated by adding errors in quadrature, and data quality (DQ) flags are combined using a bitwise OR. Parameters ---------- input : str or L2Model Input file path or data model to process. Currently, this is expected to be either a level 1 (uncalibrated) or level 2 (calibrated) data product. suffix : str or None, optional Suffix to append to the output filename. Default is ``'skysub'``. sky_fiber : int, optional Manually override the sky fiber to a specific fiber sci_fiber : int, optional Manually override the science fiber to a specific fiber Calibrations ------------ fiberflat : FiberFlatModel - selector: :class:`~hispecdrp.calibrations.selectors.FiberFlatSelector` Returns ------- output The sky-subtracted level 2 data model with updated error and data quality arrays. """ PARAMETERS = { 'suffix' : 'skysub', 'do_orders' : None, 'sky_fiber' : None, 'sci_fiber' : None, } CALIBRATIONS = { 'fiberflat': { 'selector': FiberFlatSelector, 'selector_kwargs': {} } } def _perform(self): # if isinstance(self.dataset, (datamodels.DataModel, str)): # output = None # input = [self.dataset] # elif isinstance(self.dataset, Sequence): # output = [] # input = self.dataset # Loop over inputs (now a list) #for _input in input: # Load the input model input_model = datamodels.open(self.dataset) # Init output model #output_model = self._model_types(input_model) output_model = self.get_output_model(input_model) # Which orders to process _, do_orders = self.get_do_traces(input_model, [1,2,3], self.do_orders) order_vec = input_model.make_order_vec() n_orders = len(order_vec) # Get calibrations fiberflat, _ = self.get_calibration(input_model, 'fiberflat') fiberflat_model = datamodels.open(fiberflat) spec_key = 'SPEC_UNCAL_' if isinstance(input_model, datamodels.level1.L1Model) else 'SPEC_' # get science/sky fiber numbers, sky fiber defaults to farthest fiber from tracking axis sci_fib_num, sky_fib_num = self._resolve_fibers(input_model, self.sci_fiber, self.sky_fiber) # get sky, warn if sky fiber seems unusually close or bright sky_spec = getattr(input_model, f'{spec_key}FIB{sky_fib_num}') sky_spec_err = getattr(input_model, f'{spec_key}ERR_FIB{sky_fib_num}') sky_wave = getattr(input_model, f"WAVE_FIB{sky_fib_num}") if sky_spec is None or sky_spec_err is None or sky_wave is None: raise ValueError(f"Sky fiber {sky_fib_num} data is missing. Cannot perform sky subtraction.") if input_model.meta.get(f'distance_fiber{sky_fib_num}') is not None: if input_model.meta.get(f'distance_fiber{sky_fib_num}') < 1.0: #TODO check if warning distance is appropriate number logger.warning(f"Sky fiber {sky_fib_num} is unusually close to the tracking axis. Sky subtraction may be inaccurate.") elif np.nanmedian(sky_spec) > 0.3: # arbitrary threshold for "too bright" #TODO check brightness number with observed sky brightnesses logger.warning(f"Sky fiber {sky_fib_num} seems too bright. Sky subtraction may be inaccurate.") # get science spectrum sci_spec = getattr(input_model, f'{spec_key}FIB{sci_fib_num}') sci_spec_err = getattr(input_model, f'{spec_key}ERR_FIB{sci_fib_num}') sci_wave = getattr(input_model, f"WAVE_FIB{sci_fib_num}") if sci_spec is None or sci_spec_err is None or sci_wave is None: raise ValueError(f"Science fiber {sci_fib_num} data is missing. Cannot perform sky subtraction.") # get variance and DQ arrays for science and sky fibers sci_var_poisson = output_model[sci_fib_num].spec_var_poisson sci_var_rnoise = output_model[sci_fib_num].spec_var_rnoise sci_dq = output_model[sci_fib_num].spec_dq sky_dq = output_model[sky_fib_num].spec_dq # get fiberflat spectrum from same fibers as sky/science fiberflat_sky_spec = getattr(fiberflat_model, f"SPEC_FIB{sky_fib_num}") fiberflat_sky_wave = getattr(fiberflat_model, f"WAVE_FIB{sky_fib_num}") fiberflat_sci_spec = getattr(fiberflat_model, f"SPEC_FIB{sci_fib_num}") fiberflat_sci_wave = getattr(fiberflat_model, f"WAVE_FIB{sci_fib_num}") sci_spec_sky_subtracted = np.empty_like(sci_spec) sci_spec_sky_subtracted_err = np.empty_like(sci_spec_err) sky_spec_scaled = np.empty_like(sci_spec) sky_spec_err_scaled = np.empty_like(sci_spec_err) out_dq = np.empty_like(sci_dq) for order in range(n_orders): if order_vec[order] not in do_orders: continue # get relative transmission ratio from science/sky from fiberflat calibration if fiberflat_sky_spec is None or fiberflat_sci_spec is None: logger.warning(f"Fiberflat calibration missing for science or sky fiber for order {order_vec[order]}. Sky subtraction may be inaccurate.") fiberflat_ratio = 1.0 else: fiberflat_sky_spec_interp = np.interp(fiberflat_sci_wave[order], fiberflat_sky_wave[order], fiberflat_sky_spec[order]) fiberflat_ratio = fiberflat_sci_spec[order] / fiberflat_sky_spec_interp if np.all(np.isnan(fiberflat_ratio)): logger.warning(f"Fiberflat ratio is all NaN for order {order_vec[order]}. Falling back to a ratio of 1.0. Sky subtraction may be inaccurate.") fiberflat_ratio = 1.0 sky_spec_interp = np.interp(sci_wave[order], sky_wave[order], sky_spec[order]) sky_spec_scaled[order] = sky_spec_interp * fiberflat_ratio # Interpolate sky error onto science wavelength grid and scale by fiberflat sky_spec_err_interp = np.interp(sci_wave[order], sky_wave[order], sky_spec_err[order]) sky_spec_err_scaled[order] = sky_spec_err_interp * fiberflat_ratio sci_spec_sky_subtracted[order] = sci_spec[order] - sky_spec_scaled[order] # Propagate flux errors in quadrature: err = sqrt(sci_err^2 + (scaled_sky_err)^2) sci_spec_sky_subtracted_err[order] = np.sqrt( sci_spec_err[order]**2 + sky_spec_err_scaled[order]**2 ) out_of_bounds_mask = np.logical_or((sci_wave[order] < np.nanmin(sky_wave[order])), (sci_wave[order] > np.nanmax(sky_wave[order]))) out_of_bounds_dq = np.zeros(len(sci_wave[order]), dtype=np.int32) out_of_bounds_dq[out_of_bounds_mask] = np.int32(1) out_dq[order] = sci_dq[order] | out_of_bounds_dq setattr(output_model, f'SPEC_FIB{sci_fib_num}', sci_spec_sky_subtracted) setattr(output_model, f'SPEC_ERR_FIB{sci_fib_num}', sci_spec_sky_subtracted_err) setattr(output_model, f'SPEC_VAR_POISSON_FIB{sci_fib_num}', sci_var_poisson.copy()) setattr(output_model, f'SPEC_VAR_RNOISE_FIB{sci_fib_num}', sci_var_rnoise.copy()) setattr(output_model, f'SPEC_DQ_FIB{sci_fib_num}', out_dq) setattr(output_model, f'SKY_FIB{sci_fib_num}', sky_spec_scaled) setattr(output_model, f'SKY_ERR_FIB{sci_fib_num}', sky_spec_err_scaled) # if isinstance(output, list): # output.append(output_model) # else: # output = output_model return dict(output=output_model)
[docs] def get_output_model(self, input_model, **kwargs): if isinstance(input_model, datamodels.L1Model): output_model = 'L2Model' elif isinstance(input_model, datamodels.L2Model): output_model = 'input' else: raise TypeError(f"Unsupported input model type: {type(input_model)}. Expected L1Model or L2Model.") output_model = super().get_output_model( input_model, output_model=output_model, include_data=True, **kwargs ) return output_model
[docs] def _create_preview(self): output_model = self._output['output'] # Normalize to a list if not isinstance(output_model, list): output_models = [output_model] else: output_models = output_model # Pre-compute per-model metadata so we can build traces in one pass model_meta = [] for model in output_models: sci_fib_num, sky_fib_num = self._resolve_fibers(model, self.sci_fiber, self.sky_fiber) order_vec = model.make_order_vec() filename = model.meta.get('filename', f'Model {len(model_meta) + 1}') model_meta.append(dict( model = model, sci_fib_num = sci_fib_num, sky_fib_num = sky_fib_num, order_vec = order_vec, n_orders = len(order_vec), filename = filename, )) # All models must share the same order vector for the slider to be coherent; # use the first model's order_vec as the reference. ref_order_vec = model_meta[0]['order_vec'] n_orders = model_meta[0]['n_orders'] traces_per_order = 2 # sky + sky_err band fig = go.Figure() # Build traces: outer loop = models, inner loop = orders # Layout: [model0_order0_sky, model0_order0_err, model0_order1_sky, ...] # [model1_order0_sky, model1_order0_err, ...] traces_per_model = n_orders * traces_per_order total_traces = len(output_models) * traces_per_model for m_idx, meta in enumerate(model_meta): model = meta['model'] sci_fib_num = meta['sci_fib_num'] sky_fib_num = meta['sky_fib_num'] order_vec = meta['order_vec'] sky_wave = getattr(model, f'WAVE_FIB{sci_fib_num}') sky_spec = getattr(model, f'SKY_FIB{sci_fib_num}') sky_err = getattr(model, f'SKY_ERR_FIB{sci_fib_num}') for i, order in enumerate(order_vec): # Only the first model's first order is visible on load visible = (m_idx == 0 and i == 0) w_sky = sky_wave[i] fig.add_trace(go.Scatter( x = w_sky, y = sky_spec[i], mode = 'lines', name = f'Sky (fiber {sky_fib_num})', line = dict(color='tomato', width=1.5), legendgroup = 'sky', showlegend = True, visible = visible, )) fig.add_trace(go.Scatter( x = np.concatenate([w_sky, w_sky[::-1]]), y = np.concatenate([ sky_spec[i] + sky_err[i], (sky_spec[i] - sky_err[i])[::-1] ]), fill = 'toself', fillcolor = 'rgba(255,99,71,0.15)', line = dict(color='rgba(0,0,0,0)'), name = 'Sky ±1σ', legendgroup = 'sky_err', showlegend = True, visible = visible, )) # --- Order slider --- # Each step shows the currently-selected model's traces for one order. # The active model index is tracked via a mutable closure updated by the # dropdown buttons below (Plotly sliders can't directly read button state, # so each dropdown button embeds a full visibility list for order 0 of that # model, and the slider steps are recomputed per-model inside each button). # # Strategy: slider steps always operate on ALL traces. For a given # (model_idx, order_idx) pair, exactly `traces_per_order` traces are True. def make_visibility(active_model: int, active_order: int) -> list: """Return a full visibility list with only the chosen model+order shown.""" vis = [False] * total_traces base = active_model * traces_per_model + active_order * traces_per_order for t in range(traces_per_order): vis[base + t] = True return vis # Slider steps fix model=0 by default; dropdown buttons override this by # supplying their own visibility list for every step via restyle calls. slider_steps = [] for i, order in enumerate(ref_order_vec): slider_steps.append(dict( method = 'update', label = str(order), args = [ {'visible': make_visibility(0, i)}, {'title': f'Sky — Order {order} | {model_meta[0]["filename"]}'} ], )) # --- Model dropdown --- # Each button sets visibility for order 0 of that model AND rewrites the # slider steps so subsequent slider moves stay within the same model. dropdown_buttons = [] for m_idx, meta in enumerate(model_meta): # Build slider steps for this model model_slider_steps = [] for i, order in enumerate(meta['order_vec']): model_slider_steps.append(dict( method = 'update', label = str(order), args = [ {'visible': make_visibility(m_idx, i)}, {'title': f'Sky — Order {order} | {meta["filename"]}'} ], )) dropdown_buttons.append(dict( label = meta['filename'], method = 'update', args = [ # Show order 0 for this model {'visible': make_visibility(m_idx, 0)}, { 'title': f'Sky — Order {ref_order_vec[0]} | {meta["filename"]}', # Replace the slider steps so it now controls this model 'sliders': [dict( active = 0, currentvalue = dict(prefix='Order: ', font=dict(size=14)), pad = dict(t=50), steps = model_slider_steps, )], }, ], )) fig.update_layout( title = f'Sky — Order {ref_order_vec[0]} | {model_meta[0]["filename"]}', updatemenus = [dict( buttons = dropdown_buttons, direction = 'down', showactive = True, x = 1.0, xanchor = 'right', y = 1.15, yanchor = 'top', bgcolor = '#f0f0f0', bordercolor = '#cccccc', )] if len(output_models) > 1 else [], sliders = [dict( active = 0, currentvalue = dict(prefix='Order: ', font=dict(size=14)), pad = dict(t=50), steps = slider_steps, )], xaxis = dict(title='Wavelength (micron)', showgrid=True, gridcolor='rgba(200,200,200,0.3)'), yaxis = dict(title='Flux (e-/s)', showgrid=True, gridcolor='rgba(200,200,200,0.3)'), plot_bgcolor = '#ffffff', paper_bgcolor = '#ffffff', font = dict(color='#000000'), legend = dict(bgcolor='#ffffff'), height = 550, ) # File paths — use the first model as the naming anchor base_path = output_models[0]._generate_output_path(output_dir=self.output_dir, suffix=self.suffix) output_path_html = base_path.replace('.fits', '_preview.html') output_path_png = base_path.replace('.fits', '_preview.png') if self.save_preview: fig.write_html(output_path_html, include_plotlyjs='cdn') fig.write_image(output_path_png, scale=2) if self.show_preview: fig.show() self._output['preview'] = { 'html': output_path_html, 'png': output_path_png, 'fig': fig, }
[docs] def _resolve_fibers(self, model, sci_fiber_override=None, sky_fiber_override=None): """Resolve science and sky fiber numbers from overrides or metadata.""" sci_fib_num = sci_fiber_override if sci_fib_num is not None: logger.info(f'Science fiber manually set to fiber {sci_fib_num}') else: for fiber in range(1, 4): if model.meta.get(f'object_fiber{fiber}', '').upper() == 'SCIENCE': sci_fib_num = fiber break if sci_fib_num is None: sci_fib_num = 1 logger.info(f'Could not find SCIENCE keyword in fiber meta data, science fiber set to {sci_fib_num}') sky_fib_num = sky_fiber_override if sky_fib_num is not None: logger.info(f'Sky fiber manually set to fiber {sky_fib_num}') else: sky_fib_dist = 0. for fiber in range(1, 4): fib_dist = model.meta.get(f'distance_fiber{fiber}') if fib_dist is None: logger.info(f'Could not find distance_fiber keyword in meta data for fiber {fiber}') elif fib_dist > sky_fib_dist: sky_fib_num = fiber sky_fib_dist = fib_dist if sky_fib_num is None: sky_fib_num = 3 logger.info(f'No distance_fiber keywords found in meta data for any fibers, sky fiber will be set to {sky_fib_num}.') return sci_fib_num, sky_fib_num