Source code for hispecdrp.primitives.refpix.reference_SCALES_DRP

# From SCALES DRP: Reference Pixel Correction
# Author: Athira Unni
# Primitive copied from https://github.com/scalessim/SCALES-DRP/blob/main/scalesdrp/primitives/reference.py on 2026-01-03

# import pandas as pd
import numpy as np
from astropy.io import fits
# import warnings
# warnings.filterwarnings("ignore", category=RuntimeWarning)
# import pkg_resources
import os
from scipy.optimize import minimize
import astropy.io.fits as pyfits
import scipy.sparse as sp
import matplotlib.pyplot as plt
import time
#####
# 2026-01-03 JB Ruffio: Hack to use normal numpy statistics as opposed to the robust module from the SCALES DRP
# import scalesdrp.primitives.robust as robust
import numpy as robust
#####
from scipy.optimize import leastsq
# import pkg_resources
from scipy.signal import savgol_filter



[docs] def reffix_hxrg(cube, nchans=4, in_place=False, fixcol=True, **kwargs): # Make sure we work in float32 arr = np.asarray(cube) if not np.issubdtype(arr.dtype, np.floating): # must copy when changing dtype arr = arr.astype(np.float32, copy=True) elif not in_place: arr = arr.copy() # 1) Channel bias (use altcol=False if you're doing ACN next) arr = reffix_amps(arr, nchans=nchans, in_place=True, **kwargs) # 2) ACN (even/odd row-dependent using side refs) arr = acn_filter(arr, in_place=True, **kwargs) # 3) 1/f row stripes (side refs, common-mode) if fixcol: arr = ref_filter(arr, nchans=nchans, in_place=True, **kwargs) return arr
############ ACN & CHANNEL BIAS OLD #########################
[docs] def reffix_amps_old(cube, nchans=4, in_place=True, altcol=True, supermean=False, top_ref=True, bot_ref=True, ntop=4, nbot=4, **kwargs): """ Correct amplifier offsets Matches all amplifier outputs of the detector to a common level. This routine subtracts the average of the top and bottom reference rows for each amplifier and frame individually. Parameters ---------- cube : ndarray Input datacube. Can be two or three dimensions (nz,ny,nx). nchans : int Number of output amplifier channels in the detector. Default=4. altcol : bool Calculate separate reference values for even/odd columns. supermean : bool Add back the overall mean of the reference pixels. in_place : bool Perform calculations in place. Input array is overwritten. top_ref : bool Include top reference rows when correcting channel offsets. bot_ref : bool Include bottom reference rows when correcting channel offsets. ntop : int Specify the number of top reference rows. nbot : int Specify the number of bottom reference rows. mean_func : func Function used to calculate averages. """ if not np.issubdtype(cube.dtype, np.floating): cube = cube.astype(np.float32, copy=not in_place) in_place = True if not in_place: cube = np.copy(cube) ndim = len(cube.shape) if ndim==2: ny,nx = cube.shape nz = 1 cube = cube.reshape((nz,ny,nx)) elif ndim==3: nz, ny, nx = cube.shape else: raise ValueError('Input data can only have 2 or 3 dimensions.Found {} dimensions.'.format(ndim)) chsize = int(nx / nchans) # Number of reference rows to use # Set nt or nb equal to 0 if we don't want to use either nt = ntop if top_ref else 0 nb = nbot if bot_ref else 0 if (nt+nb)==0: print("No reference pixels available for use. Returning...") return # Slice out reference pixels refs_bot = cube[:,:nb,:] refs_top = cube[:,-nt:,:] if nt==0: refs_all = refs_bot elif nb==0: refs_all = refs_top else: refs_all = np.hstack((refs_bot, refs_top)) assert refs_all.shape[1] == (nb+nt) # Supermean # the average of the average is the DC level of the output channel smean = robust.mean(refs_all) if supermean else 0.0 # Calculate avg reference values for each frame and channel #refs_all(reads,8,2048) refs_amps_avg = calc_avg_amps_old(refs_all, cube.shape, nchans=nchans, altcol=altcol, **kwargs) #(4,2, 1, 1) ==> altcol=False (channel, reads,1,1) #(2,2,4) ==> altcol=True [odd/even][channel,reads) for ch in range(nchans): # Channel indices ich1 = ch*chsize ich2 = ich1 + chsize #print(ich1) # In-place subtraction of channel medians if altcol: for i in range(nz): cube[i,:,ich1:ich2-1:2] -= refs_amps_avg[0][ch,i] cube[i,:,ich1+1:ich2:2] -= refs_amps_avg[1][ch,i] else: for i in range(nz): cube[i,:,ich1:ich2] -= refs_amps_avg[ch,i] # Add back supermean if supermean: cube += smean cube = cube.squeeze() return cube
[docs] def calc_avg_amps_old(refs_all, data_shape, nchans=4, altcol=True, mean_func=robust.mean, **kwargs): #refs_all(reads,8,2048) nz, ny, nx = data_shape chsize = int(nx / nchans) if altcol: refs_amps_avg1 = [] refs_amps_avg2 = [] for ch in range(nchans): ich1 = ch*chsize ich2 = ich1 + chsize #extracts every other column in the range [ich1, ich2) & flattens everything except nz refs_ch1 = refs_all[:,:,ich1:ich2-1:2].reshape((nz,-1)) #even refs_ch2 = refs_all[:,:,ich1+1:ich2:2].reshape((nz,-1)) #odd chavg1 = mean_func(refs_ch1,axis=1) ##one scalar per amp per frame. chavg2 = mean_func(refs_ch2,axis=1) refs_amps_avg1.append(chavg1) refs_amps_avg2.append(chavg2) # return one odd and one even value for each channel and each read return (np.array(refs_amps_avg1), np.array(refs_amps_avg2)) else: refs_amps_avg = [] for ch in range(nchans): ich1 = ch*chsize ich2 = ich1 + chsize refs_ch = refs_all[:,:,ich1:ich2].reshape((nz,-1)) chavg = mean_func(refs_ch,axis=1).reshape([-1,1,1]) refs_amps_avg.append(chavg) #return one value for each channel and each read return np.array(refs_amps_avg)
################################## NEW acn ################################
[docs] def reffix_amps(cube, nchans=4, in_place=True, altcol=False, supermean=False, top_ref=True, bot_ref=True, ntop=4, nbot=4, **kwargs): """Correct amplifier offsets Matches all amplifier outputs of the detector to a common level. This routine subtracts the average of the top and bottom reference rows for each amplifier and frame individually. Parameters ---------- cube : ndarray Input datacube. Can be two or three dimensions (nz,ny,nx). nchans : int Number of output amplifier channels in the detector. Default=4. altcol : bool Calculate separate reference values for even/odd columns. supermean : bool Add back the overall mean of the reference pixels. in_place : bool Perform calculations in place. Input array is overwritten. top_ref : bool Include top reference rows when correcting channel offsets. bot_ref : bool Include bottom reference rows when correcting channel offsets. ntop : int Specify the number of top reference rows. nbot : int Specify the number of bottom reference rows. mean_func : func Function used to calculate averages. """ if not np.issubdtype(cube.dtype, np.floating): cube = cube.astype(np.float32, copy=not in_place) in_place = True if not in_place: cube = np.copy(cube) ndim = len(cube.shape) if ndim==2: ny,nx = cube.shape nz = 1 cube = cube.reshape((nz,ny,nx)) elif ndim==3: nz, ny, nx = cube.shape else: raise ValueError('Input data can only have 2 or 3 dimensions.Found {} dimensions.'.format(ndim)) chsize = int(nx / nchans) # Number of reference rows to use # Set nt or nb equal to 0 if we don't want to use either nt = ntop if top_ref else 0 nb = nbot if bot_ref else 0 if (nt+nb)==0: print("No reference pixels available for use. Returning...") return # Slice out reference pixels refs_bot = cube[:,:nb,:] refs_top = cube[:,-nt:,:] if nt==0: refs_all = refs_bot elif nb==0: refs_all = refs_top else: refs_all = np.hstack((refs_bot, refs_top)) assert refs_all.shape[1] == (nb+nt) # Supermean # the average of the average is the DC level of the output channel smean = robust.mean(refs_all) if supermean else 0.0 # Calculate avg reference values for each frame and channel #refs_all(reads,8,2048) refs_amps_avg = calc_avg_amps(refs_all, cube.shape, nchans=nchans, altcol=altcol, **kwargs) #print(refs_amps_avg) for ch in range(nchans): # Channel indices ich1 = ch*chsize ich2 = ich1 + chsize #print(ich1) # In-place subtraction of channel medians if altcol: for i in range(nz): cube[i,:,ich1:ich2-1:2] -= refs_amps_avg[0][ch,i] cube[i,:,ich1+1:ich2:2] -= refs_amps_avg[1][ch,i] else: for i in range(nz): offs = refs_amps_avg[ch, i][np.newaxis, :] # (1, chsize) cube[i, :, ich1:ich2] -= offs # Add back supermean if supermean: cube += smean cube = cube.squeeze() return cube
[docs] def calc_avg_amps(refs_all, data_shape, nchans=4, altcol=False, mean_func=robust.mean, **kwargs): """ Compute average reference values per amplifier. Parameters ---------- refs_all : ndarray Reference pixels for bottom + top rows, shape (nz, nref, nx). data_shape : tuple Full data shape (nz, ny, nx) of the science cube. nchans : int Number of amplifier channels along the x-axis. altcol : bool If True, compute separate averages for even/odd columns (JWST-style). If False, compute a separate value for *each column* in each amp. mean_func : callable Function used to calculate averages (e.g., robust.mean, np.nanmedian). """ # Use refs_all to infer nz, nx (nref doesn't matter) nz_ref, nref, nx = refs_all.shape nz, ny, nx_full = data_shape assert nx == nx_full, "refs_all and data_shape nx mismatch" assert nz_ref == nz, "refs_all and data_shape nz mismatch" chsize = nx // nchans if altcol: # one even and one odd value per amp+frame refs_amps_avg1 = [] refs_amps_avg2 = [] for ch in range(nchans): ich1 = ch * chsize ich2 = ich1 + chsize # Even and odd columns within this channel refs_ch1 = refs_all[:, :, ich1:ich2-1:2].reshape((nz, -1)) # even refs_ch2 = refs_all[:, :, ich1+1:ich2:2].reshape((nz, -1)) # odd chavg1 = mean_func(refs_ch1, axis=1) # (nz,) chavg2 = mean_func(refs_ch2, axis=1) # (nz,) refs_amps_avg1.append(chavg1) refs_amps_avg2.append(chavg2) # Shape: (2, nchans, nz) return (np.array(refs_amps_avg1), np.array(refs_amps_avg2)) else: # --- column-wise averages per amp+frame --- # We want: refs_amps_avg[ch, i, k] = mean over ref rows for that column refs_amps_avg = np.empty((nchans, nz, chsize), dtype=np.float32) for ch in range(nchans): ich1 = ch * chsize ich2 = ich1 + chsize # Extract refs for this amp: (nz, nref, chsize) refs_ch = refs_all[:, :, ich1:ich2] # Average over reference rows axis=1 → (nz, chsize) # i.e., per frame, per column chavg = mean_func(refs_ch, axis=1) # (nz, chsize) refs_amps_avg[ch] = chavg.astype(np.float32) # Shape: (nchans, nz, chsize) return refs_amps_avg
############# additional acn ###################################
[docs] def acn_filter(cube, in_place=True, avg_type='pix', perint=False, edge_wrap=False, left_ref=True, right_ref=True, nleft=4, nright=4, mean_func=np.median, **kwargs): """ Row-dependent differences between even and odd columns (i.e. ACN that changes with row, not just a constant offset per column). They are row-series describing how the even or odd reference pixels vary vs row (and frame), after baseline removal. ACN correction using side reference columns. Separately estimates row-dependent even and odd column offsets, then subtracts them from the full image. Parameters ---------- cube : ndarray Input data: - (H, W) single frame, or - (N, H, W) stack of frames. in_place : bool If False, the input array will be copied. avg_type : {'pix', 'frame', 'int'} Baseline removal mode for reference pixels: - 'pix' : remove per-pixel mean over integration (best for many frames). - 'frame' : remove per-frame global mean of refs. - 'int' : remove a single global mean over the whole ramp. perint : bool Passed to `calc_col_smooth`: smooth per integration instead of per frame. edge_wrap : bool Passed to `calc_col_smooth`: mirror edges before smoothing to reduce ringing. left_ref, right_ref : bool Whether to use left and/or right side reference columns. nleft, nright : int Number of left/right reference columns to use. mean_func : callable Function to compute averages (e.g., np.median or robust.mean). Returns ------- out : ndarray ACN-corrected array (same shape as input, float32). """ arr = np.asarray(cube) print(arr.shape) if arr.ndim == 2: single_frame = True arr = arr[np.newaxis, ...] # (1, H, W) elif arr.ndim == 3: single_frame = False else: raise ValueError(f"acn_filter: input must be 2D or 3D, got shape {arr.shape}") if not in_place: arr = np.copy(arr) N, H, W = arr.shape # Decide how many side references we actually use nl = nleft if left_ref else 0 nr = nright if right_ref else 0 if nl < 0 or nr < 0: raise ValueError("nleft and nright must be non-negative.") if (nl + nr) == 0: print("acn_filter: No side reference columns enabled. Returning input.") return cube out = arr.astype(np.float32, copy=True) # Slice side reference columns refs_left = out[:, :, :nl] if nl > 0 else None # (N, H, nl) refs_right = out[:, :, -nr:] if nr > 0 else None # (N, H, nr) def _normalize_and_avg(refs_left, refs_right, avg_type, mean_func): """ Given left/right refs for even or odd, perform avg_type baseline removal and then average along columns to get (N, H) row-wise reference values. """ nl_flag = 0 if refs_left is None else 1 nr_flag = 0 if refs_right is None else 1 if nl_flag == 0 and nr_flag == 0: # No refs for this parity at all; return zeros return np.zeros((N, H), dtype=np.float32) # Make copies so we don't mutate outer refs if nl_flag: refs_left = np.copy(refs_left) Nloc, Hloc, Cleft = refs_left.shape if nr_flag: refs_right = np.copy(refs_right) if not nl_flag: Nloc, Hloc, Cright = refs_right.shape else: _, _, Cright = refs_right.shape if avg_type is None: mode = 'frame' else: mode = avg_type if Nloc == 1: mode = 'int' # 'int' == 'frame' when only one frame # ---- Remove intrinsic offsets depending on mode ---- if 'int' in mode: # One global scalar over the integration #compute single scalar value over frames and all left and right reference if nl_flag: refs_left -= mean_func(refs_left) if nr_flag: refs_right -= mean_func(refs_right) elif 'frame' in mode: # One scalar per frame if nl_flag: rl_flat = refs_left.reshape(Nloc, -1) rl_mean = mean_func(rl_flat, axis=1) plt.figure() plt.title('left') plt.plot(rl_mean) plt.show() for i in range(Nloc): refs_left[i] -= rl_mean[i] if nr_flag: rr_flat = refs_right.reshape(Nloc, -1) rr_mean = mean_func(rr_flat, axis=1) plt.figure() plt.title('right') plt.plot(rr_mean) plt.show() for i in range(Nloc): refs_right[i] -= rr_mean[i] elif 'pix' in mode: # Per-pixel mean over frames if nl_flag: rl_mean = mean_func(refs_left, axis=0) # (H, Cleft) #print(rl_mean.shape) #print(np.where(np.isnan(rl_mean)==True)) #plt.figure() #plt.plot(rl_mean) #plt.show() for i in range(Nloc): refs_left[i] -= rl_mean #plt.figure() #plt.title('acn using left side ref') #plt.plot(refs_left[i]) #plt.show() if nr_flag: rr_mean = mean_func(refs_right, axis=0) # (H, Cright) #print(np.where(np.isnan(rr_mean)==True)) #plt.figure() #plt.plot(rr_mean) #plt.show() for i in range(Nloc): refs_right[i] -= rr_mean #plt.figure() #plt.title('acn using right side ref') #plt.plot(refs_left[i]) #plt.show() # ---- Average left/right columns down to a single value per row ---- if nl_flag == 0: refs_side_avg = refs_right.mean(axis=2) # (N, H) elif nr_flag == 0: refs_side_avg = refs_left.mean(axis=2) # (N, H) else: # Average left and right refs_side_avg = (refs_left.mean(axis=2) + refs_right.mean(axis=2)) / 2.0 return refs_side_avg.astype(np.float32) # --- Split side refs into even/odd columns # Global column indices cols = np.arange(W) #W is 4 # Left side global indices: 0 .. nl-1 if nl > 0: left_cols = cols[:nl] left_even_mask = (left_cols % 2) == 0 left_odd_mask = ~left_even_mask refs_left_even = refs_left[:, :, left_even_mask] if left_even_mask.any() else None refs_left_odd = refs_left[:, :, left_odd_mask] if left_odd_mask.any() else None else: refs_left_even = refs_left_odd = None # Right side global indices: W-nr .. W-1 if nr > 0: right_cols = cols[-nr:] right_even_mask = (right_cols % 2) == 0 right_odd_mask = ~right_even_mask refs_right_even = refs_right[:, :, right_even_mask] if right_even_mask.any() else None refs_right_odd = refs_right[:, :, right_odd_mask] if right_odd_mask.any() else None else: refs_right_even = refs_right_odd = None # --- Compute raw row-wise ACN refs for even and odd separately --- # Shape (N, H) each ref_even = _normalize_and_avg(refs_left_even, refs_right_even, avg_type, mean_func) #(2,2048) ref_odd = _normalize_and_avg(refs_left_odd, refs_right_odd, avg_type, mean_func) #(2,2048) # --- Smooth them (FFT or SavGol), same as ref_filter / calc_col_smooth --- ref_even_sm = calc_col_smooth(ref_even, out.shape, perint=perint, edge_wrap=edge_wrap, delt=kwargs.get('delt', 5.24e-4), savgol=kwargs.get('savgol', False), winsize=kwargs.get('winsize', 31), order=kwargs.get('order', 3)) ref_odd_sm = calc_col_smooth(ref_odd, out.shape, perint=perint, edge_wrap=edge_wrap, delt=kwargs.get('delt', 5.24e-4), savgol=kwargs.get('savgol', False), winsize=kwargs.get('winsize', 31), order=kwargs.get('order', 3)) # --- Subtract from even/odd columns across the entire image --- #Each row y gets: one subtraction for even columns (ref_even_sm[:, y]), #one subtraction for odd columns (ref_odd_sm[:, y]). even_cols = cols[cols % 2 == 0] odd_cols = cols[cols % 2 == 1] if even_cols.size > 0: out[:, :, even_cols] -= ref_even_sm.reshape(N, H, 1) if odd_cols.size > 0: out[:, :, odd_cols] -= ref_odd_sm.reshape(N, H, 1) if single_frame: return out[0] return out
import numpy as np
[docs] def acn_filter_nobaserm( cube, in_place=True, perint=False, edge_wrap=False, left_ref=True, right_ref=True, nleft=4, nright=4, mean_func=np.median, # still used for averaging over columns **kwargs ): """ ACN correction using side reference columns, *without* any baseline/mean removal. This version uses the raw left/right reference pixel values (after top/bottom correction), averaged over reference columns, then smoothed vs. row, and subtracted from even/odd columns. Parameters ---------- cube : ndarray Input data: - (H, W) single frame, or - (N, H, W) stack of frames. in_place : bool If False, the input array will be copied. perint : bool Passed to `calc_col_smooth`: smooth per integration instead of per frame. edge_wrap : bool Passed to `calc_col_smooth`: mirror edges before smoothing to reduce ringing. left_ref, right_ref : bool Whether to use left and/or right side reference columns. nleft, nright : int Number of left/right reference columns to use. mean_func : callable Function to combine multiple reference columns (e.g., np.median). NOTE: this function is *not* used for baseline subtraction here, only for averaging across columns. Returns ------- out : ndarray ACN-corrected array (same shape as input, float32). """ arr = np.asarray(cube) # --- Normalize dimensionality to (N, H, W) --- if arr.ndim == 2: single_frame = True arr = arr[np.newaxis, ...] # (1, H, W) elif arr.ndim == 3: single_frame = False else: raise ValueError(f"acn_filter_nobase: input must be 2D or 3D, got shape {arr.shape}") if not in_place: arr = np.copy(arr) N, H, W = arr.shape # --- Decide how many side refs we actually use --- nl = nleft if left_ref else 0 nr = nright if right_ref else 0 if nl < 0 or nr < 0: raise ValueError("nleft and nright must be non-negative.") if (nl + nr) == 0: print("acn_filter_nobase: No side reference columns enabled. Returning input.") return cube out = arr.astype(np.float32, copy=True) # --- Slice side reference columns from the full image --- refs_left = out[:, :, :nl] if nl > 0 else None # (N, H, nl) refs_right = out[:, :, -nr:] if nr > 0 else None # (N, H, nr) def _avg_refs(refs_left, refs_right): """ Given left/right refs for a particular parity (even or odd), simply average over reference columns to get (N, H) row-wise values. NO baseline / mean removal is done here. """ nl_flag = 0 if refs_left is None else 1 nr_flag = 0 if refs_right is None else 1 if nl_flag == 0 and nr_flag == 0: # No refs for this parity at all; return zeros return np.zeros((N, H), dtype=np.float32) # Average across ref columns (axis=2) if nl_flag == 0: refs_side_avg = mean_func(refs_right, axis=2) # (N, H) elif nr_flag == 0: refs_side_avg = mean_func(refs_left, axis=2) # (N, H) else: left_avg = mean_func(refs_left, axis=2) # (N, H) right_avg = mean_func(refs_right, axis=2) # (N, H) refs_side_avg = 0.5 * (left_avg + right_avg) return refs_side_avg.astype(np.float32) # --- Split side refs into even/odd columns (global parity) --- cols = np.arange(W) # Left side global indices: 0 .. nl-1 if nl > 0: left_cols = cols[:nl] left_even_mask = (left_cols % 2) == 0 left_odd_mask = ~left_even_mask refs_left_even = refs_left[:, :, left_even_mask] if left_even_mask.any() else None refs_left_odd = refs_left[:, :, left_odd_mask] if left_odd_mask.any() else None else: refs_left_even = refs_left_odd = None # Right side global indices: W-nr .. W-1 if nr > 0: right_cols = cols[-nr:] right_even_mask = (right_cols % 2) == 0 right_odd_mask = ~right_even_mask refs_right_even = refs_right[:, :, right_even_mask] if right_even_mask.any() else None refs_right_odd = refs_right[:, :, right_odd_mask] if right_odd_mask.any() else None else: refs_right_even = refs_right_odd = None # --- Build row-wise ACN refs for even and odd, using RAW refs (no baseline removal) --- # Shapes: (N, H) ref_even = _avg_refs(refs_left_even, refs_right_even) ref_odd = _avg_refs(refs_left_odd, refs_right_odd) # --- Smooth them (FFT or SavGol), same as ref_filter / calc_col_smooth --- ref_even_sm = calc_col_smooth( ref_even, out.shape, perint=perint, edge_wrap=edge_wrap, delt=kwargs.get('delt', 5.24e-4), savgol=kwargs.get('savgol', False), winsize=kwargs.get('winsize', 31), order=kwargs.get('order', 3), ) ref_odd_sm = calc_col_smooth( ref_odd, out.shape, perint=perint, edge_wrap=edge_wrap, delt=kwargs.get('delt', 5.24e-4), savgol=kwargs.get('savgol', False), winsize=kwargs.get('winsize', 31), order=kwargs.get('order', 3), ) # --- Subtract from even/odd columns across the entire image --- even_cols = cols[cols % 2 == 0] odd_cols = cols[cols % 2 == 1] if even_cols.size > 0: out[:, :, even_cols] -= ref_even_sm.reshape(N, H, 1) if odd_cols.size > 0: out[:, :, odd_cols] -= ref_odd_sm.reshape(N, H, 1) if single_frame: return out[0] return out
############################# 1/f ################################################
[docs] def ref_filter(cube, nchans=4, in_place=True, avg_type='pix', perint=False, edge_wrap=False, left_ref=True, right_ref=True, nleft=4, nright=4, **kwargs): """ Optimal Smoothing Performs an optimal filtering of the vertical reference pixel to reduce 1/f noise (horizontal stripes). FFT method adapted from M. Robberto IDL code: http://www.stsci.edu/~robberto/Main/Software/IDL4pipeline/ Parameters ---------- cube : ndarray Input datacube. Can be two or three dimensions (nz,ny,nx). nchans : int Number of output amplifier channels in the detector. Default=4. in_place : bool Perform calculations in place. Input array is overwritten. perint : bool Smooth side reference pixel per integration, otherwise do frame-by-frame. avg_type : str Type of ref col averaging to perform. Allowed values are 'pix', 'frame', or 'int'. left_ref : bool Include left reference cols when correcting 1/f noise. right_ref : bool Include right reference cols when correcting 1/f noise. nleft : int Specify the number of left reference columns. nright : int Specify the number of right reference columns. savgol : bool Using Savitsky-Golay filter method rather than FFT. winsize : int Size of the window filter. order : int Order of the polynomial used to fit the samples. mean_func : func Function to use to calculate averages of reference columns. """ if not in_place: cube = np.copy(cube) ndim = len(cube.shape) if ndim==2: ny,nx = cube.shape nz = 1 cube = cube.reshape((nz,ny,nx)) elif ndim==3: nz, ny, nx = cube.shape else: raise ValueError('Input data can only have 2 or 3 dimensions. Found {} dimensions.'.format(ndim)) # Number of reference rows to use # Set nt or nb equal to 0 if we don't want to use either nl = nleft if left_ref else 0 nr = nright if right_ref else 0 assert nl>=0, 'Number of left reference pixels must not be negative.' assert nr>=0, 'Number of right reference pixels must not be negative.' if (nl+nr)==0: print("No reference pixels available for use. Returning...") return # Slice out reference pixel columns refs_left = cube[:,:,:nl] if nl>0 else None refs_right = cube[:,:,-nr:] if nr>0 else None refvals = calc_avg_cols(refs_left, refs_right, avg_type, **kwargs) #refvals = calc_avg_cols_no_baseline(refs_left, refs_right) # approximate x axis for smoothening delt = 10E-6 * (nx/nchans + 12.) refvals_smoothed = calc_col_smooth(refvals, cube.shape, perint=perint, edge_wrap=edge_wrap, delt=delt, **kwargs) # Final correction #for i,im in enumerate(cube): im -= refvals_smoothed[i].reshape([ny,1]) cube -= refvals_smoothed.reshape([nz,ny,1]) cube = cube.squeeze() return cube
[docs] def calc_avg_cols_no_baseline(refs_left=None, refs_right=None): """ Same output as calc_avg_cols(), but does NOT remove reference pixel means / DC offsets. Useful for comparison — this version allows reference pixel fixed-pattern biases to leak directly into the correction vector. Parameters ---------- refs_left : ndarray or None Left reference strip, shape (N, H, nleft) refs_right : ndarray or None Right reference strip, shape (N, H, nright) Returns ------- refs_side_avg : ndarray (N, H) Row-averaged reference drift without mean removal. """ nl = 0 if refs_left is None else 1 nr = 0 if refs_right is None else 1 if nl == 0 and nr == 0: return None # nothing to return # Copy so we don't modify caller data if nl > 0: refs_left = np.copy(refs_left) if nr > 0: refs_right = np.copy(refs_right) # ---- NO baseline normalization performed here ---- # Simply collapse reference pixels to one value per row if nl == 0: refs_side_avg = refs_right.mean(axis=2) # shape (N, H) elif nr == 0: refs_side_avg = refs_left.mean(axis=2) # shape (N, H) else: refs_side_avg = (refs_left.mean(axis=2) + refs_right.mean(axis=2)) / 2.0 return refs_side_avg.astype(np.float32)
[docs] def calc_avg_cols(refs_left=None, refs_right=None, avg_type='pix', mean_func=np.median, **kwargs): """ Calculate average of column references Determine the average values for the column references, which is subsequently used to estimate the 1/f noise contribution. Parameters ---------- refs_left : ndarray Left reference columns. refs_right : ndarray Right reference columns. avg_type : str Type of ref column averaging to perform to determine ref pixel variation. Allowed values are 'pixel', 'frame', or 'int'. - 'pixel' : For each ref pixel, subtract its avg value from all frames. - 'frame' : For each frame, get avg ref pixel values and subtract framewise. - 'int' : Calculate avg of all ref pixels within the ramp and subtract. mean_func : func Function to use to calculate averages of reference columns """ # Which function to use for calculating averages? # mean_func = robust.mean # mean_func = np.median # In this context, nl and nr are either 0 (False) or 1 (True) nl = 0 if refs_left is None else 1 nr = 0 if refs_right is None else 1 # Left and right reference pixels # Make a copy so as to not modify the original data if nl>0: refs_left = np.copy(refs_left) if nr>0: refs_right = np.copy(refs_right) # Set the average of left and right reference pixels to zero # By default, pixel averaging is best for large groups if avg_type is None: avg_type = 'frame' if refs_left is not None: nz, ny, nchan = refs_left.shape else: nz, ny, nchan = refs_right.shape # If there is only 1 frame, then we have to do "per frame" averaging. # Set to "per int", which produces the same result as "per frame" for nz=1. if nz==1: avg_type = 'int' # Remove average ref pixel values # Average over entire integration if 'int' in avg_type: if nl>0: refs_left -= mean_func(refs_left) #refs_left now contains only deviations from that global mean. if nr>0: refs_right -= mean_func(refs_right) # Average over each frame #'frame' mode remove a frame-by-frame DC offset, #leaving row-dependent fluctuations within each frame. # elif 'frame' in avg_type: if nl>0: refs_left_mean = mean_func(refs_left.reshape((nz,-1)), axis=1)#flatten each frame’s ref pixels to 1D, one scalar per frame. if nr>0: refs_right_mean = mean_func(refs_right.reshape((nz,-1)), axis=1) # Subtract estimate of each ref pixel "intrinsic" value for i in range(nz): if nl>0: refs_left[i] -= refs_left_mean[i] if nr>0: refs_right[i] -= refs_right_mean[i] # Take the average of each reference pixel #gives an average for each reference pixel position over all frames #Subtracting this from each frame removes each pixel’s own intrinsic bias pattern. elif 'pix' in avg_type: if nl>0: refs_left_mean = mean_func(refs_left, axis=0) #(2048, 4) if nr>0: refs_right_mean = mean_func(refs_right, axis=0) # Subtract estimate of each ref pixel "intrinsic" value for i in range(nz): if nl>0: refs_left[i] -= refs_left_mean if nr>0: refs_right[i] -= refs_right_mean if nl==0: #ref_left is none refs_side_avg = refs_right.mean(axis=2)#averages over all left reference columns elif nr==0: #ref_right is none refs_side_avg = refs_left.mean(axis=2) else: refs_side_avg = (refs_right.mean(axis=2) + refs_left.mean(axis=2)) / 2 return refs_side_avg #(2, 2048)
[docs] def calc_col_smooth(refvals, data_shape, perint=False, edge_wrap=False, delt=5.24E-4, savgol=False, winsize=31, order=3, **kwargs): """ Perform optimal smoothing of side ref pix Generates smoothed version of column reference values. Smooths values from calc_avg_cols() via FFT. Parameters ---------- refvals : ndarray Averaged column reference pixels data_shape : tuple Shape of original data (nz,ny,nx) perint : bool Smooth side reference pixel per int, otherwise per frame. edge_wrap : bool Add a partial frames to the beginning and end of each averaged time series pixels in order to get rid of edge effects. delt : float Time between reference pixel samples. savgol : bool Using Savitsky-Golay filter method rather than FFT. winsize : int Size of the window filter. order : int Order of the polynomial used to fit the samples. """ #in perint=True, you are smoothing across frames and rows together, #like treating the entire cube’s ref signal as one long time-series. nz,ny,nx = data_shape if perint: # per integration, treats the entire (nz, ny) array as one flattened 1D series if edge_wrap: # Wrap around to avoid edge effects, "mirror" the first and last frames to reduce edge ringing: #These are stacked above and below refvals #After smoothing, will strip off these mirrored sections and reshape. refvals2 = np.vstack((refvals[0][::-1], refvals, refvals[-1][::-1])) if savgol: # SavGol filter refvals_smoothed2 = savgol_filter(refvals2.ravel(), winsize, order, delta=1) else: # Or "optimal" smoothing algorithm refvals_smoothed2 = smooth_fft(refvals2, delt) refvals_smoothed = refvals_smoothed2[ny:-ny].reshape(refvals.shape) else: if savgol: refvals_smoothed = savgol_filter(refvals.ravel(), winsize, order, delta=1) else: refvals_smoothed = smooth_fft(refvals, delt) refvals_smoothed = refvals_smoothed.reshape(refvals.shape) else: #smooth each frame’s row-series separately refvals_smoothed = [] if edge_wrap: # Wrap around to avoid edge effects for ref in refvals: #(ny,) #mirror to handle FFT more gently on edges ref2 = np.concatenate((ref[:ny//2][::-1], ref, ref[ny//2:][::-1])) if savgol: ref_smth = savgol_filter(ref2, winsize, order, delta=1) else: ref_smth = smooth_fft(ref2, delt) refvals_smoothed.append(ref_smth[ny//2:ny//2+ny]) refvals_smoothed = np.array(refvals_smoothed) else: for ref in refvals: if savgol: ref_smth = savgol_filter(ref, winsize, order, delta=1) else: ref_smth = smooth_fft(ref, delt) refvals_smoothed.append(ref_smth) refvals_smoothed = np.array(refvals_smoothed) return refvals_smoothed
[docs] def smooth_fft(data, delt, first_deriv=False, second_deriv=False): """Optimal smoothing algorithm Smoothing algorithm to perform optimal filtering of the vertical reference pixel to reduce 1/f noise (horizontal stripes), based on the Kosarev & Pantos algorithm. This assumes that the data to be filtered/smoothed has been sampled evenly. If first_deriv is set, then returns two results if second_deriv is set, then returns three results. Adapted from M. Robberto IDL code: http://www.stsci.edu/~robberto/Main/Software/IDL4pipeline/ Parameters ---------- data : ndarray Signal to be filtered. delt : float Delta time between samples. first_deriv : bool Return the first derivative. second_deriv : bool Return the second derivative (along with first). """ Dat = data.flatten() N = Dat.size Pi2 = 2*np.pi OMEGA = Pi2 / (N*delt) X = np.arange(N) * delt ##------------------------------------------------ ## Center and Baselinefit of the data ##------------------------------------------------ Dat_m = Dat - np.mean(Dat) #data with its global mean removed SLOPE = (Dat_m[-1] - Dat_m[0]) / (N-2) #slope Dat_b = Dat_m - Dat_m[0] - SLOPE * X / delt #detrended, linear baseline removed ##------------------------------------------------ ## Compute fft- / power- spectrum ##------------------------------------------------ Dat_F = np.fft.rfft(Dat_b) #FFT Dat_P = np.abs(Dat_F)**2 #power spectrum ##------------------------------------------------ ## Noise spectrum from 'half' to 'full' ## Mind: half means N/4, full means N/2 ## assume that mid–high frequencies (from N/4 to N/2) are dominated by white noise, not signal. ## compute the average power in that band ==> Noise ## This is an estimate of the noise floor of the spectrum. ##------------------------------------------------ i1 = int((N-1) / 4) i2 = int((N-1) / 2) + 1 Sigma = np.sum(Dat_P[i1:i2]) Noise = Sigma / ((N-1)/2 - (N-1)/4) ##------------------------------------------------ ## Get Filtercoeff. according to Kosarev/Pantos ## Find the J0, start search at i=1 (i=0 is the mean), where the signal falls into the noise ##------------------------------------------------ J0 = 2 for i in np.arange(1, int(N/4)+1): sig0, sig1, sig2, sig3 = Dat_P[i:i+4] if (sig0<Noise) and ((sig1<Noise) or (sig2<Noise) or (sig3<Noise)): J0 = i break ##------------------------------------------------ ## Compute straight line extrapolation to log(Dat_P) ##------------------------------------------------ ii = np.arange(1,J0+1) logvals = np.log(Dat_P[1:J0+1]) XY = np.sum(ii * logvals) XX = np.sum(ii**2) S = np.sum(logvals) # Find parameters A1, B1 XM = (2. + J0) / 2 YM = S / J0 A1 = (XY - J0*XM*YM) / (XX - J0*XM*XM) B1 = YM - A1 * XM # Compute J1, the frequency for which straight # line extrapolation drops 20dB below noise J1 = int(np.ceil((np.log(0.01*Noise) - B1) / A1 )) if J1<J0: J1 = J0+1 ##------------------------------------------------ ## Compute the Kosarev-Pantos filter windows ## Frequency-ranges: 0 -- J0 | J0+1 -- J1 | J1+1 -- N2 ##------------------------------------------------ nvals = int((N-1)/2 + 1) LOPT = np.zeros_like(Dat_P) LOPT[0:J0+1] = Dat_P[0:J0+1] / (Dat_P[0:J0+1] + Noise) i_arr = np.arange(J1-J0) + J0+1 LOPT[J0+1:J1+1] = np.exp(A1*i_arr+B1) / (np.exp(A1*i_arr+B1) + Noise) ##-------------------------------------------------------------------- ## De-noise the Spectrum with the filter ## Calculate the first and second derivative (i.e. multiply by iW) ##-------------------------------------------------------------------- # first loop gives smoothed data # second loop produces first derivative # third loop produces second derivative if second_deriv: ndiff = 3 elif first_deriv: ndiff = 2 else: ndiff = 1 for diff in range(ndiff): Fltr_Spectrum = np.zeros_like(Dat_P,dtype=complex) # make the filter complex i1 = 1; n2 = int((N-1)/2)+1; i2 = i1+n2 FltrCoef = LOPT[i1:].astype(np.complex128) # differentitation in frequency domain iW = ((np.arange(n2)+i1)*OMEGA*1j)**diff # multiply spectrum with filter coefficient Fltr_Spectrum[i1:] = Dat_F[i1:] * FltrCoef * iW # Fltr_Spectrum[0] values # The derivatives of Fltr_Spectrum[0] are 0 # Mean if diff = 0 Fltr_Spectrum[0] = 0 if diff>0 else Dat_F[0] # Inverse fourier transform back in time domain Dat_T = np.fft.irfft(Fltr_Spectrum) #Dat_T[-1] = np.real(Dat_T[0]) + 1j*np.imag(Dat_T[-1]) # This ist the smoothed time series (baseline added) if diff==0: Smoothed_Data = np.real(Dat_T) + Dat[0] + SLOPE * X / delt elif diff==1: First_Diff = np.real(Dat_T) + SLOPE / delt elif diff==2: Secnd_Diff = np.real(Dat_T) if second_deriv: return Smoothed_Data, First_Diff, Secnd_Diff elif first_deriv: return Smoothed_Data, First_Diff else: return Smoothed_Data #return the original data, but with high-frequency noise stripped out,
#and only the low-frequency “shape” left, plus the original mean/trend.