Source code for hispecdrp.datamodels.meta_data
from pathlib import Path
import yaml
import copy
import numpy as np
import jsonschema
import jsonref
from typing import MutableMapping, Self
from astropy.io import fits
import os
from .utils import get_instrument_config
import logging
logger = logging.getLogger(__name__)
_META_SCHEMA_DEFAULTS = {
'title' : '', # Human-readable title, used as FITS comment
'fits_section' : 'core', # Section in FITS header
'fits_hdu' : 'PRIMARY', # FITS HDU name
'type' : 'string', # JSON schema type
'unit' : None, # Physical unit
'cal_db' : None, # Column name in calibration DB. True -> Use property name, str -> use provided string, None -> not included in DB
'merge_rule' : 'first', # Rule for merging metadata from multiple files
'include_ancestors' : False, # Include in ancestors table when merging data models?
}
# Additional JSON schema fields:
# - enum: List of allowed values
# - default: Default value
[docs]
class MetaModel(MutableMapping):
"""
Small helper class for handling metadata based on JSON schema.
Parameters
----------
schema : dict
The metadata schema.
meta : dict | None, optional
Initial metadata values. Defaults to None.
validate : bool, optional
Whether to validate metadata values against the schema. Defaults to True.
"""
def __init__(self, schema: dict, meta: dict | None = None, validate : bool = True):
self._validate = validate
self._schema = schema
self._data = {}
self._validators = {}
if meta is not None:
for k, v in meta.items():
self[k] = v
self.set_default_values()
def __getitem__(self, key):
if key in self._schema['properties']:
return self._data.get(key)
else:
raise KeyError(f"Property '{key}' not found in metadata schema.")
def __setitem__(self, key, value):
if key not in self._schema['properties']:
raise KeyError(f"Property '{key}' not found in metadata schema.")
if self._validate:
self.validate(key, value)
self._data[key] = value
def __getattr__(self, name):
try:
return self[name]
except KeyError as e:
raise AttributeError(name) from e
def __setattr__(self, name, value):
if name.startswith("_"):
object.__setattr__(self, name, value)
else:
self[name] = value
def __dir__(self):
return sorted(list(self._schema['properties'].keys()))
def __repr__(self) -> str:
return f"MetaModel({self._data})"
def __deepcopy__(self, memo) -> Self:
cls = self.__class__
result = cls.__new__(cls) # blank instance
memo[id(self)] = result
# Copy internal attributes directly
result._validate = self._validate
result._schema = copy.deepcopy(self._schema, memo)
result._data = copy.deepcopy(self._data, memo)
# Not copied: rebuilt lazily against the copied schema on first use
# (see validate()), since a Validator is a cheap-to-rebuild cache,
# not real state.
result._validators = {}
return result
def __getstate__(self) -> dict:
# _validators is a lazily-rebuilt cache (see validate()), not real
# state -- excluded here for the same reason __deepcopy__ excludes it.
return {
"_validate": self._validate,
"_schema": self._schema,
"_data": self._data,
}
def __setstate__(self, state: dict) -> None:
# Defining __setstate__ (even though this body matches the implicit
# default) is required, not cosmetic: without it, pickle's
# ``hasattr(obj, '__setstate__')`` check on the not-yet-initialized
# unpickled instance falls through to __getattr__, which reads
# self._schema before it exists -> infinite recursion. Assigning
# here goes through __setattr__, which routes "_"-prefixed names to
# object.__setattr__ directly, so this is safe pre-__init__.
self._validate = state["_validate"]
self._schema = state["_schema"]
self._data = state["_data"]
self._validators = {}
[docs]
def validate(
self,
prop_name : str,
value
):
"""
Validate a metadata field against the schema.
Parameters
----------
prop_name : str
The property name to validate.
value : Any
The value to validate.
Raises
------
ValueError
If the value does not conform to the schema.
"""
# Get the sub-schema for the property
sub_schema = self._schema['properties'].get(prop_name)
if sub_schema is None:
raise KeyError(f"Property '{prop_name}' not found in metadata schema.")
# Accept None values
if value is None:
return
try:
validator = self._validators[prop_name]
except KeyError:
# jsonschema.validate() re-resolves the validator class and
# re-checks the schema itself (check_schema) on every call --
# ~150x slower than reusing a precompiled Validator, and the
# schema for a given property never changes after load, so
# build it once here and cache it.
cls = jsonschema.validators.validator_for(sub_schema)
cls.check_schema(sub_schema)
validator = self._validators[prop_name] = cls(sub_schema)
try:
validator.validate(value)
except jsonschema.ValidationError as e:
raise ValueError(f"Invalid value for '{prop_name}': {e.message}") from None
[docs]
def to_fits_header(
self,
extname : str = 'PRIMARY',
include_missing_metadata : bool = False
) -> fits.Header:
"""
Convert the metadata to a FITS header.
Parameters
----------
extname : str, optional
The extension name for which to create the header. Defaults to 'PRIMARY'.
include_missing_metadata : bool, optional
Whether to include None (unset) metadata values in the header. Defaults to False.
Returns
-------
header : fits.Header
The FITS header containing the metadata.
"""
# Init header
header = fits.Header()
# Helper to make comments in FITS heaers
def make_comment(sub_schema : dict) -> str:
comment = sub_schema['title']
unit = sub_schema['unit']
if unit is not None:
if unit.lower() != 'none':
comment += f" ({unit})"
return comment
# Group based on section names
meta_sections = self.group_meta_by_section()
for section_name, section_attrs in meta_sections.items():
if len(section_attrs) == 0:
continue
section_vals = [self[attr] for attr in section_attrs]
if not any(section_vals) and not include_missing_metadata:
continue
# Add section header as COMMENT
header.append(('COMMENT', '=' * 40), end=True)
header.append(('COMMENT', f' {section_name}'), end=True)
header.append(('COMMENT', '=' * 40), end=True)
for attr in section_attrs:
# Sub schema and value
sub_schema = self._schema['properties'][attr]
value = self[attr]
# Skip if no value and not adding empty entries
if value is None and not include_missing_metadata:
continue
# FITS HDU
prop_ext = sub_schema['fits_hdu']
if extname != prop_ext:
continue
# Info and value
keyword = sub_schema['fits_keyword']
comment = make_comment(sub_schema)
# Add to header
header.append((keyword, value, comment), end=True)
return header
[docs]
def to_record(self) -> dict:
"""
Convert the metadata to a dictionary record.
Returns
-------
record : dict
The metadata as a dictionary.
"""
record = {}
for prop_name, sub_schema in self._schema['properties'].items():
if sub_schema['cal_db']:
db_col = sub_schema['cal_db']
if db_col is True:
db_col = prop_name
elif isinstance(db_col, str):
db_col = db_col
record[db_col] = self[prop_name]
return record
[docs]
def set_default_values(self):
"""
Set default values for metadata fields based on the schema.
"""
for prop_name, sub_schema in self._schema['properties'].items():
if self._data.get(prop_name) is None:
default_value = sub_schema.get('default')
if default_value is not None:
self[prop_name] = default_value
[docs]
def set_default(self, prop_name : str, value):
"""
Set a default value for a specific metadata field.
Parameters
----------
prop_name : str
The property name for which to set the default value.
value : Any
The default value to set if the property is not already set.
"""
if self[prop_name] is None:
self[prop_name] = value
[docs]
def group_meta_by_section(self, flat: bool = False) -> dict:
"""
Group metadata properties by their FITS section.
Parameters
----------
flat : bool, optional
Whether to return a flat dictionary instead of grouped by section. Defaults to False.
Returns
-------
dict
Dictionary of metadata grouped by FITS section.
Keys are section names and values are dictionaries of metadata properties.
Sections are ordered as they first appear in the metadata schema
YAML file (``self._schema['properties']`` preserves that order).
"""
sections = {}
for name, sub in self._schema["properties"].items():
sec = sub.get("fits_section", "Core")
sections.setdefault(sec, {})[name] = self._data.get(name)
if not flat:
return sections
# Flatten
out = {}
for sec_dict in sections.values():
out.update(sec_dict)
return out
[docs]
def boolean_columns(self) -> list[str]:
"""
Return a set of columns that are logically booleans (for PostgreSQL)
"""
return [
prop_name
for prop_name, sub_schema in self._schema["properties"].items()
if sub_schema.get("cal_db") and sub_schema.get("type") == "boolean"
]
[docs]
def pprint(self, show_none : bool = False, header_style : bool = True):
"""
Pretty-print the metadata, grouped by FITS section.
Parameters
----------
show_none : bool, optional
Whether to show metadata fields with None values. Defaults to False.
"""
for section_name, section_attrs in self.group_meta_by_section().items():
attrs = [attr for attr in section_attrs if self[attr] is not None or show_none]
if not attrs:
continue
print('=' * 40)
print(f' {section_name}')
print('=' * 40)
for attr in attrs:
sub_schema = self._schema['properties'][attr]
val = self[attr]
unit = sub_schema['unit']
if header_style:
s = f"{attr} = {val}"
if unit is not None:
s += f" ({unit})"
s += f" # {sub_schema['title']}"
print(s)
else:
if unit is not None:
print(f"{attr}: {val} {unit}")
else:
print(f"{attr}: {val}")
print(f" title: {sub_schema['title']}")
print(f" type: {sub_schema['type']}")
print(f" fits_keyword: {sub_schema['fits_keyword']}")
print(f" fits_hdu: {sub_schema['fits_hdu']}")
def __delitem__(self, key):
del self._data[key]
def __iter__(self):
return iter(self._data)
def __len__(self):
return len(self._data)
@property
def fields(self) -> list[str]:
"""
Return a view of metadata fields names, including those with None values.
"""
return self._schema['properties'].keys()
[docs]
def get_meta_schema(
instrument_name : str | None = None,
config : type | None = None,
) -> dict:
"""
Get the metadata schema for a given instrument.
Parameters
----------
instrument_name : str | None
Name of the instrument. Optional.
config : type | None
Instrument configuration class. Optional.
Returns
-------
meta : dict
The metadata schema with defaults applied.
"""
# Determine instrument config
if config is None:
assert instrument_name is not None, "Either instrument_name or config must be provided."
config = get_instrument_config(instrument_name)
# Load metadata schema for this instrument config
meta_schema = config.get_metadata_schema()
return meta_schema
[docs]
def read_meta_schema(filepath : str) -> dict:
def _loader(uri, **_):
return yaml.safe_load(
Path(
uri.replace("file://", "")
).read_text()
)
schema = jsonref.load_uri(
Path(filepath).absolute().as_uri(),
loader=_loader
)
properties = {}
if 'allOf' in schema:
parts = schema.pop('allOf')
for part in parts:
fits_section = os.path.splitext(os.path.basename(part['id']))
for k, v in part.get('properties', {}).items():
properties[k] = v
properties[k]['fits_section'] = fits_section[0].upper()
else:
raise ValueError("Metadata schema must contain 'allOf' at the top level.")
# Fix any nested allOf
for k, v in properties.items():
if 'allOf' in v:
parts = v.pop('allOf')
for part in parts:
for pk, pv in part.items():
properties[k][pk] = pv
for p in properties.values():
for k, v in _META_SCHEMA_DEFAULTS.items():
p.setdefault(k, v)
for prop_name, sub_schema in properties.items():
if sub_schema['cal_db']:
db_col = sub_schema['cal_db']
if db_col is True:
db_col = prop_name
assert isinstance(db_col, str), f"cal_db must be a string or True for property '{prop_name}'"
sub_schema['cal_db'] = db_col
schema['properties'] = properties
return schema
[docs]
def merge_meta(models: list[dict]) -> dict:
"""
Merge the meta information from multiple DataModel instances.
Parameters
----------
models : list[DataModel]
List of DataModel instances to merge meta information from.
Returns
-------
meta_merged : MetaModel
Merged metadata.
"""
merged_data = {}
# Sort models
models = sorted(models, key=lambda m: m.meta.datetime_obs)
for prop_name, sub_schema in models[0].meta._schema['properties'].items():
values = [getattr(model.meta, prop_name) for model in models]
values = [v for v in values if v is not None]
if not values:
continue
merge_rule = sub_schema['merge_rule']
if merge_rule == 'drop': # Drop the value
merged_value = None
elif merge_rule == 'first': # First value in time
merged_value = values[0]
elif merge_rule == 'last': # Last value in time
merged_value = values[-1]
elif merge_rule == 'sum': # Sum the values
merged_value = np.sum(values)
elif merge_rule == 'max': # Max of the values
merged_value = max(values)
elif merge_rule == 'min': # Min of the values
merged_value = min(values)
elif merge_rule == 'mean': # Mean of the values
merged_value = np.mean(values)
elif merge_rule == 'fiber_state': # Special rule for fibers
merged_value = _merge_fiber_state(values)
else:
raise ValueError(f"Unknown merge rule '{merge_rule}' for meta field '{prop_name}'")
merged_data[prop_name] = merged_value
meta_merged = MetaModel(
meta=merged_data,
schema=models[0].meta._schema
)
return meta_merged
_FIBER_EMPTY_STATES = ["BLOCKED", "NONE"]
[docs]
def _merge_fiber_state(values) -> str:
values = [v for v in values if v is not None]
if not values:
return "NONE"
illuminated = [v for v in values if v not in _FIBER_EMPTY_STATES]
if illuminated:
return illuminated[0]
if "BLOCKED" in values:
return "BLOCKED"
return "NONE"