Coverage for python/astro_metadata_translator/translator.py: 46%
410 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-03-20 11:09 +0000
« prev ^ index » next coverage.py v7.4.4, created at 2024-03-20 11:09 +0000
1# This file is part of astro_metadata_translator.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the LICENSE file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12"""Classes and support code for metadata translation."""
14from __future__ import annotations
16__all__ = ("MetadataTranslator", "StubTranslator", "cache_translation")
18import importlib
19import inspect
20import logging
21import math
22import numbers
23import warnings
24from abc import abstractmethod
25from collections.abc import Callable, Iterable, Iterator, Mapping, MutableMapping, Sequence
26from typing import TYPE_CHECKING, Any, ClassVar
28import astropy.io.fits.card
29import astropy.units as u
30from astropy.coordinates import Angle
32from .properties import PROPERTIES, PropertyDefinition
34if TYPE_CHECKING: 34 ↛ 35line 34 didn't jump to line 35, because the condition on line 34 was never true
35 import astropy.coordinates
36 import astropy.time
38log = logging.getLogger(__name__)
40# Location of the root of the corrections resource files
41CORRECTIONS_RESOURCE_ROOT = "corrections"
43"""Cache of version strings indexed by class."""
44_VERSION_CACHE: dict[type, str] = dict()
47def cache_translation(func: Callable, method: str | None = None) -> Callable:
48 """Cache the result of a translation method.
50 Parameters
51 ----------
52 func : `~collections.abc.Callable`
53 Translation method to cache.
54 method : `str`, optional
55 Name of the translation method to cache. Not needed if the decorator
56 is used around a normal method, but necessary when the decorator is
57 being used in a metaclass.
59 Returns
60 -------
61 wrapped : `~collections.abc.Callable`
62 Method wrapped by the caching function.
64 Notes
65 -----
66 Especially useful when a translation uses many other translation
67 methods or involves significant computation.
68 Should be used only on ``to_x()`` methods.
70 .. code-block:: python
72 @cache_translation
73 def to_detector_num(self):
74 ....
75 """
76 name = func.__name__ if method is None else method
78 def func_wrapper(self: MetadataTranslator) -> Any:
79 if name not in self._translation_cache:
80 self._translation_cache[name] = func(self)
81 return self._translation_cache[name]
83 func_wrapper.__doc__ = func.__doc__
84 func_wrapper.__name__ = f"{name}_cached"
85 return func_wrapper
88class MetadataTranslator:
89 """Per-instrument metadata translation support.
91 Parameters
92 ----------
93 header : `dict`-like
94 Representation of an instrument header that can be manipulated
95 as if it was a `dict`.
96 filename : `str`, optional
97 Name of the file whose header is being translated. For some
98 datasets with missing header information this can sometimes
99 allow for some fixups in translations.
100 """
102 # These are all deliberately empty in the base class.
103 name: str | None = None
104 """The declared name of the translator."""
106 default_search_path: Sequence[str] | None = None
107 """Default search path to use to locate header correction files."""
109 default_resource_package = __name__.split(".")[0]
110 """Module name to use to locate the correction resources."""
112 default_resource_root: str | None = None
113 """Default package resource path root to use to locate header correction
114 files within the ``default_resource_package`` package."""
116 _trivial_map: dict[str, str | list[str] | tuple[Any, ...]] = {}
117 """Dict of one-to-one mappings for header translation from standard
118 property to corresponding keyword."""
120 _const_map: dict[str, Any] = {}
121 """Dict defining a constant for specified standard properties."""
123 translators: dict[str, type[MetadataTranslator]] = dict()
124 """All registered metadata translation classes."""
126 supported_instrument: str | None = None
127 """Name of instrument understood by this translation class."""
129 all_properties: dict[str, PropertyDefinition] = {}
130 """All the valid properties for this translator including extensions."""
132 extensions: dict[str, PropertyDefinition] = {}
133 """Extension properties (`str`: `PropertyDefinition`)
135 Some instruments have important properties beyond the standard set; this is
136 the place to declare that they exist, and they will be treated in the same
137 way as the standard set, except that their names will everywhere be
138 prefixed with ``ext_``.
140 Each property is indexed by name (`str`), with a corresponding
141 `PropertyDefinition`.
142 """
144 # Static typing requires that we define the standard dynamic properties
145 # statically.
146 if TYPE_CHECKING: 146 ↛ 147line 146 didn't jump to line 147, because the condition on line 146 was never true
147 to_telescope: ClassVar[Callable[[MetadataTranslator], str]]
148 to_instrument: ClassVar[Callable[[MetadataTranslator], str]]
149 to_location: ClassVar[Callable[[MetadataTranslator], astropy.coordinates.EarthLocation]]
150 to_exposure_id: ClassVar[Callable[[MetadataTranslator], int]]
151 to_visit_id: ClassVar[Callable[[MetadataTranslator], int]]
152 to_physical_filter: ClassVar[Callable[[MetadataTranslator], str]]
153 to_datetime_begin: ClassVar[Callable[[MetadataTranslator], astropy.time.Time]]
154 to_datetime_end: ClassVar[Callable[[MetadataTranslator], astropy.time.Time]]
155 to_exposure_time: ClassVar[Callable[[MetadataTranslator], u.Quantity]]
156 to_dark_time: ClassVar[Callable[[MetadataTranslator], u.Quantity]]
157 to_boresight_airmass: ClassVar[Callable[[MetadataTranslator], float]]
158 to_boresight_rotation_angle: ClassVar[Callable[[MetadataTranslator], u.Quantity]]
159 to_boresight_rotation_coord: ClassVar[Callable[[MetadataTranslator], str]]
160 to_detector_num: ClassVar[Callable[[MetadataTranslator], int]]
161 to_detector_name: ClassVar[Callable[[MetadataTranslator], str]]
162 to_detector_serial: ClassVar[Callable[[MetadataTranslator], str]]
163 to_detector_group: ClassVar[Callable[[MetadataTranslator], str | None]]
164 to_detector_exposure_id: ClassVar[Callable[[MetadataTranslator], int]]
165 to_object: ClassVar[Callable[[MetadataTranslator], str]]
166 to_temperature: ClassVar[Callable[[MetadataTranslator], u.Quantity]]
167 to_pressure: ClassVar[Callable[[MetadataTranslator], u.Quantity]]
168 to_relative_humidity: ClassVar[Callable[[MetadataTranslator], float]]
169 to_tracking_radec: ClassVar[Callable[[MetadataTranslator], astropy.coordinates.SkyCoord]]
170 to_altaz_begin: ClassVar[Callable[[MetadataTranslator], astropy.coordinates.AltAz]]
171 to_science_program: ClassVar[Callable[[MetadataTranslator], str]]
172 to_observation_type: ClassVar[Callable[[MetadataTranslator], str]]
173 to_observation_id: ClassVar[Callable[[MetadataTranslator], str]]
175 @classmethod
176 def defined_in_this_class(cls, name: str) -> bool | None:
177 """Report if the specified class attribute is defined specifically in
178 this class.
180 Parameters
181 ----------
182 name : `str`
183 Name of the attribute to test.
185 Returns
186 -------
187 in_class : `bool`
188 `True` if there is a attribute of that name defined in this
189 specific subclass.
190 `False` if the method is not defined in this specific subclass
191 but is defined in a parent class.
192 Returns `None` if the attribute is not defined anywhere
193 in the class hierarchy (which can happen if translators have
194 typos in their mapping tables).
196 Notes
197 -----
198 Retrieves the attribute associated with the given name.
199 Then looks in all the parent classes to determine whether that
200 attribute comes from a parent class or from the current class.
201 Attributes are compared using :py:func:`id`.
202 """
203 # The attribute to compare.
204 if not hasattr(cls, name):
205 return None
206 attr_id = id(getattr(cls, name))
208 # Get all the classes in the hierarchy
209 mro = list(inspect.getmro(cls))
211 # Remove the first entry from the list since that will be the
212 # current class
213 mro.pop(0)
215 for parent in mro:
216 # Some attributes may only exist in subclasses. Skip base classes
217 # that are missing the attribute (such as object).
218 if hasattr(parent, name):
219 if id(getattr(parent, name)) == attr_id:
220 return False
221 return True
223 @classmethod
224 def _make_const_mapping(cls, property_key: str, constant: Any) -> Callable:
225 """Make a translator method that returns a constant value.
227 Parameters
228 ----------
229 property_key : `str`
230 Name of the property to be calculated (for the docstring).
231 constant : `str` or `numbers.Number`
232 Value to return for this translator.
234 Returns
235 -------
236 f : `~collections.abc.Callable`
237 Function returning the constant.
238 """
240 def constant_translator(self: MetadataTranslator) -> Any:
241 return constant
243 if property_key in cls.all_properties:
244 property_doc = cls.all_properties[property_key].doc
245 return_type = cls.all_properties[property_key].py_type
246 else:
247 return_type = type(constant)
248 property_doc = f"Returns constant value for '{property_key}' property"
250 if return_type.__module__ == "builtins":
251 full_name = return_type.__name__
252 else:
253 full_name = f"{return_type.__module__}.{return_type.__qualname__}"
255 constant_translator.__doc__ = f"""{property_doc}
257 Returns
258 -------
259 translation : `{full_name}`
260 Translated property.
261 """
262 return constant_translator
264 @classmethod
265 def _make_trivial_mapping(
266 cls,
267 property_key: str,
268 header_key: str | Sequence[str],
269 default: Any | None = None,
270 minimum: Any | None = None,
271 maximum: Any | None = None,
272 unit: astropy.unit.Unit | None = None,
273 checker: Callable | None = None,
274 ) -> Callable:
275 """Make a translator method returning a header value.
277 The header value can be converted to a `~astropy.units.Quantity`
278 if desired, and can also have its value validated.
280 See `MetadataTranslator.validate_value()` for details on the use
281 of default parameters.
283 Parameters
284 ----------
285 property_key : `str`
286 Name of the translator to be constructed (for the docstring).
287 header_key : `str` or `list` of `str`
288 Name of the key to look up in the header. If a `list` each
289 key will be tested in turn until one matches. This can deal with
290 header styles that evolve over time.
291 default : `numbers.Number` or `astropy.units.Quantity`, `str`, optional
292 If not `None`, default value to be used if the parameter read from
293 the header is not defined or if the header is missing.
294 minimum : `numbers.Number` or `astropy.units.Quantity`, optional
295 If not `None`, and if ``default`` is not `None`, minimum value
296 acceptable for this parameter.
297 maximum : `numbers.Number` or `astropy.units.Quantity`, optional
298 If not `None`, and if ``default`` is not `None`, maximum value
299 acceptable for this parameter.
300 unit : `astropy.units.Unit`, optional
301 If not `None`, the value read from the header will be converted
302 to a `~astropy.units.Quantity`. Only supported for numeric values.
303 checker : `~collections.abc.Callable`, optional
304 Callback function to be used by the translator method in case the
305 keyword is not present. Function will be executed as if it is
306 a method of the translator class. Running without raising an
307 exception will allow the default to be used. Should usually raise
308 `KeyError`.
310 Returns
311 -------
312 t : `~collections.abc.Callable`
313 Function implementing a translator with the specified
314 parameters.
315 """
316 if property_key in cls.all_properties: 316 ↛ 320line 316 didn't jump to line 320, because the condition on line 316 was never false
317 property_doc = cls.all_properties[property_key].doc
318 return_type = cls.all_properties[property_key].str_type
319 else:
320 return_type = "str` or `numbers.Number"
321 property_doc = f"Map '{header_key}' header keyword to '{property_key}' property"
323 def trivial_translator(self: MetadataTranslator) -> Any:
324 if unit is not None:
325 q = self.quantity_from_card(
326 header_key, unit, default=default, minimum=minimum, maximum=maximum, checker=checker
327 )
328 # Convert to Angle if this quantity is an angle
329 if return_type == "astropy.coordinates.Angle":
330 q = Angle(q)
331 return q
333 keywords = header_key if isinstance(header_key, list) else [header_key]
334 for key in keywords:
335 if self.is_key_ok(key):
336 value = self._header[key]
337 if default is not None and not isinstance(value, str):
338 value = self.validate_value(value, default, minimum=minimum, maximum=maximum)
339 self._used_these_cards(key)
340 break
341 else:
342 # No keywords found, use default, checking first, or raise
343 # A None default is only allowed if a checker is provided.
344 if checker is not None:
345 try:
346 checker(self)
347 except Exception:
348 raise KeyError(f"Could not find {keywords} in header")
349 return default
350 elif default is not None:
351 value = default
352 else:
353 raise KeyError(f"Could not find {keywords} in header")
355 # If we know this is meant to be a string, force to a string.
356 # Sometimes headers represent items as integers which generically
357 # we want as strings (eg OBSID). Sometimes also floats are
358 # written as "NaN" strings.
359 casts = {"str": str, "float": float, "int": int}
360 if return_type in casts and not isinstance(value, casts[return_type]) and value is not None:
361 value = casts[return_type](value)
363 return value
365 # Docstring inheritance means it is confusing to specify here
366 # exactly which header value is being used.
367 trivial_translator.__doc__ = f"""{property_doc}
369 Returns
370 -------
371 translation : `{return_type}`
372 Translated value derived from the header.
373 """
374 return trivial_translator
376 @classmethod
377 def __init_subclass__(cls, **kwargs: Any) -> None:
378 """Register all subclasses with the base class and create dynamic
379 translator methods.
381 The method provides two facilities. Firstly, every subclass
382 of `MetadataTranslator` that includes a ``name`` class property is
383 registered as a translator class that could be selected when automatic
384 header translation is attempted. Only name translator subclasses that
385 correspond to a complete instrument. Translation classes providing
386 generic translation support for multiple instrument translators should
387 not be named.
389 The second feature of this method is to convert simple translations
390 to full translator methods. Sometimes a translation is fixed (for
391 example a specific instrument name should be used) and rather than
392 provide a full ``to_property()`` translation method the mapping can be
393 defined in a class variable named ``_constMap``. Similarly, for
394 one-to-one trivial mappings from a header to a property,
395 ``_trivialMap`` can be defined. Trivial mappings are a dict mapping a
396 generic property to either a header keyword, or a tuple consisting of
397 the header keyword and a dict containing key value pairs suitable for
398 the `MetadataTranslator.quantity_from_card` method.
400 Parameters
401 ----------
402 **kwargs : `dict`
403 Arbitrary parameters passed to parent class.
404 """
405 super().__init_subclass__(**kwargs)
407 # Only register classes with declared names
408 if hasattr(cls, "name") and cls.name is not None:
409 if cls.name in MetadataTranslator.translators: 409 ↛ 410line 409 didn't jump to line 410, because the condition on line 409 was never true
410 log.warning(
411 "%s: Replacing %s translator with %s",
412 cls.name,
413 MetadataTranslator.translators[cls.name],
414 cls,
415 )
416 MetadataTranslator.translators[cls.name] = cls
418 # Check that we have not inherited constant/trivial mappings from
419 # parent class that we have already applied. Empty maps are always
420 # assumed okay
421 const_map = cls._const_map if cls._const_map and cls.defined_in_this_class("_const_map") else {}
422 trivial_map = (
423 cls._trivial_map if cls._trivial_map and cls.defined_in_this_class("_trivial_map") else {}
424 )
426 # Check for shadowing
427 trivials = set(trivial_map.keys())
428 constants = set(const_map.keys())
429 both = trivials & constants
430 if both: 430 ↛ 431line 430 didn't jump to line 431, because the condition on line 430 was never true
431 log.warning("%s: defined in both const_map and trivial_map: %s", cls.__name__, ", ".join(both))
433 all = trivials | constants
434 for name in all:
435 if cls.defined_in_this_class(f"to_{name}"): 435 ↛ 438line 435 didn't jump to line 438, because the condition on line 435 was never true
436 # Must be one of trivial or constant. If in both then constant
437 # overrides trivial.
438 location = "by _trivial_map"
439 if name in constants:
440 location = "by _const_map"
441 log.warning(
442 "%s: %s is defined explicitly but will be replaced %s", cls.__name__, name, location
443 )
445 properties = set(PROPERTIES) | {"ext_" + pp for pp in cls.extensions}
446 cls.all_properties = dict(PROPERTIES)
447 cls.all_properties.update(cls.extensions)
449 # Go through the trival mappings for this class and create
450 # corresponding translator methods
451 for property_key, header_key in trivial_map.items():
452 kwargs = {}
453 if type(header_key) is tuple:
454 kwargs = header_key[1]
455 header_key = header_key[0]
456 translator = cls._make_trivial_mapping(property_key, header_key, **kwargs)
457 method = f"to_{property_key}"
458 translator.__name__ = f"{method}_trivial_in_{cls.__name__}"
459 setattr(cls, method, cache_translation(translator, method=method))
460 if property_key not in properties: 460 ↛ 461line 460 didn't jump to line 461, because the condition on line 460 was never true
461 log.warning(f"Unexpected trivial translator for '{property_key}' defined in {cls}")
463 # Go through the constant mappings for this class and create
464 # corresponding translator methods
465 for property_key, constant in const_map.items():
466 translator = cls._make_const_mapping(property_key, constant)
467 method = f"to_{property_key}"
468 translator.__name__ = f"{method}_constant_in_{cls.__name__}"
469 setattr(cls, method, translator)
470 if property_key not in properties: 470 ↛ 471line 470 didn't jump to line 471, because the condition on line 470 was never true
471 log.warning(f"Unexpected constant translator for '{property_key}' defined in {cls}")
473 def __init__(self, header: Mapping[str, Any], filename: str | None = None) -> None:
474 self._header = header
475 self.filename = filename
476 self._used_cards: set[str] = set()
478 # Prefix to use for warnings about failed translations
479 self._log_prefix_cache: str | None = None
481 # Cache assumes header is read-only once stored in object
482 self._translation_cache: dict[str, Any] = {}
484 @classmethod
485 @abstractmethod
486 def can_translate(cls, header: Mapping[str, Any], filename: str | None = None) -> bool:
487 """Indicate whether this translation class can translate the
488 supplied header.
490 Parameters
491 ----------
492 header : `dict`-like
493 Header to convert to standardized form.
494 filename : `str`, optional
495 Name of file being translated.
497 Returns
498 -------
499 can : `bool`
500 `True` if the header is recognized by this class. `False`
501 otherwise.
502 """
503 raise NotImplementedError()
505 @classmethod
506 def can_translate_with_options(
507 cls, header: Mapping[str, Any], options: dict[str, Any], filename: str | None = None
508 ) -> bool:
509 """Determine if a header can be translated with different criteria.
511 Parameters
512 ----------
513 header : `dict`-like
514 Header to convert to standardized form.
515 options : `dict`
516 Headers to try to determine whether this header can
517 be translated by this class. If a card is found it will
518 be compared with the expected value and will return that
519 comparison. Each card will be tried in turn until one is
520 found.
521 filename : `str`, optional
522 Name of file being translated.
524 Returns
525 -------
526 can : `bool`
527 `True` if the header is recognized by this class. `False`
528 otherwise.
530 Notes
531 -----
532 Intended to be used from within `can_translate` implementations
533 for specific translators. Is not intended to be called directly
534 from `determine_translator`.
535 """
536 for card, value in options.items():
537 if card in header:
538 return header[card] == value
539 return False
541 @classmethod
542 def determine_translator(
543 cls, header: Mapping[str, Any], filename: str | None = None
544 ) -> type[MetadataTranslator]:
545 """Determine a translation class by examining the header.
547 Parameters
548 ----------
549 header : `dict`-like
550 Representation of a header.
551 filename : `str`, optional
552 Name of file being translated.
554 Returns
555 -------
556 translator : `MetadataTranslator`
557 Translation class that knows how to extract metadata from
558 the supplied header.
560 Raises
561 ------
562 ValueError
563 None of the registered translation classes understood the supplied
564 header.
565 """
566 file_msg = ""
567 if filename is not None:
568 file_msg = f" from {filename}"
569 for name, trans in cls.translators.items():
570 if trans.can_translate(header, filename=filename):
571 log.debug("Using translation class %s%s", name, file_msg)
572 return trans
573 else:
574 raise ValueError(
575 f"None of the registered translation classes {list(cls.translators.keys())}"
576 f" understood this header{file_msg}"
577 )
579 @classmethod
580 def translator_version(cls) -> str:
581 """Return the version string for this translator class.
583 Returns
584 -------
585 version : `str`
586 String identifying the version of this translator.
588 Notes
589 -----
590 Assumes that the version is available from the ``__version__``
591 variable in the parent module. If this is not the case a translator
592 should subclass this method.
593 """
594 if cls in _VERSION_CACHE:
595 return _VERSION_CACHE[cls]
597 version = "unknown"
598 module_name = cls.__module__
599 components = module_name.split(".")
600 while components:
601 # This class has already been imported so importing it
602 # should work.
603 module = importlib.import_module(".".join(components))
604 if hasattr(module, v := "__version__"):
605 version = getattr(module, v)
606 if version == "unknown":
607 # LSST software will have a fingerprint
608 if hasattr(module, v := "__fingerprint__"):
609 version = getattr(module, v)
610 break
611 else:
612 # Remove last component from module name and try again
613 components.pop()
615 _VERSION_CACHE[cls] = version
616 return version
618 @classmethod
619 def fix_header(
620 cls, header: MutableMapping[str, Any], instrument: str, obsid: str, filename: str | None = None
621 ) -> bool:
622 """Apply global fixes to a supplied header.
624 Parameters
625 ----------
626 header : `dict`
627 The header to correct. Correction is in place.
628 instrument : `str`
629 The name of the instrument.
630 obsid : `str`
631 Unique observation identifier associated with this header.
632 Will always be provided.
633 filename : `str`, optional
634 Filename associated with this header. May not be set since headers
635 can be fixed independently of any filename being known.
637 Returns
638 -------
639 modified : `bool`
640 `True` if a correction was applied.
642 Notes
643 -----
644 This method is intended to support major discrepancies in headers
645 such as:
647 * Periods of time where headers are known to be incorrect in some
648 way that can be fixed either by deriving the correct value from
649 the existing value or understanding the that correction is static
650 for the given time. This requires that the date header is
651 known.
652 * The presence of a certain value is always wrong and should be
653 corrected with a new static value regardless of date.
655 It is assumed that one off problems with headers have been applied
656 before this method is called using the per-obsid correction system.
658 Usually called from `astro_metadata_translator.fix_header`.
660 For log messages, do not assume that the filename will be present.
661 Always write log messages to fall back on using the ``obsid`` if
662 ``filename`` is `None`.
663 """
664 return False
666 @staticmethod
667 def _construct_log_prefix(obsid: str, filename: str | None = None) -> str:
668 """Construct a log prefix string from the obsid and filename.
670 Parameters
671 ----------
672 obsid : `str`
673 The observation identifier.
674 filename : `str`, optional
675 The filename associated with the header being translated.
676 Can be `None`.
677 """
678 if filename:
679 return f"{filename}({obsid})"
680 return obsid
682 @property
683 def _log_prefix(self) -> str:
684 """Return standard prefix that can be used for log messages to report
685 useful context.
687 Will be either the filename and obsid, or just the obsid depending
688 on whether a filename is known.
690 Returns
691 -------
692 prefix : `str`
693 The prefix to use.
694 """
695 if self._log_prefix_cache is None:
696 # Protect against the unfortunate event of the obsid failing to
697 # be calculated. This should be rare but should not prevent a log
698 # message from appearing.
699 try:
700 obsid = self.to_observation_id()
701 except Exception:
702 obsid = "unknown_obsid"
703 self._log_prefix_cache = self._construct_log_prefix(obsid, self.filename)
704 return self._log_prefix_cache
706 def _used_these_cards(self, *args: str) -> None:
707 """Indicate that the supplied cards have been used for translation.
709 Parameters
710 ----------
711 *args : sequence of `str`
712 Keywords used to process a translation.
713 """
714 self._used_cards.update(set(args))
716 def cards_used(self) -> frozenset[str]:
717 """Cards used during metadata extraction.
719 Returns
720 -------
721 used : `frozenset` of `str`
722 Cards used when extracting metadata.
723 """
724 return frozenset(self._used_cards)
726 @staticmethod
727 def validate_value(
728 value: float, default: float, minimum: float | None = None, maximum: float | None = None
729 ) -> float:
730 """Validate the supplied value, returning a new value if out of range.
732 Parameters
733 ----------
734 value : `float`
735 Value to be validated.
736 default : `float`
737 Default value to use if supplied value is invalid or out of range.
738 Assumed to be in the same units as the value expected in the
739 header.
740 minimum : `float`
741 Minimum possible valid value, optional. If the calculated value
742 is below this value, the default value will be used.
743 maximum : `float`
744 Maximum possible valid value, optional. If the calculated value
745 is above this value, the default value will be used.
747 Returns
748 -------
749 value : `float`
750 Either the supplied value, or a default value.
751 """
752 if value is None or math.isnan(value):
753 value = default
754 else:
755 if minimum is not None and value < minimum:
756 value = default
757 elif maximum is not None and value > maximum:
758 value = default
759 return value
761 @staticmethod
762 def is_keyword_defined(header: Mapping[str, Any], keyword: str | None) -> bool:
763 """Return `True` if the value associated with the named keyword is
764 present in the supplied header and defined.
766 Parameters
767 ----------
768 header : `dict`-lik
769 Header to use as reference.
770 keyword : `str`
771 Keyword to check against header.
773 Returns
774 -------
775 is_defined : `bool`
776 `True` if the header is present and not-`None`. `False` otherwise.
777 """
778 if keyword is None or keyword not in header:
779 return False
781 if header[keyword] is None:
782 return False
784 # Special case Astropy undefined value
785 if isinstance(header[keyword], astropy.io.fits.card.Undefined):
786 return False
788 return True
790 def resource_root(self) -> tuple[str | None, str | None]:
791 """Return package resource to use to locate correction resources within
792 an installed package.
794 Returns
795 -------
796 resource_package : `str`
797 Package resource name. `None` if no package resource are to be
798 used.
799 resource_root : `str`
800 The name of the resource root. `None` if no package resources
801 are to be used.
802 """
803 return (self.default_resource_package, self.default_resource_root)
805 def search_paths(self) -> list[str]:
806 """Search paths to use when searching for header fix up correction
807 files.
809 Returns
810 -------
811 paths : `list`
812 Directory paths to search. Can be an empty list if no special
813 directories are defined.
815 Notes
816 -----
817 Uses the classes ``default_search_path`` property if defined.
818 """
819 if self.default_search_path is not None:
820 return [p for p in self.default_search_path]
821 return []
823 def is_key_ok(self, keyword: str | None) -> bool:
824 """Return `True` if the value associated with the named keyword is
825 present in this header and defined.
827 Parameters
828 ----------
829 keyword : `str`
830 Keyword to check against header.
832 Returns
833 -------
834 is_ok : `bool`
835 `True` if the header is present and not-`None`. `False` otherwise.
836 """
837 return self.is_keyword_defined(self._header, keyword)
839 def are_keys_ok(self, keywords: Iterable[str]) -> bool:
840 """Are the supplied keys all present and defined?.
842 Parameters
843 ----------
844 keywords : iterable of `str`
845 Keywords to test.
847 Returns
848 -------
849 all_ok : `bool`
850 `True` if all supplied keys are present and defined.
851 """
852 for k in keywords:
853 if not self.is_key_ok(k):
854 return False
855 return True
857 def quantity_from_card(
858 self,
859 keywords: str | Sequence[str],
860 unit: u.Unit,
861 default: float | None = None,
862 minimum: float | None = None,
863 maximum: float | None = None,
864 checker: Callable | None = None,
865 ) -> u.Quantity:
866 """Calculate a Astropy Quantity from a header card and a unit.
868 Parameters
869 ----------
870 keywords : `str` or `list` of `str`
871 Keyword to use from header. If a list each keyword will be tried
872 in turn until one matches.
873 unit : `astropy.units.UnitBase`
874 Unit of the item in the header.
875 default : `float`, optional
876 Default value to use if the header value is invalid. Assumed
877 to be in the same units as the value expected in the header. If
878 None, no default value is used.
879 minimum : `float`, optional
880 Minimum possible valid value, optional. If the calculated value
881 is below this value, the default value will be used.
882 maximum : `float`, optional
883 Maximum possible valid value, optional. If the calculated value
884 is above this value, the default value will be used.
885 checker : `~collections.abc.Callable`, optional
886 Callback function to be used by the translator method in case the
887 keyword is not present. Function will be executed as if it is
888 a method of the translator class. Running without raising an
889 exception will allow the default to be used. Should usually raise
890 `KeyError`.
892 Returns
893 -------
894 q : `astropy.units.Quantity`
895 Quantity representing the header value.
897 Raises
898 ------
899 KeyError
900 The supplied header key is not present.
901 """
902 keyword_list = [keywords] if isinstance(keywords, str) else list(keywords)
903 for k in keyword_list:
904 if self.is_key_ok(k):
905 value = self._header[k]
906 keyword = k
907 break
908 else:
909 if checker is not None:
910 try:
911 checker(self)
912 value = default
913 if value is not None:
914 value = u.Quantity(value, unit=unit)
915 return value
916 except Exception:
917 pass
918 raise KeyError(f"Could not find {keywords} in header")
919 if isinstance(value, str):
920 # Sometimes the header has the wrong type in it but this must
921 # be a number if we are creating a quantity.
922 value = float(value)
923 self._used_these_cards(keyword)
924 if default is not None:
925 value = self.validate_value(value, default, maximum=maximum, minimum=minimum)
926 return u.Quantity(value, unit=unit)
928 def _join_keyword_values(self, keywords: Iterable[str], delim: str = "+") -> str:
929 """Join values of all defined keywords with the specified delimiter.
931 Parameters
932 ----------
933 keywords : iterable of `str`
934 Keywords to look for in header.
935 delim : `str`, optional
936 Character to use to join the values together.
938 Returns
939 -------
940 joined : `str`
941 String formed from all the keywords found in the header with
942 defined values joined by the delimiter. Empty string if no
943 defined keywords found.
944 """
945 values = []
946 for k in keywords:
947 if self.is_key_ok(k):
948 values.append(self._header[k])
949 self._used_these_cards(k)
951 if values:
952 joined = delim.join(str(v) for v in values)
953 else:
954 joined = ""
956 return joined
958 @cache_translation
959 def to_detector_unique_name(self) -> str:
960 """Return a unique name for the detector.
962 Base class implementation attempts to combine ``detector_name`` with
963 ``detector_group``. Group is only used if not `None`.
965 Can be over-ridden by specialist translator class.
967 Returns
968 -------
969 name : `str`
970 ``detector_group``_``detector_name`` if ``detector_group`` is
971 defined, else the ``detector_name`` is assumed to be unique.
972 If neither return a valid value an exception is raised.
974 Raises
975 ------
976 NotImplementedError
977 Raised if neither detector_name nor detector_group is defined.
978 """
979 name = self.to_detector_name()
980 group = self.to_detector_group()
982 if group is None and name is None:
983 raise NotImplementedError("Can not determine unique name from detector_group and detector_name")
985 if group is not None:
986 return f"{group}_{name}"
988 return name
990 @cache_translation
991 def to_exposure_group(self) -> str | None:
992 """Return the group label associated with this exposure.
994 Base class implementation returns the ``exposure_id`` in string
995 form. A subclass may do something different.
997 Returns
998 -------
999 name : `str`
1000 The ``exposure_id`` converted to a string.
1001 """
1002 exposure_id = self.to_exposure_id()
1003 if exposure_id is None:
1004 # mypy does not think this can ever happen but play it safe
1005 # with subclasses.
1006 return None # type: ignore
1007 else:
1008 return str(exposure_id)
1010 @cache_translation
1011 def to_observation_reason(self) -> str:
1012 """Return the reason this observation was taken.
1014 Base class implementation returns the ``science`` if the
1015 ``observation_type`` is science, else ``unknown``.
1016 A subclass may do something different.
1018 Returns
1019 -------
1020 name : `str`
1021 The reason for this observation.
1022 """
1023 obstype = self.to_observation_type()
1024 if obstype == "science":
1025 return "science"
1026 return "unknown"
1028 @classmethod
1029 def observing_date_to_offset(cls, observing_date: astropy.time.Time) -> astropy.time.TimeDelta | None:
1030 """Calculate the observing day offset to apply for a given observation.
1032 In some cases the definition of the observing day offset has changed
1033 during the lifetime of the instrument. For example lab data might
1034 have a different offset to that when the instrument is on the
1035 telescope.
1037 Parameters
1038 ----------
1039 observing_date : `astropy.time.Time`
1040 The observation date.
1042 Returns
1043 -------
1044 offset : `astropy.time.TimeDelta` or `None`
1045 The offset to apply when calculating the observing day for a
1046 specific time of observation. `None` implies the offset
1047 is not known for that date.
1048 """
1049 return None
1051 @classmethod
1052 def observing_date_to_observing_day(
1053 cls, observing_date: astropy.time.Time, offset: astropy.time.TimeDelta | int | None
1054 ) -> int:
1055 """Return the YYYYMMDD integer corresponding to the observing day.
1057 The offset is subtracted from the time of observation before
1058 calculating the year, month and day.
1060 Parameters
1061 ----------
1062 observing_date : `astropy.time.Time`
1063 The observation date.
1064 offset : `astropy.time.TimeDelta` | `numbers.Real` | None
1065 The offset to subtract from the observing date when calculating
1066 the observing day. If a plain number is given it is taken to be
1067 in units of seconds. If `None` no offset is applied.
1069 Returns
1070 -------
1071 day : `int`
1072 The observing day as an integer of form YYYYMMDD.
1074 Notes
1075 -----
1076 For example, if the offset is +12 hours both 2023-07-06T13:00 and
1077 2023-07-07T11:00 will return an observing day of 20230706 because
1078 the observing day goes from 2023-07-06T12:00 to 2023-07-07T12:00.
1079 """
1080 observing_date = observing_date.tai
1081 if offset:
1082 if isinstance(offset, numbers.Real):
1083 offset = astropy.time.TimeDelta(offset, format="sec", scale="tai")
1084 observing_date -= offset
1085 return int(observing_date.strftime("%Y%m%d"))
1087 @cache_translation
1088 def to_observing_day_offset(self) -> astropy.time.TimeDelta | None:
1089 """Return the offset required to calculate observing day.
1091 Base class implementation returns `None`.
1093 Returns
1094 -------
1095 offset : `astropy.time.TimeDelta` or `None`
1096 The offset to apply. Returns `None` if the offset is not defined.
1098 Notes
1099 -----
1100 This offset must be subtracted from a time of observation to calculate
1101 the observing day. This offset must be added to the YYYYMMDDT00:00
1102 observing day to calculate the time span coverage of the observing day.
1103 """
1104 datetime_begin = self.to_datetime_begin()
1105 if datetime_begin is None:
1106 return None
1107 return self.observing_date_to_offset(datetime_begin)
1109 @cache_translation
1110 def to_observing_day(self) -> int:
1111 """Return the YYYYMMDD integer corresponding to the observing day.
1113 Base class implementation uses the TAI date of the start of the
1114 observation corrected by the observing day offset. If that offset
1115 is `None` no offset will be applied.
1117 The offset is subtracted from the time of observation before
1118 calculating the year, month and day.
1120 Returns
1121 -------
1122 day : `int`
1123 The observing day as an integer of form YYYYMMDD. If the header
1124 is broken and is unable to obtain a date of observation, ``0``
1125 is returned and the assumption is made that the problem will
1126 be caught elsewhere.
1128 Notes
1129 -----
1130 For example, if the offset is +12 hours both 2023-07-06T13:00 and
1131 2023-07-07T11:00 will return an observing day of 20230706 because
1132 the observing day goes from 2023-07-06T12:00 to 2023-07-07T12:00.
1133 """
1134 datetime_begin = self.to_datetime_begin()
1135 if datetime_begin is None:
1136 return 0
1137 offset = self.to_observing_day_offset()
1138 return self.observing_date_to_observing_day(datetime_begin.tai, offset)
1140 @cache_translation
1141 def to_observation_counter(self) -> int:
1142 """Return an integer corresponding to how this observation relates
1143 to other observations.
1145 Base class implementation returns ``0`` to indicate that it is not
1146 known how an observatory will define a counter. Some observatories
1147 may not use the concept, others may use a counter that increases
1148 for every observation taken for that instrument, and others may
1149 define it to be a counter within an observing day.
1151 Returns
1152 -------
1153 sequence : `int`
1154 The observation counter. Always ``0`` for this implementation.
1155 """
1156 return 0
1158 @cache_translation
1159 def to_group_counter_start(self) -> int:
1160 """Return the observation counter of the observation that began
1161 this group.
1163 The definition of the relevant group is up to the metadata
1164 translator. It can be the first observation in the exposure_group
1165 or the first observation in the visit, but must be derivable
1166 from the metadata of this observation.
1168 Returns
1169 -------
1170 counter : `int`
1171 The observation counter for the start of the relevant group.
1172 Default implementation always returns the observation counter
1173 of this observation.
1174 """
1175 return self.to_observation_counter()
1177 @cache_translation
1178 def to_group_counter_end(self) -> int:
1179 """Return the observation counter of the observation that ends
1180 this group.
1182 The definition of the relevant group is up to the metadata
1183 translator. It can be the last observation in the exposure_group
1184 or the last observation in the visit, but must be derivable
1185 from the metadata of this observation. It is of course possible
1186 that the last observation in the group does not exist if a sequence
1187 of observations was not completed.
1189 Returns
1190 -------
1191 counter : `int`
1192 The observation counter for the end of the relevant group.
1193 Default implementation always returns the observation counter
1194 of this observation.
1195 """
1196 return self.to_observation_counter()
1198 @cache_translation
1199 def to_has_simulated_content(self) -> bool:
1200 """Return a boolean indicating whether any part of the observation
1201 was simulated.
1203 Returns
1204 -------
1205 is_simulated : `bool`
1206 `True` if this exposure has simulated content. This can be
1207 if some parts of the metadata or data were simulated. Default
1208 implementation always returns `False`.
1209 """
1210 return False
1212 @cache_translation
1213 def to_focus_z(self) -> u.Quantity:
1214 """Return a default defocal distance of 0.0 mm if there is no
1215 keyword for defocal distance in the header. The default
1216 keyword for defocal distance is ``FOCUSZ``.
1218 Returns
1219 -------
1220 focus_z: `astropy.units.Quantity`
1221 The defocal distance from header or the 0.0mm default.
1222 """
1223 return 0.0 * u.mm
1225 @classmethod
1226 def determine_translatable_headers(
1227 cls, filename: str, primary: MutableMapping[str, Any] | None = None
1228 ) -> Iterator[MutableMapping[str, Any]]:
1229 """Given a file return all the headers usable for metadata translation.
1231 This method can optionally be given a header from the file. This
1232 header will generally be the primary header or a merge of the first
1233 two headers.
1235 In the base class implementation it is assumed that
1236 this supplied header is the only useful header for metadata translation
1237 and it will be returned unchanged if given. This can avoid
1238 unnecessarily re-opening the file and re-reading the header when the
1239 content is already known.
1241 If no header is supplied, a header will be read from the supplied
1242 file using `~.file_helpers.read_basic_metadata_from_file`, allowing it
1243 to merge the primary and secondary header of a multi-extension FITS
1244 file. Subclasses can read the header from the data file using whatever
1245 technique is best for that instrument.
1247 Subclasses can return multiple headers and ignore the externally
1248 supplied header. They can also merge it with another header and return
1249 a new derived header if that is required by the particular data file.
1250 There is no requirement for the supplied header to be used.
1252 Parameters
1253 ----------
1254 filename : `str`
1255 Path to a file in a format understood by this translator.
1256 primary : `dict`-like, optional
1257 The primary header obtained by the caller. This is sometimes
1258 already known, for example if a system is trying to bootstrap
1259 without already knowing what data is in the file. For many
1260 instruments where the primary header is the only relevant
1261 header, the primary header will be returned with no further
1262 action.
1264 Yields
1265 ------
1266 headers : iterator of `dict`-like
1267 A header usable for metadata translation. For this base
1268 implementation it will be either the supplied primary header
1269 or a header read from the file. This implementation will only
1270 ever yield a single header.
1272 Notes
1273 -----
1274 Each translator class can have code specifically tailored to its
1275 own file format. It is important not to call this method with
1276 an incorrect translator class. The normal paradigm is for the
1277 caller to have read the first header and then called
1278 `determine_translator()` on the result to work out which translator
1279 class to then call to obtain the real headers to be used for
1280 translation.
1281 """
1282 if primary is not None:
1283 yield primary
1284 else:
1285 # Prevent circular import by deferring
1286 from .file_helpers import read_basic_metadata_from_file
1288 # Merge primary and secondary header if they exist.
1289 header = read_basic_metadata_from_file(filename, -1)
1290 assert header is not None # for mypy since can_raise=True
1291 yield header
1294def _make_abstract_translator_method(
1295 property: str, doc: str, return_typedoc: str, return_type: type
1296) -> Callable:
1297 """Create a an abstract translation method for this property.
1299 Parameters
1300 ----------
1301 property : `str`
1302 Name of the translator for property to be created.
1303 doc : `str`
1304 Description of the property.
1305 return_typedoc : `str`
1306 Type string of this property (used in the doc string).
1307 return_type : `type`
1308 Type of this property.
1310 Returns
1311 -------
1312 m : `~collections.abc.Callable`
1313 Translator method for this property.
1314 """
1316 def to_property(self: MetadataTranslator) -> None:
1317 raise NotImplementedError(f"Translator for '{property}' undefined.")
1319 to_property.__doc__ = f"""Return value of {property} from headers.
1321 {doc}
1323 Returns
1324 -------
1325 {property} : `{return_typedoc}`
1326 The translated property.
1327 """
1328 return to_property
1331# Make abstract methods for all the translators methods.
1332# Unfortunately registering them as abstractmethods does not work
1333# as these assignments come after the class has been created.
1334# Assigning to __abstractmethods__ directly does work but interacts
1335# poorly with the metaclass automatically generating methods from
1336# _trivialMap and _constMap.
1337# Note that subclasses that provide extension properties are assumed to not
1338# need abstract methods created for them.
1340# Allow for concrete translator methods to exist in the base class
1341# These translator methods can be defined in terms of other properties
1342CONCRETE = set()
1344for name, definition in PROPERTIES.items():
1345 method = f"to_{name}"
1346 if not MetadataTranslator.defined_in_this_class(method):
1347 setattr(
1348 MetadataTranslator,
1349 f"to_{name}",
1350 abstractmethod(
1351 _make_abstract_translator_method(
1352 name, definition.doc, definition.str_type, definition.py_type
1353 )
1354 ),
1355 )
1356 else:
1357 CONCRETE.add(method)
1360class StubTranslator(MetadataTranslator):
1361 """Translator where all the translations are stubbed out and issue
1362 warnings.
1364 This translator can be used as a base class whilst developing a new
1365 translator. It allows testing to proceed without being required to fully
1366 define all translation methods. Once complete the class should be
1367 removed from the inheritance tree.
1368 """
1370 pass
1373def _make_forwarded_stub_translator_method(
1374 cls_: type[MetadataTranslator], property: str, doc: str, return_typedoc: str, return_type: type
1375) -> Callable:
1376 """Create a stub translation method for this property that calls the
1377 base method and catches `NotImplementedError`.
1379 Parameters
1380 ----------
1381 cls_ : `type`
1382 Class to use when referencing `super()`. This would usually be
1383 `StubTranslator`.
1384 property : `str`
1385 Name of the translator for property to be created.
1386 doc : `str`
1387 Description of the property.
1388 return_typedoc : `str`
1389 Type string of this property (used in the doc string).
1390 return_type : `type`
1391 Type of this property.
1393 Returns
1394 -------
1395 m : `~collections.abc.Callable`
1396 Stub translator method for this property.
1397 """
1398 method = f"to_{property}"
1400 def to_stub(self: MetadataTranslator) -> Any:
1401 parent = getattr(super(cls_, self), method, None)
1402 try:
1403 if parent is not None:
1404 return parent()
1405 except NotImplementedError:
1406 pass
1408 warnings.warn(
1409 f"Please implement translator for property '{property}' for translator {self}", stacklevel=3
1410 )
1411 return None
1413 to_stub.__doc__ = f"""Unimplemented forwarding translator for {property}.
1415 {doc}
1417 Calls the base class translation method and if that fails with
1418 `NotImplementedError` issues a warning reminding the implementer to
1419 override this method.
1421 Returns
1422 -------
1423 {property} : `None` or `{return_typedoc}`
1424 Always returns `None`.
1425 """
1426 return to_stub
1429# Create stub translation methods for each property. These stubs warn
1430# rather than fail and should be overridden by translators.
1431for name, description in PROPERTIES.items():
1432 setattr(
1433 StubTranslator,
1434 f"to_{name}",
1435 _make_forwarded_stub_translator_method(
1436 StubTranslator, name, definition.doc, definition.str_type, definition.py_type # type: ignore
1437 ),
1438 )