Coverage for python/astro_metadata_translator/translator.py: 40%
387 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-04-07 01:03 -0700
« prev ^ index » next coverage.py v6.5.0, created at 2023-04-07 01:03 -0700
1# This file is part of astro_metadata_translator.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the LICENSE file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12"""Classes and support code for metadata translation"""
14from __future__ import annotations
16__all__ = ("MetadataTranslator", "StubTranslator", "cache_translation")
18import importlib
19import inspect
20import logging
21import math
22import warnings
23from abc import abstractmethod
24from typing import (
25 TYPE_CHECKING,
26 Any,
27 Callable,
28 ClassVar,
29 Dict,
30 FrozenSet,
31 Iterable,
32 Iterator,
33 List,
34 Mapping,
35 MutableMapping,
36 Optional,
37 Sequence,
38 Set,
39 Tuple,
40 Type,
41 Union,
42)
44import astropy.io.fits.card
45import astropy.units as u
46from astropy.coordinates import Angle
48from .properties import PROPERTIES, PropertyDefinition
50if TYPE_CHECKING: 50 ↛ 51line 50 didn't jump to line 51, because the condition on line 50 was never true
51 import astropy.coordinates
52 import astropy.time
54log = logging.getLogger(__name__)
56# Location of the root of the corrections resource files
57CORRECTIONS_RESOURCE_ROOT = "corrections"
59"""Cache of version strings indexed by class."""
60_VERSION_CACHE: Dict[Type, str] = dict()
63def cache_translation(func: Callable, method: Optional[str] = None) -> Callable:
64 """Decorator to cache the result of a translation method.
66 Especially useful when a translation uses many other translation
67 methods. Should be used only on ``to_x()`` methods.
69 Parameters
70 ----------
71 func : `function`
72 Translation method to cache.
73 method : `str`, optional
74 Name of the translation method to cache. Not needed if the decorator
75 is used around a normal method, but necessary when the decorator is
76 being used in a metaclass.
78 Returns
79 -------
80 wrapped : `function`
81 Method wrapped by the caching function.
82 """
83 name = func.__name__ if method is None else method
85 def func_wrapper(self: MetadataTranslator) -> Any:
86 if name not in self._translation_cache:
87 self._translation_cache[name] = func(self)
88 return self._translation_cache[name]
90 func_wrapper.__doc__ = func.__doc__
91 func_wrapper.__name__ = f"{name}_cached"
92 return func_wrapper
95class MetadataTranslator:
96 """Per-instrument metadata translation support
98 Parameters
99 ----------
100 header : `dict`-like
101 Representation of an instrument header that can be manipulated
102 as if it was a `dict`.
103 filename : `str`, optional
104 Name of the file whose header is being translated. For some
105 datasets with missing header information this can sometimes
106 allow for some fixups in translations.
107 """
109 # These are all deliberately empty in the base class.
110 name: Optional[str] = None
111 """The declared name of the translator."""
113 default_search_path: Optional[Sequence[str]] = None
114 """Default search path to use to locate header correction files."""
116 default_resource_package = __name__.split(".")[0]
117 """Module name to use to locate the correction resources."""
119 default_resource_root: Optional[str] = None
120 """Default package resource path root to use to locate header correction
121 files within the ``default_resource_package`` package."""
123 _trivial_map: Dict[str, Union[str, List[str], Tuple[Any, ...]]] = {}
124 """Dict of one-to-one mappings for header translation from standard
125 property to corresponding keyword."""
127 _const_map: Dict[str, Any] = {}
128 """Dict defining a constant for specified standard properties."""
130 translators: Dict[str, Type] = dict()
131 """All registered metadata translation classes."""
133 supported_instrument: Optional[str] = None
134 """Name of instrument understood by this translation class."""
136 all_properties: Dict[str, PropertyDefinition] = {}
137 """All the valid properties for this translator including extensions."""
139 extensions: Dict[str, PropertyDefinition] = {}
140 """Extension properties (`str`: `PropertyDefinition`)
142 Some instruments have important properties beyond the standard set; this is
143 the place to declare that they exist, and they will be treated in the same
144 way as the standard set, except that their names will everywhere be
145 prefixed with ``ext_``.
147 Each property is indexed by name (`str`), with a corresponding
148 `PropertyDefinition`.
149 """
151 # Static typing requires that we define the standard dynamic properties
152 # statically.
153 if TYPE_CHECKING: 153 ↛ 154line 153 didn't jump to line 154, because the condition on line 153 was never true
154 to_telescope: ClassVar[Callable[[MetadataTranslator], str]]
155 to_instrument: ClassVar[Callable[[MetadataTranslator], str]]
156 to_location: ClassVar[Callable[[MetadataTranslator], astropy.coordinates.EarthLocation]]
157 to_exposure_id: ClassVar[Callable[[MetadataTranslator], int]]
158 to_visit_id: ClassVar[Callable[[MetadataTranslator], int]]
159 to_physical_filter: ClassVar[Callable[[MetadataTranslator], str]]
160 to_datetime_begin: ClassVar[Callable[[MetadataTranslator], astropy.time.Time]]
161 to_datetime_end: ClassVar[Callable[[MetadataTranslator], astropy.time.Time]]
162 to_exposure_time: ClassVar[Callable[[MetadataTranslator], u.Quantity]]
163 to_dark_time: ClassVar[Callable[[MetadataTranslator], u.Quantity]]
164 to_boresight_airmass: ClassVar[Callable[[MetadataTranslator], float]]
165 to_boresight_rotation_angle: ClassVar[Callable[[MetadataTranslator], u.Quantity]]
166 to_boresight_rotation_coord: ClassVar[Callable[[MetadataTranslator], str]]
167 to_detector_num: ClassVar[Callable[[MetadataTranslator], int]]
168 to_detector_name: ClassVar[Callable[[MetadataTranslator], str]]
169 to_detector_serial: ClassVar[Callable[[MetadataTranslator], str]]
170 to_detector_group: ClassVar[Callable[[MetadataTranslator], Optional[str]]]
171 to_detector_exposure_id: ClassVar[Callable[[MetadataTranslator], int]]
172 to_object: ClassVar[Callable[[MetadataTranslator], str]]
173 to_temperature: ClassVar[Callable[[MetadataTranslator], u.Quantity]]
174 to_pressure: ClassVar[Callable[[MetadataTranslator], u.Quantity]]
175 to_relative_humidity: ClassVar[Callable[[MetadataTranslator], float]]
176 to_tracking_radec: ClassVar[Callable[[MetadataTranslator], astropy.coordinates.SkyCoord]]
177 to_altaz_begin: ClassVar[Callable[[MetadataTranslator], astropy.coordinates.AltAz]]
178 to_science_program: ClassVar[Callable[[MetadataTranslator], str]]
179 to_observation_type: ClassVar[Callable[[MetadataTranslator], str]]
180 to_observation_id: ClassVar[Callable[[MetadataTranslator], str]]
182 @classmethod
183 def defined_in_this_class(cls, name: str) -> Optional[bool]:
184 """Report if the specified class attribute is defined specifically in
185 this class.
187 Parameters
188 ----------
189 name : `str`
190 Name of the attribute to test.
192 Returns
193 -------
194 in_class : `bool`
195 `True` if there is a attribute of that name defined in this
196 specific subclass.
197 `False` if the method is not defined in this specific subclass
198 but is defined in a parent class.
199 Returns `None` if the attribute is not defined anywhere
200 in the class hierarchy (which can happen if translators have
201 typos in their mapping tables).
203 Notes
204 -----
205 Retrieves the attribute associated with the given name.
206 Then looks in all the parent classes to determine whether that
207 attribute comes from a parent class or from the current class.
208 Attributes are compared using `id()`.
209 """
210 # The attribute to compare.
211 if not hasattr(cls, name):
212 return None
213 attr_id = id(getattr(cls, name))
215 # Get all the classes in the hierarchy
216 mro = list(inspect.getmro(cls))
218 # Remove the first entry from the list since that will be the
219 # current class
220 mro.pop(0)
222 for parent in mro:
223 # Some attributes may only exist in subclasses. Skip base classes
224 # that are missing the attribute (such as object).
225 if hasattr(parent, name):
226 if id(getattr(parent, name)) == attr_id:
227 return False
228 return True
230 @classmethod
231 def _make_const_mapping(cls, property_key: str, constant: Any) -> Callable:
232 """Make a translator method that returns a constant value.
234 Parameters
235 ----------
236 property_key : `str`
237 Name of the property to be calculated (for the docstring).
238 constant : `str` or `numbers.Number`
239 Value to return for this translator.
241 Returns
242 -------
243 f : `function`
244 Function returning the constant.
245 """
247 def constant_translator(self: MetadataTranslator) -> Any:
248 return constant
250 if property_key in cls.all_properties:
251 property_doc = cls.all_properties[property_key].doc
252 return_type = cls.all_properties[property_key].py_type
253 else:
254 return_type = type(constant)
255 property_doc = f"Returns constant value for '{property_key}' property"
257 constant_translator.__doc__ = f"""{property_doc}
259 Returns
260 -------
261 translation : `{return_type}`
262 Translated property.
263 """
264 return constant_translator
266 @classmethod
267 def _make_trivial_mapping(
268 cls,
269 property_key: str,
270 header_key: Union[str, Sequence[str]],
271 default: Optional[Any] = None,
272 minimum: Optional[Any] = None,
273 maximum: Optional[Any] = None,
274 unit: Optional[astropy.unit.Unit] = None,
275 checker: Optional[Callable] = None,
276 ) -> Callable:
277 """Make a translator method returning a header value.
279 The header value can be converted to a `~astropy.units.Quantity`
280 if desired, and can also have its value validated.
282 See `MetadataTranslator.validate_value()` for details on the use
283 of default parameters.
285 Parameters
286 ----------
287 property_key : `str`
288 Name of the translator to be constructed (for the docstring).
289 header_key : `str` or `list` of `str`
290 Name of the key to look up in the header. If a `list` each
291 key will be tested in turn until one matches. This can deal with
292 header styles that evolve over time.
293 default : `numbers.Number` or `astropy.units.Quantity`, `str`, optional
294 If not `None`, default value to be used if the parameter read from
295 the header is not defined or if the header is missing.
296 minimum : `numbers.Number` or `astropy.units.Quantity`, optional
297 If not `None`, and if ``default`` is not `None`, minimum value
298 acceptable for this parameter.
299 maximum : `numbers.Number` or `astropy.units.Quantity`, optional
300 If not `None`, and if ``default`` is not `None`, maximum value
301 acceptable for this parameter.
302 unit : `astropy.units.Unit`, optional
303 If not `None`, the value read from the header will be converted
304 to a `~astropy.units.Quantity`. Only supported for numeric values.
305 checker : `function`, optional
306 Callback function to be used by the translator method in case the
307 keyword is not present. Function will be executed as if it is
308 a method of the translator class. Running without raising an
309 exception will allow the default to be used. Should usually raise
310 `KeyError`.
312 Returns
313 -------
314 t : `function`
315 Function implementing a translator with the specified
316 parameters.
317 """
318 if property_key in cls.all_properties: 318 ↛ 322line 318 didn't jump to line 322, because the condition on line 318 was never false
319 property_doc = cls.all_properties[property_key].doc
320 return_type = cls.all_properties[property_key].str_type
321 else:
322 return_type = "str` or `numbers.Number"
323 property_doc = f"Map '{header_key}' header keyword to '{property_key}' property"
325 def trivial_translator(self: MetadataTranslator) -> Any:
326 if unit is not None:
327 q = self.quantity_from_card(
328 header_key, unit, default=default, minimum=minimum, maximum=maximum, checker=checker
329 )
330 # Convert to Angle if this quantity is an angle
331 if return_type == "astropy.coordinates.Angle":
332 q = Angle(q)
333 return q
335 keywords = header_key if isinstance(header_key, list) else [header_key]
336 for key in keywords:
337 if self.is_key_ok(key):
338 value = self._header[key]
339 if default is not None and not isinstance(value, str):
340 value = self.validate_value(value, default, minimum=minimum, maximum=maximum)
341 self._used_these_cards(key)
342 break
343 else:
344 # No keywords found, use default, checking first, or raise
345 # A None default is only allowed if a checker is provided.
346 if checker is not None:
347 try:
348 checker(self)
349 except Exception:
350 raise KeyError(f"Could not find {keywords} in header")
351 return default
352 elif default is not None:
353 value = default
354 else:
355 raise KeyError(f"Could not find {keywords} in header")
357 # If we know this is meant to be a string, force to a string.
358 # Sometimes headers represent items as integers which generically
359 # we want as strings (eg OBSID). Sometimes also floats are
360 # written as "NaN" strings.
361 casts = {"str": str, "float": float, "int": int}
362 if return_type in casts and not isinstance(value, casts[return_type]) and value is not None:
363 value = casts[return_type](value)
365 return value
367 # Docstring inheritance means it is confusing to specify here
368 # exactly which header value is being used.
369 trivial_translator.__doc__ = f"""{property_doc}
371 Returns
372 -------
373 translation : `{return_type}`
374 Translated value derived from the header.
375 """
376 return trivial_translator
378 @classmethod
379 def __init_subclass__(cls, **kwargs: Any) -> None:
380 """Register all subclasses with the base class and create dynamic
381 translator methods.
383 The method provides two facilities. Firstly, every subclass
384 of `MetadataTranslator` that includes a ``name`` class property is
385 registered as a translator class that could be selected when automatic
386 header translation is attempted. Only name translator subclasses that
387 correspond to a complete instrument. Translation classes providing
388 generic translation support for multiple instrument translators should
389 not be named.
391 The second feature of this method is to convert simple translations
392 to full translator methods. Sometimes a translation is fixed (for
393 example a specific instrument name should be used) and rather than
394 provide a full ``to_property()`` translation method the mapping can be
395 defined in a class variable named ``_constMap``. Similarly, for
396 one-to-one trivial mappings from a header to a property,
397 ``_trivialMap`` can be defined. Trivial mappings are a dict mapping a
398 generic property to either a header keyword, or a tuple consisting of
399 the header keyword and a dict containing key value pairs suitable for
400 the `MetadataTranslator.quantity_from_card()` method.
401 """
402 super().__init_subclass__(**kwargs)
404 # Only register classes with declared names
405 if hasattr(cls, "name") and cls.name is not None:
406 if cls.name in MetadataTranslator.translators: 406 ↛ 407line 406 didn't jump to line 407, because the condition on line 406 was never true
407 log.warning(
408 "%s: Replacing %s translator with %s",
409 cls.name,
410 MetadataTranslator.translators[cls.name],
411 cls,
412 )
413 MetadataTranslator.translators[cls.name] = cls
415 # Check that we have not inherited constant/trivial mappings from
416 # parent class that we have already applied. Empty maps are always
417 # assumed okay
418 const_map = cls._const_map if cls._const_map and cls.defined_in_this_class("_const_map") else {}
419 trivial_map = (
420 cls._trivial_map if cls._trivial_map and cls.defined_in_this_class("_trivial_map") else {}
421 )
423 # Check for shadowing
424 trivials = set(trivial_map.keys())
425 constants = set(const_map.keys())
426 both = trivials & constants
427 if both: 427 ↛ 428line 427 didn't jump to line 428, because the condition on line 427 was never true
428 log.warning("%s: defined in both const_map and trivial_map: %s", cls.__name__, ", ".join(both))
430 all = trivials | constants
431 for name in all:
432 if cls.defined_in_this_class(f"to_{name}"): 432 ↛ 435line 432 didn't jump to line 435, because the condition on line 432 was never true
433 # Must be one of trivial or constant. If in both then constant
434 # overrides trivial.
435 location = "by _trivial_map"
436 if name in constants:
437 location = "by _const_map"
438 log.warning(
439 "%s: %s is defined explicitly but will be replaced %s", cls.__name__, name, location
440 )
442 properties = set(PROPERTIES) | set(("ext_" + pp for pp in cls.extensions))
443 cls.all_properties = dict(PROPERTIES)
444 cls.all_properties.update(cls.extensions)
446 # Go through the trival mappings for this class and create
447 # corresponding translator methods
448 for property_key, header_key in trivial_map.items():
449 kwargs = {}
450 if type(header_key) == tuple:
451 kwargs = header_key[1]
452 header_key = header_key[0]
453 translator = cls._make_trivial_mapping(property_key, header_key, **kwargs)
454 method = f"to_{property_key}"
455 translator.__name__ = f"{method}_trivial_in_{cls.__name__}"
456 setattr(cls, method, cache_translation(translator, method=method))
457 if property_key not in properties: 457 ↛ 458line 457 didn't jump to line 458, because the condition on line 457 was never true
458 log.warning(f"Unexpected trivial translator for '{property_key}' defined in {cls}")
460 # Go through the constant mappings for this class and create
461 # corresponding translator methods
462 for property_key, constant in const_map.items():
463 translator = cls._make_const_mapping(property_key, constant)
464 method = f"to_{property_key}"
465 translator.__name__ = f"{method}_constant_in_{cls.__name__}"
466 setattr(cls, method, translator)
467 if property_key not in properties: 467 ↛ 468line 467 didn't jump to line 468, because the condition on line 467 was never true
468 log.warning(f"Unexpected constant translator for '{property_key}' defined in {cls}")
470 def __init__(self, header: Mapping[str, Any], filename: Optional[str] = None) -> None:
471 self._header = header
472 self.filename = filename
473 self._used_cards: Set[str] = set()
475 # Prefix to use for warnings about failed translations
476 self._log_prefix_cache: Optional[str] = None
478 # Cache assumes header is read-only once stored in object
479 self._translation_cache: Dict[str, Any] = {}
481 @classmethod
482 @abstractmethod
483 def can_translate(cls, header: MutableMapping[str, Any], filename: Optional[str] = None) -> bool:
484 """Indicate whether this translation class can translate the
485 supplied header.
487 Parameters
488 ----------
489 header : `dict`-like
490 Header to convert to standardized form.
491 filename : `str`, optional
492 Name of file being translated.
494 Returns
495 -------
496 can : `bool`
497 `True` if the header is recognized by this class. `False`
498 otherwise.
499 """
500 raise NotImplementedError()
502 @classmethod
503 def can_translate_with_options(
504 cls, header: Mapping[str, Any], options: Dict[str, Any], filename: Optional[str] = None
505 ) -> bool:
506 """Helper method for `can_translate` allowing options.
508 Parameters
509 ----------
510 header : `dict`-like
511 Header to convert to standardized form.
512 options : `dict`
513 Headers to try to determine whether this header can
514 be translated by this class. If a card is found it will
515 be compared with the expected value and will return that
516 comparison. Each card will be tried in turn until one is
517 found.
518 filename : `str`, optional
519 Name of file being translated.
521 Returns
522 -------
523 can : `bool`
524 `True` if the header is recognized by this class. `False`
525 otherwise.
527 Notes
528 -----
529 Intended to be used from within `can_translate` implementations
530 for specific translators. Is not intended to be called directly
531 from `determine_translator`.
532 """
533 for card, value in options.items():
534 if card in header:
535 return header[card] == value
536 return False
538 @classmethod
539 def determine_translator(
540 cls, header: Mapping[str, Any], filename: Optional[str] = None
541 ) -> Type[MetadataTranslator]:
542 """Determine a translation class by examining the header
544 Parameters
545 ----------
546 header : `dict`-like
547 Representation of a header.
548 filename : `str`, optional
549 Name of file being translated.
551 Returns
552 -------
553 translator : `MetadataTranslator`
554 Translation class that knows how to extract metadata from
555 the supplied header.
557 Raises
558 ------
559 ValueError
560 None of the registered translation classes understood the supplied
561 header.
562 """
563 file_msg = ""
564 if filename is not None:
565 file_msg = f" from {filename}"
566 for name, trans in cls.translators.items():
567 if trans.can_translate(header, filename=filename):
568 log.debug("Using translation class %s%s", name, file_msg)
569 return trans
570 else:
571 raise ValueError(
572 f"None of the registered translation classes {list(cls.translators.keys())}"
573 f" understood this header{file_msg}"
574 )
576 @classmethod
577 def translator_version(cls) -> str:
578 """Return the version string for this translator class.
580 Returns
581 -------
582 version : `str`
583 String identifying the version of this translator.
585 Notes
586 -----
587 Assumes that the version is available from the ``__version__``
588 variable in the parent module. If this is not the case a translator
589 should subclass this method.
590 """
591 if cls in _VERSION_CACHE:
592 return _VERSION_CACHE[cls]
594 version = "unknown"
595 module_name = cls.__module__
596 components = module_name.split(".")
597 while components:
598 # This class has already been imported so importing it
599 # should work.
600 module = importlib.import_module(".".join(components))
601 if hasattr(module, v := "__version__"):
602 version = getattr(module, v)
603 if version == "unknown":
604 # LSST software will have a fingerprint
605 if hasattr(module, v := "__fingerprint__"):
606 version = getattr(module, v)
607 break
608 else:
609 # Remove last component from module name and try again
610 components.pop()
612 _VERSION_CACHE[cls] = version
613 return version
615 @classmethod
616 def fix_header(
617 cls, header: MutableMapping[str, Any], instrument: str, obsid: str, filename: Optional[str] = None
618 ) -> bool:
619 """Apply global fixes to a supplied header.
621 Parameters
622 ----------
623 header : `dict`
624 The header to correct. Correction is in place.
625 instrument : `str`
626 The name of the instrument.
627 obsid : `str`
628 Unique observation identifier associated with this header.
629 Will always be provided.
630 filename : `str`, optional
631 Filename associated with this header. May not be set since headers
632 can be fixed independently of any filename being known.
634 Returns
635 -------
636 modified : `bool`
637 `True` if a correction was applied.
639 Notes
640 -----
641 This method is intended to support major discrepancies in headers
642 such as:
644 * Periods of time where headers are known to be incorrect in some
645 way that can be fixed either by deriving the correct value from
646 the existing value or understanding the that correction is static
647 for the given time. This requires that the date header is
648 known.
649 * The presence of a certain value is always wrong and should be
650 corrected with a new static value regardless of date.
652 It is assumed that one off problems with headers have been applied
653 before this method is called using the per-obsid correction system.
655 Usually called from `astro_metadata_translator.fix_header`.
657 For log messages, do not assume that the filename will be present.
658 Always write log messages to fall back on using the ``obsid`` if
659 ``filename`` is `None`.
660 """
661 return False
663 @staticmethod
664 def _construct_log_prefix(obsid: str, filename: Optional[str] = None) -> str:
665 """Construct a log prefix string from the obsid and filename.
667 Parameters
668 ----------
669 obsid : `str`
670 The observation identifier.
671 filename : `str`, optional
672 The filename associated with the header being translated.
673 Can be `None`.
674 """
675 if filename:
676 return f"{filename}({obsid})"
677 return obsid
679 @property
680 def _log_prefix(self) -> str:
681 """Standard prefix that can be used for log messages to report
682 useful context.
684 Will be either the filename and obsid, or just the obsid depending
685 on whether a filename is known.
687 Returns
688 -------
689 prefix : `str`
690 The prefix to use.
691 """
692 if self._log_prefix_cache is None:
693 # Protect against the unfortunate event of the obsid failing to
694 # be calculated. This should be rare but should not prevent a log
695 # message from appearing.
696 try:
697 obsid = self.to_observation_id()
698 except Exception:
699 obsid = "unknown_obsid"
700 self._log_prefix_cache = self._construct_log_prefix(obsid, self.filename)
701 return self._log_prefix_cache
703 def _used_these_cards(self, *args: str) -> None:
704 """Indicate that the supplied cards have been used for translation.
706 Parameters
707 ----------
708 args : sequence of `str`
709 Keywords used to process a translation.
710 """
711 self._used_cards.update(set(args))
713 def cards_used(self) -> FrozenSet[str]:
714 """Cards used during metadata extraction.
716 Returns
717 -------
718 used : `frozenset` of `str`
719 Cards used when extracting metadata.
720 """
721 return frozenset(self._used_cards)
723 @staticmethod
724 def validate_value(
725 value: float, default: float, minimum: Optional[float] = None, maximum: Optional[float] = None
726 ) -> float:
727 """Validate the supplied value, returning a new value if out of range
729 Parameters
730 ----------
731 value : `float`
732 Value to be validated.
733 default : `float`
734 Default value to use if supplied value is invalid or out of range.
735 Assumed to be in the same units as the value expected in the
736 header.
737 minimum : `float`
738 Minimum possible valid value, optional. If the calculated value
739 is below this value, the default value will be used.
740 maximum : `float`
741 Maximum possible valid value, optional. If the calculated value
742 is above this value, the default value will be used.
744 Returns
745 -------
746 value : `float`
747 Either the supplied value, or a default value.
748 """
749 if value is None or math.isnan(value):
750 value = default
751 else:
752 if minimum is not None and value < minimum:
753 value = default
754 elif maximum is not None and value > maximum:
755 value = default
756 return value
758 @staticmethod
759 def is_keyword_defined(header: Mapping[str, Any], keyword: Optional[str]) -> bool:
760 """Return `True` if the value associated with the named keyword is
761 present in the supplied header and defined.
763 Parameters
764 ----------
765 header : `dict`-lik
766 Header to use as reference.
767 keyword : `str`
768 Keyword to check against header.
770 Returns
771 -------
772 is_defined : `bool`
773 `True` if the header is present and not-`None`. `False` otherwise.
774 """
775 if keyword is None or keyword not in header:
776 return False
778 if header[keyword] is None:
779 return False
781 # Special case Astropy undefined value
782 if isinstance(header[keyword], astropy.io.fits.card.Undefined):
783 return False
785 return True
787 def resource_root(self) -> Tuple[Optional[str], Optional[str]]:
788 """Package resource to use to locate correction resources within an
789 installed package.
791 Returns
792 -------
793 resource_package : `str`
794 Package resource name. `None` if no package resource are to be
795 used.
796 resource_root : `str`
797 The name of the resource root. `None` if no package resources
798 are to be used.
799 """
800 return (self.default_resource_package, self.default_resource_root)
802 def search_paths(self) -> List[str]:
803 """Search paths to use when searching for header fix up correction
804 files.
806 Returns
807 -------
808 paths : `list`
809 Directory paths to search. Can be an empty list if no special
810 directories are defined.
812 Notes
813 -----
814 Uses the classes ``default_search_path`` property if defined.
815 """
816 if self.default_search_path is not None:
817 return [p for p in self.default_search_path]
818 return []
820 def is_key_ok(self, keyword: Optional[str]) -> bool:
821 """Return `True` if the value associated with the named keyword is
822 present in this header and defined.
824 Parameters
825 ----------
826 keyword : `str`
827 Keyword to check against header.
829 Returns
830 -------
831 is_ok : `bool`
832 `True` if the header is present and not-`None`. `False` otherwise.
833 """
834 return self.is_keyword_defined(self._header, keyword)
836 def are_keys_ok(self, keywords: Iterable[str]) -> bool:
837 """Are the supplied keys all present and defined?
839 Parameters
840 ----------
841 keywords : iterable of `str`
842 Keywords to test.
844 Returns
845 -------
846 all_ok : `bool`
847 `True` if all supplied keys are present and defined.
848 """
849 for k in keywords:
850 if not self.is_key_ok(k):
851 return False
852 return True
854 def quantity_from_card(
855 self,
856 keywords: Union[str, Sequence[str]],
857 unit: u.Unit,
858 default: Optional[float] = None,
859 minimum: Optional[float] = None,
860 maximum: Optional[float] = None,
861 checker: Optional[Callable] = None,
862 ) -> u.Quantity:
863 """Calculate a Astropy Quantity from a header card and a unit.
865 Parameters
866 ----------
867 keywords : `str` or `list` of `str`
868 Keyword to use from header. If a list each keyword will be tried
869 in turn until one matches.
870 unit : `astropy.units.UnitBase`
871 Unit of the item in the header.
872 default : `float`, optional
873 Default value to use if the header value is invalid. Assumed
874 to be in the same units as the value expected in the header. If
875 None, no default value is used.
876 minimum : `float`, optional
877 Minimum possible valid value, optional. If the calculated value
878 is below this value, the default value will be used.
879 maximum : `float`, optional
880 Maximum possible valid value, optional. If the calculated value
881 is above this value, the default value will be used.
882 checker : `function`, optional
883 Callback function to be used by the translator method in case the
884 keyword is not present. Function will be executed as if it is
885 a method of the translator class. Running without raising an
886 exception will allow the default to be used. Should usually raise
887 `KeyError`.
889 Returns
890 -------
891 q : `astropy.units.Quantity`
892 Quantity representing the header value.
894 Raises
895 ------
896 KeyError
897 The supplied header key is not present.
898 """
899 keyword_list = [keywords] if isinstance(keywords, str) else list(keywords)
900 for k in keyword_list:
901 if self.is_key_ok(k):
902 value = self._header[k]
903 keyword = k
904 break
905 else:
906 if checker is not None:
907 try:
908 checker(self)
909 value = default
910 if value is not None:
911 value = u.Quantity(value, unit=unit)
912 return value
913 except Exception:
914 pass
915 raise KeyError(f"Could not find {keywords} in header")
916 if isinstance(value, str):
917 # Sometimes the header has the wrong type in it but this must
918 # be a number if we are creating a quantity.
919 value = float(value)
920 self._used_these_cards(keyword)
921 if default is not None:
922 value = self.validate_value(value, default, maximum=maximum, minimum=minimum)
923 return u.Quantity(value, unit=unit)
925 def _join_keyword_values(self, keywords: Iterable[str], delim: str = "+") -> str:
926 """Join values of all defined keywords with the specified delimiter.
928 Parameters
929 ----------
930 keywords : iterable of `str`
931 Keywords to look for in header.
932 delim : `str`, optional
933 Character to use to join the values together.
935 Returns
936 -------
937 joined : `str`
938 String formed from all the keywords found in the header with
939 defined values joined by the delimiter. Empty string if no
940 defined keywords found.
941 """
942 values = []
943 for k in keywords:
944 if self.is_key_ok(k):
945 values.append(self._header[k])
946 self._used_these_cards(k)
948 if values:
949 joined = delim.join(str(v) for v in values)
950 else:
951 joined = ""
953 return joined
955 @cache_translation
956 def to_detector_unique_name(self) -> str:
957 """Return a unique name for the detector.
959 Base class implementation attempts to combine ``detector_name`` with
960 ``detector_group``. Group is only used if not `None`.
962 Can be over-ridden by specialist translator class.
964 Returns
965 -------
966 name : `str`
967 ``detector_group``_``detector_name`` if ``detector_group`` is
968 defined, else the ``detector_name`` is assumed to be unique.
969 If neither return a valid value an exception is raised.
971 Raises
972 ------
973 NotImplementedError
974 Raised if neither detector_name nor detector_group is defined.
975 """
976 name = self.to_detector_name()
977 group = self.to_detector_group()
979 if group is None and name is None:
980 raise NotImplementedError("Can not determine unique name from detector_group and detector_name")
982 if group is not None:
983 return f"{group}_{name}"
985 return name
987 @cache_translation
988 def to_exposure_group(self) -> Optional[str]:
989 """Return the group label associated with this exposure.
991 Base class implementation returns the ``exposure_id`` in string
992 form. A subclass may do something different.
994 Returns
995 -------
996 name : `str`
997 The ``exposure_id`` converted to a string.
998 """
999 exposure_id = self.to_exposure_id()
1000 if exposure_id is None:
1001 # mypy does not think this can ever happen but play it safe
1002 # with subclasses.
1003 return None # type: ignore
1004 else:
1005 return str(exposure_id)
1007 @cache_translation
1008 def to_observation_reason(self) -> str:
1009 """Return the reason this observation was taken.
1011 Base class implementation returns the ``science`` if the
1012 ``observation_type`` is science, else ``unknown``.
1013 A subclass may do something different.
1015 Returns
1016 -------
1017 name : `str`
1018 The reason for this observation.
1019 """
1020 obstype = self.to_observation_type()
1021 if obstype == "science":
1022 return "science"
1023 return "unknown"
1025 @cache_translation
1026 def to_observing_day(self) -> int:
1027 """Return the YYYYMMDD integer corresponding to the observing day.
1029 Base class implementation uses the TAI date of the start of the
1030 observation.
1032 Returns
1033 -------
1034 day : `int`
1035 The observing day as an integer of form YYYYMMDD. If the header
1036 is broken and is unable to obtain a date of observation, ``0``
1037 is returned and the assumption is made that the problem will
1038 be caught elsewhere.
1039 """
1040 datetime_begin = self.to_datetime_begin()
1041 if datetime_begin is None:
1042 return 0
1043 return int(datetime_begin.tai.strftime("%Y%m%d"))
1045 @cache_translation
1046 def to_observation_counter(self) -> int:
1047 """Return an integer corresponding to how this observation relates
1048 to other observations.
1050 Base class implementation returns ``0`` to indicate that it is not
1051 known how an observatory will define a counter. Some observatories
1052 may not use the concept, others may use a counter that increases
1053 for every observation taken for that instrument, and others may
1054 define it to be a counter within an observing day.
1056 Returns
1057 -------
1058 sequence : `int`
1059 The observation counter. Always ``0`` for this implementation.
1060 """
1061 return 0
1063 @cache_translation
1064 def to_group_counter_start(self) -> int:
1065 """Return the observation counter of the observation that began
1066 this group.
1068 The definition of the relevant group is up to the metadata
1069 translator. It can be the first observation in the exposure_group
1070 or the first observation in the visit, but must be derivable
1071 from the metadata of this observation.
1073 Returns
1074 -------
1075 counter : `int`
1076 The observation counter for the start of the relevant group.
1077 Default implementation always returns the observation counter
1078 of this observation.
1079 """
1080 return self.to_observation_counter()
1082 @cache_translation
1083 def to_group_counter_end(self) -> int:
1084 """Return the observation counter of the observation that ends
1085 this group.
1087 The definition of the relevant group is up to the metadata
1088 translator. It can be the last observation in the exposure_group
1089 or the last observation in the visit, but must be derivable
1090 from the metadata of this observation. It is of course possible
1091 that the last observation in the group does not exist if a sequence
1092 of observations was not completed.
1094 Returns
1095 -------
1096 counter : `int`
1097 The observation counter for the end of the relevant group.
1098 Default implementation always returns the observation counter
1099 of this observation.
1100 """
1101 return self.to_observation_counter()
1103 @cache_translation
1104 def to_has_simulated_content(self) -> bool:
1105 """Return a boolean indicating whether any part of the observation
1106 was simulated.
1108 Returns
1109 -------
1110 is_simulated : `bool`
1111 `True` if this exposure has simulated content. This can be
1112 if some parts of the metadata or data were simulated. Default
1113 implementation always returns `False`.
1114 """
1115 return False
1117 @cache_translation
1118 def to_focus_z(self) -> u.Quantity:
1119 """Return a default defocal distance of 0.0 mm if there is no
1120 keyword for defocal distance in the header. The default
1121 keyword for defocal distance is ``FOCUSZ``.
1123 Returns
1124 -------
1125 focus_z: `astropy.units.Quantity`
1126 The defocal distance from header or the 0.0mm default
1127 """
1128 return 0.0 * u.mm
1130 @classmethod
1131 def determine_translatable_headers(
1132 cls, filename: str, primary: Optional[MutableMapping[str, Any]] = None
1133 ) -> Iterator[MutableMapping[str, Any]]:
1134 """Given a file return all the headers usable for metadata translation.
1136 This method can optionally be given a header from the file. This
1137 header will generally be the primary header or a merge of the first
1138 two headers.
1140 In the base class implementation it is assumed that
1141 this supplied header is the only useful header for metadata translation
1142 and it will be returned unchanged if given. This can avoid
1143 unnecesarily re-opening the file and re-reading the header when the
1144 content is already known.
1146 If no header is supplied, a header will be read from the supplied
1147 file using `read_basic_metadata_from_file`, allowing it to merge
1148 the primary and secondary header of a multi-extension FITS file.
1149 Subclasses can read the header from the data file using whatever
1150 technique is best for that instrument.
1152 Subclasses can return multiple headers and ignore the externally
1153 supplied header. They can also merge it with another header and return
1154 a new derived header if that is required by the particular data file.
1155 There is no requirement for the supplied header to be used.
1157 Parameters
1158 ----------
1159 filename : `str`
1160 Path to a file in a format understood by this translator.
1161 primary : `dict`-like, optional
1162 The primary header obtained by the caller. This is sometimes
1163 already known, for example if a system is trying to bootstrap
1164 without already knowing what data is in the file. For many
1165 instruments where the primary header is the only relevant
1166 header, the primary header will be returned with no further
1167 action.
1169 Yields
1170 ------
1171 headers : iterator of `dict`-like
1172 A header usable for metadata translation. For this base
1173 implementation it will be either the supplied primary header
1174 or a header read from the file. This implementation will only
1175 ever yield a single header.
1177 Notes
1178 -----
1179 Each translator class can have code specifically tailored to its
1180 own file format. It is important not to call this method with
1181 an incorrect translator class. The normal paradigm is for the
1182 caller to have read the first header and then called
1183 `determine_translator()` on the result to work out which translator
1184 class to then call to obtain the real headers to be used for
1185 translation.
1186 """
1187 if primary is not None:
1188 yield primary
1189 else:
1190 # Prevent circular import by deferring
1191 from .file_helpers import read_basic_metadata_from_file
1193 # Merge primary and secondary header if they exist.
1194 header = read_basic_metadata_from_file(filename, -1)
1195 assert header is not None # for mypy since can_raise=True
1196 yield header
1199def _make_abstract_translator_method(
1200 property: str, doc: str, return_typedoc: str, return_type: Type
1201) -> Callable:
1202 """Create a an abstract translation method for this property.
1204 Parameters
1205 ----------
1206 property : `str`
1207 Name of the translator for property to be created.
1208 doc : `str`
1209 Description of the property.
1210 return_typedoc : `str`
1211 Type string of this property (used in the doc string).
1212 return_type : `class`
1213 Type of this property.
1215 Returns
1216 -------
1217 m : `function`
1218 Translator method for this property.
1219 """
1221 def to_property(self: MetadataTranslator) -> None:
1222 raise NotImplementedError(f"Translator for '{property}' undefined.")
1224 to_property.__doc__ = f"""Return value of {property} from headers.
1226 {doc}
1228 Returns
1229 -------
1230 {property} : `{return_typedoc}`
1231 The translated property.
1232 """
1233 return to_property
1236# Make abstract methods for all the translators methods.
1237# Unfortunately registering them as abstractmethods does not work
1238# as these assignments come after the class has been created.
1239# Assigning to __abstractmethods__ directly does work but interacts
1240# poorly with the metaclass automatically generating methods from
1241# _trivialMap and _constMap.
1242# Note that subclasses that provide extension properties are assumed to not
1243# need abstract methods created for them.
1245# Allow for concrete translator methods to exist in the base class
1246# These translator methods can be defined in terms of other properties
1247CONCRETE = set()
1249for name, definition in PROPERTIES.items():
1250 method = f"to_{name}"
1251 if not MetadataTranslator.defined_in_this_class(method):
1252 setattr(
1253 MetadataTranslator,
1254 f"to_{name}",
1255 abstractmethod(
1256 _make_abstract_translator_method(
1257 name, definition.doc, definition.str_type, definition.py_type
1258 )
1259 ),
1260 )
1261 else:
1262 CONCRETE.add(method)
1265class StubTranslator(MetadataTranslator):
1266 """Translator where all the translations are stubbed out and issue
1267 warnings.
1269 This translator can be used as a base class whilst developing a new
1270 translator. It allows testing to proceed without being required to fully
1271 define all translation methods. Once complete the class should be
1272 removed from the inheritance tree.
1274 """
1276 pass
1279def _make_forwarded_stub_translator_method(
1280 cls: Type[MetadataTranslator], property: str, doc: str, return_typedoc: str, return_type: Type
1281) -> Callable:
1282 """Create a stub translation method for this property that calls the
1283 base method and catches `NotImplementedError`.
1285 Parameters
1286 ----------
1287 cls : `class`
1288 Class to use when referencing `super()`. This would usually be
1289 `StubTranslator`.
1290 property : `str`
1291 Name of the translator for property to be created.
1292 doc : `str`
1293 Description of the property.
1294 return_typedoc : `str`
1295 Type string of this property (used in the doc string).
1296 return_type : `class`
1297 Type of this property.
1299 Returns
1300 -------
1301 m : `function`
1302 Stub translator method for this property.
1303 """
1304 method = f"to_{property}"
1306 def to_stub(self: MetadataTranslator) -> Any:
1307 parent = getattr(super(cls, self), method, None)
1308 try:
1309 if parent is not None:
1310 return parent()
1311 except NotImplementedError:
1312 pass
1314 warnings.warn(
1315 f"Please implement translator for property '{property}' for translator {self}", stacklevel=3
1316 )
1317 return None
1319 to_stub.__doc__ = f"""Unimplemented forwarding translator for {property}.
1321 {doc}
1323 Calls the base class translation method and if that fails with
1324 `NotImplementedError` issues a warning reminding the implementer to
1325 override this method.
1327 Returns
1328 -------
1329 {property} : `None` or `{return_typedoc}`
1330 Always returns `None`.
1331 """
1332 return to_stub
1335# Create stub translation methods for each property. These stubs warn
1336# rather than fail and should be overridden by translators.
1337for name, description in PROPERTIES.items():
1338 setattr(
1339 StubTranslator,
1340 f"to_{name}",
1341 _make_forwarded_stub_translator_method(
1342 StubTranslator, name, definition.doc, definition.str_type, definition.py_type # type: ignore
1343 ),
1344 )