Coverage for python/astro_metadata_translator/observationInfo.py: 15%
268 statements
« prev ^ index » next coverage.py v6.4, created at 2022-06-02 03:27 -0700
« prev ^ index » next coverage.py v6.4, created at 2022-06-02 03:27 -0700
1# This file is part of astro_metadata_translator.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the LICENSE file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12"""Represent standard metadata from instrument headers"""
14from __future__ import annotations
16__all__ = ("ObservationInfo", "makeObservationInfo")
18import copy
19import itertools
20import json
21import logging
22import math
23from typing import (
24 TYPE_CHECKING,
25 Any,
26 Callable,
27 Dict,
28 FrozenSet,
29 MutableMapping,
30 Optional,
31 Sequence,
32 Set,
33 Tuple,
34 Type,
35)
37import astropy.time
38from astropy.coordinates import AltAz, SkyCoord
40from .headers import fix_header
41from .properties import PROPERTIES, PropertyDefinition
42from .translator import MetadataTranslator
44if TYPE_CHECKING: 44 ↛ 45line 44 didn't jump to line 45, because the condition on line 44 was never true
45 import astropy.coordinates
46 import astropy.units
48log = logging.getLogger(__name__)
51class ObservationInfo:
52 """Standardized representation of an instrument header for a single
53 exposure observation.
55 There is a core set of instrumental properties that are pre-defined.
56 Additional properties may be defined, either through the
57 ``makeObservationInfo`` factory function by providing the ``extensions``
58 definitions, or through the regular ``ObservationInfo`` constructor when
59 the extensions have been defined in the ``MetadataTranslator`` for the
60 instrument of interest (or in the provided ``translator_class``).
62 Parameters
63 ----------
64 header : `dict`-like
65 Representation of an instrument header accessible as a `dict`.
66 May be updated with header corrections if corrections are found.
67 filename : `str`, optional
68 Name of the file whose header is being translated. For some
69 datasets with missing header information this can sometimes
70 allow for some fixups in translations.
71 translator_class : `MetadataTranslator`-class, optional
72 If not `None`, the class to use to translate the supplied headers
73 into standard form. Otherwise each registered translator class will
74 be asked in turn if it knows how to translate the supplied header.
75 pedantic : `bool`, optional
76 If True the translation must succeed for all properties. If False
77 individual property translations must all be implemented but can fail
78 and a warning will be issued.
79 search_path : iterable, optional
80 Override search paths to use during header fix up.
81 required : `set`, optional
82 This parameter can be used to confirm that all properties contained
83 in the set must translate correctly and also be non-None. For the case
84 where ``pedantic`` is `True` this will still check that the resulting
85 value is not `None`.
86 subset : `set`, optional
87 If not `None`, controls the translations that will be performed
88 during construction. This can be useful if the caller is only
89 interested in a subset of the properties and knows that some of
90 the others might be slow to compute (for example the airmass if it
91 has to be derived).
93 Raises
94 ------
95 ValueError
96 Raised if the supplied header was not recognized by any of the
97 registered translators. Also raised if the request property subset
98 is not a subset of the known properties.
99 TypeError
100 Raised if the supplied translator class was not a MetadataTranslator.
101 KeyError
102 Raised if a required property cannot be calculated, or if pedantic
103 mode is enabled and any translations fails.
104 NotImplementedError
105 Raised if the selected translator does not support a required
106 property.
108 Notes
109 -----
110 Headers will be corrected if correction files are located and this will
111 modify the header provided to the constructor.
113 Values of the properties are read-only.
114 """
116 # Static typing requires that we define the standard dynamic properties
117 # statically.
118 if TYPE_CHECKING: 118 ↛ 119line 118 didn't jump to line 119, because the condition on line 118 was never true
119 telescope: int
120 instrument: str
121 location: astropy.coordinates.EarthLocation
122 exposure_id: int
123 visit_id: int
124 physical_filter: str
125 datetime_begin: astropy.time.Time
126 datetime_end: astropy.time.Time
127 exposure_group: str
128 exposure_time: astropy.units.Quantity
129 dark_time: astropy.units.Quantity
130 boresight_airmass: float
131 boresight_rotation_angle: astropy.units.Quantity
132 boresight_rotation_coord: str
133 detector_num: int
134 detector_name: str
135 detector_serial: str
136 detector_group: str
137 detector_exposure_id: int
138 object: str
139 temperature: astropy.units.Quantity
140 pressure: astropy.units.Quantity
141 relative_humidity: float
142 tracking_radec: astropy.coordinates.SkyCoord
143 altaz_begin: astropy.coordinates.AltAz
144 science_program: str
145 observation_counter: int
146 observation_reason: str
147 observation_type: str
148 observation_id: str
149 observing_day: int
150 group_counter_start: int
151 group_counter_end: int
152 has_simulated_content: bool
154 def __init__(
155 self,
156 header: Optional[MutableMapping[str, Any]],
157 filename: Optional[str] = None,
158 translator_class: Optional[Type[MetadataTranslator]] = None,
159 pedantic: bool = False,
160 search_path: Optional[Sequence[str]] = None,
161 required: Optional[Set[str]] = None,
162 subset: Optional[Set[str]] = None,
163 ) -> None:
165 # Initialize the empty object
166 self._header: MutableMapping[str, Any] = {}
167 self.filename = filename
168 self._translator = None
169 self.translator_class_name = "<None>"
171 # To allow makeObservationInfo to work, we special case a None
172 # header
173 if header is None:
174 return
176 # Fix up the header (if required)
177 fix_header(header, translator_class=translator_class, filename=filename, search_path=search_path)
179 # Store the supplied header for later stripping
180 self._header = header
182 if translator_class is None:
183 translator_class = MetadataTranslator.determine_translator(header, filename=filename)
184 elif not issubclass(translator_class, MetadataTranslator):
185 raise TypeError(f"Translator class must be a MetadataTranslator, not {translator_class}")
187 self._declare_extensions(translator_class.extensions)
189 # Create an instance for this header
190 translator = translator_class(header, filename=filename)
192 # Store the translator
193 self._translator = translator
194 self.translator_class_name = translator_class.__name__
196 # Form file information string in case we need an error message
197 if filename:
198 file_info = f" and file {filename}"
199 else:
200 file_info = ""
202 # Determine the properties of interest
203 full_set = set(self.all_properties)
204 if subset is not None:
205 if not subset:
206 raise ValueError("Cannot request no properties be calculated.")
207 if not subset.issubset(full_set):
208 raise ValueError(
209 "Requested subset is not a subset of known properties. " f"Got extra: {subset - full_set}"
210 )
211 properties = subset
212 else:
213 properties = full_set
215 if required is None:
216 required = set()
217 else:
218 if not required.issubset(full_set):
219 raise ValueError("Requested required properties include unknowns: " f"{required - full_set}")
221 # Loop over each property and request the translated form
222 for t in properties:
223 # prototype code
224 method = f"to_{t}"
225 property = f"_{t}" if not t.startswith("ext_") else t
227 try:
228 value = getattr(translator, method)()
229 except NotImplementedError as e:
230 raise NotImplementedError(
231 f"No translation exists for property '{t}'" f" using translator {translator.__class__}"
232 ) from e
233 except KeyError as e:
234 err_msg = (
235 f"Error calculating property '{t}' using translator {translator.__class__}" f"{file_info}"
236 )
237 if pedantic or t in required:
238 raise KeyError(err_msg) from e
239 else:
240 log.debug("Calculation of property '%s' failed with header: %s", t, header)
241 log.warning(f"Ignoring {err_msg}: {e}")
242 continue
244 definition = self.all_properties[t]
245 if not self._is_property_ok(definition, value):
246 err_msg = (
247 f"Value calculated for property '{t}' is wrong type "
248 f"({type(value)} != {definition.str_type}) using translator {translator.__class__}"
249 f"{file_info}"
250 )
251 if pedantic or t in required:
252 raise TypeError(err_msg)
253 else:
254 log.debug("Calcuation of property '%s' had unexpected type with header: %s", t, header)
255 log.warning(f"Ignoring {err_msg}")
257 if value is None and t in required:
258 raise KeyError(f"Calculation of required property {t} resulted in a value of None")
260 super().__setattr__(property, value) # allows setting even write-protected extensions
262 @staticmethod
263 def _get_all_properties(
264 extensions: Optional[Dict[str, PropertyDefinition]] = None
265 ) -> Dict[str, PropertyDefinition]:
266 """Return the definitions of all properties
268 Parameters
269 ----------
270 extensions : `dict` [`str`: `PropertyDefinition`]
271 List of extension property definitions, indexed by name (with no
272 "ext_" prefix).
274 Returns
275 -------
276 properties : `dict` [`str`: `PropertyDefinition`]
277 Merged list of all property definitions, indexed by name. Extension
278 properties will be listed with an ``ext_`` prefix.
279 """
280 properties = dict(PROPERTIES)
281 if extensions:
282 properties.update({"ext_" + pp: dd for pp, dd in extensions.items()})
283 return properties
285 def _declare_extensions(self, extensions: Optional[Dict[str, PropertyDefinition]]) -> None:
286 """Declare and set up extension properties
288 This should always be called internally as part of the creation of a
289 new `ObservationInfo`.
291 The core set of properties each have a python ``property`` that makes
292 them read-only, and serves as a useful place to hang the docstring.
293 However, the core set are set up at compile time, whereas the extension
294 properties have to be configured at run time (because we don't know
295 what they will be until we look at the header and figure out what
296 instrument we're dealing with) when we have an instance rather than a
297 class (and python ``property`` doesn't work on instances; only on
298 classes). We therefore use a separate scheme for the extension
299 properties: we write them directly to their associated instance
300 variable, and we use ``__setattr__`` to protect them as read-only.
301 Unfortunately, with this scheme, we can't give extension properties a
302 docstring; but we're setting them up at runtime, so maybe that's not
303 terribly important.
305 Parameters
306 ----------
307 extensions : `dict` [`str`: `PropertyDefinition`]
308 List of extension property definitions, indexed by name (with no
309 "ext_" prefix).
310 """
311 if not extensions:
312 extensions = {}
313 for name in extensions:
314 super().__setattr__("ext_" + name, None)
315 self.extensions = extensions
316 self.all_properties = self._get_all_properties(extensions)
318 def __setattr__(self, name: str, value: Any) -> Any:
319 """Set attribute
321 This provides read-only protection for the extension properties. The
322 core set of properties have read-only protection via the use of the
323 python ``property``.
324 """
325 if hasattr(self, "extensions") and name.startswith("ext_") and name[4:] in self.extensions:
326 raise AttributeError(f"Attribute {name} is read-only")
327 return super().__setattr__(name, value)
329 @classmethod
330 def _is_property_ok(cls, definition: PropertyDefinition, value: Any) -> bool:
331 """Compare the supplied value against the expected type as defined
332 for the corresponding property.
334 Parameters
335 ----------
336 definition : `PropertyDefinition`
337 Property definition.
338 value : `object`
339 Value of the property to validate.
341 Returns
342 -------
343 is_ok : `bool`
344 `True` if the value is of an appropriate type.
346 Notes
347 -----
348 Currently only the type of the property is validated. There is no
349 attempt to check bounds or determine that a Quantity is compatible
350 with the property.
351 """
352 if value is None:
353 return True
355 # For AltAz coordinates, they can either arrive as AltAz or
356 # as SkyCoord(frame=AltAz) so try to find the frame inside
357 # the SkyCoord.
358 if issubclass(definition.py_type, AltAz) and isinstance(value, SkyCoord):
359 value = value.frame
361 if not isinstance(value, definition.py_type):
362 return False
364 return True
366 @property
367 def cards_used(self) -> FrozenSet[str]:
368 """Header cards used for the translation.
370 Returns
371 -------
372 used : `frozenset` of `str`
373 Set of card used.
374 """
375 if not self._translator:
376 return frozenset()
377 return self._translator.cards_used()
379 def stripped_header(self) -> MutableMapping[str, Any]:
380 """Return a copy of the supplied header with used keywords removed.
382 Returns
383 -------
384 stripped : `dict`-like
385 Same class as header supplied to constructor, but with the
386 headers used to calculate the generic information removed.
387 """
388 hdr = copy.copy(self._header)
389 used = self.cards_used
390 for c in used:
391 del hdr[c]
392 return hdr
394 def __str__(self) -> str:
395 # Put more interesting answers at front of list
396 # and then do remainder
397 priority = ("instrument", "telescope", "datetime_begin")
398 properties = sorted(set(self.all_properties) - set(priority))
400 result = ""
401 for p in itertools.chain(priority, properties):
402 value = getattr(self, p)
403 if isinstance(value, astropy.time.Time):
404 value.format = "isot"
405 value = str(value.value)
406 result += f"{p}: {value}\n"
408 return result
410 def __eq__(self, other: Any) -> bool:
411 """Compares equal if standard properties are equal"""
412 if not isinstance(other, ObservationInfo):
413 return NotImplemented
415 # Compare simplified forms.
416 # Cannot compare directly because nan will not equate as equal
417 # whereas they should be equal for our purposes
418 self_simple = self.to_simple()
419 other_simple = other.to_simple()
421 # We don't care about the translator internal detail
422 self_simple.pop("_translator", None)
423 other_simple.pop("_translator", None)
425 for k, self_value in self_simple.items():
426 other_value = other_simple[k]
427 if self_value != other_value:
428 if math.isnan(self_value) and math.isnan(other_value):
429 # If both are nan this is fine
430 continue
431 return False
432 return True
434 def __lt__(self, other: Any) -> bool:
435 if not isinstance(other, ObservationInfo):
436 return NotImplemented
437 return self.datetime_begin < other.datetime_begin
439 def __gt__(self, other: Any) -> bool:
440 if not isinstance(other, ObservationInfo):
441 return NotImplemented
442 return self.datetime_begin > other.datetime_begin
444 def __getstate__(self) -> Tuple[Any, ...]:
445 """Get pickleable state
447 Returns the properties. Deliberately does not preserve the full
448 current state; in particular, does not return the full header or
449 translator.
451 Returns
452 -------
453 state : `tuple`
454 Pickled state.
455 """
456 state = dict()
457 for p in self.all_properties:
458 state[p] = getattr(self, p)
460 return state, self.extensions
462 def __setstate__(self, state: Tuple[Any, ...]) -> None:
463 """Set object state from pickle
465 Parameters
466 ----------
467 state : `tuple`
468 Pickled state.
469 """
470 try:
471 state, extensions = state
472 except ValueError:
473 # Backwards compatibility for pickles generated before DM-34175
474 extensions = {}
475 self._declare_extensions(extensions)
476 for p in self.all_properties:
477 if p.startswith("ext_"):
478 # allows setting even write-protected extensions
479 super().__setattr__(p, state[p]) # type: ignore
480 else:
481 property = f"_{p}"
482 setattr(self, property, state[p]) # type: ignore
484 def to_simple(self) -> MutableMapping[str, Any]:
485 """Convert the contents of this object to simple dict form.
487 The keys of the dict are the standard properties but the values
488 can be simplified to support JSON serialization. For example a
489 SkyCoord might be represented as an ICRS RA/Dec tuple rather than
490 a full SkyCoord representation.
492 Any properties with `None` value will be skipped.
494 Can be converted back to an `ObservationInfo` using `from_simple()`.
496 Returns
497 -------
498 simple : `dict` of [`str`, `Any`]
499 Simple dict of all properties.
501 Notes
502 -----
503 Round-tripping of extension properties requires that the
504 `ObservationInfo` was created with the help of a registered
505 `MetadataTranslator` (which contains the extension property
506 definitions).
507 """
508 simple = {}
509 if hasattr(self, "_translator") and self._translator and self._translator.name:
510 simple["_translator"] = self._translator.name
512 for p in self.all_properties:
513 property = f"_{p}" if not p.startswith("ext_") else p
514 value = getattr(self, property)
515 if value is None:
516 continue
518 # Access the function to simplify the property
519 simplifier = self.all_properties[p].to_simple
521 if simplifier is None:
522 simple[p] = value
523 continue
525 simple[p] = simplifier(value)
527 return simple
529 def to_json(self) -> str:
530 """Serialize the object to JSON string.
532 Returns
533 -------
534 j : `str`
535 The properties of the ObservationInfo in JSON string form.
537 Notes
538 -----
539 Round-tripping of extension properties requires that the
540 `ObservationInfo` was created with the help of a registered
541 `MetadataTranslator` (which contains the extension property
542 definitions).
543 """
544 return json.dumps(self.to_simple())
546 @classmethod
547 def from_simple(cls, simple: MutableMapping[str, Any]) -> ObservationInfo:
548 """Convert the entity returned by `to_simple` back into an
549 `ObservationInfo`.
551 Parameters
552 ----------
553 simple : `dict` [`str`, `Any`]
554 The dict returned by `to_simple()`
556 Returns
557 -------
558 obsinfo : `ObservationInfo`
559 New object constructed from the dict.
561 Notes
562 -----
563 Round-tripping of extension properties requires that the
564 `ObservationInfo` was created with the help of a registered
565 `MetadataTranslator` (which contains the extension property
566 definitions).
567 """
568 extensions = {}
569 translator = simple.pop("_translator", None)
570 if translator:
571 if translator not in MetadataTranslator.translators:
572 raise KeyError(f"Unrecognised translator: {translator}")
573 extensions = MetadataTranslator.translators[translator].extensions
575 properties = cls._get_all_properties(extensions)
577 processed: Dict[str, Any] = {}
578 for k, v in simple.items():
580 if v is None:
581 continue
583 # Access the function to convert from simple form
584 complexifier = properties[k].from_simple
586 if complexifier is not None:
587 v = complexifier(v, **processed)
589 processed[k] = v
591 return cls.makeObservationInfo(extensions=extensions, **processed)
593 @classmethod
594 def from_json(cls, json_str: str) -> ObservationInfo:
595 """Create `ObservationInfo` from JSON string.
597 Parameters
598 ----------
599 json_str : `str`
600 The JSON representation.
602 Returns
603 -------
604 obsinfo : `ObservationInfo`
605 Reconstructed object.
607 Notes
608 -----
609 Round-tripping of extension properties requires that the
610 `ObservationInfo` was created with the help of a registered
611 `MetadataTranslator` (which contains the extension property
612 definitions).
613 """
614 simple = json.loads(json_str)
615 return cls.from_simple(simple)
617 @classmethod
618 def makeObservationInfo( # noqa: N802
619 cls, *, extensions: Optional[Dict[str, PropertyDefinition]] = None, **kwargs: Any
620 ) -> ObservationInfo:
621 """Construct an `ObservationInfo` from the supplied parameters.
623 Parameters
624 ----------
625 extensions : `dict` [`str`: `PropertyDefinition`], optional
626 Optional extension definitions, indexed by extension name (without
627 the ``ext_`` prefix, which will be added by `ObservationInfo`).
628 **kwargs
629 Name-value pairs for any properties to be set. In the case of
630 extension properties, the names should include the ``ext_`` prefix.
632 Notes
633 -----
634 The supplied parameters should use names matching the property.
635 The type of the supplied value will be checked against the property.
636 Any properties not supplied will be assigned a value of `None`.
638 Raises
639 ------
640 KeyError
641 Raised if a supplied parameter key is not a known property.
642 TypeError
643 Raised if a supplied value does not match the expected type
644 of the property.
645 """
647 obsinfo = cls(None)
648 obsinfo._declare_extensions(extensions)
650 unused = set(kwargs)
652 for p in obsinfo.all_properties:
653 if p in kwargs:
654 property = f"_{p}" if not p.startswith("ext_") else p
655 value = kwargs[p]
656 definition = obsinfo.all_properties[p]
657 if not cls._is_property_ok(definition, value):
658 raise TypeError(
659 f"Supplied value {value} for property {p} "
660 f"should be of class {definition.str_type} not {value.__class__}"
661 )
662 super(cls, obsinfo).__setattr__(property, value) # allows setting write-protected extensions
663 unused.remove(p)
665 # Recent additions to ObservationInfo may not be present in
666 # serializations. In theory they can be derived from other
667 # values in the default case. This might not be the right thing
668 # to do.
669 for k in ("group_counter_start", "group_counter_end"):
670 if k not in kwargs and "observation_counter" in kwargs:
671 super(cls, obsinfo).__setattr__(f"_{k}", obsinfo.observation_counter)
672 if (k := "has_simulated_content") not in kwargs:
673 super(cls, obsinfo).__setattr__(f"_{k}", False)
675 if unused:
676 n = len(unused)
677 raise KeyError(f"Unrecognized propert{'y' if n == 1 else 'ies'} provided: {', '.join(unused)}")
679 return obsinfo
682# Method to add the standard properties
683def _make_property(property: str, doc: str, return_typedoc: str, return_type: Type) -> Callable:
684 """Create a getter method with associated docstring.
686 Parameters
687 ----------
688 property : `str`
689 Name of the property getter to be created.
690 doc : `str`
691 Description of this property.
692 return_typedoc : `str`
693 Type string of this property (used in the doc string).
694 return_type : `class`
695 Type of this property.
697 Returns
698 -------
699 p : `function`
700 Getter method for this property.
701 """
703 def getter(self: ObservationInfo) -> Any:
704 return getattr(self, f"_{property}")
706 getter.__doc__ = f"""{doc}
708 Returns
709 -------
710 {property} : `{return_typedoc}`
711 Access the property.
712 """
713 return getter
716# Set up the core set of properties
717# In order to provide read-only protection, each attribute is hidden behind a
718# python "property" wrapper.
719for name, definition in PROPERTIES.items():
720 setattr(ObservationInfo, f"_{name}", None)
721 setattr(
722 ObservationInfo,
723 name,
724 property(_make_property(name, definition.doc, definition.str_type, definition.py_type)),
725 )
728def makeObservationInfo( # noqa: N802
729 *, extensions: Optional[Dict[str, PropertyDefinition]] = None, **kwargs: Any
730) -> ObservationInfo:
731 """Construct an `ObservationInfo` from the supplied parameters.
733 Parameters
734 ----------
735 extensions : `dict` [`str`: `PropertyDefinition`], optional
736 Optional extension definitions, indexed by extension name (without
737 the ``ext_`` prefix, which will be added by `ObservationInfo`).
738 **kwargs
739 Name-value pairs for any properties to be set. In the case of
740 extension properties, the names should include the ``ext_`` prefix.
742 Notes
743 -----
744 The supplied parameters should use names matching the property.
745 The type of the supplied value will be checked against the property.
746 Any properties not supplied will be assigned a value of `None`.
748 Raises
749 ------
750 KeyError
751 Raised if a supplied parameter key is not a known property.
752 TypeError
753 Raised if a supplied value does not match the expected type
754 of the property.
755 """
756 return ObservationInfo.makeObservationInfo(extensions=extensions, **kwargs)