Coverage for python/astro_metadata_translator/observationInfo.py: 14%
269 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-04-07 01:03 -0700
« prev ^ index » next coverage.py v6.5.0, created at 2023-04-07 01:03 -0700
1# This file is part of astro_metadata_translator.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the LICENSE file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12"""Represent standard metadata from instrument headers"""
14from __future__ import annotations
16__all__ = ("ObservationInfo", "makeObservationInfo")
18import copy
19import itertools
20import json
21import logging
22import math
23from typing import (
24 TYPE_CHECKING,
25 Any,
26 Callable,
27 Dict,
28 FrozenSet,
29 MutableMapping,
30 Optional,
31 Sequence,
32 Set,
33 Tuple,
34 Type,
35)
37import astropy.time
38from astropy.coordinates import AltAz, SkyCoord
40from .headers import fix_header
41from .properties import PROPERTIES, PropertyDefinition
42from .translator import MetadataTranslator
44if TYPE_CHECKING: 44 ↛ 45line 44 didn't jump to line 45, because the condition on line 44 was never true
45 import astropy.coordinates
46 import astropy.units
48log = logging.getLogger(__name__)
51class ObservationInfo:
52 """Standardized representation of an instrument header for a single
53 exposure observation.
55 There is a core set of instrumental properties that are pre-defined.
56 Additional properties may be defined, either through the
57 ``makeObservationInfo`` factory function by providing the ``extensions``
58 definitions, or through the regular ``ObservationInfo`` constructor when
59 the extensions have been defined in the ``MetadataTranslator`` for the
60 instrument of interest (or in the provided ``translator_class``).
62 Parameters
63 ----------
64 header : `dict`-like
65 Representation of an instrument header accessible as a `dict`.
66 May be updated with header corrections if corrections are found.
67 filename : `str`, optional
68 Name of the file whose header is being translated. For some
69 datasets with missing header information this can sometimes
70 allow for some fixups in translations.
71 translator_class : `MetadataTranslator`-class, optional
72 If not `None`, the class to use to translate the supplied headers
73 into standard form. Otherwise each registered translator class will
74 be asked in turn if it knows how to translate the supplied header.
75 pedantic : `bool`, optional
76 If True the translation must succeed for all properties. If False
77 individual property translations must all be implemented but can fail
78 and a warning will be issued.
79 search_path : iterable, optional
80 Override search paths to use during header fix up.
81 required : `set`, optional
82 This parameter can be used to confirm that all properties contained
83 in the set must translate correctly and also be non-None. For the case
84 where ``pedantic`` is `True` this will still check that the resulting
85 value is not `None`.
86 subset : `set`, optional
87 If not `None`, controls the translations that will be performed
88 during construction. This can be useful if the caller is only
89 interested in a subset of the properties and knows that some of
90 the others might be slow to compute (for example the airmass if it
91 has to be derived).
93 Raises
94 ------
95 ValueError
96 Raised if the supplied header was not recognized by any of the
97 registered translators. Also raised if the request property subset
98 is not a subset of the known properties.
99 TypeError
100 Raised if the supplied translator class was not a MetadataTranslator.
101 KeyError
102 Raised if a required property cannot be calculated, or if pedantic
103 mode is enabled and any translations fails.
104 NotImplementedError
105 Raised if the selected translator does not support a required
106 property.
108 Notes
109 -----
110 Headers will be corrected if correction files are located and this will
111 modify the header provided to the constructor.
113 Values of the properties are read-only.
114 """
116 # Static typing requires that we define the standard dynamic properties
117 # statically.
118 if TYPE_CHECKING: 118 ↛ 119line 118 didn't jump to line 119, because the condition on line 118 was never true
119 telescope: int
120 instrument: str
121 location: astropy.coordinates.EarthLocation
122 exposure_id: int
123 visit_id: int
124 physical_filter: str
125 datetime_begin: astropy.time.Time
126 datetime_end: astropy.time.Time
127 exposure_group: str
128 exposure_time: astropy.units.Quantity
129 dark_time: astropy.units.Quantity
130 boresight_airmass: float
131 boresight_rotation_angle: astropy.units.Quantity
132 boresight_rotation_coord: str
133 detector_num: int
134 detector_name: str
135 detector_serial: str
136 detector_group: str
137 detector_exposure_id: int
138 focus_z: astropy.units.Quantity
139 object: str
140 temperature: astropy.units.Quantity
141 pressure: astropy.units.Quantity
142 relative_humidity: float
143 tracking_radec: astropy.coordinates.SkyCoord
144 altaz_begin: astropy.coordinates.AltAz
145 science_program: str
146 observation_counter: int
147 observation_reason: str
148 observation_type: str
149 observation_id: str
150 observing_day: int
151 group_counter_start: int
152 group_counter_end: int
153 has_simulated_content: bool
155 def __init__(
156 self,
157 header: Optional[MutableMapping[str, Any]],
158 filename: Optional[str] = None,
159 translator_class: Optional[Type[MetadataTranslator]] = None,
160 pedantic: bool = False,
161 search_path: Optional[Sequence[str]] = None,
162 required: Optional[Set[str]] = None,
163 subset: Optional[Set[str]] = None,
164 ) -> None:
165 # Initialize the empty object
166 self._header: MutableMapping[str, Any] = {}
167 self.filename = filename
168 self._translator = None
169 self.translator_class_name = "<None>"
171 # To allow makeObservationInfo to work, we special case a None
172 # header
173 if header is None:
174 return
176 # Fix up the header (if required)
177 fix_header(header, translator_class=translator_class, filename=filename, search_path=search_path)
179 # Store the supplied header for later stripping
180 self._header = header
182 if translator_class is None:
183 translator_class = MetadataTranslator.determine_translator(header, filename=filename)
184 elif not issubclass(translator_class, MetadataTranslator):
185 raise TypeError(f"Translator class must be a MetadataTranslator, not {translator_class}")
187 self._declare_extensions(translator_class.extensions)
189 # Create an instance for this header
190 translator = translator_class(header, filename=filename)
192 # Store the translator
193 self._translator = translator
194 self.translator_class_name = translator_class.__name__
196 # Form file information string in case we need an error message
197 if filename:
198 file_info = f" and file {filename}"
199 else:
200 file_info = ""
202 # Determine the properties of interest
203 full_set = set(self.all_properties)
204 if subset is not None:
205 if not subset:
206 raise ValueError("Cannot request no properties be calculated.")
207 if not subset.issubset(full_set):
208 raise ValueError(
209 "Requested subset is not a subset of known properties. " f"Got extra: {subset - full_set}"
210 )
211 properties = subset
212 else:
213 properties = full_set
215 if required is None:
216 required = set()
217 else:
218 if not required.issubset(full_set):
219 raise ValueError("Requested required properties include unknowns: " f"{required - full_set}")
221 # Loop over each property and request the translated form
222 for t in properties:
223 # prototype code
224 method = f"to_{t}"
225 property = f"_{t}" if not t.startswith("ext_") else t
227 try:
228 value = getattr(translator, method)()
229 except NotImplementedError as e:
230 raise NotImplementedError(
231 f"No translation exists for property '{t}'" f" using translator {translator.__class__}"
232 ) from e
233 except Exception as e:
234 err_msg = (
235 f"Error calculating property '{t}' using translator {translator.__class__}" f"{file_info}"
236 )
237 if pedantic or t in required:
238 raise KeyError(err_msg) from e
239 else:
240 log.debug("Calculation of property '%s' failed with header: %s", t, header)
241 log.warning(f"Ignoring {err_msg}: {e}")
242 continue
244 definition = self.all_properties[t]
245 if not self._is_property_ok(definition, value):
246 err_msg = (
247 f"Value calculated for property '{t}' is wrong type "
248 f"({type(value)} != {definition.str_type}) using translator {translator.__class__}"
249 f"{file_info}"
250 )
251 if pedantic or t in required:
252 raise TypeError(err_msg)
253 else:
254 log.debug("Calcuation of property '%s' had unexpected type with header: %s", t, header)
255 log.warning(f"Ignoring {err_msg}")
257 if value is None and t in required:
258 raise KeyError(f"Calculation of required property {t} resulted in a value of None")
260 super().__setattr__(property, value) # allows setting even write-protected extensions
262 @staticmethod
263 def _get_all_properties(
264 extensions: Optional[Dict[str, PropertyDefinition]] = None
265 ) -> Dict[str, PropertyDefinition]:
266 """Return the definitions of all properties
268 Parameters
269 ----------
270 extensions : `dict` [`str`: `PropertyDefinition`]
271 List of extension property definitions, indexed by name (with no
272 "ext_" prefix).
274 Returns
275 -------
276 properties : `dict` [`str`: `PropertyDefinition`]
277 Merged list of all property definitions, indexed by name. Extension
278 properties will be listed with an ``ext_`` prefix.
279 """
280 properties = dict(PROPERTIES)
281 if extensions:
282 properties.update({"ext_" + pp: dd for pp, dd in extensions.items()})
283 return properties
285 def _declare_extensions(self, extensions: Optional[Dict[str, PropertyDefinition]]) -> None:
286 """Declare and set up extension properties
288 This should always be called internally as part of the creation of a
289 new `ObservationInfo`.
291 The core set of properties each have a python ``property`` that makes
292 them read-only, and serves as a useful place to hang the docstring.
293 However, the core set are set up at compile time, whereas the extension
294 properties have to be configured at run time (because we don't know
295 what they will be until we look at the header and figure out what
296 instrument we're dealing with) when we have an instance rather than a
297 class (and python ``property`` doesn't work on instances; only on
298 classes). We therefore use a separate scheme for the extension
299 properties: we write them directly to their associated instance
300 variable, and we use ``__setattr__`` to protect them as read-only.
301 Unfortunately, with this scheme, we can't give extension properties a
302 docstring; but we're setting them up at runtime, so maybe that's not
303 terribly important.
305 Parameters
306 ----------
307 extensions : `dict` [`str`: `PropertyDefinition`]
308 List of extension property definitions, indexed by name (with no
309 "ext_" prefix).
310 """
311 if not extensions:
312 extensions = {}
313 for name in extensions:
314 super().__setattr__("ext_" + name, None)
315 self.extensions = extensions
316 self.all_properties = self._get_all_properties(extensions)
318 def __setattr__(self, name: str, value: Any) -> Any:
319 """Set attribute
321 This provides read-only protection for the extension properties. The
322 core set of properties have read-only protection via the use of the
323 python ``property``.
324 """
325 if hasattr(self, "extensions") and name.startswith("ext_") and name[4:] in self.extensions:
326 raise AttributeError(f"Attribute {name} is read-only")
327 return super().__setattr__(name, value)
329 @classmethod
330 def _is_property_ok(cls, definition: PropertyDefinition, value: Any) -> bool:
331 """Compare the supplied value against the expected type as defined
332 for the corresponding property.
334 Parameters
335 ----------
336 definition : `PropertyDefinition`
337 Property definition.
338 value : `object`
339 Value of the property to validate.
341 Returns
342 -------
343 is_ok : `bool`
344 `True` if the value is of an appropriate type.
346 Notes
347 -----
348 Currently only the type of the property is validated. There is no
349 attempt to check bounds or determine that a Quantity is compatible
350 with the property.
351 """
352 if value is None:
353 return True
355 # For AltAz coordinates, they can either arrive as AltAz or
356 # as SkyCoord(frame=AltAz) so try to find the frame inside
357 # the SkyCoord.
358 if issubclass(definition.py_type, AltAz) and isinstance(value, SkyCoord):
359 value = value.frame
361 if not isinstance(value, definition.py_type):
362 return False
364 return True
366 @property
367 def cards_used(self) -> FrozenSet[str]:
368 """Header cards used for the translation.
370 Returns
371 -------
372 used : `frozenset` of `str`
373 Set of card used.
374 """
375 if not self._translator:
376 return frozenset()
377 return self._translator.cards_used()
379 def stripped_header(self) -> MutableMapping[str, Any]:
380 """Return a copy of the supplied header with used keywords removed.
382 Returns
383 -------
384 stripped : `dict`-like
385 Same class as header supplied to constructor, but with the
386 headers used to calculate the generic information removed.
387 """
388 hdr = copy.copy(self._header)
389 used = self.cards_used
390 for c in used:
391 del hdr[c]
392 return hdr
394 def __str__(self) -> str:
395 # Put more interesting answers at front of list
396 # and then do remainder
397 priority = ("instrument", "telescope", "datetime_begin")
398 properties = sorted(set(self.all_properties) - set(priority))
400 result = ""
401 for p in itertools.chain(priority, properties):
402 value = getattr(self, p)
403 if isinstance(value, astropy.time.Time):
404 value.format = "isot"
405 value = str(value.value)
406 result += f"{p}: {value}\n"
408 return result
410 def __eq__(self, other: Any) -> bool:
411 """Compares equal if standard properties are equal"""
412 if not isinstance(other, ObservationInfo):
413 return NotImplemented
415 # Compare simplified forms.
416 # Cannot compare directly because nan will not equate as equal
417 # whereas they should be equal for our purposes
418 self_simple = self.to_simple()
419 other_simple = other.to_simple()
421 # We don't care about the translator internal detail
422 self_simple.pop("_translator", None)
423 other_simple.pop("_translator", None)
425 for k, self_value in self_simple.items():
426 other_value = other_simple[k]
427 if self_value != other_value:
428 if math.isnan(self_value) and math.isnan(other_value):
429 # If both are nan this is fine
430 continue
431 return False
432 return True
434 def __lt__(self, other: Any) -> bool:
435 if not isinstance(other, ObservationInfo):
436 return NotImplemented
437 return self.datetime_begin < other.datetime_begin
439 def __gt__(self, other: Any) -> bool:
440 if not isinstance(other, ObservationInfo):
441 return NotImplemented
442 return self.datetime_begin > other.datetime_begin
444 def __getstate__(self) -> Tuple[Any, ...]:
445 """Get pickleable state
447 Returns the properties. Deliberately does not preserve the full
448 current state; in particular, does not return the full header or
449 translator.
451 Returns
452 -------
453 state : `tuple`
454 Pickled state.
455 """
456 state = dict()
457 for p in self.all_properties:
458 state[p] = getattr(self, p)
460 return state, self.extensions
462 def __setstate__(self, state: Tuple[Any, ...]) -> None:
463 """Set object state from pickle
465 Parameters
466 ----------
467 state : `tuple`
468 Pickled state.
469 """
470 try:
471 state, extensions = state
472 except ValueError:
473 # Backwards compatibility for pickles generated before DM-34175
474 extensions = {}
475 self._declare_extensions(extensions)
476 for p in self.all_properties:
477 if p.startswith("ext_"):
478 # allows setting even write-protected extensions
479 super().__setattr__(p, state[p]) # type: ignore
480 else:
481 property = f"_{p}"
482 setattr(self, property, state[p]) # type: ignore
484 def to_simple(self) -> MutableMapping[str, Any]:
485 """Convert the contents of this object to simple dict form.
487 The keys of the dict are the standard properties but the values
488 can be simplified to support JSON serialization. For example a
489 SkyCoord might be represented as an ICRS RA/Dec tuple rather than
490 a full SkyCoord representation.
492 Any properties with `None` value will be skipped.
494 Can be converted back to an `ObservationInfo` using `from_simple()`.
496 Returns
497 -------
498 simple : `dict` of [`str`, `Any`]
499 Simple dict of all properties.
501 Notes
502 -----
503 Round-tripping of extension properties requires that the
504 `ObservationInfo` was created with the help of a registered
505 `MetadataTranslator` (which contains the extension property
506 definitions).
507 """
508 simple = {}
509 if hasattr(self, "_translator") and self._translator and self._translator.name:
510 simple["_translator"] = self._translator.name
512 for p in self.all_properties:
513 property = f"_{p}" if not p.startswith("ext_") else p
514 value = getattr(self, property)
515 if value is None:
516 continue
518 # Access the function to simplify the property
519 simplifier = self.all_properties[p].to_simple
521 if simplifier is None:
522 simple[p] = value
523 continue
525 simple[p] = simplifier(value)
527 return simple
529 def to_json(self) -> str:
530 """Serialize the object to JSON string.
532 Returns
533 -------
534 j : `str`
535 The properties of the ObservationInfo in JSON string form.
537 Notes
538 -----
539 Round-tripping of extension properties requires that the
540 `ObservationInfo` was created with the help of a registered
541 `MetadataTranslator` (which contains the extension property
542 definitions).
543 """
544 return json.dumps(self.to_simple())
546 @classmethod
547 def from_simple(cls, simple: MutableMapping[str, Any]) -> ObservationInfo:
548 """Convert the entity returned by `to_simple` back into an
549 `ObservationInfo`.
551 Parameters
552 ----------
553 simple : `dict` [`str`, `Any`]
554 The dict returned by `to_simple()`
556 Returns
557 -------
558 obsinfo : `ObservationInfo`
559 New object constructed from the dict.
561 Notes
562 -----
563 Round-tripping of extension properties requires that the
564 `ObservationInfo` was created with the help of a registered
565 `MetadataTranslator` (which contains the extension property
566 definitions).
567 """
568 extensions = {}
569 translator = simple.pop("_translator", None)
570 if translator:
571 if translator not in MetadataTranslator.translators:
572 raise KeyError(f"Unrecognised translator: {translator}")
573 extensions = MetadataTranslator.translators[translator].extensions
575 properties = cls._get_all_properties(extensions)
577 processed: Dict[str, Any] = {}
578 for k, v in simple.items():
579 if v is None:
580 continue
582 # Access the function to convert from simple form
583 complexifier = properties[k].from_simple
585 if complexifier is not None:
586 v = complexifier(v, **processed)
588 processed[k] = v
590 return cls.makeObservationInfo(extensions=extensions, **processed)
592 @classmethod
593 def from_json(cls, json_str: str) -> ObservationInfo:
594 """Create `ObservationInfo` from JSON string.
596 Parameters
597 ----------
598 json_str : `str`
599 The JSON representation.
601 Returns
602 -------
603 obsinfo : `ObservationInfo`
604 Reconstructed object.
606 Notes
607 -----
608 Round-tripping of extension properties requires that the
609 `ObservationInfo` was created with the help of a registered
610 `MetadataTranslator` (which contains the extension property
611 definitions).
612 """
613 simple = json.loads(json_str)
614 return cls.from_simple(simple)
616 @classmethod
617 def makeObservationInfo( # noqa: N802
618 cls, *, extensions: Optional[Dict[str, PropertyDefinition]] = None, **kwargs: Any
619 ) -> ObservationInfo:
620 """Construct an `ObservationInfo` from the supplied parameters.
622 Parameters
623 ----------
624 extensions : `dict` [`str`: `PropertyDefinition`], optional
625 Optional extension definitions, indexed by extension name (without
626 the ``ext_`` prefix, which will be added by `ObservationInfo`).
627 **kwargs
628 Name-value pairs for any properties to be set. In the case of
629 extension properties, the names should include the ``ext_`` prefix.
631 Notes
632 -----
633 The supplied parameters should use names matching the property.
634 The type of the supplied value will be checked against the property.
635 Any properties not supplied will be assigned a value of `None`.
637 Raises
638 ------
639 KeyError
640 Raised if a supplied parameter key is not a known property.
641 TypeError
642 Raised if a supplied value does not match the expected type
643 of the property.
644 """
646 obsinfo = cls(None)
647 obsinfo._declare_extensions(extensions)
649 unused = set(kwargs)
651 for p in obsinfo.all_properties:
652 if p in kwargs:
653 property = f"_{p}" if not p.startswith("ext_") else p
654 value = kwargs[p]
655 definition = obsinfo.all_properties[p]
656 if not cls._is_property_ok(definition, value):
657 raise TypeError(
658 f"Supplied value {value} for property {p} "
659 f"should be of class {definition.str_type} not {value.__class__}"
660 )
661 super(cls, obsinfo).__setattr__(property, value) # allows setting write-protected extensions
662 unused.remove(p)
664 # Recent additions to ObservationInfo may not be present in
665 # serializations. In theory they can be derived from other
666 # values in the default case. This might not be the right thing
667 # to do.
668 for k in ("group_counter_start", "group_counter_end"):
669 if k not in kwargs and "observation_counter" in kwargs:
670 super(cls, obsinfo).__setattr__(f"_{k}", obsinfo.observation_counter)
671 if (k := "has_simulated_content") not in kwargs:
672 super(cls, obsinfo).__setattr__(f"_{k}", False)
674 if unused:
675 n = len(unused)
676 raise KeyError(f"Unrecognized propert{'y' if n == 1 else 'ies'} provided: {', '.join(unused)}")
678 return obsinfo
681# Method to add the standard properties
682def _make_property(property: str, doc: str, return_typedoc: str, return_type: Type) -> Callable:
683 """Create a getter method with associated docstring.
685 Parameters
686 ----------
687 property : `str`
688 Name of the property getter to be created.
689 doc : `str`
690 Description of this property.
691 return_typedoc : `str`
692 Type string of this property (used in the doc string).
693 return_type : `class`
694 Type of this property.
696 Returns
697 -------
698 p : `function`
699 Getter method for this property.
700 """
702 def getter(self: ObservationInfo) -> Any:
703 return getattr(self, f"_{property}")
705 getter.__doc__ = f"""{doc}
707 Returns
708 -------
709 {property} : `{return_typedoc}`
710 Access the property.
711 """
712 return getter
715# Set up the core set of properties
716# In order to provide read-only protection, each attribute is hidden behind a
717# python "property" wrapper.
718for name, definition in PROPERTIES.items():
719 setattr(ObservationInfo, f"_{name}", None)
720 setattr(
721 ObservationInfo,
722 name,
723 property(_make_property(name, definition.doc, definition.str_type, definition.py_type)),
724 )
727def makeObservationInfo( # noqa: N802
728 *, extensions: Optional[Dict[str, PropertyDefinition]] = None, **kwargs: Any
729) -> ObservationInfo:
730 """Construct an `ObservationInfo` from the supplied parameters.
732 Parameters
733 ----------
734 extensions : `dict` [`str`: `PropertyDefinition`], optional
735 Optional extension definitions, indexed by extension name (without
736 the ``ext_`` prefix, which will be added by `ObservationInfo`).
737 **kwargs
738 Name-value pairs for any properties to be set. In the case of
739 extension properties, the names should include the ``ext_`` prefix.
741 Notes
742 -----
743 The supplied parameters should use names matching the property.
744 The type of the supplied value will be checked against the property.
745 Any properties not supplied will be assigned a value of `None`.
747 Raises
748 ------
749 KeyError
750 Raised if a supplied parameter key is not a known property.
751 TypeError
752 Raised if a supplied value does not match the expected type
753 of the property.
754 """
755 return ObservationInfo.makeObservationInfo(extensions=extensions, **kwargs)