Coverage for python/astro_metadata_translator/observationInfo.py: 14%
269 statements
« prev ^ index » next coverage.py v7.2.1, created at 2023-03-13 02:40 +0000
« prev ^ index » next coverage.py v7.2.1, created at 2023-03-13 02:40 +0000
1# This file is part of astro_metadata_translator.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the LICENSE file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12"""Represent standard metadata from instrument headers"""
14from __future__ import annotations
16__all__ = ("ObservationInfo", "makeObservationInfo")
18import copy
19import itertools
20import json
21import logging
22import math
23from typing import (
24 TYPE_CHECKING,
25 Any,
26 Callable,
27 Dict,
28 FrozenSet,
29 MutableMapping,
30 Optional,
31 Sequence,
32 Set,
33 Tuple,
34 Type,
35)
37import astropy.time
38from astropy.coordinates import AltAz, SkyCoord
40from .headers import fix_header
41from .properties import PROPERTIES, PropertyDefinition
42from .translator import MetadataTranslator
44if TYPE_CHECKING: 44 ↛ 45line 44 didn't jump to line 45, because the condition on line 44 was never true
45 import astropy.coordinates
46 import astropy.units
48log = logging.getLogger(__name__)
51class ObservationInfo:
52 """Standardized representation of an instrument header for a single
53 exposure observation.
55 There is a core set of instrumental properties that are pre-defined.
56 Additional properties may be defined, either through the
57 ``makeObservationInfo`` factory function by providing the ``extensions``
58 definitions, or through the regular ``ObservationInfo`` constructor when
59 the extensions have been defined in the ``MetadataTranslator`` for the
60 instrument of interest (or in the provided ``translator_class``).
62 Parameters
63 ----------
64 header : `dict`-like
65 Representation of an instrument header accessible as a `dict`.
66 May be updated with header corrections if corrections are found.
67 filename : `str`, optional
68 Name of the file whose header is being translated. For some
69 datasets with missing header information this can sometimes
70 allow for some fixups in translations.
71 translator_class : `MetadataTranslator`-class, optional
72 If not `None`, the class to use to translate the supplied headers
73 into standard form. Otherwise each registered translator class will
74 be asked in turn if it knows how to translate the supplied header.
75 pedantic : `bool`, optional
76 If True the translation must succeed for all properties. If False
77 individual property translations must all be implemented but can fail
78 and a warning will be issued.
79 search_path : iterable, optional
80 Override search paths to use during header fix up.
81 required : `set`, optional
82 This parameter can be used to confirm that all properties contained
83 in the set must translate correctly and also be non-None. For the case
84 where ``pedantic`` is `True` this will still check that the resulting
85 value is not `None`.
86 subset : `set`, optional
87 If not `None`, controls the translations that will be performed
88 during construction. This can be useful if the caller is only
89 interested in a subset of the properties and knows that some of
90 the others might be slow to compute (for example the airmass if it
91 has to be derived).
93 Raises
94 ------
95 ValueError
96 Raised if the supplied header was not recognized by any of the
97 registered translators. Also raised if the request property subset
98 is not a subset of the known properties.
99 TypeError
100 Raised if the supplied translator class was not a MetadataTranslator.
101 KeyError
102 Raised if a required property cannot be calculated, or if pedantic
103 mode is enabled and any translations fails.
104 NotImplementedError
105 Raised if the selected translator does not support a required
106 property.
108 Notes
109 -----
110 Headers will be corrected if correction files are located and this will
111 modify the header provided to the constructor.
113 Values of the properties are read-only.
114 """
116 # Static typing requires that we define the standard dynamic properties
117 # statically.
118 if TYPE_CHECKING: 118 ↛ 119line 118 didn't jump to line 119, because the condition on line 118 was never true
119 telescope: int
120 instrument: str
121 location: astropy.coordinates.EarthLocation
122 exposure_id: int
123 visit_id: int
124 physical_filter: str
125 datetime_begin: astropy.time.Time
126 datetime_end: astropy.time.Time
127 exposure_group: str
128 exposure_time: astropy.units.Quantity
129 dark_time: astropy.units.Quantity
130 boresight_airmass: float
131 boresight_rotation_angle: astropy.units.Quantity
132 boresight_rotation_coord: str
133 detector_num: int
134 detector_name: str
135 detector_serial: str
136 detector_group: str
137 detector_exposure_id: int
138 focus_z: astropy.units.Quantity
139 object: str
140 temperature: astropy.units.Quantity
141 pressure: astropy.units.Quantity
142 relative_humidity: float
143 tracking_radec: astropy.coordinates.SkyCoord
144 altaz_begin: astropy.coordinates.AltAz
145 science_program: str
146 observation_counter: int
147 observation_reason: str
148 observation_type: str
149 observation_id: str
150 observing_day: int
151 group_counter_start: int
152 group_counter_end: int
153 has_simulated_content: bool
155 def __init__(
156 self,
157 header: Optional[MutableMapping[str, Any]],
158 filename: Optional[str] = None,
159 translator_class: Optional[Type[MetadataTranslator]] = None,
160 pedantic: bool = False,
161 search_path: Optional[Sequence[str]] = None,
162 required: Optional[Set[str]] = None,
163 subset: Optional[Set[str]] = None,
164 ) -> None:
166 # Initialize the empty object
167 self._header: MutableMapping[str, Any] = {}
168 self.filename = filename
169 self._translator = None
170 self.translator_class_name = "<None>"
172 # To allow makeObservationInfo to work, we special case a None
173 # header
174 if header is None:
175 return
177 # Fix up the header (if required)
178 fix_header(header, translator_class=translator_class, filename=filename, search_path=search_path)
180 # Store the supplied header for later stripping
181 self._header = header
183 if translator_class is None:
184 translator_class = MetadataTranslator.determine_translator(header, filename=filename)
185 elif not issubclass(translator_class, MetadataTranslator):
186 raise TypeError(f"Translator class must be a MetadataTranslator, not {translator_class}")
188 self._declare_extensions(translator_class.extensions)
190 # Create an instance for this header
191 translator = translator_class(header, filename=filename)
193 # Store the translator
194 self._translator = translator
195 self.translator_class_name = translator_class.__name__
197 # Form file information string in case we need an error message
198 if filename:
199 file_info = f" and file {filename}"
200 else:
201 file_info = ""
203 # Determine the properties of interest
204 full_set = set(self.all_properties)
205 if subset is not None:
206 if not subset:
207 raise ValueError("Cannot request no properties be calculated.")
208 if not subset.issubset(full_set):
209 raise ValueError(
210 "Requested subset is not a subset of known properties. " f"Got extra: {subset - full_set}"
211 )
212 properties = subset
213 else:
214 properties = full_set
216 if required is None:
217 required = set()
218 else:
219 if not required.issubset(full_set):
220 raise ValueError("Requested required properties include unknowns: " f"{required - full_set}")
222 # Loop over each property and request the translated form
223 for t in properties:
224 # prototype code
225 method = f"to_{t}"
226 property = f"_{t}" if not t.startswith("ext_") else t
228 try:
229 value = getattr(translator, method)()
230 except NotImplementedError as e:
231 raise NotImplementedError(
232 f"No translation exists for property '{t}'" f" using translator {translator.__class__}"
233 ) from e
234 except Exception as e:
235 err_msg = (
236 f"Error calculating property '{t}' using translator {translator.__class__}" f"{file_info}"
237 )
238 if pedantic or t in required:
239 raise KeyError(err_msg) from e
240 else:
241 log.debug("Calculation of property '%s' failed with header: %s", t, header)
242 log.warning(f"Ignoring {err_msg}: {e}")
243 continue
245 definition = self.all_properties[t]
246 if not self._is_property_ok(definition, value):
247 err_msg = (
248 f"Value calculated for property '{t}' is wrong type "
249 f"({type(value)} != {definition.str_type}) using translator {translator.__class__}"
250 f"{file_info}"
251 )
252 if pedantic or t in required:
253 raise TypeError(err_msg)
254 else:
255 log.debug("Calcuation of property '%s' had unexpected type with header: %s", t, header)
256 log.warning(f"Ignoring {err_msg}")
258 if value is None and t in required:
259 raise KeyError(f"Calculation of required property {t} resulted in a value of None")
261 super().__setattr__(property, value) # allows setting even write-protected extensions
263 @staticmethod
264 def _get_all_properties(
265 extensions: Optional[Dict[str, PropertyDefinition]] = None
266 ) -> Dict[str, PropertyDefinition]:
267 """Return the definitions of all properties
269 Parameters
270 ----------
271 extensions : `dict` [`str`: `PropertyDefinition`]
272 List of extension property definitions, indexed by name (with no
273 "ext_" prefix).
275 Returns
276 -------
277 properties : `dict` [`str`: `PropertyDefinition`]
278 Merged list of all property definitions, indexed by name. Extension
279 properties will be listed with an ``ext_`` prefix.
280 """
281 properties = dict(PROPERTIES)
282 if extensions:
283 properties.update({"ext_" + pp: dd for pp, dd in extensions.items()})
284 return properties
286 def _declare_extensions(self, extensions: Optional[Dict[str, PropertyDefinition]]) -> None:
287 """Declare and set up extension properties
289 This should always be called internally as part of the creation of a
290 new `ObservationInfo`.
292 The core set of properties each have a python ``property`` that makes
293 them read-only, and serves as a useful place to hang the docstring.
294 However, the core set are set up at compile time, whereas the extension
295 properties have to be configured at run time (because we don't know
296 what they will be until we look at the header and figure out what
297 instrument we're dealing with) when we have an instance rather than a
298 class (and python ``property`` doesn't work on instances; only on
299 classes). We therefore use a separate scheme for the extension
300 properties: we write them directly to their associated instance
301 variable, and we use ``__setattr__`` to protect them as read-only.
302 Unfortunately, with this scheme, we can't give extension properties a
303 docstring; but we're setting them up at runtime, so maybe that's not
304 terribly important.
306 Parameters
307 ----------
308 extensions : `dict` [`str`: `PropertyDefinition`]
309 List of extension property definitions, indexed by name (with no
310 "ext_" prefix).
311 """
312 if not extensions:
313 extensions = {}
314 for name in extensions:
315 super().__setattr__("ext_" + name, None)
316 self.extensions = extensions
317 self.all_properties = self._get_all_properties(extensions)
319 def __setattr__(self, name: str, value: Any) -> Any:
320 """Set attribute
322 This provides read-only protection for the extension properties. The
323 core set of properties have read-only protection via the use of the
324 python ``property``.
325 """
326 if hasattr(self, "extensions") and name.startswith("ext_") and name[4:] in self.extensions:
327 raise AttributeError(f"Attribute {name} is read-only")
328 return super().__setattr__(name, value)
330 @classmethod
331 def _is_property_ok(cls, definition: PropertyDefinition, value: Any) -> bool:
332 """Compare the supplied value against the expected type as defined
333 for the corresponding property.
335 Parameters
336 ----------
337 definition : `PropertyDefinition`
338 Property definition.
339 value : `object`
340 Value of the property to validate.
342 Returns
343 -------
344 is_ok : `bool`
345 `True` if the value is of an appropriate type.
347 Notes
348 -----
349 Currently only the type of the property is validated. There is no
350 attempt to check bounds or determine that a Quantity is compatible
351 with the property.
352 """
353 if value is None:
354 return True
356 # For AltAz coordinates, they can either arrive as AltAz or
357 # as SkyCoord(frame=AltAz) so try to find the frame inside
358 # the SkyCoord.
359 if issubclass(definition.py_type, AltAz) and isinstance(value, SkyCoord):
360 value = value.frame
362 if not isinstance(value, definition.py_type):
363 return False
365 return True
367 @property
368 def cards_used(self) -> FrozenSet[str]:
369 """Header cards used for the translation.
371 Returns
372 -------
373 used : `frozenset` of `str`
374 Set of card used.
375 """
376 if not self._translator:
377 return frozenset()
378 return self._translator.cards_used()
380 def stripped_header(self) -> MutableMapping[str, Any]:
381 """Return a copy of the supplied header with used keywords removed.
383 Returns
384 -------
385 stripped : `dict`-like
386 Same class as header supplied to constructor, but with the
387 headers used to calculate the generic information removed.
388 """
389 hdr = copy.copy(self._header)
390 used = self.cards_used
391 for c in used:
392 del hdr[c]
393 return hdr
395 def __str__(self) -> str:
396 # Put more interesting answers at front of list
397 # and then do remainder
398 priority = ("instrument", "telescope", "datetime_begin")
399 properties = sorted(set(self.all_properties) - set(priority))
401 result = ""
402 for p in itertools.chain(priority, properties):
403 value = getattr(self, p)
404 if isinstance(value, astropy.time.Time):
405 value.format = "isot"
406 value = str(value.value)
407 result += f"{p}: {value}\n"
409 return result
411 def __eq__(self, other: Any) -> bool:
412 """Compares equal if standard properties are equal"""
413 if not isinstance(other, ObservationInfo):
414 return NotImplemented
416 # Compare simplified forms.
417 # Cannot compare directly because nan will not equate as equal
418 # whereas they should be equal for our purposes
419 self_simple = self.to_simple()
420 other_simple = other.to_simple()
422 # We don't care about the translator internal detail
423 self_simple.pop("_translator", None)
424 other_simple.pop("_translator", None)
426 for k, self_value in self_simple.items():
427 other_value = other_simple[k]
428 if self_value != other_value:
429 if math.isnan(self_value) and math.isnan(other_value):
430 # If both are nan this is fine
431 continue
432 return False
433 return True
435 def __lt__(self, other: Any) -> bool:
436 if not isinstance(other, ObservationInfo):
437 return NotImplemented
438 return self.datetime_begin < other.datetime_begin
440 def __gt__(self, other: Any) -> bool:
441 if not isinstance(other, ObservationInfo):
442 return NotImplemented
443 return self.datetime_begin > other.datetime_begin
445 def __getstate__(self) -> Tuple[Any, ...]:
446 """Get pickleable state
448 Returns the properties. Deliberately does not preserve the full
449 current state; in particular, does not return the full header or
450 translator.
452 Returns
453 -------
454 state : `tuple`
455 Pickled state.
456 """
457 state = dict()
458 for p in self.all_properties:
459 state[p] = getattr(self, p)
461 return state, self.extensions
463 def __setstate__(self, state: Tuple[Any, ...]) -> None:
464 """Set object state from pickle
466 Parameters
467 ----------
468 state : `tuple`
469 Pickled state.
470 """
471 try:
472 state, extensions = state
473 except ValueError:
474 # Backwards compatibility for pickles generated before DM-34175
475 extensions = {}
476 self._declare_extensions(extensions)
477 for p in self.all_properties:
478 if p.startswith("ext_"):
479 # allows setting even write-protected extensions
480 super().__setattr__(p, state[p]) # type: ignore
481 else:
482 property = f"_{p}"
483 setattr(self, property, state[p]) # type: ignore
485 def to_simple(self) -> MutableMapping[str, Any]:
486 """Convert the contents of this object to simple dict form.
488 The keys of the dict are the standard properties but the values
489 can be simplified to support JSON serialization. For example a
490 SkyCoord might be represented as an ICRS RA/Dec tuple rather than
491 a full SkyCoord representation.
493 Any properties with `None` value will be skipped.
495 Can be converted back to an `ObservationInfo` using `from_simple()`.
497 Returns
498 -------
499 simple : `dict` of [`str`, `Any`]
500 Simple dict of all properties.
502 Notes
503 -----
504 Round-tripping of extension properties requires that the
505 `ObservationInfo` was created with the help of a registered
506 `MetadataTranslator` (which contains the extension property
507 definitions).
508 """
509 simple = {}
510 if hasattr(self, "_translator") and self._translator and self._translator.name:
511 simple["_translator"] = self._translator.name
513 for p in self.all_properties:
514 property = f"_{p}" if not p.startswith("ext_") else p
515 value = getattr(self, property)
516 if value is None:
517 continue
519 # Access the function to simplify the property
520 simplifier = self.all_properties[p].to_simple
522 if simplifier is None:
523 simple[p] = value
524 continue
526 simple[p] = simplifier(value)
528 return simple
530 def to_json(self) -> str:
531 """Serialize the object to JSON string.
533 Returns
534 -------
535 j : `str`
536 The properties of the ObservationInfo in JSON string form.
538 Notes
539 -----
540 Round-tripping of extension properties requires that the
541 `ObservationInfo` was created with the help of a registered
542 `MetadataTranslator` (which contains the extension property
543 definitions).
544 """
545 return json.dumps(self.to_simple())
547 @classmethod
548 def from_simple(cls, simple: MutableMapping[str, Any]) -> ObservationInfo:
549 """Convert the entity returned by `to_simple` back into an
550 `ObservationInfo`.
552 Parameters
553 ----------
554 simple : `dict` [`str`, `Any`]
555 The dict returned by `to_simple()`
557 Returns
558 -------
559 obsinfo : `ObservationInfo`
560 New object constructed from the dict.
562 Notes
563 -----
564 Round-tripping of extension properties requires that the
565 `ObservationInfo` was created with the help of a registered
566 `MetadataTranslator` (which contains the extension property
567 definitions).
568 """
569 extensions = {}
570 translator = simple.pop("_translator", None)
571 if translator:
572 if translator not in MetadataTranslator.translators:
573 raise KeyError(f"Unrecognised translator: {translator}")
574 extensions = MetadataTranslator.translators[translator].extensions
576 properties = cls._get_all_properties(extensions)
578 processed: Dict[str, Any] = {}
579 for k, v in simple.items():
581 if v is None:
582 continue
584 # Access the function to convert from simple form
585 complexifier = properties[k].from_simple
587 if complexifier is not None:
588 v = complexifier(v, **processed)
590 processed[k] = v
592 return cls.makeObservationInfo(extensions=extensions, **processed)
594 @classmethod
595 def from_json(cls, json_str: str) -> ObservationInfo:
596 """Create `ObservationInfo` from JSON string.
598 Parameters
599 ----------
600 json_str : `str`
601 The JSON representation.
603 Returns
604 -------
605 obsinfo : `ObservationInfo`
606 Reconstructed object.
608 Notes
609 -----
610 Round-tripping of extension properties requires that the
611 `ObservationInfo` was created with the help of a registered
612 `MetadataTranslator` (which contains the extension property
613 definitions).
614 """
615 simple = json.loads(json_str)
616 return cls.from_simple(simple)
618 @classmethod
619 def makeObservationInfo( # noqa: N802
620 cls, *, extensions: Optional[Dict[str, PropertyDefinition]] = None, **kwargs: Any
621 ) -> ObservationInfo:
622 """Construct an `ObservationInfo` from the supplied parameters.
624 Parameters
625 ----------
626 extensions : `dict` [`str`: `PropertyDefinition`], optional
627 Optional extension definitions, indexed by extension name (without
628 the ``ext_`` prefix, which will be added by `ObservationInfo`).
629 **kwargs
630 Name-value pairs for any properties to be set. In the case of
631 extension properties, the names should include the ``ext_`` prefix.
633 Notes
634 -----
635 The supplied parameters should use names matching the property.
636 The type of the supplied value will be checked against the property.
637 Any properties not supplied will be assigned a value of `None`.
639 Raises
640 ------
641 KeyError
642 Raised if a supplied parameter key is not a known property.
643 TypeError
644 Raised if a supplied value does not match the expected type
645 of the property.
646 """
648 obsinfo = cls(None)
649 obsinfo._declare_extensions(extensions)
651 unused = set(kwargs)
653 for p in obsinfo.all_properties:
654 if p in kwargs:
655 property = f"_{p}" if not p.startswith("ext_") else p
656 value = kwargs[p]
657 definition = obsinfo.all_properties[p]
658 if not cls._is_property_ok(definition, value):
659 raise TypeError(
660 f"Supplied value {value} for property {p} "
661 f"should be of class {definition.str_type} not {value.__class__}"
662 )
663 super(cls, obsinfo).__setattr__(property, value) # allows setting write-protected extensions
664 unused.remove(p)
666 # Recent additions to ObservationInfo may not be present in
667 # serializations. In theory they can be derived from other
668 # values in the default case. This might not be the right thing
669 # to do.
670 for k in ("group_counter_start", "group_counter_end"):
671 if k not in kwargs and "observation_counter" in kwargs:
672 super(cls, obsinfo).__setattr__(f"_{k}", obsinfo.observation_counter)
673 if (k := "has_simulated_content") not in kwargs:
674 super(cls, obsinfo).__setattr__(f"_{k}", False)
676 if unused:
677 n = len(unused)
678 raise KeyError(f"Unrecognized propert{'y' if n == 1 else 'ies'} provided: {', '.join(unused)}")
680 return obsinfo
683# Method to add the standard properties
684def _make_property(property: str, doc: str, return_typedoc: str, return_type: Type) -> Callable:
685 """Create a getter method with associated docstring.
687 Parameters
688 ----------
689 property : `str`
690 Name of the property getter to be created.
691 doc : `str`
692 Description of this property.
693 return_typedoc : `str`
694 Type string of this property (used in the doc string).
695 return_type : `class`
696 Type of this property.
698 Returns
699 -------
700 p : `function`
701 Getter method for this property.
702 """
704 def getter(self: ObservationInfo) -> Any:
705 return getattr(self, f"_{property}")
707 getter.__doc__ = f"""{doc}
709 Returns
710 -------
711 {property} : `{return_typedoc}`
712 Access the property.
713 """
714 return getter
717# Set up the core set of properties
718# In order to provide read-only protection, each attribute is hidden behind a
719# python "property" wrapper.
720for name, definition in PROPERTIES.items():
721 setattr(ObservationInfo, f"_{name}", None)
722 setattr(
723 ObservationInfo,
724 name,
725 property(_make_property(name, definition.doc, definition.str_type, definition.py_type)),
726 )
729def makeObservationInfo( # noqa: N802
730 *, extensions: Optional[Dict[str, PropertyDefinition]] = None, **kwargs: Any
731) -> ObservationInfo:
732 """Construct an `ObservationInfo` from the supplied parameters.
734 Parameters
735 ----------
736 extensions : `dict` [`str`: `PropertyDefinition`], optional
737 Optional extension definitions, indexed by extension name (without
738 the ``ext_`` prefix, which will be added by `ObservationInfo`).
739 **kwargs
740 Name-value pairs for any properties to be set. In the case of
741 extension properties, the names should include the ``ext_`` prefix.
743 Notes
744 -----
745 The supplied parameters should use names matching the property.
746 The type of the supplied value will be checked against the property.
747 Any properties not supplied will be assigned a value of `None`.
749 Raises
750 ------
751 KeyError
752 Raised if a supplied parameter key is not a known property.
753 TypeError
754 Raised if a supplied value does not match the expected type
755 of the property.
756 """
757 return ObservationInfo.makeObservationInfo(extensions=extensions, **kwargs)