Coverage for python/astro_metadata_translator/observationInfo.py: 17%
270 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-08-05 01:30 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-08-05 01:30 +0000
1# This file is part of astro_metadata_translator.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the LICENSE file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12"""Represent standard metadata from instrument headers."""
14from __future__ import annotations
16__all__ = ("ObservationInfo", "makeObservationInfo")
18import copy
19import itertools
20import json
21import logging
22import math
23from collections.abc import Callable, MutableMapping, Sequence
24from typing import TYPE_CHECKING, Any
26import astropy.time
27from astropy.coordinates import AltAz, SkyCoord
29from .headers import fix_header
30from .properties import PROPERTIES, PropertyDefinition
31from .translator import MetadataTranslator
33if TYPE_CHECKING: 33 ↛ 34line 33 didn't jump to line 34, because the condition on line 33 was never true
34 import astropy.coordinates
35 import astropy.units
37log = logging.getLogger(__name__)
40class ObservationInfo:
41 """Standardized representation of an instrument header for a single
42 exposure observation.
44 There is a core set of instrumental properties that are pre-defined.
45 Additional properties may be defined, either through the
46 ``makeObservationInfo`` factory function by providing the ``extensions``
47 definitions, or through the regular ``ObservationInfo`` constructor when
48 the extensions have been defined in the ``MetadataTranslator`` for the
49 instrument of interest (or in the provided ``translator_class``).
51 Parameters
52 ----------
53 header : `dict`-like
54 Representation of an instrument header accessible as a `dict`.
55 May be updated with header corrections if corrections are found.
56 filename : `str`, optional
57 Name of the file whose header is being translated. For some
58 datasets with missing header information this can sometimes
59 allow for some fixups in translations.
60 translator_class : `MetadataTranslator`-class, optional
61 If not `None`, the class to use to translate the supplied headers
62 into standard form. Otherwise each registered translator class will
63 be asked in turn if it knows how to translate the supplied header.
64 pedantic : `bool`, optional
65 If True the translation must succeed for all properties. If False
66 individual property translations must all be implemented but can fail
67 and a warning will be issued.
68 search_path : iterable, optional
69 Override search paths to use during header fix up.
70 required : `set`, optional
71 This parameter can be used to confirm that all properties contained
72 in the set must translate correctly and also be non-None. For the case
73 where ``pedantic`` is `True` this will still check that the resulting
74 value is not `None`.
75 subset : `set`, optional
76 If not `None`, controls the translations that will be performed
77 during construction. This can be useful if the caller is only
78 interested in a subset of the properties and knows that some of
79 the others might be slow to compute (for example the airmass if it
80 has to be derived).
82 Raises
83 ------
84 ValueError
85 Raised if the supplied header was not recognized by any of the
86 registered translators. Also raised if the request property subset
87 is not a subset of the known properties.
88 TypeError
89 Raised if the supplied translator class was not a MetadataTranslator.
90 KeyError
91 Raised if a required property cannot be calculated, or if pedantic
92 mode is enabled and any translations fails.
93 NotImplementedError
94 Raised if the selected translator does not support a required
95 property.
97 Notes
98 -----
99 Headers will be corrected if correction files are located and this will
100 modify the header provided to the constructor.
102 Values of the properties are read-only.
103 """
105 # Static typing requires that we define the standard dynamic properties
106 # statically.
107 if TYPE_CHECKING: 107 ↛ 108line 107 didn't jump to line 108, because the condition on line 107 was never true
108 telescope: int
109 instrument: str
110 location: astropy.coordinates.EarthLocation
111 exposure_id: int
112 visit_id: int
113 physical_filter: str
114 datetime_begin: astropy.time.Time
115 datetime_end: astropy.time.Time
116 exposure_group: str
117 exposure_time: astropy.units.Quantity
118 dark_time: astropy.units.Quantity
119 boresight_airmass: float
120 boresight_rotation_angle: astropy.units.Quantity
121 boresight_rotation_coord: str
122 detector_num: int
123 detector_name: str
124 detector_serial: str
125 detector_group: str
126 detector_exposure_id: int
127 focus_z: astropy.units.Quantity
128 object: str
129 temperature: astropy.units.Quantity
130 pressure: astropy.units.Quantity
131 relative_humidity: float
132 tracking_radec: astropy.coordinates.SkyCoord
133 altaz_begin: astropy.coordinates.AltAz
134 science_program: str
135 observation_counter: int
136 observation_reason: str
137 observation_type: str
138 observation_id: str
139 observing_day: int
140 group_counter_start: int
141 group_counter_end: int
142 has_simulated_content: bool
144 def __init__(
145 self,
146 header: MutableMapping[str, Any] | None,
147 filename: str | None = None,
148 translator_class: type[MetadataTranslator] | None = None,
149 pedantic: bool = False,
150 search_path: Sequence[str] | None = None,
151 required: set[str] | None = None,
152 subset: set[str] | None = None,
153 ) -> None:
154 # Initialize the empty object
155 self._header: MutableMapping[str, Any] = {}
156 self.filename = filename
157 self._translator = None
158 self.translator_class_name = "<None>"
160 # To allow makeObservationInfo to work, we special case a None
161 # header
162 if header is None:
163 return
165 # Fix up the header (if required)
166 fix_header(header, translator_class=translator_class, filename=filename, search_path=search_path)
168 # Store the supplied header for later stripping
169 self._header = header
171 if translator_class is None:
172 translator_class = MetadataTranslator.determine_translator(header, filename=filename)
173 elif not issubclass(translator_class, MetadataTranslator):
174 raise TypeError(f"Translator class must be a MetadataTranslator, not {translator_class}")
176 self._declare_extensions(translator_class.extensions)
178 # Create an instance for this header
179 translator = translator_class(header, filename=filename)
181 # Store the translator
182 self._translator = translator
183 self.translator_class_name = translator_class.__name__
185 # Form file information string in case we need an error message
186 if filename:
187 file_info = f" and file {filename}"
188 else:
189 file_info = ""
191 # Determine the properties of interest
192 full_set = set(self.all_properties)
193 if subset is not None:
194 if not subset:
195 raise ValueError("Cannot request no properties be calculated.")
196 if not subset.issubset(full_set):
197 raise ValueError(
198 "Requested subset is not a subset of known properties. " f"Got extra: {subset - full_set}"
199 )
200 properties = subset
201 else:
202 properties = full_set
204 if required is None:
205 required = set()
206 else:
207 if not required.issubset(full_set):
208 raise ValueError("Requested required properties include unknowns: " f"{required - full_set}")
210 # Loop over each property and request the translated form
211 for t in properties:
212 # prototype code
213 method = f"to_{t}"
214 property = f"_{t}" if not t.startswith("ext_") else t
216 try:
217 value = getattr(translator, method)()
218 except NotImplementedError as e:
219 raise NotImplementedError(
220 f"No translation exists for property '{t}'" f" using translator {translator.__class__}"
221 ) from e
222 except Exception as e:
223 err_msg = (
224 f"Error calculating property '{t}' using translator {translator.__class__}" f"{file_info}"
225 )
226 if pedantic or t in required:
227 raise KeyError(err_msg) from e
228 else:
229 log.debug("Calculation of property '%s' failed with header: %s", t, header)
230 log.warning(f"Ignoring {err_msg}: {e}")
231 continue
233 definition = self.all_properties[t]
234 if not self._is_property_ok(definition, value):
235 err_msg = (
236 f"Value calculated for property '{t}' is wrong type "
237 f"({type(value)} != {definition.str_type}) using translator {translator.__class__}"
238 f"{file_info}"
239 )
240 if pedantic or t in required:
241 raise TypeError(err_msg)
242 else:
243 log.debug("Calcuation of property '%s' had unexpected type with header: %s", t, header)
244 log.warning(f"Ignoring {err_msg}")
246 if value is None and t in required:
247 raise KeyError(f"Calculation of required property {t} resulted in a value of None")
249 super().__setattr__(property, value) # allows setting even write-protected extensions
251 @staticmethod
252 def _get_all_properties(
253 extensions: dict[str, PropertyDefinition] | None = None
254 ) -> dict[str, PropertyDefinition]:
255 """Return the definitions of all properties.
257 Parameters
258 ----------
259 extensions : `dict` [`str`: `PropertyDefinition`]
260 List of extension property definitions, indexed by name (with no
261 "ext_" prefix).
263 Returns
264 -------
265 properties : `dict` [`str`: `PropertyDefinition`]
266 Merged list of all property definitions, indexed by name. Extension
267 properties will be listed with an ``ext_`` prefix.
268 """
269 properties = dict(PROPERTIES)
270 if extensions:
271 properties.update({"ext_" + pp: dd for pp, dd in extensions.items()})
272 return properties
274 def _declare_extensions(self, extensions: dict[str, PropertyDefinition] | None) -> None:
275 """Declare and set up extension properties.
277 This should always be called internally as part of the creation of a
278 new `ObservationInfo`.
280 The core set of properties each have a python ``property`` that makes
281 them read-only, and serves as a useful place to hang the docstring.
282 However, the core set are set up at compile time, whereas the extension
283 properties have to be configured at run time (because we don't know
284 what they will be until we look at the header and figure out what
285 instrument we're dealing with) when we have an instance rather than a
286 class (and python ``property`` doesn't work on instances; only on
287 classes). We therefore use a separate scheme for the extension
288 properties: we write them directly to their associated instance
289 variable, and we use ``__setattr__`` to protect them as read-only.
290 Unfortunately, with this scheme, we can't give extension properties a
291 docstring; but we're setting them up at runtime, so maybe that's not
292 terribly important.
294 Parameters
295 ----------
296 extensions : `dict` [`str`: `PropertyDefinition`]
297 List of extension property definitions, indexed by name (with no
298 "ext_" prefix).
299 """
300 if not extensions:
301 extensions = {}
302 for name in extensions:
303 super().__setattr__("ext_" + name, None)
304 self.extensions = extensions
305 self.all_properties = self._get_all_properties(extensions)
307 def __setattr__(self, name: str, value: Any) -> Any:
308 """Set attribute.
310 This provides read-only protection for the extension properties. The
311 core set of properties have read-only protection via the use of the
312 python ``property``.
313 """
314 if hasattr(self, "extensions") and name.startswith("ext_") and name[4:] in self.extensions:
315 raise AttributeError(f"Attribute {name} is read-only")
316 return super().__setattr__(name, value)
318 @classmethod
319 def _is_property_ok(cls, definition: PropertyDefinition, value: Any) -> bool:
320 """Compare the supplied value against the expected type as defined
321 for the corresponding property.
323 Parameters
324 ----------
325 definition : `PropertyDefinition`
326 Property definition.
327 value : `object`
328 Value of the property to validate.
330 Returns
331 -------
332 is_ok : `bool`
333 `True` if the value is of an appropriate type.
335 Notes
336 -----
337 Currently only the type of the property is validated. There is no
338 attempt to check bounds or determine that a Quantity is compatible
339 with the property.
340 """
341 if value is None:
342 return True
344 # For AltAz coordinates, they can either arrive as AltAz or
345 # as SkyCoord(frame=AltAz) so try to find the frame inside
346 # the SkyCoord.
347 if issubclass(definition.py_type, AltAz) and isinstance(value, SkyCoord):
348 value = value.frame
350 if not isinstance(value, definition.py_type):
351 return False
353 return True
355 @property
356 def cards_used(self) -> frozenset[str]:
357 """Header cards used for the translation.
359 Returns
360 -------
361 used : `frozenset` of `str`
362 Set of card used.
363 """
364 if not self._translator:
365 return frozenset()
366 return self._translator.cards_used()
368 def stripped_header(self) -> MutableMapping[str, Any]:
369 """Return a copy of the supplied header with used keywords removed.
371 Returns
372 -------
373 stripped : `dict`-like
374 Same class as header supplied to constructor, but with the
375 headers used to calculate the generic information removed.
376 """
377 hdr = copy.copy(self._header)
378 used = self.cards_used
379 for c in used:
380 del hdr[c]
381 return hdr
383 def __str__(self) -> str:
384 # Put more interesting answers at front of list
385 # and then do remainder
386 priority = ("instrument", "telescope", "datetime_begin")
387 properties = sorted(set(self.all_properties) - set(priority))
389 result = ""
390 for p in itertools.chain(priority, properties):
391 value = getattr(self, p)
392 if isinstance(value, astropy.time.Time):
393 value.format = "isot"
394 value = str(value.value)
395 result += f"{p}: {value}\n"
397 return result
399 def __eq__(self, other: Any) -> bool:
400 """Check equality with another object.
402 Compares equal if standard properties are equal.
403 """
404 if not isinstance(other, ObservationInfo):
405 return NotImplemented
407 # Compare simplified forms.
408 # Cannot compare directly because nan will not equate as equal
409 # whereas they should be equal for our purposes
410 self_simple = self.to_simple()
411 other_simple = other.to_simple()
413 # We don't care about the translator internal detail
414 self_simple.pop("_translator", None)
415 other_simple.pop("_translator", None)
417 for k, self_value in self_simple.items():
418 other_value = other_simple[k]
419 if self_value != other_value:
420 if math.isnan(self_value) and math.isnan(other_value):
421 # If both are nan this is fine
422 continue
423 return False
424 return True
426 def __lt__(self, other: Any) -> bool:
427 if not isinstance(other, ObservationInfo):
428 return NotImplemented
429 return self.datetime_begin < other.datetime_begin
431 def __gt__(self, other: Any) -> bool:
432 if not isinstance(other, ObservationInfo):
433 return NotImplemented
434 return self.datetime_begin > other.datetime_begin
436 def __getstate__(self) -> tuple[Any, ...]:
437 """Get pickleable state.
439 Returns the properties. Deliberately does not preserve the full
440 current state; in particular, does not return the full header or
441 translator.
443 Returns
444 -------
445 state : `tuple`
446 Pickled state.
447 """
448 state = dict()
449 for p in self.all_properties:
450 state[p] = getattr(self, p)
452 return state, self.extensions
454 def __setstate__(self, state: tuple[Any, ...]) -> None:
455 """Set object state from pickle.
457 Parameters
458 ----------
459 state : `tuple`
460 Pickled state.
461 """
462 try:
463 state, extensions = state
464 except ValueError:
465 # Backwards compatibility for pickles generated before DM-34175
466 extensions = {}
467 self._declare_extensions(extensions)
468 for p in self.all_properties:
469 if p.startswith("ext_"):
470 # allows setting even write-protected extensions
471 super().__setattr__(p, state[p]) # type: ignore
472 else:
473 property = f"_{p}"
474 setattr(self, property, state[p]) # type: ignore
476 def to_simple(self) -> MutableMapping[str, Any]:
477 """Convert the contents of this object to simple dict form.
479 The keys of the dict are the standard properties but the values
480 can be simplified to support JSON serialization. For example a
481 SkyCoord might be represented as an ICRS RA/Dec tuple rather than
482 a full SkyCoord representation.
484 Any properties with `None` value will be skipped.
486 Can be converted back to an `ObservationInfo` using `from_simple()`.
488 Returns
489 -------
490 simple : `dict` of [`str`, `~typing.Any`]
491 Simple dict of all properties.
493 Notes
494 -----
495 Round-tripping of extension properties requires that the
496 `ObservationInfo` was created with the help of a registered
497 `MetadataTranslator` (which contains the extension property
498 definitions).
499 """
500 simple = {}
501 if hasattr(self, "_translator") and self._translator and self._translator.name:
502 simple["_translator"] = self._translator.name
504 for p in self.all_properties:
505 property = f"_{p}" if not p.startswith("ext_") else p
506 value = getattr(self, property)
507 if value is None:
508 continue
510 # Access the function to simplify the property
511 simplifier = self.all_properties[p].to_simple
513 if simplifier is None:
514 simple[p] = value
515 continue
517 simple[p] = simplifier(value)
519 return simple
521 def to_json(self) -> str:
522 """Serialize the object to JSON string.
524 Returns
525 -------
526 j : `str`
527 The properties of the ObservationInfo in JSON string form.
529 Notes
530 -----
531 Round-tripping of extension properties requires that the
532 `ObservationInfo` was created with the help of a registered
533 `MetadataTranslator` (which contains the extension property
534 definitions).
535 """
536 return json.dumps(self.to_simple())
538 @classmethod
539 def from_simple(cls, simple: MutableMapping[str, Any]) -> ObservationInfo:
540 """Convert the entity returned by `to_simple` back into an
541 `ObservationInfo`.
543 Parameters
544 ----------
545 simple : `dict` [`str`, `~typing.Any`]
546 The dict returned by `to_simple()`
548 Returns
549 -------
550 obsinfo : `ObservationInfo`
551 New object constructed from the dict.
553 Notes
554 -----
555 Round-tripping of extension properties requires that the
556 `ObservationInfo` was created with the help of a registered
557 `MetadataTranslator` (which contains the extension property
558 definitions).
559 """
560 extensions = {}
561 translator = simple.pop("_translator", None)
562 if translator:
563 if translator not in MetadataTranslator.translators:
564 raise KeyError(f"Unrecognised translator: {translator}")
565 extensions = MetadataTranslator.translators[translator].extensions
567 properties = cls._get_all_properties(extensions)
569 processed: dict[str, Any] = {}
570 for k, v in simple.items():
571 if v is None:
572 continue
574 # Access the function to convert from simple form
575 complexifier = properties[k].from_simple
577 if complexifier is not None:
578 v = complexifier(v, **processed)
580 processed[k] = v
582 return cls.makeObservationInfo(extensions=extensions, **processed)
584 @classmethod
585 def from_json(cls, json_str: str) -> ObservationInfo:
586 """Create `ObservationInfo` from JSON string.
588 Parameters
589 ----------
590 json_str : `str`
591 The JSON representation.
593 Returns
594 -------
595 obsinfo : `ObservationInfo`
596 Reconstructed object.
598 Notes
599 -----
600 Round-tripping of extension properties requires that the
601 `ObservationInfo` was created with the help of a registered
602 `MetadataTranslator` (which contains the extension property
603 definitions).
604 """
605 simple = json.loads(json_str)
606 return cls.from_simple(simple)
608 @classmethod
609 def makeObservationInfo( # noqa: N802
610 cls, *, extensions: dict[str, PropertyDefinition] | None = None, **kwargs: Any
611 ) -> ObservationInfo:
612 """Construct an `ObservationInfo` from the supplied parameters.
614 Parameters
615 ----------
616 extensions : `dict` [`str`: `PropertyDefinition`], optional
617 Optional extension definitions, indexed by extension name (without
618 the ``ext_`` prefix, which will be added by `ObservationInfo`).
619 **kwargs
620 Name-value pairs for any properties to be set. In the case of
621 extension properties, the names should include the ``ext_`` prefix.
623 Notes
624 -----
625 The supplied parameters should use names matching the property.
626 The type of the supplied value will be checked against the property.
627 Any properties not supplied will be assigned a value of `None`.
629 Raises
630 ------
631 KeyError
632 Raised if a supplied parameter key is not a known property.
633 TypeError
634 Raised if a supplied value does not match the expected type
635 of the property.
636 """
637 obsinfo = cls(None)
638 obsinfo._declare_extensions(extensions)
640 unused = set(kwargs)
642 for p in obsinfo.all_properties:
643 if p in kwargs:
644 property = f"_{p}" if not p.startswith("ext_") else p
645 value = kwargs[p]
646 definition = obsinfo.all_properties[p]
647 if not cls._is_property_ok(definition, value):
648 raise TypeError(
649 f"Supplied value {value} for property {p} "
650 f"should be of class {definition.str_type} not {value.__class__}"
651 )
652 super(cls, obsinfo).__setattr__(property, value) # allows setting write-protected extensions
653 unused.remove(p)
655 # Recent additions to ObservationInfo may not be present in
656 # serializations. In theory they can be derived from other
657 # values in the default case. This might not be the right thing
658 # to do.
659 for k in ("group_counter_start", "group_counter_end"):
660 if k not in kwargs and "observation_counter" in kwargs:
661 super(cls, obsinfo).__setattr__(f"_{k}", obsinfo.observation_counter)
662 if (k := "has_simulated_content") not in kwargs:
663 super(cls, obsinfo).__setattr__(f"_{k}", False)
665 if unused:
666 n = len(unused)
667 raise KeyError(f"Unrecognized propert{'y' if n == 1 else 'ies'} provided: {', '.join(unused)}")
669 return obsinfo
672# Method to add the standard properties
673def _make_property(property: str, doc: str, return_typedoc: str, return_type: type) -> Callable:
674 """Create a getter method with associated docstring.
676 Parameters
677 ----------
678 property : `str`
679 Name of the property getter to be created.
680 doc : `str`
681 Description of this property.
682 return_typedoc : `str`
683 Type string of this property (used in the doc string).
684 return_type : `class`
685 Type of this property.
687 Returns
688 -------
689 p : `function`
690 Getter method for this property.
691 """
693 def getter(self: ObservationInfo) -> Any:
694 return getattr(self, f"_{property}")
696 getter.__doc__ = f"""{doc}
698 Returns
699 -------
700 {property} : `{return_typedoc}`
701 Access the property.
702 """
703 return getter
706# Set up the core set of properties
707# In order to provide read-only protection, each attribute is hidden behind a
708# python "property" wrapper.
709for name, definition in PROPERTIES.items():
710 setattr(ObservationInfo, f"_{name}", None)
711 setattr(
712 ObservationInfo,
713 name,
714 property(_make_property(name, definition.doc, definition.str_type, definition.py_type)),
715 )
718def makeObservationInfo( # noqa: N802
719 *, extensions: dict[str, PropertyDefinition] | None = None, **kwargs: Any
720) -> ObservationInfo:
721 """Construct an `ObservationInfo` from the supplied parameters.
723 Parameters
724 ----------
725 extensions : `dict` [`str`: `PropertyDefinition`], optional
726 Optional extension definitions, indexed by extension name (without
727 the ``ext_`` prefix, which will be added by `ObservationInfo`).
728 **kwargs
729 Name-value pairs for any properties to be set. In the case of
730 extension properties, the names should include the ``ext_`` prefix.
732 Notes
733 -----
734 The supplied parameters should use names matching the property.
735 The type of the supplied value will be checked against the property.
736 Any properties not supplied will be assigned a value of `None`.
738 Raises
739 ------
740 KeyError
741 Raised if a supplied parameter key is not a known property.
742 TypeError
743 Raised if a supplied value does not match the expected type
744 of the property.
745 """
746 return ObservationInfo.makeObservationInfo(extensions=extensions, **kwargs)