Coverage for python/astro_metadata_translator/observationInfo.py: 14%
270 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-27 02:38 -0700
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-27 02:38 -0700
1# This file is part of astro_metadata_translator.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the LICENSE file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12"""Represent standard metadata from instrument headers"""
14from __future__ import annotations
16__all__ = ("ObservationInfo", "makeObservationInfo")
18import copy
19import itertools
20import json
21import logging
22import math
23from collections.abc import Callable, MutableMapping, Sequence
24from typing import TYPE_CHECKING, Any
26import astropy.time
27from astropy.coordinates import AltAz, SkyCoord
29from .headers import fix_header
30from .properties import PROPERTIES, PropertyDefinition
31from .translator import MetadataTranslator
33if TYPE_CHECKING: 33 ↛ 34line 33 didn't jump to line 34, because the condition on line 33 was never true
34 import astropy.coordinates
35 import astropy.units
37log = logging.getLogger(__name__)
40class ObservationInfo:
41 """Standardized representation of an instrument header for a single
42 exposure observation.
44 There is a core set of instrumental properties that are pre-defined.
45 Additional properties may be defined, either through the
46 ``makeObservationInfo`` factory function by providing the ``extensions``
47 definitions, or through the regular ``ObservationInfo`` constructor when
48 the extensions have been defined in the ``MetadataTranslator`` for the
49 instrument of interest (or in the provided ``translator_class``).
51 Parameters
52 ----------
53 header : `dict`-like
54 Representation of an instrument header accessible as a `dict`.
55 May be updated with header corrections if corrections are found.
56 filename : `str`, optional
57 Name of the file whose header is being translated. For some
58 datasets with missing header information this can sometimes
59 allow for some fixups in translations.
60 translator_class : `MetadataTranslator`-class, optional
61 If not `None`, the class to use to translate the supplied headers
62 into standard form. Otherwise each registered translator class will
63 be asked in turn if it knows how to translate the supplied header.
64 pedantic : `bool`, optional
65 If True the translation must succeed for all properties. If False
66 individual property translations must all be implemented but can fail
67 and a warning will be issued.
68 search_path : iterable, optional
69 Override search paths to use during header fix up.
70 required : `set`, optional
71 This parameter can be used to confirm that all properties contained
72 in the set must translate correctly and also be non-None. For the case
73 where ``pedantic`` is `True` this will still check that the resulting
74 value is not `None`.
75 subset : `set`, optional
76 If not `None`, controls the translations that will be performed
77 during construction. This can be useful if the caller is only
78 interested in a subset of the properties and knows that some of
79 the others might be slow to compute (for example the airmass if it
80 has to be derived).
82 Raises
83 ------
84 ValueError
85 Raised if the supplied header was not recognized by any of the
86 registered translators. Also raised if the request property subset
87 is not a subset of the known properties.
88 TypeError
89 Raised if the supplied translator class was not a MetadataTranslator.
90 KeyError
91 Raised if a required property cannot be calculated, or if pedantic
92 mode is enabled and any translations fails.
93 NotImplementedError
94 Raised if the selected translator does not support a required
95 property.
97 Notes
98 -----
99 Headers will be corrected if correction files are located and this will
100 modify the header provided to the constructor.
102 Values of the properties are read-only.
103 """
105 # Static typing requires that we define the standard dynamic properties
106 # statically.
107 if TYPE_CHECKING: 107 ↛ 108line 107 didn't jump to line 108, because the condition on line 107 was never true
108 telescope: int
109 instrument: str
110 location: astropy.coordinates.EarthLocation
111 exposure_id: int
112 visit_id: int
113 physical_filter: str
114 datetime_begin: astropy.time.Time
115 datetime_end: astropy.time.Time
116 exposure_group: str
117 exposure_time: astropy.units.Quantity
118 dark_time: astropy.units.Quantity
119 boresight_airmass: float
120 boresight_rotation_angle: astropy.units.Quantity
121 boresight_rotation_coord: str
122 detector_num: int
123 detector_name: str
124 detector_serial: str
125 detector_group: str
126 detector_exposure_id: int
127 focus_z: astropy.units.Quantity
128 object: str
129 temperature: astropy.units.Quantity
130 pressure: astropy.units.Quantity
131 relative_humidity: float
132 tracking_radec: astropy.coordinates.SkyCoord
133 altaz_begin: astropy.coordinates.AltAz
134 science_program: str
135 observation_counter: int
136 observation_reason: str
137 observation_type: str
138 observation_id: str
139 observing_day: int
140 group_counter_start: int
141 group_counter_end: int
142 has_simulated_content: bool
144 def __init__(
145 self,
146 header: MutableMapping[str, Any] | None,
147 filename: str | None = None,
148 translator_class: type[MetadataTranslator] | None = None,
149 pedantic: bool = False,
150 search_path: Sequence[str] | None = None,
151 required: set[str] | None = None,
152 subset: set[str] | None = None,
153 ) -> None:
154 # Initialize the empty object
155 self._header: MutableMapping[str, Any] = {}
156 self.filename = filename
157 self._translator = None
158 self.translator_class_name = "<None>"
160 # To allow makeObservationInfo to work, we special case a None
161 # header
162 if header is None:
163 return
165 # Fix up the header (if required)
166 fix_header(header, translator_class=translator_class, filename=filename, search_path=search_path)
168 # Store the supplied header for later stripping
169 self._header = header
171 if translator_class is None:
172 translator_class = MetadataTranslator.determine_translator(header, filename=filename)
173 elif not issubclass(translator_class, MetadataTranslator):
174 raise TypeError(f"Translator class must be a MetadataTranslator, not {translator_class}")
176 self._declare_extensions(translator_class.extensions)
178 # Create an instance for this header
179 translator = translator_class(header, filename=filename)
181 # Store the translator
182 self._translator = translator
183 self.translator_class_name = translator_class.__name__
185 # Form file information string in case we need an error message
186 if filename:
187 file_info = f" and file {filename}"
188 else:
189 file_info = ""
191 # Determine the properties of interest
192 full_set = set(self.all_properties)
193 if subset is not None:
194 if not subset:
195 raise ValueError("Cannot request no properties be calculated.")
196 if not subset.issubset(full_set):
197 raise ValueError(
198 "Requested subset is not a subset of known properties. " f"Got extra: {subset - full_set}"
199 )
200 properties = subset
201 else:
202 properties = full_set
204 if required is None:
205 required = set()
206 else:
207 if not required.issubset(full_set):
208 raise ValueError("Requested required properties include unknowns: " f"{required - full_set}")
210 # Loop over each property and request the translated form
211 for t in properties:
212 # prototype code
213 method = f"to_{t}"
214 property = f"_{t}" if not t.startswith("ext_") else t
216 try:
217 value = getattr(translator, method)()
218 except NotImplementedError as e:
219 raise NotImplementedError(
220 f"No translation exists for property '{t}'" f" using translator {translator.__class__}"
221 ) from e
222 except Exception as e:
223 err_msg = (
224 f"Error calculating property '{t}' using translator {translator.__class__}" f"{file_info}"
225 )
226 if pedantic or t in required:
227 raise KeyError(err_msg) from e
228 else:
229 log.debug("Calculation of property '%s' failed with header: %s", t, header)
230 log.warning(f"Ignoring {err_msg}: {e}")
231 continue
233 definition = self.all_properties[t]
234 if not self._is_property_ok(definition, value):
235 err_msg = (
236 f"Value calculated for property '{t}' is wrong type "
237 f"({type(value)} != {definition.str_type}) using translator {translator.__class__}"
238 f"{file_info}"
239 )
240 if pedantic or t in required:
241 raise TypeError(err_msg)
242 else:
243 log.debug("Calcuation of property '%s' had unexpected type with header: %s", t, header)
244 log.warning(f"Ignoring {err_msg}")
246 if value is None and t in required:
247 raise KeyError(f"Calculation of required property {t} resulted in a value of None")
249 super().__setattr__(property, value) # allows setting even write-protected extensions
251 @staticmethod
252 def _get_all_properties(
253 extensions: dict[str, PropertyDefinition] | None = None
254 ) -> dict[str, PropertyDefinition]:
255 """Return the definitions of all properties
257 Parameters
258 ----------
259 extensions : `dict` [`str`: `PropertyDefinition`]
260 List of extension property definitions, indexed by name (with no
261 "ext_" prefix).
263 Returns
264 -------
265 properties : `dict` [`str`: `PropertyDefinition`]
266 Merged list of all property definitions, indexed by name. Extension
267 properties will be listed with an ``ext_`` prefix.
268 """
269 properties = dict(PROPERTIES)
270 if extensions:
271 properties.update({"ext_" + pp: dd for pp, dd in extensions.items()})
272 return properties
274 def _declare_extensions(self, extensions: dict[str, PropertyDefinition] | None) -> None:
275 """Declare and set up extension properties
277 This should always be called internally as part of the creation of a
278 new `ObservationInfo`.
280 The core set of properties each have a python ``property`` that makes
281 them read-only, and serves as a useful place to hang the docstring.
282 However, the core set are set up at compile time, whereas the extension
283 properties have to be configured at run time (because we don't know
284 what they will be until we look at the header and figure out what
285 instrument we're dealing with) when we have an instance rather than a
286 class (and python ``property`` doesn't work on instances; only on
287 classes). We therefore use a separate scheme for the extension
288 properties: we write them directly to their associated instance
289 variable, and we use ``__setattr__`` to protect them as read-only.
290 Unfortunately, with this scheme, we can't give extension properties a
291 docstring; but we're setting them up at runtime, so maybe that's not
292 terribly important.
294 Parameters
295 ----------
296 extensions : `dict` [`str`: `PropertyDefinition`]
297 List of extension property definitions, indexed by name (with no
298 "ext_" prefix).
299 """
300 if not extensions:
301 extensions = {}
302 for name in extensions:
303 super().__setattr__("ext_" + name, None)
304 self.extensions = extensions
305 self.all_properties = self._get_all_properties(extensions)
307 def __setattr__(self, name: str, value: Any) -> Any:
308 """Set attribute
310 This provides read-only protection for the extension properties. The
311 core set of properties have read-only protection via the use of the
312 python ``property``.
313 """
314 if hasattr(self, "extensions") and name.startswith("ext_") and name[4:] in self.extensions:
315 raise AttributeError(f"Attribute {name} is read-only")
316 return super().__setattr__(name, value)
318 @classmethod
319 def _is_property_ok(cls, definition: PropertyDefinition, value: Any) -> bool:
320 """Compare the supplied value against the expected type as defined
321 for the corresponding property.
323 Parameters
324 ----------
325 definition : `PropertyDefinition`
326 Property definition.
327 value : `object`
328 Value of the property to validate.
330 Returns
331 -------
332 is_ok : `bool`
333 `True` if the value is of an appropriate type.
335 Notes
336 -----
337 Currently only the type of the property is validated. There is no
338 attempt to check bounds or determine that a Quantity is compatible
339 with the property.
340 """
341 if value is None:
342 return True
344 # For AltAz coordinates, they can either arrive as AltAz or
345 # as SkyCoord(frame=AltAz) so try to find the frame inside
346 # the SkyCoord.
347 if issubclass(definition.py_type, AltAz) and isinstance(value, SkyCoord):
348 value = value.frame
350 if not isinstance(value, definition.py_type):
351 return False
353 return True
355 @property
356 def cards_used(self) -> frozenset[str]:
357 """Header cards used for the translation.
359 Returns
360 -------
361 used : `frozenset` of `str`
362 Set of card used.
363 """
364 if not self._translator:
365 return frozenset()
366 return self._translator.cards_used()
368 def stripped_header(self) -> MutableMapping[str, Any]:
369 """Return a copy of the supplied header with used keywords removed.
371 Returns
372 -------
373 stripped : `dict`-like
374 Same class as header supplied to constructor, but with the
375 headers used to calculate the generic information removed.
376 """
377 hdr = copy.copy(self._header)
378 used = self.cards_used
379 for c in used:
380 del hdr[c]
381 return hdr
383 def __str__(self) -> str:
384 # Put more interesting answers at front of list
385 # and then do remainder
386 priority = ("instrument", "telescope", "datetime_begin")
387 properties = sorted(set(self.all_properties) - set(priority))
389 result = ""
390 for p in itertools.chain(priority, properties):
391 value = getattr(self, p)
392 if isinstance(value, astropy.time.Time):
393 value.format = "isot"
394 value = str(value.value)
395 result += f"{p}: {value}\n"
397 return result
399 def __eq__(self, other: Any) -> bool:
400 """Compares equal if standard properties are equal"""
401 if not isinstance(other, ObservationInfo):
402 return NotImplemented
404 # Compare simplified forms.
405 # Cannot compare directly because nan will not equate as equal
406 # whereas they should be equal for our purposes
407 self_simple = self.to_simple()
408 other_simple = other.to_simple()
410 # We don't care about the translator internal detail
411 self_simple.pop("_translator", None)
412 other_simple.pop("_translator", None)
414 for k, self_value in self_simple.items():
415 other_value = other_simple[k]
416 if self_value != other_value:
417 if math.isnan(self_value) and math.isnan(other_value):
418 # If both are nan this is fine
419 continue
420 return False
421 return True
423 def __lt__(self, other: Any) -> bool:
424 if not isinstance(other, ObservationInfo):
425 return NotImplemented
426 return self.datetime_begin < other.datetime_begin
428 def __gt__(self, other: Any) -> bool:
429 if not isinstance(other, ObservationInfo):
430 return NotImplemented
431 return self.datetime_begin > other.datetime_begin
433 def __getstate__(self) -> tuple[Any, ...]:
434 """Get pickleable state
436 Returns the properties. Deliberately does not preserve the full
437 current state; in particular, does not return the full header or
438 translator.
440 Returns
441 -------
442 state : `tuple`
443 Pickled state.
444 """
445 state = dict()
446 for p in self.all_properties:
447 state[p] = getattr(self, p)
449 return state, self.extensions
451 def __setstate__(self, state: tuple[Any, ...]) -> None:
452 """Set object state from pickle
454 Parameters
455 ----------
456 state : `tuple`
457 Pickled state.
458 """
459 try:
460 state, extensions = state
461 except ValueError:
462 # Backwards compatibility for pickles generated before DM-34175
463 extensions = {}
464 self._declare_extensions(extensions)
465 for p in self.all_properties:
466 if p.startswith("ext_"):
467 # allows setting even write-protected extensions
468 super().__setattr__(p, state[p]) # type: ignore
469 else:
470 property = f"_{p}"
471 setattr(self, property, state[p]) # type: ignore
473 def to_simple(self) -> MutableMapping[str, Any]:
474 """Convert the contents of this object to simple dict form.
476 The keys of the dict are the standard properties but the values
477 can be simplified to support JSON serialization. For example a
478 SkyCoord might be represented as an ICRS RA/Dec tuple rather than
479 a full SkyCoord representation.
481 Any properties with `None` value will be skipped.
483 Can be converted back to an `ObservationInfo` using `from_simple()`.
485 Returns
486 -------
487 simple : `dict` of [`str`, `Any`]
488 Simple dict of all properties.
490 Notes
491 -----
492 Round-tripping of extension properties requires that the
493 `ObservationInfo` was created with the help of a registered
494 `MetadataTranslator` (which contains the extension property
495 definitions).
496 """
497 simple = {}
498 if hasattr(self, "_translator") and self._translator and self._translator.name:
499 simple["_translator"] = self._translator.name
501 for p in self.all_properties:
502 property = f"_{p}" if not p.startswith("ext_") else p
503 value = getattr(self, property)
504 if value is None:
505 continue
507 # Access the function to simplify the property
508 simplifier = self.all_properties[p].to_simple
510 if simplifier is None:
511 simple[p] = value
512 continue
514 simple[p] = simplifier(value)
516 return simple
518 def to_json(self) -> str:
519 """Serialize the object to JSON string.
521 Returns
522 -------
523 j : `str`
524 The properties of the ObservationInfo in JSON string form.
526 Notes
527 -----
528 Round-tripping of extension properties requires that the
529 `ObservationInfo` was created with the help of a registered
530 `MetadataTranslator` (which contains the extension property
531 definitions).
532 """
533 return json.dumps(self.to_simple())
535 @classmethod
536 def from_simple(cls, simple: MutableMapping[str, Any]) -> ObservationInfo:
537 """Convert the entity returned by `to_simple` back into an
538 `ObservationInfo`.
540 Parameters
541 ----------
542 simple : `dict` [`str`, `Any`]
543 The dict returned by `to_simple()`
545 Returns
546 -------
547 obsinfo : `ObservationInfo`
548 New object constructed from the dict.
550 Notes
551 -----
552 Round-tripping of extension properties requires that the
553 `ObservationInfo` was created with the help of a registered
554 `MetadataTranslator` (which contains the extension property
555 definitions).
556 """
557 extensions = {}
558 translator = simple.pop("_translator", None)
559 if translator:
560 if translator not in MetadataTranslator.translators:
561 raise KeyError(f"Unrecognised translator: {translator}")
562 extensions = MetadataTranslator.translators[translator].extensions
564 properties = cls._get_all_properties(extensions)
566 processed: dict[str, Any] = {}
567 for k, v in simple.items():
568 if v is None:
569 continue
571 # Access the function to convert from simple form
572 complexifier = properties[k].from_simple
574 if complexifier is not None:
575 v = complexifier(v, **processed)
577 processed[k] = v
579 return cls.makeObservationInfo(extensions=extensions, **processed)
581 @classmethod
582 def from_json(cls, json_str: str) -> ObservationInfo:
583 """Create `ObservationInfo` from JSON string.
585 Parameters
586 ----------
587 json_str : `str`
588 The JSON representation.
590 Returns
591 -------
592 obsinfo : `ObservationInfo`
593 Reconstructed object.
595 Notes
596 -----
597 Round-tripping of extension properties requires that the
598 `ObservationInfo` was created with the help of a registered
599 `MetadataTranslator` (which contains the extension property
600 definitions).
601 """
602 simple = json.loads(json_str)
603 return cls.from_simple(simple)
605 @classmethod
606 def makeObservationInfo( # noqa: N802
607 cls, *, extensions: dict[str, PropertyDefinition] | None = None, **kwargs: Any
608 ) -> ObservationInfo:
609 """Construct an `ObservationInfo` from the supplied parameters.
611 Parameters
612 ----------
613 extensions : `dict` [`str`: `PropertyDefinition`], optional
614 Optional extension definitions, indexed by extension name (without
615 the ``ext_`` prefix, which will be added by `ObservationInfo`).
616 **kwargs
617 Name-value pairs for any properties to be set. In the case of
618 extension properties, the names should include the ``ext_`` prefix.
620 Notes
621 -----
622 The supplied parameters should use names matching the property.
623 The type of the supplied value will be checked against the property.
624 Any properties not supplied will be assigned a value of `None`.
626 Raises
627 ------
628 KeyError
629 Raised if a supplied parameter key is not a known property.
630 TypeError
631 Raised if a supplied value does not match the expected type
632 of the property.
633 """
635 obsinfo = cls(None)
636 obsinfo._declare_extensions(extensions)
638 unused = set(kwargs)
640 for p in obsinfo.all_properties:
641 if p in kwargs:
642 property = f"_{p}" if not p.startswith("ext_") else p
643 value = kwargs[p]
644 definition = obsinfo.all_properties[p]
645 if not cls._is_property_ok(definition, value):
646 raise TypeError(
647 f"Supplied value {value} for property {p} "
648 f"should be of class {definition.str_type} not {value.__class__}"
649 )
650 super(cls, obsinfo).__setattr__(property, value) # allows setting write-protected extensions
651 unused.remove(p)
653 # Recent additions to ObservationInfo may not be present in
654 # serializations. In theory they can be derived from other
655 # values in the default case. This might not be the right thing
656 # to do.
657 for k in ("group_counter_start", "group_counter_end"):
658 if k not in kwargs and "observation_counter" in kwargs:
659 super(cls, obsinfo).__setattr__(f"_{k}", obsinfo.observation_counter)
660 if (k := "has_simulated_content") not in kwargs:
661 super(cls, obsinfo).__setattr__(f"_{k}", False)
663 if unused:
664 n = len(unused)
665 raise KeyError(f"Unrecognized propert{'y' if n == 1 else 'ies'} provided: {', '.join(unused)}")
667 return obsinfo
670# Method to add the standard properties
671def _make_property(property: str, doc: str, return_typedoc: str, return_type: type) -> Callable:
672 """Create a getter method with associated docstring.
674 Parameters
675 ----------
676 property : `str`
677 Name of the property getter to be created.
678 doc : `str`
679 Description of this property.
680 return_typedoc : `str`
681 Type string of this property (used in the doc string).
682 return_type : `class`
683 Type of this property.
685 Returns
686 -------
687 p : `function`
688 Getter method for this property.
689 """
691 def getter(self: ObservationInfo) -> Any:
692 return getattr(self, f"_{property}")
694 getter.__doc__ = f"""{doc}
696 Returns
697 -------
698 {property} : `{return_typedoc}`
699 Access the property.
700 """
701 return getter
704# Set up the core set of properties
705# In order to provide read-only protection, each attribute is hidden behind a
706# python "property" wrapper.
707for name, definition in PROPERTIES.items():
708 setattr(ObservationInfo, f"_{name}", None)
709 setattr(
710 ObservationInfo,
711 name,
712 property(_make_property(name, definition.doc, definition.str_type, definition.py_type)),
713 )
716def makeObservationInfo( # noqa: N802
717 *, extensions: dict[str, PropertyDefinition] | None = None, **kwargs: Any
718) -> ObservationInfo:
719 """Construct an `ObservationInfo` from the supplied parameters.
721 Parameters
722 ----------
723 extensions : `dict` [`str`: `PropertyDefinition`], optional
724 Optional extension definitions, indexed by extension name (without
725 the ``ext_`` prefix, which will be added by `ObservationInfo`).
726 **kwargs
727 Name-value pairs for any properties to be set. In the case of
728 extension properties, the names should include the ``ext_`` prefix.
730 Notes
731 -----
732 The supplied parameters should use names matching the property.
733 The type of the supplied value will be checked against the property.
734 Any properties not supplied will be assigned a value of `None`.
736 Raises
737 ------
738 KeyError
739 Raised if a supplied parameter key is not a known property.
740 TypeError
741 Raised if a supplied value does not match the expected type
742 of the property.
743 """
744 return ObservationInfo.makeObservationInfo(extensions=extensions, **kwargs)