Coverage for python/lsst/obs/lsst/translators/lsst.py : 19%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is currently part of obs_lsst but is written to allow it
2# to be migrated to the astro_metadata_translator package at a later date.
3#
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the LICENSE file in this directory for details of code ownership.
7#
8# Use of this source code is governed by a 3-clause BSD-style
9# license that can be found in the LICENSE file.
11"""Metadata translation support code for LSST headers"""
13__all__ = ("TZERO", "SIMONYI_LOCATION", "read_detector_ids",
14 "compute_detector_exposure_id_generic", "LsstBaseTranslator",
15 "SIMONYI_TELESCOPE")
17import os.path
18import yaml
19import logging
20import re
21import datetime
22import hashlib
24import astropy.coordinates
25import astropy.units as u
26from astropy.time import Time, TimeDelta
27from astropy.coordinates import EarthLocation
29from lsst.utils import getPackageDir
31from astro_metadata_translator import cache_translation, FitsTranslator
32from astro_metadata_translator.translators.helpers import tracking_from_degree_headers, \
33 altaz_from_degree_headers
36TZERO = Time("2015-01-01T00:00", format="isot", scale="utc")
37TZERO_DATETIME = TZERO.to_datetime()
39# Delimiter to use for multiple filters/gratings
40FILTER_DELIMITER = "~"
42# Regex to use for parsing a GROUPID string
43GROUP_RE = re.compile(r"^(\d\d\d\d\-\d\d\-\d\dT\d\d:\d\d:\d\d)\.(\d\d\d)(?:[\+#](\d+))?$")
45# LSST Default location in the absence of headers
46SIMONYI_LOCATION = EarthLocation.from_geodetic(-70.749417, -30.244639, 2663.0)
48# Name of the main survey telescope
49SIMONYI_TELESCOPE = "Simonyi Survey Telescope"
51obs_lsst_packageDir = getPackageDir("obs_lsst")
53log = logging.getLogger(__name__)
56def read_detector_ids(policyFile):
57 """Read a camera policy file and retrieve the mapping from CCD name
58 to ID.
60 Parameters
61 ----------
62 policyFile : `str`
63 Name of YAML policy file to read, relative to the obs_lsst
64 package.
66 Returns
67 -------
68 mapping : `dict` of `str` to (`int`, `str`)
69 A `dict` with keys being the full names of the detectors, and the
70 value is a `tuple` containing the integer detector number and the
71 detector serial number.
73 Notes
74 -----
75 Reads the camera YAML definition file directly and extracts just the
76 IDs and serials. This routine does not use the standard
77 `~lsst.obs.base.yamlCamera.YAMLCamera` infrastructure or
78 `lsst.afw.cameraGeom`. This is because the translators are intended to
79 have minimal dependencies on LSST infrastructure.
80 """
82 file = os.path.join(obs_lsst_packageDir, policyFile)
83 try:
84 with open(file) as fh:
85 # Use the fast parser since these files are large
86 camera = yaml.load(fh, Loader=yaml.CSafeLoader)
87 except OSError as e:
88 raise ValueError(f"Could not load camera policy file {file}") from e
90 mapping = {}
91 for ccd, value in camera["CCDs"].items():
92 mapping[ccd] = (int(value["id"]), value["serial"])
94 return mapping
97def compute_detector_exposure_id_generic(exposure_id, detector_num, max_num=1000, mode="concat"):
98 """Compute the detector_exposure_id from the exposure id and the
99 detector number.
101 Parameters
102 ----------
103 exposure_id : `int`
104 The exposure ID.
105 detector_num : `int`
106 The detector number.
107 max_num : `int`, optional
108 Maximum number of detectors to make space for. Defaults to 1000.
109 mode : `str`, optional
110 Computation mode. Defaults to "concat".
111 - concat : Concatenate the exposure ID and detector number, making
112 sure that there is space for max_num and zero padding.
113 - multiply : Multiply the exposure ID by the maximum detector
114 number and add the detector number.
116 Returns
117 -------
118 detector_exposure_id : `int`
119 Computed ID.
121 Raises
122 ------
123 ValueError
124 The detector number is out of range.
125 """
127 if detector_num is None:
128 raise ValueError("Detector number must be defined.")
129 if detector_num > max_num or detector_num < 0:
130 raise ValueError(f"Detector number out of range 0 <= {detector_num} <= {max_num}")
132 if mode == "concat":
133 npad = len(str(max_num))
134 return int(f"{exposure_id}{detector_num:0{npad}d}")
135 elif mode == "multiply":
136 return max_num*exposure_id + detector_num
137 else:
138 raise ValueError(f"Computation mode of '{mode}' is not understood")
141class LsstBaseTranslator(FitsTranslator):
142 """Translation methods useful for all LSST-style headers."""
144 _const_map = {}
145 _trivial_map = {}
147 # Do not specify a name for this translator
148 cameraPolicyFile = None
149 """Path to policy file relative to obs_lsst root."""
151 detectorMapping = None
152 """Mapping of detector name to detector number and serial."""
154 detectorSerials = None
155 """Mapping of detector serial number to raft, number, and name."""
157 DETECTOR_MAX = 999
158 """Maximum number of detectors to use when calculating the
159 detector_exposure_id."""
161 _DEFAULT_LOCATION = SIMONYI_LOCATION
162 """Default telescope location in absence of relevant FITS headers."""
164 _ROLLOVER_TIME = TimeDelta(12*60*60, scale="tai", format="sec")
165 """Time delta for the definition of a Rubin Observatory start of day.
166 Used when the header is missing. See LSE-400 for details."""
168 @classmethod
169 def __init_subclass__(cls, **kwargs):
170 """Ensure that subclasses clear their own detector mapping entries
171 such that subclasses of translators that use detector mappings
172 do not pick up the incorrect values from a parent."""
174 cls.detectorMapping = None
175 cls.detectorSerials = None
177 super().__init_subclass__(**kwargs)
179 def search_paths(self):
180 """Search paths to use for LSST data when looking for header correction
181 files.
183 Returns
184 -------
185 path : `list`
186 List with a single element containing the full path to the
187 ``corrections`` directory within the ``obs_lsst`` package.
188 """
189 return [os.path.join(obs_lsst_packageDir, "corrections")]
191 @classmethod
192 def compute_detector_exposure_id(cls, exposure_id, detector_num):
193 """Compute the detector exposure ID from detector number and
194 exposure ID.
196 This is a helper method to allow code working outside the translator
197 infrastructure to use the same algorithm.
199 Parameters
200 ----------
201 exposure_id : `int`
202 Unique exposure ID.
203 detector_num : `int`
204 Detector number.
206 Returns
207 -------
208 detector_exposure_id : `int`
209 The calculated ID.
210 """
211 return compute_detector_exposure_id_generic(exposure_id, detector_num,
212 max_num=cls.DETECTOR_MAX,
213 mode="concat")
215 @classmethod
216 def max_detector_exposure_id(cls):
217 """The maximum detector exposure ID expected to be generated by
218 this instrument.
220 Returns
221 -------
222 max_id : `int`
223 The maximum value.
224 """
225 max_exposure_id = cls.max_exposure_id()
226 return cls.compute_detector_exposure_id(max_exposure_id, cls.DETECTOR_MAX)
228 @classmethod
229 def max_exposure_id(cls):
230 """The maximum exposure ID expected from this instrument.
232 Returns
233 -------
234 max_exposure_id : `int`
235 The maximum value.
236 """
237 max_date = "2050-12-31T23:59.999"
238 max_seqnum = 99_999
239 max_controller = "C" # This controller triggers the largest numbers
240 return cls.compute_exposure_id(max_date, max_seqnum, max_controller)
242 @classmethod
243 def detector_mapping(cls):
244 """Returns the mapping of full name to detector ID and serial.
246 Returns
247 -------
248 mapping : `dict` of `str`:`tuple`
249 Returns the mapping of full detector name (group+detector)
250 to detector number and serial.
252 Raises
253 ------
254 ValueError
255 Raised if no camera policy file has been registered with this
256 translation class.
258 Notes
259 -----
260 Will construct the mapping if none has previously been constructed.
261 """
262 if cls.cameraPolicyFile is not None:
263 if cls.detectorMapping is None:
264 cls.detectorMapping = read_detector_ids(cls.cameraPolicyFile)
265 else:
266 raise ValueError(f"Translation class '{cls.__name__}' has no registered camera policy file")
268 return cls.detectorMapping
270 @classmethod
271 def detector_serials(cls):
272 """Obtain the mapping of detector serial to detector group, name,
273 and number.
275 Returns
276 -------
277 info : `dict` of `tuple` of (`str`, `str`, `int`)
278 A `dict` with the serial numbers as keys and values of detector
279 group, name, and number.
280 """
281 if cls.detectorSerials is None:
282 detector_mapping = cls.detector_mapping()
284 if detector_mapping is not None:
285 # Form mapping to go from serial number to names/numbers
286 serials = {}
287 for fullname, (id, serial) in cls.detectorMapping.items():
288 raft, detector_name = fullname.split("_")
289 if serial in serials:
290 raise RuntimeError(f"Serial {serial} is defined in multiple places")
291 serials[serial] = (raft, detector_name, id)
292 cls.detectorSerials = serials
293 else:
294 raise RuntimeError("Unable to obtain detector mapping information")
296 return cls.detectorSerials
298 @classmethod
299 def compute_detector_num_from_name(cls, detector_group, detector_name):
300 """Helper method to return the detector number from the name.
302 Parameters
303 ----------
304 detector_group : `str`
305 Name of the detector grouping. This is generally the raft name.
306 detector_name : `str`
307 Detector name.
309 Returns
310 -------
311 num : `int`
312 Detector number.
313 """
314 fullname = f"{detector_group}_{detector_name}"
316 num = None
317 detector_mapping = cls.detector_mapping()
318 if detector_mapping is None:
319 raise RuntimeError("Unable to obtain detector mapping information")
321 if fullname in detector_mapping:
322 num = detector_mapping[fullname]
323 else:
324 log.warning(f"Unable to determine detector number from detector name {fullname}")
325 return None
327 return num[0]
329 @classmethod
330 def compute_detector_info_from_serial(cls, detector_serial):
331 """Helper method to return the detector information from the serial.
333 Parameters
334 ----------
335 detector_serial : `str`
336 Detector serial ID.
338 Returns
339 -------
340 info : `tuple` of (`str`, `str`, `int`)
341 Detector group, name, and number.
342 """
343 serial_mapping = cls.detector_serials()
344 if serial_mapping is None:
345 raise RuntimeError("Unable to obtain serial mapping information")
347 if detector_serial in serial_mapping:
348 info = serial_mapping[detector_serial]
349 else:
350 raise RuntimeError("Unable to determine detector information from detector serial"
351 f" {detector_serial}")
353 return info
355 @staticmethod
356 def compute_exposure_id(dayobs, seqnum, controller=None):
357 """Helper method to calculate the exposure_id.
359 Parameters
360 ----------
361 dayobs : `str`
362 Day of observation in either YYYYMMDD or YYYY-MM-DD format.
363 If the string looks like ISO format it will be truncated before the
364 ``T`` before being handled.
365 seqnum : `int` or `str`
366 Sequence number.
367 controller : `str`, optional
368 Controller to use. If this is "O", no change is made to the
369 exposure ID. If it is "C" a 1000 is added to the year component
370 of the exposure ID.
371 `None` indicates that the controller is not relevant to the
372 exposure ID calculation (generally this is the case for test
373 stand data).
375 Returns
376 -------
377 exposure_id : `int`
378 Exposure ID in form YYYYMMDDnnnnn form.
379 """
380 if "T" in dayobs:
381 dayobs = dayobs[:dayobs.find("T")]
383 dayobs = dayobs.replace("-", "")
385 if len(dayobs) != 8:
386 raise ValueError(f"Malformed dayobs: {dayobs}")
388 # Expect no more than 99,999 exposures in a day
389 maxdigits = 5
390 if seqnum >= 10**maxdigits:
391 raise ValueError(f"Sequence number ({seqnum}) exceeds limit")
393 # Camera control changes the exposure ID
394 if controller is not None:
395 if controller == "O":
396 pass
397 elif controller == "C":
398 # Add 1000 to the year component
399 dayobs = int(dayobs)
400 dayobs += 1000_00_00
401 else:
402 raise ValueError(f"Supplied controller, '{controller}' is neither 'O' nor 'C'")
404 # Form the number as a string zero padding the sequence number
405 idstr = f"{dayobs}{seqnum:0{maxdigits}d}"
407 # Exposure ID has to be an integer
408 return int(idstr)
410 def _is_on_mountain(self):
411 """Indicate whether these data are coming from the instrument
412 installed on the mountain.
414 Returns
415 -------
416 is : `bool`
417 `True` if instrument is on the mountain.
418 """
419 if "TSTAND" in self._header:
420 return False
421 return True
423 def is_on_sky(self):
424 """Determine if this is an on-sky observation.
426 Returns
427 -------
428 is_on_sky : `bool`
429 Returns True if this is a observation on sky on the
430 summit.
431 """
432 # For LSST we think on sky unless tracksys is local
433 if self.is_key_ok("TRACKSYS"):
434 if self._header["TRACKSYS"].lower() == "local":
435 # not on sky
436 return False
438 # These are obviously not on sky
439 if self.to_observation_type() in ("bias", "dark", "flat"):
440 return False
442 return self._is_on_mountain()
444 @cache_translation
445 def to_location(self):
446 # Docstring will be inherited. Property defined in properties.py
447 if not self._is_on_mountain():
448 return None
449 try:
450 # Try standard FITS headers
451 return super().to_location()
452 except KeyError:
453 return self._DEFAULT_LOCATION
455 @cache_translation
456 def to_datetime_begin(self):
457 # Docstring will be inherited. Property defined in properties.py
458 self._used_these_cards("MJD-OBS")
459 return Time(self._header["MJD-OBS"], scale="tai", format="mjd")
461 @cache_translation
462 def to_datetime_end(self):
463 # Docstring will be inherited. Property defined in properties.py
464 if self.is_key_ok("DATE-END"):
465 return super().to_datetime_end()
467 return self.to_datetime_begin() + self.to_exposure_time()
469 @cache_translation
470 def to_detector_num(self):
471 # Docstring will be inherited. Property defined in properties.py
472 raft = self.to_detector_group()
473 detector = self.to_detector_name()
474 return self.compute_detector_num_from_name(raft, detector)
476 @cache_translation
477 def to_detector_exposure_id(self):
478 # Docstring will be inherited. Property defined in properties.py
479 exposure_id = self.to_exposure_id()
480 num = self.to_detector_num()
481 return self.compute_detector_exposure_id(exposure_id, num)
483 @cache_translation
484 def to_observation_type(self):
485 # Docstring will be inherited. Property defined in properties.py
486 obstype = self._header["IMGTYPE"]
487 self._used_these_cards("IMGTYPE")
488 obstype = obstype.lower()
489 if obstype in ("skyexp", "object"):
490 obstype = "science"
491 return obstype
493 @cache_translation
494 def to_observation_reason(self):
495 # Docstring will be inherited. Property defined in properties.py
496 if self.is_key_ok("TESTTYPE"):
497 reason = self._header["TESTTYPE"]
498 self._used_these_cards("TESTTYPE")
499 return reason.lower()
500 # no specific header present so use the default translation
501 return super().to_observation_reason()
503 @cache_translation
504 def to_dark_time(self):
505 """Calculate the dark time.
507 If a DARKTIME header is not found, the value is assumed to be
508 identical to the exposure time.
510 Returns
511 -------
512 dark : `astropy.units.Quantity`
513 The dark time in seconds.
514 """
515 if self.is_key_ok("DARKTIME"):
516 darktime = self._header["DARKTIME"]*u.s
517 self._used_these_cards("DARKTIME")
518 else:
519 log.warning("%s: Unable to determine dark time. Setting from exposure time.",
520 self._log_prefix)
521 darktime = self.to_exposure_time()
522 return darktime
524 @cache_translation
525 def to_exposure_id(self):
526 """Generate a unique exposure ID number
528 This is a combination of DAYOBS and SEQNUM, and optionally
529 CONTRLLR.
531 Returns
532 -------
533 exposure_id : `int`
534 Unique exposure number.
535 """
536 if "CALIB_ID" in self._header:
537 self._used_these_cards("CALIB_ID")
538 return None
540 dayobs = self._header["DAYOBS"]
541 seqnum = self._header["SEQNUM"]
542 self._used_these_cards("DAYOBS", "SEQNUM")
544 if self.is_key_ok("CONTRLLR"):
545 controller = self._header["CONTRLLR"]
546 self._used_these_cards("CONTRLLR")
547 else:
548 controller = None
550 return self.compute_exposure_id(dayobs, seqnum, controller=controller)
552 @cache_translation
553 def to_visit_id(self):
554 """Calculate the visit associated with this exposure.
556 Notes
557 -----
558 For LATISS and LSSTCam the default visit is derived from the
559 exposure group. For other instruments we return the exposure_id.
560 """
562 exposure_group = self.to_exposure_group()
563 # If the group is an int we return it
564 try:
565 visit_id = int(exposure_group)
566 return visit_id
567 except ValueError:
568 pass
570 # A Group is defined as ISO date with an extension
571 # The integer must be the same for a given group so we can never
572 # use datetime_begin.
573 # Nominally a GROUPID looks like "ISODATE+N" where the +N is
574 # optional. This can be converted to seconds since epoch with
575 # an adjustment for N.
576 # For early data lacking that form we hash the group and return
577 # the int.
578 matches_date = GROUP_RE.match(exposure_group)
579 if matches_date:
580 iso_str = matches_date.group(1)
581 fraction = matches_date.group(2)
582 n = matches_date.group(3)
583 if n is not None:
584 n = int(n)
585 else:
586 n = 0
587 iso = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%S")
589 tdelta = iso - TZERO_DATETIME
590 epoch = int(tdelta.total_seconds())
592 # Form the integer from EPOCH + 3 DIGIT FRAC + 0-pad N
593 visit_id = int(f"{epoch}{fraction}{n:04d}")
594 else:
595 # Non-standard string so convert to numbers
596 # using a hash function. Use the first N hex digits
597 group_bytes = exposure_group.encode("us-ascii")
598 hasher = hashlib.blake2b(group_bytes)
599 # Need to be big enough it does not possibly clash with the
600 # date-based version above
601 digest = hasher.hexdigest()[:14]
602 visit_id = int(digest, base=16)
604 # To help with hash collision, append the string length
605 visit_id = int(f"{visit_id}{len(exposure_group):02d}")
607 return visit_id
609 @cache_translation
610 def to_physical_filter(self):
611 """Calculate the physical filter name.
613 Returns
614 -------
615 filter : `str`
616 Name of filter. Can be a combination of FILTER, FILTER1 and FILTER2
617 headers joined by a "~". Returns "unknown" if no filter is declared
618 """
619 joined = self._join_keyword_values(["FILTER", "FILTER1", "FILTER2"], delim=FILTER_DELIMITER)
620 if not joined:
621 joined = "unknown"
623 return joined
625 @cache_translation
626 def to_tracking_radec(self):
627 if not self.is_on_sky():
628 return None
630 # RA/DEC are *derived* headers and for the case where the DATE-BEG
631 # is 1970 they are garbage and should not be used.
632 if self._header["DATE-OBS"] == self._header["DATE"]:
633 # A fixed up date -- use AZEL as source of truth
634 altaz = self.to_altaz_begin()
635 radec = astropy.coordinates.SkyCoord(altaz.transform_to(astropy.coordinates.ICRS),
636 obstime=altaz.obstime,
637 location=altaz.location)
638 else:
639 radecsys = ("RADESYS",)
640 radecpairs = (("RASTART", "DECSTART"), ("RA", "DEC"))
641 radec = tracking_from_degree_headers(self, radecsys, radecpairs)
643 return radec
645 @cache_translation
646 def to_altaz_begin(self):
647 if not self._is_on_mountain():
648 return None
650 # ALTAZ always relevant unless bias or dark
651 if self.to_observation_type() in ("bias", "dark"):
652 return None
654 return altaz_from_degree_headers(self, (("ELSTART", "AZSTART"),),
655 self.to_datetime_begin(), is_zd=False)
657 @cache_translation
658 def to_exposure_group(self):
659 """Calculate the exposure group string.
661 For LSSTCam and LATISS this is read from the ``GROUPID`` header.
662 If that header is missing the exposure_id is returned instead as
663 a string.
664 """
665 if self.is_key_ok("GROUPID"):
666 exposure_group = self._header["GROUPID"]
667 self._used_these_cards("GROUPID")
668 return exposure_group
669 return super().to_exposure_group()
671 @staticmethod
672 def _is_filter_empty(filter):
673 """Return true if the supplied filter indicates an empty filter slot
675 Parameters
676 ----------
677 filter : `str`
678 The filter string to check.
680 Returns
681 -------
682 is_empty : `bool`
683 `True` if the filter string looks like it is referring to an
684 empty filter slot. For example this can be if the filter is
685 "empty" or "empty_2".
686 """
687 return bool(re.match(r"empty_?\d*$", filter.lower()))
689 def _determine_primary_filter(self):
690 """Determine the primary filter from the ``FILTER`` header.
692 Returns
693 -------
694 filter : `str`
695 The contents of the ``FILTER`` header with some appropriate
696 defaulting.
697 """
699 if self.is_key_ok("FILTER"):
700 physical_filter = self._header["FILTER"]
701 self._used_these_cards("FILTER")
703 if self._is_filter_empty(physical_filter):
704 physical_filter = "empty"
705 else:
706 # Be explicit about having no knowledge of the filter
707 # by setting it to "unknown". It should always have a value.
708 physical_filter = "unknown"
710 # Warn if the filter being unknown is important
711 obstype = self.to_observation_type()
712 if obstype not in ("bias", "dark"):
713 log.warning("%s: Unable to determine the filter",
714 self._log_prefix)
716 return physical_filter
718 @cache_translation
719 def to_observing_day(self):
720 """Return the day of observation as YYYYMMDD integer.
722 For LSSTCam and other compliant instruments this is the value
723 of the DAYOBS header.
725 Returns
726 -------
727 obs_day : `int`
728 The day of observation.
729 """
730 if self.is_key_ok("DAYOBS"):
731 self._used_these_cards("DAYOBS")
732 return int(self._header["DAYOBS"])
734 # Calculate it ourselves correcting for the Rubin offset
735 date = self.to_datetime_begin().tai
736 date -= self._ROLLOVER_TIME
737 return int(date.strftime("%Y%m%d"))
739 @cache_translation
740 def to_observation_counter(self):
741 """Return the sequence number within the observing day.
743 Returns
744 -------
745 counter : `int`
746 The sequence number for this day.
747 """
748 if self.is_key_ok("SEQNUM"):
749 # Some older LATISS data may not have the header
750 # but this is corrected in fix_header for LATISS.
751 self._used_these_cards("SEQNUM")
752 return int(self._header["SEQNUM"])
754 # This indicates a problem so we warn and return a 0
755 log.warning("%s: Unable to determine the observation counter so returning 0",
756 self._log_prefix)
757 return 0