Coverage for python/lsst/obs/lsst/translators/lsst.py : 20%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is currently part of obs_lsst but is written to allow it
2# to be migrated to the astro_metadata_translator package at a later date.
3#
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the LICENSE file in this directory for details of code ownership.
7#
8# Use of this source code is governed by a 3-clause BSD-style
9# license that can be found in the LICENSE file.
11"""Metadata translation support code for LSST headers"""
13__all__ = ("ROLLOVERTIME", "TZERO", "SIMONYI_LOCATION", "read_detector_ids",
14 "compute_detector_exposure_id_generic", "LsstBaseTranslator",
15 "SIMONYI_TELESCOPE")
17import os.path
18import yaml
19import logging
20import re
21import datetime
22import hashlib
24import astropy.coordinates
25import astropy.units as u
26from astropy.time import Time, TimeDelta
27from astropy.coordinates import EarthLocation
29from lsst.utils import getPackageDir
31from astro_metadata_translator import cache_translation, FitsTranslator
32from astro_metadata_translator.translators.helpers import tracking_from_degree_headers, \
33 altaz_from_degree_headers
35# LSST day clock starts at UTC+8
36ROLLOVERTIME = TimeDelta(8*60*60, scale="tai", format="sec")
37TZERO = Time("2015-01-01T00:00", format="isot", scale="utc")
38TZERO_DATETIME = TZERO.to_datetime()
40# Delimiter to use for multiple filters/gratings
41FILTER_DELIMITER = "~"
43# Regex to use for parsing a GROUPID string
44GROUP_RE = re.compile(r"^(\d\d\d\d\-\d\d\-\d\dT\d\d:\d\d:\d\d)\.(\d\d\d)(?:[\+#](\d+))?$")
46# LSST Default location in the absence of headers
47SIMONYI_LOCATION = EarthLocation.from_geodetic(-70.749417, -30.244639, 2663.0)
49# Name of the main survey telescope
50SIMONYI_TELESCOPE = "Simonyi Survey Telescope"
52obs_lsst_packageDir = getPackageDir("obs_lsst")
54log = logging.getLogger(__name__)
57def read_detector_ids(policyFile):
58 """Read a camera policy file and retrieve the mapping from CCD name
59 to ID.
61 Parameters
62 ----------
63 policyFile : `str`
64 Name of YAML policy file to read, relative to the obs_lsst
65 package.
67 Returns
68 -------
69 mapping : `dict` of `str` to (`int`, `str`)
70 A `dict` with keys being the full names of the detectors, and the
71 value is a `tuple` containing the integer detector number and the
72 detector serial number.
74 Notes
75 -----
76 Reads the camera YAML definition file directly and extracts just the
77 IDs and serials. This routine does not use the standard
78 `~lsst.obs.base.yamlCamera.YAMLCamera` infrastructure or
79 `lsst.afw.cameraGeom`. This is because the translators are intended to
80 have minimal dependencies on LSST infrastructure.
81 """
83 file = os.path.join(obs_lsst_packageDir, policyFile)
84 try:
85 with open(file) as fh:
86 # Use the fast parser since these files are large
87 camera = yaml.load(fh, Loader=yaml.CSafeLoader)
88 except OSError as e:
89 raise ValueError(f"Could not load camera policy file {file}") from e
91 mapping = {}
92 for ccd, value in camera["CCDs"].items():
93 mapping[ccd] = (int(value["id"]), value["serial"])
95 return mapping
98def compute_detector_exposure_id_generic(exposure_id, detector_num, max_num=1000, mode="concat"):
99 """Compute the detector_exposure_id from the exposure id and the
100 detector number.
102 Parameters
103 ----------
104 exposure_id : `int`
105 The exposure ID.
106 detector_num : `int`
107 The detector number.
108 max_num : `int`, optional
109 Maximum number of detectors to make space for. Defaults to 1000.
110 mode : `str`, optional
111 Computation mode. Defaults to "concat".
112 - concat : Concatenate the exposure ID and detector number, making
113 sure that there is space for max_num and zero padding.
114 - multiply : Multiply the exposure ID by the maximum detector
115 number and add the detector number.
117 Returns
118 -------
119 detector_exposure_id : `int`
120 Computed ID.
122 Raises
123 ------
124 ValueError
125 The detector number is out of range.
126 """
128 if detector_num is None:
129 raise ValueError("Detector number must be defined.")
130 if detector_num > max_num or detector_num < 0:
131 raise ValueError(f"Detector number out of range 0 <= {detector_num} <= {max_num}")
133 if mode == "concat":
134 npad = len(str(max_num))
135 return int(f"{exposure_id}{detector_num:0{npad}d}")
136 elif mode == "multiply":
137 return max_num*exposure_id + detector_num
138 else:
139 raise ValueError(f"Computation mode of '{mode}' is not understood")
142class LsstBaseTranslator(FitsTranslator):
143 """Translation methods useful for all LSST-style headers."""
145 _const_map = {}
146 _trivial_map = {}
148 # Do not specify a name for this translator
149 cameraPolicyFile = None
150 """Path to policy file relative to obs_lsst root."""
152 detectorMapping = None
153 """Mapping of detector name to detector number and serial."""
155 detectorSerials = None
156 """Mapping of detector serial number to raft, number, and name."""
158 DETECTOR_MAX = 999
159 """Maximum number of detectors to use when calculating the
160 detector_exposure_id."""
162 _DEFAULT_LOCATION = SIMONYI_LOCATION
163 """Default telescope location in absence of relevant FITS headers."""
165 @classmethod
166 def __init_subclass__(cls, **kwargs):
167 """Ensure that subclasses clear their own detector mapping entries
168 such that subclasses of translators that use detector mappings
169 do not pick up the incorrect values from a parent."""
171 cls.detectorMapping = None
172 cls.detectorSerials = None
174 super().__init_subclass__(**kwargs)
176 def search_paths(self):
177 """Search paths to use for LSST data when looking for header correction
178 files.
180 Returns
181 -------
182 path : `list`
183 List with a single element containing the full path to the
184 ``corrections`` directory within the ``obs_lsst`` package.
185 """
186 return [os.path.join(obs_lsst_packageDir, "corrections")]
188 @classmethod
189 def compute_detector_exposure_id(cls, exposure_id, detector_num):
190 """Compute the detector exposure ID from detector number and
191 exposure ID.
193 This is a helper method to allow code working outside the translator
194 infrastructure to use the same algorithm.
196 Parameters
197 ----------
198 exposure_id : `int`
199 Unique exposure ID.
200 detector_num : `int`
201 Detector number.
203 Returns
204 -------
205 detector_exposure_id : `int`
206 The calculated ID.
207 """
208 return compute_detector_exposure_id_generic(exposure_id, detector_num,
209 max_num=cls.DETECTOR_MAX,
210 mode="concat")
212 @classmethod
213 def max_detector_exposure_id(cls):
214 """The maximum detector exposure ID expected to be generated by
215 this instrument.
217 Returns
218 -------
219 max_id : `int`
220 The maximum value.
221 """
222 max_exposure_id = cls.max_exposure_id()
223 return cls.compute_detector_exposure_id(max_exposure_id, cls.DETECTOR_MAX)
225 @classmethod
226 def max_exposure_id(cls):
227 """The maximum exposure ID expected from this instrument.
229 Returns
230 -------
231 max_exposure_id : `int`
232 The maximum value.
233 """
234 max_date = "2050-12-31T23:59.999"
235 max_seqnum = 99_999
236 max_controller = "C" # This controller triggers the largest numbers
237 return cls.compute_exposure_id(max_date, max_seqnum, max_controller)
239 @classmethod
240 def detector_mapping(cls):
241 """Returns the mapping of full name to detector ID and serial.
243 Returns
244 -------
245 mapping : `dict` of `str`:`tuple`
246 Returns the mapping of full detector name (group+detector)
247 to detector number and serial.
249 Raises
250 ------
251 ValueError
252 Raised if no camera policy file has been registered with this
253 translation class.
255 Notes
256 -----
257 Will construct the mapping if none has previously been constructed.
258 """
259 if cls.cameraPolicyFile is not None:
260 if cls.detectorMapping is None:
261 cls.detectorMapping = read_detector_ids(cls.cameraPolicyFile)
262 else:
263 raise ValueError(f"Translation class '{cls.__name__}' has no registered camera policy file")
265 return cls.detectorMapping
267 @classmethod
268 def detector_serials(cls):
269 """Obtain the mapping of detector serial to detector group, name,
270 and number.
272 Returns
273 -------
274 info : `dict` of `tuple` of (`str`, `str`, `int`)
275 A `dict` with the serial numbers as keys and values of detector
276 group, name, and number.
277 """
278 if cls.detectorSerials is None:
279 detector_mapping = cls.detector_mapping()
281 if detector_mapping is not None:
282 # Form mapping to go from serial number to names/numbers
283 serials = {}
284 for fullname, (id, serial) in cls.detectorMapping.items():
285 raft, detector_name = fullname.split("_")
286 if serial in serials:
287 raise RuntimeError(f"Serial {serial} is defined in multiple places")
288 serials[serial] = (raft, detector_name, id)
289 cls.detectorSerials = serials
290 else:
291 raise RuntimeError("Unable to obtain detector mapping information")
293 return cls.detectorSerials
295 @classmethod
296 def compute_detector_num_from_name(cls, detector_group, detector_name):
297 """Helper method to return the detector number from the name.
299 Parameters
300 ----------
301 detector_group : `str`
302 Name of the detector grouping. This is generally the raft name.
303 detector_name : `str`
304 Detector name.
306 Returns
307 -------
308 num : `int`
309 Detector number.
310 """
311 fullname = f"{detector_group}_{detector_name}"
313 num = None
314 detector_mapping = cls.detector_mapping()
315 if detector_mapping is None:
316 raise RuntimeError("Unable to obtain detector mapping information")
318 if fullname in detector_mapping:
319 num = detector_mapping[fullname]
320 else:
321 log.warning(f"Unable to determine detector number from detector name {fullname}")
322 return None
324 return num[0]
326 @classmethod
327 def compute_detector_info_from_serial(cls, detector_serial):
328 """Helper method to return the detector information from the serial.
330 Parameters
331 ----------
332 detector_serial : `str`
333 Detector serial ID.
335 Returns
336 -------
337 info : `tuple` of (`str`, `str`, `int`)
338 Detector group, name, and number.
339 """
340 serial_mapping = cls.detector_serials()
341 if serial_mapping is None:
342 raise RuntimeError("Unable to obtain serial mapping information")
344 if detector_serial in serial_mapping:
345 info = serial_mapping[detector_serial]
346 else:
347 raise RuntimeError("Unable to determine detector information from detector serial"
348 f" {detector_serial}")
350 return info
352 @staticmethod
353 def compute_exposure_id(dayobs, seqnum, controller=None):
354 """Helper method to calculate the exposure_id.
356 Parameters
357 ----------
358 dayobs : `str`
359 Day of observation in either YYYYMMDD or YYYY-MM-DD format.
360 If the string looks like ISO format it will be truncated before the
361 ``T`` before being handled.
362 seqnum : `int` or `str`
363 Sequence number.
364 controller : `str`, optional
365 Controller to use. If this is "O", no change is made to the
366 exposure ID. If it is "C" a 1000 is added to the year component
367 of the exposure ID.
368 `None` indicates that the controller is not relevant to the
369 exposure ID calculation (generally this is the case for test
370 stand data).
372 Returns
373 -------
374 exposure_id : `int`
375 Exposure ID in form YYYYMMDDnnnnn form.
376 """
377 if "T" in dayobs:
378 dayobs = dayobs[:dayobs.find("T")]
380 dayobs = dayobs.replace("-", "")
382 if len(dayobs) != 8:
383 raise ValueError(f"Malformed dayobs: {dayobs}")
385 # Expect no more than 99,999 exposures in a day
386 maxdigits = 5
387 if seqnum >= 10**maxdigits:
388 raise ValueError(f"Sequence number ({seqnum}) exceeds limit")
390 # Camera control changes the exposure ID
391 if controller is not None:
392 if controller == "O":
393 pass
394 elif controller == "C":
395 # Add 1000 to the year component
396 dayobs = int(dayobs)
397 dayobs += 1000_00_00
398 else:
399 raise ValueError(f"Supplied controller, '{controller}' is neither 'O' nor 'C'")
401 # Form the number as a string zero padding the sequence number
402 idstr = f"{dayobs}{seqnum:0{maxdigits}d}"
404 # Exposure ID has to be an integer
405 return int(idstr)
407 def _is_on_mountain(self):
408 """Indicate whether these data are coming from the instrument
409 installed on the mountain.
411 Returns
412 -------
413 is : `bool`
414 `True` if instrument is on the mountain.
415 """
416 if "TSTAND" in self._header:
417 return False
418 return True
420 def is_on_sky(self):
421 """Determine if this is an on-sky observation.
423 Returns
424 -------
425 is_on_sky : `bool`
426 Returns True if this is a observation on sky on the
427 summit.
428 """
429 # For LSST we think on sky unless tracksys is local
430 if self.is_key_ok("TRACKSYS"):
431 if self._header["TRACKSYS"].lower() == "local":
432 # not on sky
433 return False
435 # These are obviously not on sky
436 if self.to_observation_type() in ("bias", "dark", "flat"):
437 return False
439 return self._is_on_mountain()
441 @cache_translation
442 def to_location(self):
443 # Docstring will be inherited. Property defined in properties.py
444 if not self._is_on_mountain():
445 return None
446 try:
447 # Try standard FITS headers
448 return super().to_location()
449 except KeyError:
450 return self._DEFAULT_LOCATION
452 @cache_translation
453 def to_datetime_begin(self):
454 # Docstring will be inherited. Property defined in properties.py
455 self._used_these_cards("MJD-OBS")
456 return Time(self._header["MJD-OBS"], scale="tai", format="mjd")
458 @cache_translation
459 def to_datetime_end(self):
460 # Docstring will be inherited. Property defined in properties.py
461 if self.is_key_ok("DATE-END"):
462 return super().to_datetime_end()
464 return self.to_datetime_begin() + self.to_exposure_time()
466 @cache_translation
467 def to_detector_num(self):
468 # Docstring will be inherited. Property defined in properties.py
469 raft = self.to_detector_group()
470 detector = self.to_detector_name()
471 return self.compute_detector_num_from_name(raft, detector)
473 @cache_translation
474 def to_detector_exposure_id(self):
475 # Docstring will be inherited. Property defined in properties.py
476 exposure_id = self.to_exposure_id()
477 num = self.to_detector_num()
478 return self.compute_detector_exposure_id(exposure_id, num)
480 @cache_translation
481 def to_observation_type(self):
482 # Docstring will be inherited. Property defined in properties.py
483 obstype = self._header["IMGTYPE"]
484 self._used_these_cards("IMGTYPE")
485 obstype = obstype.lower()
486 if obstype in ("skyexp", "object"):
487 obstype = "science"
488 return obstype
490 @cache_translation
491 def to_observation_reason(self):
492 # Docstring will be inherited. Property defined in properties.py
493 if self.is_key_ok("TESTTYPE"):
494 reason = self._header["TESTTYPE"]
495 self._used_these_cards("TESTTYPE")
496 return reason.lower()
497 # no specific header present so use the default translation
498 return super().to_observation_reason()
500 @cache_translation
501 def to_dark_time(self):
502 """Calculate the dark time.
504 If a DARKTIME header is not found, the value is assumed to be
505 identical to the exposure time.
507 Returns
508 -------
509 dark : `astropy.units.Quantity`
510 The dark time in seconds.
511 """
512 if self.is_key_ok("DARKTIME"):
513 darktime = self._header["DARKTIME"]*u.s
514 self._used_these_cards("DARKTIME")
515 else:
516 log.warning("%s: Unable to determine dark time. Setting from exposure time.",
517 self.to_observation_id())
518 darktime = self.to_exposure_time()
519 return darktime
521 @cache_translation
522 def to_exposure_id(self):
523 """Generate a unique exposure ID number
525 This is a combination of DAYOBS and SEQNUM, and optionally
526 CONTRLLR.
528 Returns
529 -------
530 exposure_id : `int`
531 Unique exposure number.
532 """
533 if "CALIB_ID" in self._header:
534 self._used_these_cards("CALIB_ID")
535 return None
537 dayobs = self._header["DAYOBS"]
538 seqnum = self._header["SEQNUM"]
539 self._used_these_cards("DAYOBS", "SEQNUM")
541 if self.is_key_ok("CONTRLLR"):
542 controller = self._header["CONTRLLR"]
543 self._used_these_cards("CONTRLLR")
544 else:
545 controller = None
547 return self.compute_exposure_id(dayobs, seqnum, controller=controller)
549 @cache_translation
550 def to_visit_id(self):
551 """Calculate the visit associated with this exposure.
553 Notes
554 -----
555 For LATISS and LSSTCam the default visit is derived from the
556 exposure group. For other instruments we return the exposure_id.
557 """
559 exposure_group = self.to_exposure_group()
560 # If the group is an int we return it
561 try:
562 visit_id = int(exposure_group)
563 return visit_id
564 except ValueError:
565 pass
567 # A Group is defined as ISO date with an extension
568 # The integer must be the same for a given group so we can never
569 # use datetime_begin.
570 # Nominally a GROUPID looks like "ISODATE+N" where the +N is
571 # optional. This can be converted to seconds since epoch with
572 # an adjustment for N.
573 # For early data lacking that form we hash the group and return
574 # the int.
575 matches_date = GROUP_RE.match(exposure_group)
576 if matches_date:
577 iso_str = matches_date.group(1)
578 fraction = matches_date.group(2)
579 n = matches_date.group(3)
580 if n is not None:
581 n = int(n)
582 else:
583 n = 0
584 iso = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%S")
586 tdelta = iso - TZERO_DATETIME
587 epoch = int(tdelta.total_seconds())
589 # Form the integer from EPOCH + 3 DIGIT FRAC + 0-pad N
590 visit_id = int(f"{epoch}{fraction}{n:04d}")
591 else:
592 # Non-standard string so convert to numbers
593 # using a hash function. Use the first N hex digits
594 group_bytes = exposure_group.encode("us-ascii")
595 hasher = hashlib.blake2b(group_bytes)
596 # Need to be big enough it does not possibly clash with the
597 # date-based version above
598 digest = hasher.hexdigest()[:14]
599 visit_id = int(digest, base=16)
601 # To help with hash collision, append the string length
602 visit_id = int(f"{visit_id}{len(exposure_group):02d}")
604 return visit_id
606 @cache_translation
607 def to_physical_filter(self):
608 """Calculate the physical filter name.
610 Returns
611 -------
612 filter : `str`
613 Name of filter. Can be a combination of FILTER, FILTER1 and FILTER2
614 headers joined by a "~". Returns "UNKNOWN" if no filter is declared
615 """
616 joined = self._join_keyword_values(["FILTER", "FILTER1", "FILTER2"], delim=FILTER_DELIMITER)
617 if not joined:
618 joined = "UNKNOWN"
620 return joined
622 @cache_translation
623 def to_tracking_radec(self):
624 if not self.is_on_sky():
625 return None
627 # RA/DEC are *derived* headers and for the case where the DATE-BEG
628 # is 1970 they are garbage and should not be used.
629 if self._header["DATE-OBS"] == self._header["DATE"]:
630 # A fixed up date -- use AZEL as source of truth
631 altaz = self.to_altaz_begin()
632 radec = astropy.coordinates.SkyCoord(altaz.transform_to(astropy.coordinates.ICRS),
633 obstime=altaz.obstime,
634 location=altaz.location)
635 else:
636 radecsys = ("RADESYS",)
637 radecpairs = (("RASTART", "DECSTART"), ("RA", "DEC"))
638 radec = tracking_from_degree_headers(self, radecsys, radecpairs)
640 return radec
642 @cache_translation
643 def to_altaz_begin(self):
644 if not self._is_on_mountain():
645 return None
647 # ALTAZ always relevant unless bias or dark
648 if self.to_observation_type() in ("bias", "dark"):
649 return None
651 return altaz_from_degree_headers(self, (("ELSTART", "AZSTART"),),
652 self.to_datetime_begin(), is_zd=False)
654 @cache_translation
655 def to_exposure_group(self):
656 """Calculate the exposure group string.
658 For LSSTCam and LATISS this is read from the ``GROUPID`` header.
659 If that header is missing the exposure_id is returned instead as
660 a string.
661 """
662 if self.is_key_ok("GROUPID"):
663 exposure_group = self._header["GROUPID"]
664 self._used_these_cards("GROUPID")
665 return exposure_group
666 return super().to_exposure_group()