Coverage for python/lsst/obs/lsst/translators/lsst.py : 20%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is currently part of obs_lsst but is written to allow it
2# to be migrated to the astro_metadata_translator package at a later date.
3#
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the LICENSE file in this directory for details of code ownership.
7#
8# Use of this source code is governed by a 3-clause BSD-style
9# license that can be found in the LICENSE file.
11"""Metadata translation support code for LSST headers"""
13__all__ = ("ROLLOVERTIME", "TZERO", "LSST_LOCATION", "read_detector_ids",
14 "compute_detector_exposure_id_generic", "LsstBaseTranslator")
16import os.path
17import yaml
18import logging
19import re
20import datetime
21import hashlib
23import astropy.coordinates
24import astropy.units as u
25from astropy.time import Time, TimeDelta
26from astropy.coordinates import EarthLocation
28from lsst.utils import getPackageDir
30from astro_metadata_translator import cache_translation, FitsTranslator
31from astro_metadata_translator.translators.helpers import tracking_from_degree_headers, \
32 altaz_from_degree_headers
34# LSST day clock starts at UTC+8
35ROLLOVERTIME = TimeDelta(8*60*60, scale="tai", format="sec")
36TZERO = Time("2015-01-01T00:00", format="isot", scale="utc")
37TZERO_DATETIME = TZERO.to_datetime()
39# Delimiter to use for multiple filters/gratings
40FILTER_DELIMITER = "~"
42# Regex to use for parsing a GROUPID string
43GROUP_RE = re.compile(r"^(\d\d\d\d\-\d\d\-\d\dT\d\d:\d\d:\d\d)\.(\d\d\d)(?:[\+#](\d+))?$")
45# LSST Default location in the absence of headers
46LSST_LOCATION = EarthLocation.from_geodetic(-70.749417, -30.244639, 2663.0)
48obs_lsst_packageDir = getPackageDir("obs_lsst")
50log = logging.getLogger(__name__)
53def read_detector_ids(policyFile):
54 """Read a camera policy file and retrieve the mapping from CCD name
55 to ID.
57 Parameters
58 ----------
59 policyFile : `str`
60 Name of YAML policy file to read, relative to the obs_lsst
61 package.
63 Returns
64 -------
65 mapping : `dict` of `str` to (`int`, `str`)
66 A `dict` with keys being the full names of the detectors, and the
67 value is a `tuple` containing the integer detector number and the
68 detector serial number.
70 Notes
71 -----
72 Reads the camera YAML definition file directly and extracts just the
73 IDs and serials. This routine does not use the standard
74 `~lsst.obs.base.yamlCamera.YAMLCamera` infrastructure or
75 `lsst.afw.cameraGeom`. This is because the translators are intended to
76 have minimal dependencies on LSST infrastructure.
77 """
79 file = os.path.join(obs_lsst_packageDir, policyFile)
80 try:
81 with open(file) as fh:
82 # Use the fast parser since these files are large
83 camera = yaml.load(fh, Loader=yaml.CSafeLoader)
84 except OSError as e:
85 raise ValueError(f"Could not load camera policy file {file}") from e
87 mapping = {}
88 for ccd, value in camera["CCDs"].items():
89 mapping[ccd] = (int(value["id"]), value["serial"])
91 return mapping
94def compute_detector_exposure_id_generic(exposure_id, detector_num, max_num=1000, mode="concat"):
95 """Compute the detector_exposure_id from the exposure id and the
96 detector number.
98 Parameters
99 ----------
100 exposure_id : `int`
101 The exposure ID.
102 detector_num : `int`
103 The detector number.
104 max_num : `int`, optional
105 Maximum number of detectors to make space for. Defaults to 1000.
106 mode : `str`, optional
107 Computation mode. Defaults to "concat".
108 - concat : Concatenate the exposure ID and detector number, making
109 sure that there is space for max_num and zero padding.
110 - multiply : Multiply the exposure ID by the maximum detector
111 number and add the detector number.
113 Returns
114 -------
115 detector_exposure_id : `int`
116 Computed ID.
118 Raises
119 ------
120 ValueError
121 The detector number is out of range.
122 """
124 if detector_num is None:
125 raise ValueError("Detector number must be defined.")
126 if detector_num > max_num or detector_num < 0:
127 raise ValueError(f"Detector number out of range 0 <= {detector_num} <= {max_num}")
129 if mode == "concat":
130 npad = len(str(max_num))
131 return int(f"{exposure_id}{detector_num:0{npad}d}")
132 elif mode == "multiply":
133 return max_num*exposure_id + detector_num
134 else:
135 raise ValueError(f"Computation mode of '{mode}' is not understood")
138class LsstBaseTranslator(FitsTranslator):
139 """Translation methods useful for all LSST-style headers."""
141 _const_map = {}
142 _trivial_map = {}
144 # Do not specify a name for this translator
145 cameraPolicyFile = None
146 """Path to policy file relative to obs_lsst root."""
148 detectorMapping = None
149 """Mapping of detector name to detector number and serial."""
151 detectorSerials = None
152 """Mapping of detector serial number to raft, number, and name."""
154 DETECTOR_MAX = 999
155 """Maximum number of detectors to use when calculating the
156 detector_exposure_id."""
158 _DEFAULT_LOCATION = LSST_LOCATION
159 """Default telescope location in absence of relevant FITS headers."""
161 @classmethod
162 def __init_subclass__(cls, **kwargs):
163 """Ensure that subclasses clear their own detector mapping entries
164 such that subclasses of translators that use detector mappings
165 do not pick up the incorrect values from a parent."""
167 cls.detectorMapping = None
168 cls.detectorSerials = None
170 super().__init_subclass__(**kwargs)
172 def search_paths(self):
173 """Search paths to use for LSST data when looking for header correction
174 files.
176 Returns
177 -------
178 path : `list`
179 List with a single element containing the full path to the
180 ``corrections`` directory within the ``obs_lsst`` package.
181 """
182 return [os.path.join(obs_lsst_packageDir, "corrections")]
184 @classmethod
185 def compute_detector_exposure_id(cls, exposure_id, detector_num):
186 """Compute the detector exposure ID from detector number and
187 exposure ID.
189 This is a helper method to allow code working outside the translator
190 infrastructure to use the same algorithm.
192 Parameters
193 ----------
194 exposure_id : `int`
195 Unique exposure ID.
196 detector_num : `int`
197 Detector number.
199 Returns
200 -------
201 detector_exposure_id : `int`
202 The calculated ID.
203 """
204 return compute_detector_exposure_id_generic(exposure_id, detector_num,
205 max_num=cls.DETECTOR_MAX,
206 mode="concat")
208 @classmethod
209 def max_detector_exposure_id(cls):
210 """The maximum detector exposure ID expected to be generated by
211 this instrument.
213 Returns
214 -------
215 max_id : `int`
216 The maximum value.
217 """
218 max_exposure_id = cls.max_exposure_id()
219 return cls.compute_detector_exposure_id(max_exposure_id, cls.DETECTOR_MAX)
221 @classmethod
222 def max_exposure_id(cls):
223 """The maximum exposure ID expected from this instrument.
225 Returns
226 -------
227 max_exposure_id : `int`
228 The maximum value.
229 """
230 max_date = "2050-12-31T23:59.999"
231 max_seqnum = 99_999
232 max_controller = "C" # This controller triggers the largest numbers
233 return cls.compute_exposure_id(max_date, max_seqnum, max_controller)
235 @classmethod
236 def detector_mapping(cls):
237 """Returns the mapping of full name to detector ID and serial.
239 Returns
240 -------
241 mapping : `dict` of `str`:`tuple`
242 Returns the mapping of full detector name (group+detector)
243 to detector number and serial.
245 Raises
246 ------
247 ValueError
248 Raised if no camera policy file has been registered with this
249 translation class.
251 Notes
252 -----
253 Will construct the mapping if none has previously been constructed.
254 """
255 if cls.cameraPolicyFile is not None:
256 if cls.detectorMapping is None:
257 cls.detectorMapping = read_detector_ids(cls.cameraPolicyFile)
258 else:
259 raise ValueError(f"Translation class '{cls.__name__}' has no registered camera policy file")
261 return cls.detectorMapping
263 @classmethod
264 def detector_serials(cls):
265 """Obtain the mapping of detector serial to detector group, name,
266 and number.
268 Returns
269 -------
270 info : `dict` of `tuple` of (`str`, `str`, `int`)
271 A `dict` with the serial numbers as keys and values of detector
272 group, name, and number.
273 """
274 if cls.detectorSerials is None:
275 detector_mapping = cls.detector_mapping()
277 if detector_mapping is not None:
278 # Form mapping to go from serial number to names/numbers
279 serials = {}
280 for fullname, (id, serial) in cls.detectorMapping.items():
281 raft, detector_name = fullname.split("_")
282 if serial in serials:
283 raise RuntimeError(f"Serial {serial} is defined in multiple places")
284 serials[serial] = (raft, detector_name, id)
285 cls.detectorSerials = serials
286 else:
287 raise RuntimeError("Unable to obtain detector mapping information")
289 return cls.detectorSerials
291 @classmethod
292 def compute_detector_num_from_name(cls, detector_group, detector_name):
293 """Helper method to return the detector number from the name.
295 Parameters
296 ----------
297 detector_group : `str`
298 Name of the detector grouping. This is generally the raft name.
299 detector_name : `str`
300 Detector name.
302 Returns
303 -------
304 num : `int`
305 Detector number.
306 """
307 fullname = f"{detector_group}_{detector_name}"
309 num = None
310 detector_mapping = cls.detector_mapping()
311 if detector_mapping is None:
312 raise RuntimeError("Unable to obtain detector mapping information")
314 if fullname in detector_mapping:
315 num = detector_mapping[fullname]
316 else:
317 log.warning(f"Unable to determine detector number from detector name {fullname}")
318 return None
320 return num[0]
322 @classmethod
323 def compute_detector_info_from_serial(cls, detector_serial):
324 """Helper method to return the detector information from the serial.
326 Parameters
327 ----------
328 detector_serial : `str`
329 Detector serial ID.
331 Returns
332 -------
333 info : `tuple` of (`str`, `str`, `int`)
334 Detector group, name, and number.
335 """
336 serial_mapping = cls.detector_serials()
337 if serial_mapping is None:
338 raise RuntimeError("Unable to obtain serial mapping information")
340 if detector_serial in serial_mapping:
341 info = serial_mapping[detector_serial]
342 else:
343 raise RuntimeError("Unable to determine detector information from detector serial"
344 f" {detector_serial}")
346 return info
348 @staticmethod
349 def compute_exposure_id(dayobs, seqnum, controller=None):
350 """Helper method to calculate the exposure_id.
352 Parameters
353 ----------
354 dayobs : `str`
355 Day of observation in either YYYYMMDD or YYYY-MM-DD format.
356 If the string looks like ISO format it will be truncated before the
357 ``T`` before being handled.
358 seqnum : `int` or `str`
359 Sequence number.
360 controller : `str`, optional
361 Controller to use. If this is "O", no change is made to the
362 exposure ID. If it is "C" a 1000 is added to the year component
363 of the exposure ID.
364 `None` indicates that the controller is not relevant to the
365 exposure ID calculation (generally this is the case for test
366 stand data).
368 Returns
369 -------
370 exposure_id : `int`
371 Exposure ID in form YYYYMMDDnnnnn form.
372 """
373 if "T" in dayobs:
374 dayobs = dayobs[:dayobs.find("T")]
376 dayobs = dayobs.replace("-", "")
378 if len(dayobs) != 8:
379 raise ValueError(f"Malformed dayobs: {dayobs}")
381 # Expect no more than 99,999 exposures in a day
382 maxdigits = 5
383 if seqnum >= 10**maxdigits:
384 raise ValueError(f"Sequence number ({seqnum}) exceeds limit")
386 # Camera control changes the exposure ID
387 if controller is not None:
388 if controller == "O":
389 pass
390 elif controller == "C":
391 # Add 1000 to the year component
392 dayobs = int(dayobs)
393 dayobs += 1000_00_00
394 else:
395 raise ValueError(f"Supplied controller, '{controller}' is neither 'O' nor 'C'")
397 # Form the number as a string zero padding the sequence number
398 idstr = f"{dayobs}{seqnum:0{maxdigits}d}"
400 # Exposure ID has to be an integer
401 return int(idstr)
403 def _is_on_mountain(self):
404 """Indicate whether these data are coming from the instrument
405 installed on the mountain.
407 Returns
408 -------
409 is : `bool`
410 `True` if instrument is on the mountain.
411 """
412 if "TSTAND" in self._header:
413 return False
414 return True
416 def is_on_sky(self):
417 """Determine if this is an on-sky observation.
419 Returns
420 -------
421 is_on_sky : `bool`
422 Returns True if this is a observation on sky on the
423 summit.
424 """
425 # For LSST we think on sky unless tracksys is local
426 if self.is_key_ok("TRACKSYS"):
427 if self._header["TRACKSYS"].lower() == "local":
428 # not on sky
429 return False
431 # These are obviously not on sky
432 if self.to_observation_type() in ("bias", "dark", "flat"):
433 return False
435 return self._is_on_mountain()
437 @cache_translation
438 def to_location(self):
439 # Docstring will be inherited. Property defined in properties.py
440 if not self._is_on_mountain():
441 return None
442 try:
443 # Try standard FITS headers
444 return super().to_location()
445 except KeyError:
446 return self._DEFAULT_LOCATION
448 @cache_translation
449 def to_datetime_begin(self):
450 # Docstring will be inherited. Property defined in properties.py
451 self._used_these_cards("MJD-OBS")
452 return Time(self._header["MJD-OBS"], scale="tai", format="mjd")
454 @cache_translation
455 def to_datetime_end(self):
456 # Docstring will be inherited. Property defined in properties.py
457 if self.is_key_ok("DATE-END"):
458 return super().to_datetime_end()
460 return self.to_datetime_begin() + self.to_exposure_time()
462 @cache_translation
463 def to_detector_num(self):
464 # Docstring will be inherited. Property defined in properties.py
465 raft = self.to_detector_group()
466 detector = self.to_detector_name()
467 return self.compute_detector_num_from_name(raft, detector)
469 @cache_translation
470 def to_detector_exposure_id(self):
471 # Docstring will be inherited. Property defined in properties.py
472 exposure_id = self.to_exposure_id()
473 num = self.to_detector_num()
474 return self.compute_detector_exposure_id(exposure_id, num)
476 @cache_translation
477 def to_observation_type(self):
478 # Docstring will be inherited. Property defined in properties.py
479 obstype = self._header["IMGTYPE"]
480 self._used_these_cards("IMGTYPE")
481 obstype = obstype.lower()
482 if obstype in ("skyexp", "object"):
483 obstype = "science"
484 return obstype
486 @cache_translation
487 def to_dark_time(self):
488 """Calculate the dark time.
490 If a DARKTIME header is not found, the value is assumed to be
491 identical to the exposure time.
493 Returns
494 -------
495 dark : `astropy.units.Quantity`
496 The dark time in seconds.
497 """
498 if self.is_key_ok("DARKTIME"):
499 darktime = self._header["DARKTIME"]*u.s
500 self._used_these_cards("DARKTIME")
501 else:
502 log.warning("%s: Unable to determine dark time. Setting from exposure time.",
503 self.to_observation_id())
504 darktime = self.to_exposure_time()
505 return darktime
507 @cache_translation
508 def to_exposure_id(self):
509 """Generate a unique exposure ID number
511 This is a combination of DAYOBS and SEQNUM, and optionally
512 CONTRLLR.
514 Returns
515 -------
516 exposure_id : `int`
517 Unique exposure number.
518 """
519 if "CALIB_ID" in self._header:
520 self._used_these_cards("CALIB_ID")
521 return None
523 dayobs = self._header["DAYOBS"]
524 seqnum = self._header["SEQNUM"]
525 self._used_these_cards("DAYOBS", "SEQNUM")
527 if self.is_key_ok("CONTRLLR"):
528 controller = self._header["CONTRLLR"]
529 self._used_these_cards("CONTRLLR")
530 else:
531 controller = None
533 return self.compute_exposure_id(dayobs, seqnum, controller=controller)
535 @cache_translation
536 def to_visit_id(self):
537 """Calculate the visit associated with this exposure.
539 Notes
540 -----
541 For LATISS and LSSTCam the default visit is derived from the
542 exposure group. For other instruments we return the exposure_id.
543 """
545 exposure_group = self.to_exposure_group()
546 # If the group is an int we return it
547 try:
548 visit_id = int(exposure_group)
549 return visit_id
550 except ValueError:
551 pass
553 # A Group is defined as ISO date with an extension
554 # The integer must be the same for a given group so we can never
555 # use datetime_begin.
556 # Nominally a GROUPID looks like "ISODATE+N" where the +N is
557 # optional. This can be converted to seconds since epoch with
558 # an adjustment for N.
559 # For early data lacking that form we hash the group and return
560 # the int.
561 matches_date = GROUP_RE.match(exposure_group)
562 if matches_date:
563 iso_str = matches_date.group(1)
564 fraction = matches_date.group(2)
565 n = matches_date.group(3)
566 if n is not None:
567 n = int(n)
568 else:
569 n = 0
570 iso = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%S")
572 tdelta = iso - TZERO_DATETIME
573 epoch = int(tdelta.total_seconds())
575 # Form the integer from EPOCH + 3 DIGIT FRAC + 0-pad N
576 visit_id = int(f"{epoch}{fraction}{n:04d}")
577 else:
578 # Non-standard string so convert to numbers
579 # using a hash function. Use the first N hex digits
580 group_bytes = exposure_group.encode("us-ascii")
581 hasher = hashlib.blake2b(group_bytes)
582 # Need to be big enough it does not possibly clash with the
583 # date-based version above
584 digest = hasher.hexdigest()[:14]
585 visit_id = int(digest, base=16)
587 # To help with hash collision, append the string length
588 visit_id = int(f"{visit_id}{len(exposure_group):02d}")
590 return visit_id
592 @cache_translation
593 def to_physical_filter(self):
594 """Calculate the physical filter name.
596 Returns
597 -------
598 filter : `str`
599 Name of filter. Can be a combination of FILTER, FILTER1 and FILTER2
600 headers joined by a "~". Returns "NONE" if no filter is declared.
601 """
602 joined = self._join_keyword_values(["FILTER", "FILTER1", "FILTER2"], delim=FILTER_DELIMITER)
603 if not joined:
604 joined = "NONE"
606 return joined
608 @cache_translation
609 def to_tracking_radec(self):
610 if not self.is_on_sky():
611 return None
613 # RA/DEC are *derived* headers and for the case where the DATE-BEG
614 # is 1970 they are garbage and should not be used.
615 if self._header["DATE-OBS"] == self._header["DATE"]:
616 # A fixed up date -- use AZEL as source of truth
617 altaz = self.to_altaz_begin()
618 radec = astropy.coordinates.SkyCoord(altaz.transform_to(astropy.coordinates.ICRS),
619 obstime=altaz.obstime,
620 location=altaz.location)
621 else:
622 radecsys = ("RADESYS",)
623 radecpairs = (("RASTART", "DECSTART"), ("RA", "DEC"))
624 radec = tracking_from_degree_headers(self, radecsys, radecpairs)
626 return radec
628 @cache_translation
629 def to_altaz_begin(self):
630 if not self._is_on_mountain():
631 return None
633 # ALTAZ always relevant unless bias or dark
634 if self.to_observation_type() in ("bias", "dark"):
635 return None
637 return altaz_from_degree_headers(self, (("ELSTART", "AZSTART"),),
638 self.to_datetime_begin(), is_zd=False)
640 @cache_translation
641 def to_exposure_group(self):
642 """Calculate the exposure group string.
644 For LSSTCam and LATISS this is read from the ``GROUPID`` header.
645 If that header is missing the exposure_id is returned instead as
646 a string.
647 """
648 if self.is_key_ok("GROUPID"):
649 exposure_group = self._header["GROUPID"]
650 self._used_these_cards("GROUPID")
651 return exposure_group
652 return super().to_exposure_group()