Coverage for python/lsst/obs/lsst/translators/latiss.py: 17%
239 statements
« prev ^ index » next coverage.py v6.4, created at 2022-06-01 03:57 -0700
« prev ^ index » next coverage.py v6.4, created at 2022-06-01 03:57 -0700
1# This file is currently part of obs_lsst but is written to allow it
2# to be migrated to the astro_metadata_translator package at a later date.
3#
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the LICENSE file in this directory for details of code ownership.
7#
8# Use of this source code is governed by a 3-clause BSD-style
9# license that can be found in the LICENSE file.
11"""Metadata translation code for LSST LATISS headers"""
13__all__ = ("LatissTranslator", )
15import logging
16import math
18import astropy.units as u
19from astropy.time import Time
20from astropy.coordinates import EarthLocation
22from astro_metadata_translator import cache_translation
23from astro_metadata_translator.translators.helpers import is_non_science
24from .lsst import LsstBaseTranslator, FILTER_DELIMITER
26log = logging.getLogger(__name__)
29# AuxTel is not the same place as LSST
30# These coordinates read from Apple Maps
31AUXTEL_LOCATION = EarthLocation.from_geodetic(-70.747698, -30.244728, 2663.0)
33# Date instrument is taking data at telescope
34# Prior to this date many parameters are automatically nulled out
35# since the headers have not historically been reliable
36TSTART = Time("2020-01-01T00:00", format="isot", scale="utc")
38# Define the sensor and group name for AuxTel globally so that it can be used
39# in multiple places. There is no raft but for consistency with other LSST
40# cameras we define one.
41_DETECTOR_GROUP_NAME = "RXX"
42_DETECTOR_NAME = "S00"
44# Date 068 detector was put in LATISS
45DETECTOR_068_DATE = Time("2019-06-24T00:00", format="isot", scale="utc")
47# IMGTYPE header is filled in after this date
48IMGTYPE_OKAY_DATE = Time("2019-11-07T00:00", format="isot", scale="utc")
50# OBJECT IMGTYPE really means ENGTEST until this date
51OBJECT_IS_ENGTEST = Time("2020-01-27T20:00", format="isot", scale="utc")
53# RA and DEC headers are in radians until this date
54RADEC_IS_RADIANS = Time("2020-01-28T22:00", format="isot", scale="utc")
56# RASTART/DECSTART/RAEND/DECEND used wrong telescope location before this
57# 2020-02-01T00:00 we fixed the telescope location, but RASTART is still
58# in mount coordinates, so off by pointing model.
59RASTART_IS_BAD = Time("2020-05-01T00:00", format="isot", scale="utc")
61# Between RASTART_IS_BAD and this time the RASTART header uses hours
62# instead of degrees.
63RASTART_IS_HOURS = Time("2021-02-11T18:45", format="isot", scale="utc")
65# From this date RASTART is correct as-is.
66RASTART_IS_OKAY = Time("2021-02-12T00:00", format="isot", scale="utc")
68# DATE-END is not to be trusted before this date
69DATE_END_IS_BAD = Time("2020-02-01T00:00", format="isot", scale="utc")
71# The convention for the reporting of ROTPA changed by 180 here
72ROTPA_CONVENTION_180_SWITCH1 = Time("2020-11-19T00:00", format="isot", scale="utc")
73ROTPA_CONVENTION_180_SWITCH2 = Time("2021-10-29T00:00", format="isot", scale="utc")
75# Scaling factor radians to degrees. Keep it simple.
76RAD2DEG = 180.0 / math.pi
79def is_non_science_or_lab(self):
80 """Pseudo method to determine whether this is a lab or non-science
81 header.
83 Raises
84 ------
85 KeyError
86 If this is a science observation and on the mountain.
87 """
88 # Return without raising if this is not a science observation
89 # since the defaults are fine.
90 try:
91 # This will raise if it is a science observation
92 is_non_science(self)
93 return
94 except KeyError:
95 pass
97 # We are still in the lab, return and use the default
98 if not self._is_on_mountain():
99 return
101 # This is a science observation on the mountain so we should not
102 # use defaults
103 raise KeyError(f"{self._log_prefix}: Required key is missing and this is a mountain science observation")
106class LatissTranslator(LsstBaseTranslator):
107 """Metadata translator for LSST LATISS data from AuxTel.
109 For lab measurements many values are masked out.
110 """
112 name = "LSST_LATISS"
113 """Name of this translation class"""
115 supported_instrument = "LATISS"
116 """Supports the LATISS instrument."""
118 _const_map = {
119 "instrument": "LATISS",
120 "telescope": "Rubin Auxiliary Telescope",
121 "detector_group": _DETECTOR_GROUP_NAME,
122 "detector_num": 0,
123 "detector_name": _DETECTOR_NAME, # Single sensor
124 "relative_humidity": None,
125 "pressure": None,
126 "temperature": None,
127 }
129 _trivial_map = {
130 "observation_id": (["OBSID", "IMGNAME"], dict(default=None, checker=is_non_science)),
131 "detector_serial": ["LSST_NUM", "DETSER"],
132 "object": ("OBJECT", dict(checker=is_non_science_or_lab, default="UNKNOWN")),
133 "boresight_rotation_angle": (["ROTPA", "ROTANGLE"], dict(checker=is_non_science_or_lab,
134 default=float("nan"), unit=u.deg)),
135 "science_program": ("PROGRAM", dict(default="unknown")),
136 }
138 DETECTOR_GROUP_NAME = _DETECTOR_GROUP_NAME
139 """Fixed name of detector group."""
141 DETECTOR_NAME = _DETECTOR_NAME
142 """Fixed name of single sensor."""
144 DETECTOR_MAX = 0
145 """Maximum number of detectors to use when calculating the
146 detector_exposure_id."""
148 _DEFAULT_LOCATION = AUXTEL_LOCATION
149 """Default telescope location in absence of relevant FITS headers."""
151 @classmethod
152 def can_translate(cls, header, filename=None):
153 """Indicate whether this translation class can translate the
154 supplied header.
156 Parameters
157 ----------
158 header : `dict`-like
159 Header to convert to standardized form.
160 filename : `str`, optional
161 Name of file being translated.
163 Returns
164 -------
165 can : `bool`
166 `True` if the header is recognized by this class. `False`
167 otherwise.
168 """
169 # INSTRUME keyword might be of two types
170 if "INSTRUME" in header:
171 instrume = header["INSTRUME"]
172 for v in ("LSST_ATISS", "LATISS"):
173 if instrume == v:
174 return True
175 # Calibration files strip important headers at the moment so guess
176 if "DETNAME" in header and header["DETNAME"] == "RXX_S00":
177 return True
178 return False
180 @classmethod
181 def fix_header(cls, header, instrument, obsid, filename=None):
182 """Fix an incorrect LATISS header.
184 Parameters
185 ----------
186 header : `dict`
187 The header to update. Updates are in place.
188 instrument : `str`
189 The name of the instrument.
190 obsid : `str`
191 Unique observation identifier associated with this header.
192 Will always be provided.
193 filename : `str`, optional
194 Filename associated with this header. May not be set since headers
195 can be fixed independently of any filename being known.
197 Returns
198 -------
199 modified = `bool`
200 Returns `True` if the header was updated.
202 Notes
203 -----
204 This method does not apply per-obsid corrections. The following
205 corrections are applied:
207 * On June 24th 2019 the detector was changed from ITL-3800C-098
208 to ITL-3800C-068. The header is intended to be correct in the
209 future.
210 * In late 2019 the DATE-OBS and MJD-OBS headers were reporting
211 1970 dates. To correct, the DATE/MJD headers are copied in to
212 replace them and the -END headers are cleared.
213 * Until November 2019 the IMGTYPE was set in the GROUPID header.
214 The value is moved to IMGTYPE.
215 * SHUTTIME is always forced to be `None`.
217 Corrections are reported as debug level log messages.
219 See `~astro_metadata_translator.fix_header` for details of the general
220 process.
221 """
222 modified = False
224 # Calculate the standard label to use for log messages
225 log_label = cls._construct_log_prefix(obsid, filename)
227 if "OBSID" not in header:
228 # Very old data used IMGNAME
229 header["OBSID"] = obsid
230 modified = True
231 # We are reporting the OBSID so no need to repeat it at start
232 # of log message. Use filename if we have it.
233 log_prefix = f"{filename}: " if filename else ""
234 log.debug("%sAssigning OBSID to a value of '%s'", log_prefix, header["OBSID"])
236 if "DAYOBS" not in header:
237 # OBS-NITE could have the value for DAYOBS but it is safer
238 # for older data to set it from the OBSID. Fall back to OBS-NITE
239 # if we have no alternative
240 dayObs = None
241 try:
242 dayObs = obsid.split("_", 3)[2]
243 except (AttributeError, ValueError):
244 # did not split as expected
245 pass
246 if dayObs is None or len(dayObs) != 8:
247 if "OBS-NITE" in header:
248 dayObs = header["OBS-NITE"]
249 log.debug("%s: Setting DAYOBS to '%s' from OBS-NITE header", log_label, dayObs)
250 else:
251 log.debug("%s: Unable to determine DAYOBS from header", log_label)
252 else:
253 log.debug("%s: Setting DAYOBS to '%s' from OBSID", log_label, dayObs)
254 if dayObs:
255 header["DAYOBS"] = dayObs
256 modified = True
258 if "SEQNUM" not in header:
259 try:
260 seqnum = obsid.split("_", 3)[3]
261 except (AttributeError, ValueError):
262 # did not split as expected
263 pass
264 else:
265 header["SEQNUM"] = int(seqnum)
266 modified = True
267 log.debug("%s: Extracting SEQNUM of '%s' from OBSID", log_label, header["SEQNUM"])
269 # The DATE-OBS / MJD-OBS keys can be 1970
270 if "DATE-OBS" in header and header["DATE-OBS"].startswith("1970"):
271 # Copy the headers from the DATE and MJD since we have no other
272 # choice.
273 header["DATE-OBS"] = header["DATE"]
274 header["DATE-BEG"] = header["DATE-OBS"]
275 header["MJD-OBS"] = header["MJD"]
276 header["MJD-BEG"] = header["MJD-OBS"]
278 # And clear the DATE-END and MJD-END -- the translator will use
279 # EXPTIME instead.
280 header["DATE-END"] = None
281 header["MJD-END"] = None
283 log.debug("%s: Forcing 1970 dates to '%s'", log_label, header["DATE"])
284 modified = True
286 # Create a translator since we need the date
287 translator = cls(header)
288 date = translator.to_datetime_begin()
289 if date > DETECTOR_068_DATE:
290 header["LSST_NUM"] = "ITL-3800C-068"
291 log.debug("%s: Forcing detector serial to %s", log_label, header["LSST_NUM"])
292 modified = True
294 if date < DATE_END_IS_BAD:
295 # DATE-END may or may not be in TAI and may or may not be
296 # before DATE-BEG. Simpler to clear it
297 if header.get("DATE-END"):
298 header["DATE-END"] = None
299 header["MJD-END"] = None
301 log.debug("%s: Clearing DATE-END as being untrustworthy", log_label)
302 modified = True
304 # Up until a certain date GROUPID was the IMGTYPE
305 if date < IMGTYPE_OKAY_DATE:
306 groupId = header.get("GROUPID")
307 if groupId and not groupId.startswith("test"):
308 imgType = header.get("IMGTYPE")
309 if not imgType:
310 if "_" in groupId:
311 # Sometimes have the form dark_0001_0002
312 # in this case we pull the IMGTYPE off the front and
313 # do not clear groupId (although groupId may now
314 # repeat on different days).
315 groupId, _ = groupId.split("_", 1)
316 elif groupId.upper() != "FOCUS" and groupId.upper().startswith("FOCUS"):
317 # If it is exactly FOCUS we want groupId cleared
318 groupId = "FOCUS"
319 else:
320 header["GROUPID"] = None
321 header["IMGTYPE"] = groupId
322 log.debug("%s: Setting IMGTYPE to '%s' from GROUPID", log_label, header["IMGTYPE"])
323 modified = True
324 else:
325 # Someone could be fixing headers in old data
326 # and we do not want GROUPID == IMGTYPE
327 if imgType == groupId:
328 # Clear the group so we default to original
329 header["GROUPID"] = None
331 # We were using OBJECT for engineering observations early on
332 if date < OBJECT_IS_ENGTEST:
333 imgType = header.get("IMGTYPE")
334 if imgType == "OBJECT":
335 header["IMGTYPE"] = "ENGTEST"
336 log.debug("%s: Changing OBJECT observation type to %s",
337 log_label, header["IMGTYPE"])
338 modified = True
340 # Early on the RA/DEC headers were stored in radians
341 if date < RADEC_IS_RADIANS:
342 if header.get("RA") is not None:
343 header["RA"] *= RAD2DEG
344 log.debug("%s: Changing RA header to degrees", log_label)
345 modified = True
346 if header.get("DEC") is not None:
347 header["DEC"] *= RAD2DEG
348 log.debug("%s: Changing DEC header to degrees", log_label)
349 modified = True
351 if header.get("SHUTTIME"):
352 log.debug("%s: Forcing SHUTTIME header to be None", log_label)
353 header["SHUTTIME"] = None
354 modified = True
356 if "OBJECT" not in header:
357 # Only patch OBJECT IMGTYPE
358 if "IMGTYPE" in header and header["IMGTYPE"] == "OBJECT":
359 log.debug("%s: Forcing OBJECT header to exist", log_label)
360 header["OBJECT"] = "NOTSET"
361 modified = True
363 if "RADESYS" in header:
364 if header["RADESYS"] == "":
365 # Default to ICRS
366 header["RADESYS"] = "ICRS"
367 log.debug("%s: Forcing blank RADESYS to '%s'", log_label, header["RADESYS"])
368 modified = True
370 if date < RASTART_IS_HOURS:
371 # Avoid two checks for case where RASTART is fine
372 if date < RASTART_IS_BAD:
373 # The wrong telescope position was used. Unsetting these will
374 # force the RA/DEC demand headers to be used instead.
375 for h in ("RASTART", "DECSTART", "RAEND", "DECEND"):
376 header[h] = None
377 log.debug("%s: Forcing derived RA/Dec headers to undefined", log_label)
378 modified = True
379 else:
380 # Correct hours to degrees
381 for h in ("RASTART", "RAEND"):
382 if header[h]:
383 header[h] *= 15.0
384 log.debug("%s: Correcting RASTART/END from hours to degrees", log_label)
385 modified = True
387 # RASTART/END headers have a TAI/UTC confusion causing an offset
388 # of 37 seconds for a period of time.
389 if RASTART_IS_BAD < date < RASTART_IS_OKAY:
390 modified = True
391 offset = (37.0 / 3600.0) * 15.0
392 for epoch in ("START", "END"):
393 h = "RA" + epoch
394 if header[h]:
395 header[h] += offset
397 if date < ROTPA_CONVENTION_180_SWITCH2 and date > ROTPA_CONVENTION_180_SWITCH1:
398 header['ROTPA'] = header['ROTPA'] - 180
399 modified = True
401 return modified
403 def _is_on_mountain(self):
404 date = self.to_datetime_begin()
405 if date > TSTART:
406 return True
407 return False
409 @staticmethod
410 def compute_detector_exposure_id(exposure_id, detector_num):
411 """Compute the detector exposure ID from detector number and
412 exposure ID.
414 This is a helper method to allow code working outside the translator
415 infrastructure to use the same algorithm.
417 Parameters
418 ----------
419 exposure_id : `int`
420 Unique exposure ID.
421 detector_num : `int`
422 Detector number.
424 Returns
425 -------
426 detector_exposure_id : `int`
427 The calculated ID.
428 """
429 if detector_num != 0:
430 log.warning("Unexpected non-zero detector number for LATISS")
431 return exposure_id
433 @cache_translation
434 def to_dark_time(self):
435 # Docstring will be inherited. Property defined in properties.py
437 # Always compare with exposure time
438 # We may revisit this later if there is a cutoff date where we
439 # can always trust the header.
440 exptime = self.to_exposure_time()
442 if self.is_key_ok("DARKTIME"):
443 darktime = self.quantity_from_card("DARKTIME", u.s)
444 if darktime >= exptime:
445 return darktime
446 reason = "Dark time less than exposure time."
447 else:
448 reason = "Dark time not defined."
450 log.warning("%s: %s Setting dark time to the exposure time.",
451 self._log_prefix, reason)
452 return exptime
454 @cache_translation
455 def to_exposure_time(self):
456 # Docstring will be inherited. Property defined in properties.py
457 # Some data is missing a value for EXPTIME.
458 # Have to be careful we do not have circular logic when trying to
459 # guess
460 if self.is_key_ok("EXPTIME"):
461 return self.quantity_from_card("EXPTIME", u.s)
463 # A missing or undefined EXPTIME is problematic. Set to -1
464 # to indicate that none was found.
465 log.warning("%s: Insufficient information to derive exposure time. Setting to -1.0s",
466 self._log_prefix)
467 return -1.0 * u.s
469 @cache_translation
470 def to_observation_type(self):
471 """Determine the observation type.
473 In the absence of an ``IMGTYPE`` header, assumes lab data is always a
474 dark if exposure time is non-zero, else bias.
476 Returns
477 -------
478 obstype : `str`
479 Observation type.
480 """
482 # LATISS observation type is documented to appear in OBSTYPE
483 # but for historical reasons prefers IMGTYPE.
484 # Test the keys in order until we find one that contains a
485 # defined value.
486 obstype_keys = ["OBSTYPE", "IMGTYPE"]
488 obstype = None
489 for k in obstype_keys:
490 if self.is_key_ok(k):
491 obstype = self._header[k]
492 self._used_these_cards(k)
493 obstype = obstype.lower()
494 break
496 if obstype is not None:
497 if obstype == "object" and not self._is_on_mountain():
498 # Do not map object to science in lab since most
499 # code assume science is on sky with RA/Dec.
500 obstype = "labobject"
501 elif obstype in ("skyexp", "object"):
502 obstype = "science"
504 return obstype
506 # In the absence of any observation type information, return
507 # unknown unless we think it might be a bias.
508 exptime = self.to_exposure_time()
509 if exptime == 0.0:
510 obstype = "bias"
511 else:
512 obstype = "unknown"
513 log.warning("%s: Unable to determine observation type. Guessing '%s'",
514 self._log_prefix, obstype)
515 return obstype
517 @cache_translation
518 def to_physical_filter(self):
519 """Calculate the physical filter name.
521 Returns
522 -------
523 filter : `str`
524 Name of filter. A combination of FILTER and GRATING
525 headers joined by a "~". The filter and grating are always
526 combined. The filter or grating part will be "none" if no value
527 is specified. Uses "empty" if any of the filters or gratings
528 indicate an "empty_N" name. "unknown" indicates that the filter is
529 not defined anywhere but we think it should be. "none" indicates
530 that the filter was not defined but the observation is a dark
531 or bias.
532 """
534 physical_filter = self._determine_primary_filter()
536 if self.is_key_ok("GRATING"):
537 grating = self._header["GRATING"]
538 self._used_these_cards("GRATING")
540 if not grating or grating.lower().startswith("empty"):
541 grating = "empty"
542 else:
543 # Be explicit about having no knowledge of the grating
544 grating = "unknown"
546 physical_filter = f"{physical_filter}{FILTER_DELIMITER}{grating}"
548 return physical_filter