Coverage for python/lsst/obs/lsst/translators/latiss.py: 15%
228 statements
« prev ^ index » next coverage.py v6.4.1, created at 2022-07-11 08:24 +0000
« prev ^ index » next coverage.py v6.4.1, created at 2022-07-11 08:24 +0000
1# This file is currently part of obs_lsst but is written to allow it
2# to be migrated to the astro_metadata_translator package at a later date.
3#
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the LICENSE file in this directory for details of code ownership.
7#
8# Use of this source code is governed by a 3-clause BSD-style
9# license that can be found in the LICENSE file.
11"""Metadata translation code for LSST LATISS headers"""
13__all__ = ("LatissTranslator", )
15import logging
16import math
18import astropy.units as u
19from astropy.time import Time
20from astropy.coordinates import EarthLocation
22from astro_metadata_translator import cache_translation
23from astro_metadata_translator.translators.helpers import is_non_science
24from .lsst import LsstBaseTranslator, FILTER_DELIMITER
26log = logging.getLogger(__name__)
29# AuxTel is not the same place as LSST
30# These coordinates read from Apple Maps
31AUXTEL_LOCATION = EarthLocation.from_geodetic(-70.747698, -30.244728, 2663.0)
33# Date instrument is taking data at telescope
34# Prior to this date many parameters are automatically nulled out
35# since the headers have not historically been reliable
36TSTART = Time("2020-01-01T00:00", format="isot", scale="utc")
38# Define the sensor and group name for AuxTel globally so that it can be used
39# in multiple places. There is no raft but for consistency with other LSST
40# cameras we define one.
41_DETECTOR_GROUP_NAME = "RXX"
42_DETECTOR_NAME = "S00"
44# Date 068 detector was put in LATISS
45DETECTOR_068_DATE = Time("2019-06-24T00:00", format="isot", scale="utc")
47# IMGTYPE header is filled in after this date
48IMGTYPE_OKAY_DATE = Time("2019-11-07T00:00", format="isot", scale="utc")
50# OBJECT IMGTYPE really means ENGTEST until this date
51OBJECT_IS_ENGTEST = Time("2020-01-27T20:00", format="isot", scale="utc")
53# RA and DEC headers are in radians until this date
54RADEC_IS_RADIANS = Time("2020-01-28T22:00", format="isot", scale="utc")
56# RASTART/DECSTART/RAEND/DECEND used wrong telescope location before this
57# 2020-02-01T00:00 we fixed the telescope location, but RASTART is still
58# in mount coordinates, so off by pointing model.
59RASTART_IS_BAD = Time("2020-05-01T00:00", format="isot", scale="utc")
61# Between RASTART_IS_BAD and this time the RASTART header uses hours
62# instead of degrees.
63RASTART_IS_HOURS = Time("2021-02-11T18:45", format="isot", scale="utc")
65# From this date RASTART is correct as-is.
66RASTART_IS_OKAY = Time("2021-02-12T00:00", format="isot", scale="utc")
68# DATE-END is not to be trusted before this date
69DATE_END_IS_BAD = Time("2020-02-01T00:00", format="isot", scale="utc")
71# Scaling factor radians to degrees. Keep it simple.
72RAD2DEG = 180.0 / math.pi
75def is_non_science_or_lab(self):
76 """Pseudo method to determine whether this is a lab or non-science
77 header.
79 Raises
80 ------
81 KeyError
82 If this is a science observation and on the mountain.
83 """
84 # Return without raising if this is not a science observation
85 # since the defaults are fine.
86 try:
87 # This will raise if it is a science observation
88 is_non_science(self)
89 return
90 except KeyError:
91 pass
93 # We are still in the lab, return and use the default
94 if not self._is_on_mountain():
95 return
97 # This is a science observation on the mountain so we should not
98 # use defaults
99 raise KeyError(f"{self._log_prefix}: Required key is missing and this is a mountain science observation")
102class LatissTranslator(LsstBaseTranslator):
103 """Metadata translator for LSST LATISS data from AuxTel.
105 For lab measurements many values are masked out.
106 """
108 name = "LSST_LATISS"
109 """Name of this translation class"""
111 supported_instrument = "LATISS"
112 """Supports the LATISS instrument."""
114 _const_map = {
115 "instrument": "LATISS",
116 "telescope": "Rubin Auxiliary Telescope",
117 "detector_group": _DETECTOR_GROUP_NAME,
118 "detector_num": 0,
119 "detector_name": _DETECTOR_NAME, # Single sensor
120 "science_program": "unknown",
121 "relative_humidity": None,
122 "pressure": None,
123 "temperature": None,
124 }
126 _trivial_map = {
127 "observation_id": (["OBSID", "IMGNAME"], dict(default=None, checker=is_non_science)),
128 "detector_serial": ["LSST_NUM", "DETSER"],
129 "object": ("OBJECT", dict(checker=is_non_science_or_lab, default="UNKNOWN")),
130 "boresight_rotation_angle": (["ROTPA", "ROTANGLE"], dict(checker=is_non_science_or_lab,
131 default=float("nan"), unit=u.deg)),
132 }
134 DETECTOR_GROUP_NAME = _DETECTOR_GROUP_NAME
135 """Fixed name of detector group."""
137 DETECTOR_NAME = _DETECTOR_NAME
138 """Fixed name of single sensor."""
140 DETECTOR_MAX = 0
141 """Maximum number of detectors to use when calculating the
142 detector_exposure_id."""
144 _DEFAULT_LOCATION = AUXTEL_LOCATION
145 """Default telescope location in absence of relevant FITS headers."""
147 @classmethod
148 def can_translate(cls, header, filename=None):
149 """Indicate whether this translation class can translate the
150 supplied header.
152 Parameters
153 ----------
154 header : `dict`-like
155 Header to convert to standardized form.
156 filename : `str`, optional
157 Name of file being translated.
159 Returns
160 -------
161 can : `bool`
162 `True` if the header is recognized by this class. `False`
163 otherwise.
164 """
165 # INSTRUME keyword might be of two types
166 if "INSTRUME" in header:
167 instrume = header["INSTRUME"]
168 for v in ("LSST_ATISS", "LATISS"):
169 if instrume == v:
170 return True
171 # Calibration files strip important headers at the moment so guess
172 if "DETNAME" in header and header["DETNAME"] == "RXX_S00":
173 return True
174 return False
176 @classmethod
177 def fix_header(cls, header, instrument, obsid, filename=None):
178 """Fix an incorrect LATISS header.
180 Parameters
181 ----------
182 header : `dict`
183 The header to update. Updates are in place.
184 instrument : `str`
185 The name of the instrument.
186 obsid : `str`
187 Unique observation identifier associated with this header.
188 Will always be provided.
189 filename : `str`, optional
190 Filename associated with this header. May not be set since headers
191 can be fixed independently of any filename being known.
193 Returns
194 -------
195 modified = `bool`
196 Returns `True` if the header was updated.
198 Notes
199 -----
200 This method does not apply per-obsid corrections. The following
201 corrections are applied:
203 * On June 24th 2019 the detector was changed from ITL-3800C-098
204 to ITL-3800C-068. The header is intended to be correct in the
205 future.
206 * In late 2019 the DATE-OBS and MJD-OBS headers were reporting
207 1970 dates. To correct, the DATE/MJD headers are copied in to
208 replace them and the -END headers are cleared.
209 * Until November 2019 the IMGTYPE was set in the GROUPID header.
210 The value is moved to IMGTYPE.
211 * SHUTTIME is always forced to be `None`.
213 Corrections are reported as debug level log messages.
215 See `~astro_metadata_translator.fix_header` for details of the general
216 process.
217 """
218 modified = False
220 # Calculate the standard label to use for log messages
221 log_label = cls._construct_log_prefix(obsid, filename)
223 if "OBSID" not in header:
224 # Very old data used IMGNAME
225 header["OBSID"] = obsid
226 modified = True
227 # We are reporting the OBSID so no need to repeat it at start
228 # of log message. Use filename if we have it.
229 log_prefix = f"{filename}: " if filename else ""
230 log.debug("%sAssigning OBSID to a value of '%s'", log_prefix, header["OBSID"])
232 if "DAYOBS" not in header:
233 # OBS-NITE could have the value for DAYOBS but it is safer
234 # for older data to set it from the OBSID. Fall back to OBS-NITE
235 # if we have no alternative
236 dayObs = None
237 try:
238 dayObs = obsid.split("_", 3)[2]
239 except (AttributeError, ValueError):
240 # did not split as expected
241 pass
242 if dayObs is None or len(dayObs) != 8:
243 if "OBS-NITE" in header:
244 dayObs = header["OBS-NITE"]
245 log.debug("%s: Setting DAYOBS to '%s' from OBS-NITE header", log_label, dayObs)
246 else:
247 log.debug("%s: Unable to determine DAYOBS from header", log_label)
248 else:
249 log.debug("%s: Setting DAYOBS to '%s' from OBSID", log_label, dayObs)
250 if dayObs:
251 header["DAYOBS"] = dayObs
252 modified = True
254 if "SEQNUM" not in header:
255 try:
256 seqnum = obsid.split("_", 3)[3]
257 except (AttributeError, ValueError):
258 # did not split as expected
259 pass
260 else:
261 header["SEQNUM"] = int(seqnum)
262 modified = True
263 log.debug("%s: Extracting SEQNUM of '%s' from OBSID", log_label, header["SEQNUM"])
265 # The DATE-OBS / MJD-OBS keys can be 1970
266 if "DATE-OBS" in header and header["DATE-OBS"].startswith("1970"):
267 # Copy the headers from the DATE and MJD since we have no other
268 # choice.
269 header["DATE-OBS"] = header["DATE"]
270 header["DATE-BEG"] = header["DATE-OBS"]
271 header["MJD-OBS"] = header["MJD"]
272 header["MJD-BEG"] = header["MJD-OBS"]
274 # And clear the DATE-END and MJD-END -- the translator will use
275 # EXPTIME instead.
276 header["DATE-END"] = None
277 header["MJD-END"] = None
279 log.debug("%s: Forcing 1970 dates to '%s'", log_label, header["DATE"])
280 modified = True
282 # Create a translator since we need the date
283 translator = cls(header)
284 date = translator.to_datetime_begin()
285 if date > DETECTOR_068_DATE:
286 header["LSST_NUM"] = "ITL-3800C-068"
287 log.debug("%s: Forcing detector serial to %s", log_label, header["LSST_NUM"])
288 modified = True
290 if date < DATE_END_IS_BAD:
291 # DATE-END may or may not be in TAI and may or may not be
292 # before DATE-BEG. Simpler to clear it
293 if header.get("DATE-END"):
294 header["DATE-END"] = None
295 header["MJD-END"] = None
297 log.debug("%s: Clearing DATE-END as being untrustworthy", log_label)
298 modified = True
300 # Up until a certain date GROUPID was the IMGTYPE
301 if date < IMGTYPE_OKAY_DATE:
302 groupId = header.get("GROUPID")
303 if groupId and not groupId.startswith("test"):
304 imgType = header.get("IMGTYPE")
305 if not imgType:
306 if "_" in groupId:
307 # Sometimes have the form dark_0001_0002
308 # in this case we pull the IMGTYPE off the front and
309 # do not clear groupId (although groupId may now
310 # repeat on different days).
311 groupId, _ = groupId.split("_", 1)
312 elif groupId.upper() != "FOCUS" and groupId.upper().startswith("FOCUS"):
313 # If it is exactly FOCUS we want groupId cleared
314 groupId = "FOCUS"
315 else:
316 header["GROUPID"] = None
317 header["IMGTYPE"] = groupId
318 log.debug("%s: Setting IMGTYPE to '%s' from GROUPID", log_label, header["IMGTYPE"])
319 modified = True
320 else:
321 # Someone could be fixing headers in old data
322 # and we do not want GROUPID == IMGTYPE
323 if imgType == groupId:
324 # Clear the group so we default to original
325 header["GROUPID"] = None
327 # We were using OBJECT for engineering observations early on
328 if date < OBJECT_IS_ENGTEST:
329 imgType = header.get("IMGTYPE")
330 if imgType == "OBJECT":
331 header["IMGTYPE"] = "ENGTEST"
332 log.debug("%s: Changing OBJECT observation type to %s",
333 log_label, header["IMGTYPE"])
334 modified = True
336 # Early on the RA/DEC headers were stored in radians
337 if date < RADEC_IS_RADIANS:
338 if header.get("RA") is not None:
339 header["RA"] *= RAD2DEG
340 log.debug("%s: Changing RA header to degrees", log_label)
341 modified = True
342 if header.get("DEC") is not None:
343 header["DEC"] *= RAD2DEG
344 log.debug("%s: Changing DEC header to degrees", log_label)
345 modified = True
347 if header.get("SHUTTIME"):
348 log.debug("%s: Forcing SHUTTIME header to be None", log_label)
349 header["SHUTTIME"] = None
350 modified = True
352 if "OBJECT" not in header:
353 # Only patch OBJECT IMGTYPE
354 if "IMGTYPE" in header and header["IMGTYPE"] == "OBJECT":
355 log.debug("%s: Forcing OBJECT header to exist", log_label)
356 header["OBJECT"] = "NOTSET"
357 modified = True
359 if "RADESYS" in header:
360 if header["RADESYS"] == "":
361 # Default to ICRS
362 header["RADESYS"] = "ICRS"
363 log.debug("%s: Forcing blank RADESYS to '%s'", log_label, header["RADESYS"])
364 modified = True
366 if date < RASTART_IS_HOURS:
367 # Avoid two checks for case where RASTART is fine
368 if date < RASTART_IS_BAD:
369 # The wrong telescope position was used. Unsetting these will
370 # force the RA/DEC demand headers to be used instead.
371 for h in ("RASTART", "DECSTART", "RAEND", "DECEND"):
372 header[h] = None
373 log.debug("%s: Forcing derived RA/Dec headers to undefined", log_label)
374 modified = True
375 else:
376 # Correct hours to degrees
377 for h in ("RASTART", "RAEND"):
378 if header[h]:
379 header[h] *= 15.0
380 log.debug("%s: Correcting RASTART/END from hours to degrees", log_label)
381 modified = True
383 # RASTART/END headers have a TAI/UTC confusion causing an offset
384 # of 37 seconds for a period of time.
385 if RASTART_IS_BAD < date < RASTART_IS_OKAY:
386 modified = True
387 offset = (37.0 / 3600.0) * 15.0
388 for epoch in ("START", "END"):
389 h = "RA" + epoch
390 if header[h]:
391 header[h] += offset
393 return modified
395 def _is_on_mountain(self):
396 date = self.to_datetime_begin()
397 if date > TSTART:
398 return True
399 return False
401 @staticmethod
402 def compute_detector_exposure_id(exposure_id, detector_num):
403 """Compute the detector exposure ID from detector number and
404 exposure ID.
406 This is a helper method to allow code working outside the translator
407 infrastructure to use the same algorithm.
409 Parameters
410 ----------
411 exposure_id : `int`
412 Unique exposure ID.
413 detector_num : `int`
414 Detector number.
416 Returns
417 -------
418 detector_exposure_id : `int`
419 The calculated ID.
420 """
421 if detector_num != 0:
422 log.warning("Unexpected non-zero detector number for LATISS")
423 return exposure_id
425 @cache_translation
426 def to_dark_time(self):
427 # Docstring will be inherited. Property defined in properties.py
429 # Always compare with exposure time
430 # We may revisit this later if there is a cutoff date where we
431 # can always trust the header.
432 exptime = self.to_exposure_time()
434 if self.is_key_ok("DARKTIME"):
435 darktime = self.quantity_from_card("DARKTIME", u.s)
436 if darktime >= exptime:
437 return darktime
438 reason = "Dark time less than exposure time."
439 else:
440 reason = "Dark time not defined."
442 log.warning("%s: %s Setting dark time to the exposure time.",
443 self._log_prefix, reason)
444 return exptime
446 @cache_translation
447 def to_exposure_time(self):
448 # Docstring will be inherited. Property defined in properties.py
449 # Some data is missing a value for EXPTIME.
450 # Have to be careful we do not have circular logic when trying to
451 # guess
452 if self.is_key_ok("EXPTIME"):
453 return self.quantity_from_card("EXPTIME", u.s)
455 # A missing or undefined EXPTIME is problematic. Set to -1
456 # to indicate that none was found.
457 log.warning("%s: Insufficient information to derive exposure time. Setting to -1.0s",
458 self._log_prefix)
459 return -1.0 * u.s
461 @cache_translation
462 def to_observation_type(self):
463 """Determine the observation type.
465 In the absence of an ``IMGTYPE`` header, assumes lab data is always a
466 dark if exposure time is non-zero, else bias.
468 Returns
469 -------
470 obstype : `str`
471 Observation type.
472 """
474 # LATISS observation type is documented to appear in OBSTYPE
475 # but for historical reasons prefers IMGTYPE.
476 # Test the keys in order until we find one that contains a
477 # defined value.
478 obstype_keys = ["OBSTYPE", "IMGTYPE"]
480 obstype = None
481 for k in obstype_keys:
482 if self.is_key_ok(k):
483 obstype = self._header[k]
484 self._used_these_cards(k)
485 obstype = obstype.lower()
486 break
488 if obstype is not None:
489 if obstype == "object" and not self._is_on_mountain():
490 # Do not map object to science in lab since most
491 # code assume science is on sky with RA/Dec.
492 obstype = "labobject"
493 elif obstype in ("skyexp", "object"):
494 obstype = "science"
496 return obstype
498 # In the absence of any observation type information, return
499 # unknown unless we think it might be a bias.
500 exptime = self.to_exposure_time()
501 if exptime == 0.0:
502 obstype = "bias"
503 else:
504 obstype = "unknown"
505 log.warning("%s: Unable to determine observation type. Guessing '%s'",
506 self._log_prefix, obstype)
507 return obstype
509 @cache_translation
510 def to_physical_filter(self):
511 """Calculate the physical filter name.
513 Returns
514 -------
515 filter : `str`
516 Name of filter. A combination of FILTER and GRATING
517 headers joined by a "~". The filter and grating are always
518 combined. The filter or grating part will be "none" if no value
519 is specified. Uses "empty" if any of the filters or gratings
520 indicate an "empty_N" name. "unknown" indicates that the filter is
521 not defined anywhere but we think it should be. "none" indicates
522 that the filter was not defined but the observation is a dark
523 or bias.
524 """
526 physical_filter = self._determine_primary_filter()
528 if self.is_key_ok("GRATING"):
529 grating = self._header["GRATING"]
530 self._used_these_cards("GRATING")
532 if not grating or grating.lower().startswith("empty"):
533 grating = "empty"
534 else:
535 # Be explicit about having no knowledge of the grating
536 grating = "unknown"
538 physical_filter = f"{physical_filter}{FILTER_DELIMITER}{grating}"
540 return physical_filter