Coverage for python/lsst/obs/lsst/translators/latiss.py: 16%
244 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-06 22:09 +0000
1# This file is currently part of obs_lsst but is written to allow it
2# to be migrated to the astro_metadata_translator package at a later date.
3#
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the LICENSE file in this directory for details of code ownership.
7#
8# Use of this source code is governed by a 3-clause BSD-style
9# license that can be found in the LICENSE file.
11"""Metadata translation code for LSST LATISS headers"""
13__all__ = ("LatissTranslator", )
15import logging
16import math
18import astropy.units as u
19from astropy.time import Time
20from astropy.coordinates import EarthLocation
22from astro_metadata_translator import cache_translation
23from astro_metadata_translator.translators.helpers import is_non_science
24from .lsst import LsstBaseTranslator, FILTER_DELIMITER
26log = logging.getLogger(__name__)
29# AuxTel is not the same place as LSST
30# These coordinates read from Apple Maps
31AUXTEL_LOCATION = EarthLocation.from_geodetic(-70.747698, -30.244728, 2663.0)
33# Date instrument is taking data at telescope
34# Prior to this date many parameters are automatically nulled out
35# since the headers have not historically been reliable
36TSTART = Time("2020-01-01T00:00", format="isot", scale="utc")
38# Define the sensor and group name for AuxTel globally so that it can be used
39# in multiple places. There is no raft but for consistency with other LSST
40# cameras we define one.
41_DETECTOR_GROUP_NAME = "RXX"
42_DETECTOR_NAME = "S00"
44# Date 068 detector was put in LATISS
45DETECTOR_068_DATE = Time("2019-06-24T00:00", format="isot", scale="utc")
47# IMGTYPE header is filled in after this date
48IMGTYPE_OKAY_DATE = Time("2019-11-07T00:00", format="isot", scale="utc")
50# OBJECT IMGTYPE really means ENGTEST until this date
51OBJECT_IS_ENGTEST = Time("2020-01-27T20:00", format="isot", scale="utc")
53# RA and DEC headers are in radians until this date
54RADEC_IS_RADIANS = Time("2020-01-28T22:00", format="isot", scale="utc")
56# RASTART/DECSTART/RAEND/DECEND used wrong telescope location before this
57# 2020-02-01T00:00 we fixed the telescope location, but RASTART is still
58# in mount coordinates, so off by pointing model.
59RASTART_IS_BAD = Time("2020-05-01T00:00", format="isot", scale="utc")
61# Between RASTART_IS_BAD and this time the RASTART header uses hours
62# instead of degrees.
63RASTART_IS_HOURS = Time("2021-02-11T18:45", format="isot", scale="utc")
65# From this date RASTART is correct as-is.
66RASTART_IS_OKAY = Time("2021-02-12T00:00", format="isot", scale="utc")
68# DATE-END is not to be trusted before this date
69DATE_END_IS_BAD = Time("2020-02-01T00:00", format="isot", scale="utc")
71# The convention for the reporting of ROTPA changed by 180 here
72ROTPA_CONVENTION_180_SWITCH1 = Time("2020-11-19T00:00", format="isot", scale="utc")
73ROTPA_CONVENTION_180_SWITCH2 = Time("2021-10-29T00:00", format="isot", scale="utc")
75# TARGET is set to start with 'spec:' for dispsered images before this date
76TARGET_STARTS_SPECCOLON = Time("2022-07-10T00:00", format="isot", scale="utc")
78# Scaling factor radians to degrees. Keep it simple.
79RAD2DEG = 180.0 / math.pi
82def is_non_science_or_lab(self):
83 """Pseudo method to determine whether this is a lab or non-science
84 header.
86 Raises
87 ------
88 KeyError
89 If this is a science observation and on the mountain.
90 """
91 # Return without raising if this is not a science observation
92 # since the defaults are fine.
93 try:
94 # This will raise if it is a science observation
95 is_non_science(self)
96 return
97 except KeyError:
98 pass
100 # We are still in the lab, return and use the default
101 if not self._is_on_mountain():
102 return
104 # This is a science observation on the mountain so we should not
105 # use defaults
106 raise KeyError(f"{self._log_prefix}: Required key is missing and this is a mountain science observation")
109class LatissTranslator(LsstBaseTranslator):
110 """Metadata translator for LSST LATISS data from AuxTel.
112 For lab measurements many values are masked out.
113 """
115 name = "LSST_LATISS"
116 """Name of this translation class"""
118 supported_instrument = "LATISS"
119 """Supports the LATISS instrument."""
121 _const_map = {
122 "instrument": "LATISS",
123 "telescope": "Rubin Auxiliary Telescope",
124 "detector_group": _DETECTOR_GROUP_NAME,
125 "detector_num": 0,
126 "detector_name": _DETECTOR_NAME, # Single sensor
127 "relative_humidity": None,
128 "pressure": None,
129 "temperature": None,
130 }
132 _trivial_map = {
133 "observation_id": (["OBSID", "IMGNAME"], dict(default=None, checker=is_non_science)),
134 "detector_serial": ["LSST_NUM", "DETSER"],
135 "object": ("OBJECT", dict(checker=is_non_science_or_lab, default="UNKNOWN")),
136 "boresight_rotation_angle": (["ROTPA", "ROTANGLE"], dict(checker=is_non_science_or_lab,
137 default=float("nan"), unit=u.deg)),
138 "science_program": ("PROGRAM", dict(default="unknown")),
139 }
141 DETECTOR_GROUP_NAME = _DETECTOR_GROUP_NAME
142 """Fixed name of detector group."""
144 DETECTOR_NAME = _DETECTOR_NAME
145 """Fixed name of single sensor."""
147 DETECTOR_MAX = 0
148 """Maximum number of detectors to use when calculating the
149 detector_exposure_id."""
151 _DEFAULT_LOCATION = AUXTEL_LOCATION
152 """Default telescope location in absence of relevant FITS headers."""
154 @classmethod
155 def can_translate(cls, header, filename=None):
156 """Indicate whether this translation class can translate the
157 supplied header.
159 Parameters
160 ----------
161 header : `dict`-like
162 Header to convert to standardized form.
163 filename : `str`, optional
164 Name of file being translated.
166 Returns
167 -------
168 can : `bool`
169 `True` if the header is recognized by this class. `False`
170 otherwise.
171 """
172 # INSTRUME keyword might be of two types
173 if "INSTRUME" in header:
174 instrume = header["INSTRUME"]
175 for v in ("LSST_ATISS", "LATISS"):
176 if instrume == v:
177 return True
178 # Calibration files strip important headers at the moment so guess
179 if "DETNAME" in header and header["DETNAME"] == "RXX_S00":
180 return True
181 return False
183 @classmethod
184 def fix_header(cls, header, instrument, obsid, filename=None):
185 """Fix an incorrect LATISS header.
187 Parameters
188 ----------
189 header : `dict`
190 The header to update. Updates are in place.
191 instrument : `str`
192 The name of the instrument.
193 obsid : `str`
194 Unique observation identifier associated with this header.
195 Will always be provided.
196 filename : `str`, optional
197 Filename associated with this header. May not be set since headers
198 can be fixed independently of any filename being known.
200 Returns
201 -------
202 modified = `bool`
203 Returns `True` if the header was updated.
205 Notes
206 -----
207 This method does not apply per-obsid corrections. The following
208 corrections are applied:
210 * On June 24th 2019 the detector was changed from ITL-3800C-098
211 to ITL-3800C-068. The header is intended to be correct in the
212 future.
213 * In late 2019 the DATE-OBS and MJD-OBS headers were reporting
214 1970 dates. To correct, the DATE/MJD headers are copied in to
215 replace them and the -END headers are cleared.
216 * Until November 2019 the IMGTYPE was set in the GROUPID header.
217 The value is moved to IMGTYPE.
218 * SHUTTIME is always forced to be `None`.
220 Corrections are reported as debug level log messages.
222 See `~astro_metadata_translator.fix_header` for details of the general
223 process.
224 """
225 modified = False
227 # Calculate the standard label to use for log messages
228 log_label = cls._construct_log_prefix(obsid, filename)
230 if "OBSID" not in header:
231 # Very old data used IMGNAME
232 header["OBSID"] = obsid
233 modified = True
234 # We are reporting the OBSID so no need to repeat it at start
235 # of log message. Use filename if we have it.
236 log_prefix = f"{filename}: " if filename else ""
237 log.debug("%sAssigning OBSID to a value of '%s'", log_prefix, header["OBSID"])
239 if "DAYOBS" not in header:
240 # OBS-NITE could have the value for DAYOBS but it is safer
241 # for older data to set it from the OBSID. Fall back to OBS-NITE
242 # if we have no alternative
243 dayObs = None
244 try:
245 dayObs = obsid.split("_", 3)[2]
246 except (AttributeError, ValueError):
247 # did not split as expected
248 pass
249 if dayObs is None or len(dayObs) != 8:
250 if "OBS-NITE" in header:
251 dayObs = header["OBS-NITE"]
252 log.debug("%s: Setting DAYOBS to '%s' from OBS-NITE header", log_label, dayObs)
253 else:
254 log.debug("%s: Unable to determine DAYOBS from header", log_label)
255 else:
256 log.debug("%s: Setting DAYOBS to '%s' from OBSID", log_label, dayObs)
257 if dayObs:
258 header["DAYOBS"] = dayObs
259 modified = True
261 if "SEQNUM" not in header:
262 try:
263 seqnum = obsid.split("_", 3)[3]
264 except (AttributeError, ValueError):
265 # did not split as expected
266 pass
267 else:
268 header["SEQNUM"] = int(seqnum)
269 modified = True
270 log.debug("%s: Extracting SEQNUM of '%s' from OBSID", log_label, header["SEQNUM"])
272 # The DATE-OBS / MJD-OBS keys can be 1970
273 if "DATE-OBS" in header and header["DATE-OBS"].startswith("1970"):
274 # Copy the headers from the DATE and MJD since we have no other
275 # choice.
276 header["DATE-OBS"] = header["DATE"]
277 header["DATE-BEG"] = header["DATE-OBS"]
278 header["MJD-OBS"] = header["MJD"]
279 header["MJD-BEG"] = header["MJD-OBS"]
281 # And clear the DATE-END and MJD-END -- the translator will use
282 # EXPTIME instead.
283 header["DATE-END"] = None
284 header["MJD-END"] = None
286 log.debug("%s: Forcing 1970 dates to '%s'", log_label, header["DATE"])
287 modified = True
289 # Create a translator since we need the date
290 translator = cls(header)
291 date = translator.to_datetime_begin()
292 if date > DETECTOR_068_DATE:
293 header["LSST_NUM"] = "ITL-3800C-068"
294 log.debug("%s: Forcing detector serial to %s", log_label, header["LSST_NUM"])
295 modified = True
297 if date < DATE_END_IS_BAD:
298 # DATE-END may or may not be in TAI and may or may not be
299 # before DATE-BEG. Simpler to clear it
300 if header.get("DATE-END"):
301 header["DATE-END"] = None
302 header["MJD-END"] = None
304 log.debug("%s: Clearing DATE-END as being untrustworthy", log_label)
305 modified = True
307 # Up until a certain date GROUPID was the IMGTYPE
308 if date < IMGTYPE_OKAY_DATE:
309 groupId = header.get("GROUPID")
310 if groupId and not groupId.startswith("test"):
311 imgType = header.get("IMGTYPE")
312 if not imgType:
313 if "_" in groupId:
314 # Sometimes have the form dark_0001_0002
315 # in this case we pull the IMGTYPE off the front and
316 # do not clear groupId (although groupId may now
317 # repeat on different days).
318 groupId, _ = groupId.split("_", 1)
319 elif groupId.upper() != "FOCUS" and groupId.upper().startswith("FOCUS"):
320 # If it is exactly FOCUS we want groupId cleared
321 groupId = "FOCUS"
322 else:
323 header["GROUPID"] = None
324 header["IMGTYPE"] = groupId
325 log.debug("%s: Setting IMGTYPE to '%s' from GROUPID", log_label, header["IMGTYPE"])
326 modified = True
327 else:
328 # Someone could be fixing headers in old data
329 # and we do not want GROUPID == IMGTYPE
330 if imgType == groupId:
331 # Clear the group so we default to original
332 header["GROUPID"] = None
334 # We were using OBJECT for engineering observations early on
335 if date < OBJECT_IS_ENGTEST:
336 imgType = header.get("IMGTYPE")
337 if imgType == "OBJECT":
338 header["IMGTYPE"] = "ENGTEST"
339 log.debug("%s: Changing OBJECT observation type to %s",
340 log_label, header["IMGTYPE"])
341 modified = True
343 # Early on the RA/DEC headers were stored in radians
344 if date < RADEC_IS_RADIANS:
345 if header.get("RA") is not None:
346 header["RA"] *= RAD2DEG
347 log.debug("%s: Changing RA header to degrees", log_label)
348 modified = True
349 if header.get("DEC") is not None:
350 header["DEC"] *= RAD2DEG
351 log.debug("%s: Changing DEC header to degrees", log_label)
352 modified = True
354 if header.get("SHUTTIME"):
355 log.debug("%s: Forcing SHUTTIME header to be None", log_label)
356 header["SHUTTIME"] = None
357 modified = True
359 if "OBJECT" not in header:
360 # Only patch OBJECT IMGTYPE
361 if "IMGTYPE" in header and header["IMGTYPE"] == "OBJECT":
362 log.debug("%s: Forcing OBJECT header to exist", log_label)
363 header["OBJECT"] = "NOTSET"
364 modified = True
366 if date < TARGET_STARTS_SPECCOLON:
367 if "OBJECT" in header:
368 header["OBJECT"] = header['OBJECT'].replace('spec:', '')
369 modified = True
371 if "RADESYS" in header:
372 if header["RADESYS"] == "":
373 # Default to ICRS
374 header["RADESYS"] = "ICRS"
375 log.debug("%s: Forcing blank RADESYS to '%s'", log_label, header["RADESYS"])
376 modified = True
378 if date < RASTART_IS_HOURS:
379 # Avoid two checks for case where RASTART is fine
380 if date < RASTART_IS_BAD:
381 # The wrong telescope position was used. Unsetting these will
382 # force the RA/DEC demand headers to be used instead.
383 for h in ("RASTART", "DECSTART", "RAEND", "DECEND"):
384 header[h] = None
385 log.debug("%s: Forcing derived RA/Dec headers to undefined", log_label)
386 modified = True
387 else:
388 # Correct hours to degrees
389 for h in ("RASTART", "RAEND"):
390 if header[h]:
391 header[h] *= 15.0
392 log.debug("%s: Correcting RASTART/END from hours to degrees", log_label)
393 modified = True
395 # RASTART/END headers have a TAI/UTC confusion causing an offset
396 # of 37 seconds for a period of time.
397 if RASTART_IS_BAD < date < RASTART_IS_OKAY:
398 modified = True
399 offset = (37.0 / 3600.0) * 15.0
400 for epoch in ("START", "END"):
401 h = "RA" + epoch
402 if header[h]:
403 header[h] += offset
405 if date < ROTPA_CONVENTION_180_SWITCH2 and date > ROTPA_CONVENTION_180_SWITCH1:
406 header['ROTPA'] = header['ROTPA'] - 180
407 modified = True
409 return modified
411 def _is_on_mountain(self):
412 date = self.to_datetime_begin()
413 if date > TSTART:
414 return True
415 return False
417 @staticmethod
418 def compute_detector_exposure_id(exposure_id, detector_num):
419 """Compute the detector exposure ID from detector number and
420 exposure ID.
422 This is a helper method to allow code working outside the translator
423 infrastructure to use the same algorithm.
425 Parameters
426 ----------
427 exposure_id : `int`
428 Unique exposure ID.
429 detector_num : `int`
430 Detector number.
432 Returns
433 -------
434 detector_exposure_id : `int`
435 The calculated ID.
436 """
437 if detector_num != 0:
438 log.warning("Unexpected non-zero detector number for LATISS")
439 return exposure_id
441 @cache_translation
442 def to_dark_time(self):
443 # Docstring will be inherited. Property defined in properties.py
445 # Always compare with exposure time
446 # We may revisit this later if there is a cutoff date where we
447 # can always trust the header.
448 exptime = self.to_exposure_time()
450 if self.is_key_ok("DARKTIME"):
451 darktime = self.quantity_from_card("DARKTIME", u.s)
452 if darktime >= exptime:
453 return darktime
454 reason = "Dark time less than exposure time."
455 else:
456 reason = "Dark time not defined."
458 log.warning("%s: %s Setting dark time to the exposure time.",
459 self._log_prefix, reason)
460 return exptime
462 @cache_translation
463 def to_exposure_time(self):
464 # Docstring will be inherited. Property defined in properties.py
465 # Some data is missing a value for EXPTIME.
466 # Have to be careful we do not have circular logic when trying to
467 # guess
468 if self.is_key_ok("EXPTIME"):
469 return self.quantity_from_card("EXPTIME", u.s)
471 # A missing or undefined EXPTIME is problematic. Set to -1
472 # to indicate that none was found.
473 log.warning("%s: Insufficient information to derive exposure time. Setting to -1.0s",
474 self._log_prefix)
475 return -1.0 * u.s
477 @cache_translation
478 def to_observation_type(self):
479 """Determine the observation type.
481 In the absence of an ``IMGTYPE`` header, assumes lab data is always a
482 dark if exposure time is non-zero, else bias.
484 Returns
485 -------
486 obstype : `str`
487 Observation type.
488 """
490 # LATISS observation type is documented to appear in OBSTYPE
491 # but for historical reasons prefers IMGTYPE.
492 # Test the keys in order until we find one that contains a
493 # defined value.
494 obstype_keys = ["OBSTYPE", "IMGTYPE"]
496 obstype = None
497 for k in obstype_keys:
498 if self.is_key_ok(k):
499 obstype = self._header[k]
500 self._used_these_cards(k)
501 obstype = obstype.lower()
502 break
504 if obstype is not None:
505 if obstype == "object" and not self._is_on_mountain():
506 # Do not map object to science in lab since most
507 # code assume science is on sky with RA/Dec.
508 obstype = "labobject"
509 elif obstype in ("skyexp", "object"):
510 obstype = "science"
512 return obstype
514 # In the absence of any observation type information, return
515 # unknown unless we think it might be a bias.
516 exptime = self.to_exposure_time()
517 if exptime == 0.0:
518 obstype = "bias"
519 else:
520 obstype = "unknown"
521 log.warning("%s: Unable to determine observation type. Guessing '%s'",
522 self._log_prefix, obstype)
523 return obstype
525 @cache_translation
526 def to_physical_filter(self):
527 """Calculate the physical filter name.
529 Returns
530 -------
531 filter : `str`
532 Name of filter. A combination of FILTER and GRATING
533 headers joined by a "~". The filter and grating are always
534 combined. The filter or grating part will be "none" if no value
535 is specified. Uses "empty" if any of the filters or gratings
536 indicate an "empty_N" name. "unknown" indicates that the filter is
537 not defined anywhere but we think it should be. "none" indicates
538 that the filter was not defined but the observation is a dark
539 or bias.
540 """
542 physical_filter = self._determine_primary_filter()
544 if self.is_key_ok("GRATING"):
545 grating = self._header["GRATING"]
546 self._used_these_cards("GRATING")
548 if not grating or grating.lower().startswith("empty"):
549 grating = "empty"
550 else:
551 # Be explicit about having no knowledge of the grating
552 grating = "unknown"
554 physical_filter = f"{physical_filter}{FILTER_DELIMITER}{grating}"
556 return physical_filter