Coverage for python/lsst/obs/lsst/translators/latiss.py : 15%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is currently part of obs_lsst but is written to allow it
2# to be migrated to the astro_metadata_translator package at a later date.
3#
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the LICENSE file in this directory for details of code ownership.
7#
8# Use of this source code is governed by a 3-clause BSD-style
9# license that can be found in the LICENSE file.
11"""Metadata translation code for LSST LATISS headers"""
13__all__ = ("LatissTranslator", )
15import logging
16import math
18import astropy.units as u
19from astropy.time import Time
20from astropy.coordinates import EarthLocation
22from astro_metadata_translator import cache_translation
23from astro_metadata_translator.translators.helpers import is_non_science
24from .lsst import LsstBaseTranslator, FILTER_DELIMITER
26log = logging.getLogger(__name__)
29# AuxTel is not the same place as LSST
30# These coordinates read from Apple Maps
31AUXTEL_LOCATION = EarthLocation.from_geodetic(-70.747698, -30.244728, 2663.0)
33# Date instrument is taking data at telescope
34# Prior to this date many parameters are automatically nulled out
35# since the headers have not historically been reliable
36TSTART = Time("2020-01-01T00:00", format="isot", scale="utc")
38# Define the sensor and group name for AuxTel globally so that it can be used
39# in multiple places. There is no raft but for consistency with other LSST
40# cameras we define one.
41_DETECTOR_GROUP_NAME = "RXX"
42_DETECTOR_NAME = "S00"
44# Date 068 detector was put in LATISS
45DETECTOR_068_DATE = Time("2019-06-24T00:00", format="isot", scale="utc")
47# IMGTYPE header is filled in after this date
48IMGTYPE_OKAY_DATE = Time("2019-11-07T00:00", format="isot", scale="utc")
50# OBJECT IMGTYPE really means ENGTEST until this date
51OBJECT_IS_ENGTEST = Time("2020-01-27T20:00", format="isot", scale="utc")
53# RA and DEC headers are in radians until this date
54RADEC_IS_RADIANS = Time("2020-01-28T22:00", format="isot", scale="utc")
56# RASTART/DECSTART/RAEND/DECEND used wrong telescope location before this
57# 2020-02-01T00:00 we fixed the telescope location, but RASTART is still
58# in mount coordinates, so off by pointing model.
59RASTART_IS_BAD = Time("2020-05-01T00:00", format="isot", scale="utc")
61# Between RASTART_IS_BAD and this time the RASTART header uses hours
62# instead of degrees.
63RASTART_IS_HOURS = Time("2021-02-11T18:45", format="isot", scale="utc")
65# DATE-END is not to be trusted before this date
66DATE_END_IS_BAD = Time("2020-02-01T00:00", format="isot", scale="utc")
68# Scaling factor radians to degrees. Keep it simple.
69RAD2DEG = 180.0 / math.pi
72def is_non_science_or_lab(self):
73 """Pseudo method to determine whether this is a lab or non-science
74 header.
76 Raises
77 ------
78 KeyError
79 If this is a science observation and on the mountain.
80 """
81 # Return without raising if this is not a science observation
82 # since the defaults are fine.
83 try:
84 # This will raise if it is a science observation
85 is_non_science(self)
86 return
87 except KeyError:
88 pass
90 # We are still in the lab, return and use the default
91 if not self._is_on_mountain():
92 return
94 # This is a science observation on the mountain so we should not
95 # use defaults
96 raise KeyError(f"{self._log_prefix}: Required key is missing and this is a mountain science observation")
99class LatissTranslator(LsstBaseTranslator):
100 """Metadata translator for LSST LATISS data from AuxTel.
102 For lab measurements many values are masked out.
103 """
105 name = "LSST_LATISS"
106 """Name of this translation class"""
108 supported_instrument = "LATISS"
109 """Supports the LATISS instrument."""
111 _const_map = {
112 "instrument": "LATISS",
113 "telescope": "Rubin Auxiliary Telescope",
114 "detector_group": _DETECTOR_GROUP_NAME,
115 "detector_num": 0,
116 "detector_name": _DETECTOR_NAME, # Single sensor
117 "science_program": "unknown",
118 "relative_humidity": None,
119 "pressure": None,
120 "temperature": None,
121 }
123 _trivial_map = {
124 "observation_id": (["OBSID", "IMGNAME"], dict(default=None, checker=is_non_science)),
125 "detector_serial": ["LSST_NUM", "DETSER"],
126 "object": ("OBJECT", dict(checker=is_non_science_or_lab, default="UNKNOWN")),
127 "boresight_rotation_angle": (["ROTPA", "ROTANGLE"], dict(checker=is_non_science_or_lab,
128 default=float("nan"), unit=u.deg)),
129 }
131 DETECTOR_GROUP_NAME = _DETECTOR_GROUP_NAME
132 """Fixed name of detector group."""
134 DETECTOR_NAME = _DETECTOR_NAME
135 """Fixed name of single sensor."""
137 DETECTOR_MAX = 0
138 """Maximum number of detectors to use when calculating the
139 detector_exposure_id."""
141 _DEFAULT_LOCATION = AUXTEL_LOCATION
142 """Default telescope location in absence of relevant FITS headers."""
144 @classmethod
145 def can_translate(cls, header, filename=None):
146 """Indicate whether this translation class can translate the
147 supplied header.
149 Parameters
150 ----------
151 header : `dict`-like
152 Header to convert to standardized form.
153 filename : `str`, optional
154 Name of file being translated.
156 Returns
157 -------
158 can : `bool`
159 `True` if the header is recognized by this class. `False`
160 otherwise.
161 """
162 # INSTRUME keyword might be of two types
163 if "INSTRUME" in header:
164 instrume = header["INSTRUME"]
165 for v in ("LSST_ATISS", "LATISS"):
166 if instrume == v:
167 return True
168 # Calibration files strip important headers at the moment so guess
169 if "DETNAME" in header and header["DETNAME"] == "RXX_S00":
170 return True
171 return False
173 @classmethod
174 def fix_header(cls, header, instrument, obsid, filename=None):
175 """Fix an incorrect LATISS header.
177 Parameters
178 ----------
179 header : `dict`
180 The header to update. Updates are in place.
181 instrument : `str`
182 The name of the instrument.
183 obsid : `str`
184 Unique observation identifier associated with this header.
185 Will always be provided.
186 filename : `str`, optional
187 Filename associated with this header. May not be set since headers
188 can be fixed independently of any filename being known.
190 Returns
191 -------
192 modified = `bool`
193 Returns `True` if the header was updated.
195 Notes
196 -----
197 This method does not apply per-obsid corrections. The following
198 corrections are applied:
200 * On June 24th 2019 the detector was changed from ITL-3800C-098
201 to ITL-3800C-068. The header is intended to be correct in the
202 future.
203 * In late 2019 the DATE-OBS and MJD-OBS headers were reporting
204 1970 dates. To correct, the DATE/MJD headers are copied in to
205 replace them and the -END headers are cleared.
206 * Until November 2019 the IMGTYPE was set in the GROUPID header.
207 The value is moved to IMGTYPE.
208 * SHUTTIME is always forced to be `None`.
210 Corrections are reported as debug level log messages.
212 See `~astro_metadata_translator.fix_header` for details of the general
213 process.
214 """
215 modified = False
217 # Calculate the standard label to use for log messages
218 log_label = cls._construct_log_prefix(obsid, filename)
220 if "OBSID" not in header:
221 # Very old data used IMGNAME
222 header["OBSID"] = obsid
223 modified = True
224 # We are reporting the OBSID so no need to repeat it at start
225 # of log message. Use filename if we have it.
226 log_prefix = f"{filename}: " if filename else ""
227 log.debug("%sAssigning OBSID to a value of '%s'", log_prefix, header["OBSID"])
229 if "DAYOBS" not in header:
230 # OBS-NITE could have the value for DAYOBS but it is safer
231 # for older data to set it from the OBSID. Fall back to OBS-NITE
232 # if we have no alternative
233 dayObs = None
234 try:
235 dayObs = obsid.split("_", 3)[2]
236 except (AttributeError, ValueError):
237 # did not split as expected
238 pass
239 if dayObs is None or len(dayObs) != 8:
240 if "OBS-NITE" in header:
241 dayObs = header["OBS-NITE"]
242 log.debug("%s: Setting DAYOBS to '%s' from OBS-NITE header", log_label, dayObs)
243 else:
244 log.debug("%s: Unable to determine DAYOBS from header", log_label)
245 else:
246 log.debug("%s: Setting DAYOBS to '%s' from OBSID", log_label, dayObs)
247 if dayObs:
248 header["DAYOBS"] = dayObs
249 modified = True
251 if "SEQNUM" not in header:
252 try:
253 seqnum = obsid.split("_", 3)[3]
254 except (AttributeError, ValueError):
255 # did not split as expected
256 pass
257 else:
258 header["SEQNUM"] = int(seqnum)
259 modified = True
260 log.debug("%s: Extracting SEQNUM of '%s' from OBSID", log_label, header["SEQNUM"])
262 # The DATE-OBS / MJD-OBS keys can be 1970
263 if "DATE-OBS" in header and header["DATE-OBS"].startswith("1970"):
264 # Copy the headers from the DATE and MJD since we have no other
265 # choice.
266 header["DATE-OBS"] = header["DATE"]
267 header["DATE-BEG"] = header["DATE-OBS"]
268 header["MJD-OBS"] = header["MJD"]
269 header["MJD-BEG"] = header["MJD-OBS"]
271 # And clear the DATE-END and MJD-END -- the translator will use
272 # EXPTIME instead.
273 header["DATE-END"] = None
274 header["MJD-END"] = None
276 log.debug("%s: Forcing 1970 dates to '%s'", log_label, header["DATE"])
277 modified = True
279 # Create a translator since we need the date
280 translator = cls(header)
281 date = translator.to_datetime_begin()
282 if date > DETECTOR_068_DATE:
283 header["LSST_NUM"] = "ITL-3800C-068"
284 log.debug("%s: Forcing detector serial to %s", log_label, header["LSST_NUM"])
285 modified = True
287 if date < DATE_END_IS_BAD:
288 # DATE-END may or may not be in TAI and may or may not be
289 # before DATE-BEG. Simpler to clear it
290 if header.get("DATE-END"):
291 header["DATE-END"] = None
292 header["MJD-END"] = None
294 log.debug("%s: Clearing DATE-END as being untrustworthy", log_label)
295 modified = True
297 # Up until a certain date GROUPID was the IMGTYPE
298 if date < IMGTYPE_OKAY_DATE:
299 groupId = header.get("GROUPID")
300 if groupId and not groupId.startswith("test"):
301 imgType = header.get("IMGTYPE")
302 if not imgType:
303 if "_" in groupId:
304 # Sometimes have the form dark_0001_0002
305 # in this case we pull the IMGTYPE off the front and
306 # do not clear groupId (although groupId may now
307 # repeat on different days).
308 groupId, _ = groupId.split("_", 1)
309 elif groupId.upper() != "FOCUS" and groupId.upper().startswith("FOCUS"):
310 # If it is exactly FOCUS we want groupId cleared
311 groupId = "FOCUS"
312 else:
313 header["GROUPID"] = None
314 header["IMGTYPE"] = groupId
315 log.debug("%s: Setting IMGTYPE to '%s' from GROUPID", log_label, header["IMGTYPE"])
316 modified = True
317 else:
318 # Someone could be fixing headers in old data
319 # and we do not want GROUPID == IMGTYPE
320 if imgType == groupId:
321 # Clear the group so we default to original
322 header["GROUPID"] = None
324 # We were using OBJECT for engineering observations early on
325 if date < OBJECT_IS_ENGTEST:
326 imgType = header.get("IMGTYPE")
327 if imgType == "OBJECT":
328 header["IMGTYPE"] = "ENGTEST"
329 log.debug("%s: Changing OBJECT observation type to %s",
330 log_label, header["IMGTYPE"])
331 modified = True
333 # Early on the RA/DEC headers were stored in radians
334 if date < RADEC_IS_RADIANS:
335 if header.get("RA") is not None:
336 header["RA"] *= RAD2DEG
337 log.debug("%s: Changing RA header to degrees", log_label)
338 modified = True
339 if header.get("DEC") is not None:
340 header["DEC"] *= RAD2DEG
341 log.debug("%s: Changing DEC header to degrees", log_label)
342 modified = True
344 if header.get("SHUTTIME"):
345 log.debug("%s: Forcing SHUTTIME header to be None", log_label)
346 header["SHUTTIME"] = None
347 modified = True
349 if "OBJECT" not in header:
350 # Only patch OBJECT IMGTYPE
351 if "IMGTYPE" in header and header["IMGTYPE"] == "OBJECT":
352 log.debug("%s: Forcing OBJECT header to exist", log_label)
353 header["OBJECT"] = "NOTSET"
354 modified = True
356 if "RADESYS" in header:
357 if header["RADESYS"] == "":
358 # Default to ICRS
359 header["RADESYS"] = "ICRS"
360 log.debug("%s: Forcing blank RADESYS to '%s'", log_label, header["RADESYS"])
361 modified = True
363 if date < RASTART_IS_HOURS:
364 # Avoid two checks for case where RASTART is fine
365 if date < RASTART_IS_BAD:
366 # The wrong telescope position was used. Unsetting these will
367 # force the RA/DEC demand headers to be used instead.
368 for h in ("RASTART", "DECSTART", "RAEND", "DECEND"):
369 header[h] = None
370 log.debug("%s: Forcing derived RA/Dec headers to undefined", log_label)
371 modified = True
372 else:
373 # Correct hours to degrees
374 for h in ("RASTART", "RAEND"):
375 if header[h]:
376 header[h] *= 15.0
377 log.debug("%s: Correcting RASTART/END from hours to degrees", log_label)
378 modified = True
380 # RASTART/END headers have a TAI/UTC confusion causing an offset
381 # of 37 seconds. Once this is fixed in the acquisition system
382 # the correction will have an upper date bound.
383 if date > RASTART_IS_BAD:
384 offset = (37.0 / 3600.0) * 15.0
385 for epoch in ("START", "END"):
386 h = "RA" + epoch
387 if header[h]:
388 header[h] += offset
390 return modified
392 def _is_on_mountain(self):
393 date = self.to_datetime_begin()
394 if date > TSTART:
395 return True
396 return False
398 @staticmethod
399 def compute_detector_exposure_id(exposure_id, detector_num):
400 """Compute the detector exposure ID from detector number and
401 exposure ID.
403 This is a helper method to allow code working outside the translator
404 infrastructure to use the same algorithm.
406 Parameters
407 ----------
408 exposure_id : `int`
409 Unique exposure ID.
410 detector_num : `int`
411 Detector number.
413 Returns
414 -------
415 detector_exposure_id : `int`
416 The calculated ID.
417 """
418 if detector_num != 0:
419 log.warning("Unexpected non-zero detector number for LATISS")
420 return exposure_id
422 @cache_translation
423 def to_dark_time(self):
424 # Docstring will be inherited. Property defined in properties.py
426 # Always compare with exposure time
427 # We may revisit this later if there is a cutoff date where we
428 # can always trust the header.
429 exptime = self.to_exposure_time()
431 if self.is_key_ok("DARKTIME"):
432 darktime = self.quantity_from_card("DARKTIME", u.s)
433 if darktime >= exptime:
434 return darktime
435 reason = "Dark time less than exposure time."
436 else:
437 reason = "Dark time not defined."
439 log.warning("%s: %s Setting dark time to the exposure time.",
440 self._log_prefix, reason)
441 return exptime
443 @cache_translation
444 def to_exposure_time(self):
445 # Docstring will be inherited. Property defined in properties.py
446 # Some data is missing a value for EXPTIME.
447 # Have to be careful we do not have circular logic when trying to
448 # guess
449 if self.is_key_ok("EXPTIME"):
450 return self.quantity_from_card("EXPTIME", u.s)
452 # A missing or undefined EXPTIME is problematic. Set to -1
453 # to indicate that none was found.
454 log.warning("%s: Insufficient information to derive exposure time. Setting to -1.0s",
455 self._log_prefix)
456 return -1.0 * u.s
458 @cache_translation
459 def to_observation_type(self):
460 """Determine the observation type.
462 In the absence of an ``IMGTYPE`` header, assumes lab data is always a
463 dark if exposure time is non-zero, else bias.
465 Returns
466 -------
467 obstype : `str`
468 Observation type.
469 """
471 # LATISS observation type is documented to appear in OBSTYPE
472 # but for historical reasons prefers IMGTYPE.
473 # Test the keys in order until we find one that contains a
474 # defined value.
475 obstype_keys = ["OBSTYPE", "IMGTYPE"]
477 obstype = None
478 for k in obstype_keys:
479 if self.is_key_ok(k):
480 obstype = self._header[k]
481 self._used_these_cards(k)
482 obstype = obstype.lower()
483 break
485 if obstype is not None:
486 if obstype == "object" and not self._is_on_mountain():
487 # Do not map object to science in lab since most
488 # code assume science is on sky with RA/Dec.
489 obstype = "labobject"
490 elif obstype in ("skyexp", "object"):
491 obstype = "science"
493 return obstype
495 # In the absence of any observation type information, return
496 # unknown unless we think it might be a bias.
497 exptime = self.to_exposure_time()
498 if exptime == 0.0:
499 obstype = "bias"
500 else:
501 obstype = "unknown"
502 log.warning("%s: Unable to determine observation type. Guessing '%s'",
503 self._log_prefix, obstype)
504 return obstype
506 @cache_translation
507 def to_physical_filter(self):
508 """Calculate the physical filter name.
510 Returns
511 -------
512 filter : `str`
513 Name of filter. A combination of FILTER and GRATING
514 headers joined by a "~". The filter and grating are always
515 combined. The filter or grating part will be "none" if no value
516 is specified. Uses "empty" if any of the filters or gratings
517 indicate an "empty_N" name. "unknown" indicates that the filter is
518 not defined anywhere but we think it should be. "none" indicates
519 that the filter was not defined but the observation is a dark
520 or bias.
521 """
523 physical_filter = self._determine_primary_filter()
525 if self.is_key_ok("GRATING"):
526 grating = self._header["GRATING"]
527 self._used_these_cards("GRATING")
529 if not grating or grating.lower().startswith("empty"):
530 grating = "empty"
531 else:
532 # Be explicit about having no knowledge of the grating
533 grating = "unknown"
535 physical_filter = f"{physical_filter}{FILTER_DELIMITER}{grating}"
537 return physical_filter