Coverage for python/lsst/obs/lsst/translators/latiss.py: 17%
236 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-10-20 10:51 +0000
1# This file is currently part of obs_lsst but is written to allow it
2# to be migrated to the astro_metadata_translator package at a later date.
3#
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the LICENSE file in this directory for details of code ownership.
7#
8# Use of this source code is governed by a 3-clause BSD-style
9# license that can be found in the LICENSE file.
11"""Metadata translation code for LSST LATISS headers"""
13__all__ = ("LatissTranslator", )
15import logging
16import math
18import astropy.units as u
19from astropy.time import Time
20from astropy.coordinates import EarthLocation
22from astro_metadata_translator import cache_translation
23from astro_metadata_translator.translators.helpers import is_non_science
24from .lsst import LsstBaseTranslator, FILTER_DELIMITER
25from .lsstCam import is_non_science_or_lab
27log = logging.getLogger(__name__)
30# AuxTel is not the same place as LSST
31# These coordinates read from Apple Maps
32AUXTEL_LOCATION = EarthLocation.from_geodetic(-70.747698, -30.244728, 2663.0)
34# Date instrument is taking data at telescope
35# Prior to this date many parameters are automatically nulled out
36# since the headers have not historically been reliable
37TSTART = Time("2020-01-01T00:00", format="isot", scale="utc")
39# Define the sensor and group name for AuxTel globally so that it can be used
40# in multiple places. There is no raft but for consistency with other LSST
41# cameras we define one.
42_DETECTOR_GROUP_NAME = "RXX"
43_DETECTOR_NAME = "S00"
45# Date 068 detector was put in LATISS
46DETECTOR_068_DATE = Time("2019-06-24T00:00", format="isot", scale="utc")
48# IMGTYPE header is filled in after this date
49IMGTYPE_OKAY_DATE = Time("2019-11-07T00:00", format="isot", scale="utc")
51# OBJECT IMGTYPE really means ENGTEST until this date
52OBJECT_IS_ENGTEST = Time("2020-01-27T20:00", format="isot", scale="utc")
54# RA and DEC headers are in radians until this date
55RADEC_IS_RADIANS = Time("2020-01-28T22:00", format="isot", scale="utc")
57# RASTART/DECSTART/RAEND/DECEND used wrong telescope location before this
58# 2020-02-01T00:00 we fixed the telescope location, but RASTART is still
59# in mount coordinates, so off by pointing model.
60RASTART_IS_BAD = Time("2020-05-01T00:00", format="isot", scale="utc")
62# Between RASTART_IS_BAD and this time the RASTART header uses hours
63# instead of degrees.
64RASTART_IS_HOURS = Time("2021-02-11T18:45", format="isot", scale="utc")
66# From this date RASTART is correct as-is.
67RASTART_IS_OKAY = Time("2021-02-12T00:00", format="isot", scale="utc")
69# DATE-END is not to be trusted before this date
70DATE_END_IS_BAD = Time("2020-02-01T00:00", format="isot", scale="utc")
72# The convention for the reporting of ROTPA changed by 180 here
73ROTPA_CONVENTION_180_SWITCH1 = Time("2020-11-19T00:00", format="isot", scale="utc")
74ROTPA_CONVENTION_180_SWITCH2 = Time("2021-10-29T00:00", format="isot", scale="utc")
76# TARGET is set to start with 'spec:' for dispsered images before this date
77TARGET_STARTS_SPECCOLON = Time("2022-07-10T00:00", format="isot", scale="utc")
79# Scaling factor radians to degrees. Keep it simple.
80RAD2DEG = 180.0 / math.pi
83class LatissTranslator(LsstBaseTranslator):
84 """Metadata translator for LSST LATISS data from AuxTel.
86 For lab measurements many values are masked out.
87 """
89 name = "LSST_LATISS"
90 """Name of this translation class"""
92 supported_instrument = "LATISS"
93 """Supports the LATISS instrument."""
95 _const_map = {
96 "instrument": "LATISS",
97 "telescope": "Rubin Auxiliary Telescope",
98 "detector_group": _DETECTOR_GROUP_NAME,
99 "detector_num": 0,
100 "detector_name": _DETECTOR_NAME, # Single sensor
101 "relative_humidity": None,
102 "pressure": None,
103 "temperature": None,
104 }
106 _trivial_map = {
107 "observation_id": (["OBSID", "IMGNAME"], dict(default=None, checker=is_non_science)),
108 "detector_serial": ["LSST_NUM", "DETSER"],
109 "object": ("OBJECT", dict(checker=is_non_science_or_lab, default="UNKNOWN")),
110 "boresight_rotation_angle": (["ROTPA", "ROTANGLE"], dict(checker=is_non_science_or_lab,
111 default=float("nan"), unit=u.deg)),
112 "science_program": ("PROGRAM", dict(default="unknown")),
113 }
115 DETECTOR_GROUP_NAME = _DETECTOR_GROUP_NAME
116 """Fixed name of detector group."""
118 DETECTOR_NAME = _DETECTOR_NAME
119 """Fixed name of single sensor."""
121 DETECTOR_MAX = 0
122 """Maximum number of detectors to use when calculating the
123 detector_exposure_id."""
125 _DEFAULT_LOCATION = AUXTEL_LOCATION
126 """Default telescope location in absence of relevant FITS headers."""
128 @classmethod
129 def can_translate(cls, header, filename=None):
130 """Indicate whether this translation class can translate the
131 supplied header.
133 Parameters
134 ----------
135 header : `dict`-like
136 Header to convert to standardized form.
137 filename : `str`, optional
138 Name of file being translated.
140 Returns
141 -------
142 can : `bool`
143 `True` if the header is recognized by this class. `False`
144 otherwise.
145 """
146 # INSTRUME keyword might be of two types
147 if "INSTRUME" in header:
148 instrume = header["INSTRUME"]
149 for v in ("LSST_ATISS", "LATISS"):
150 if instrume == v:
151 return True
152 # Calibration files strip important headers at the moment so guess
153 if "DETNAME" in header and header["DETNAME"] == "RXX_S00":
154 return True
155 return False
157 @classmethod
158 def fix_header(cls, header, instrument, obsid, filename=None):
159 """Fix an incorrect LATISS header.
161 Parameters
162 ----------
163 header : `dict`
164 The header to update. Updates are in place.
165 instrument : `str`
166 The name of the instrument.
167 obsid : `str`
168 Unique observation identifier associated with this header.
169 Will always be provided.
170 filename : `str`, optional
171 Filename associated with this header. May not be set since headers
172 can be fixed independently of any filename being known.
174 Returns
175 -------
176 modified = `bool`
177 Returns `True` if the header was updated.
179 Notes
180 -----
181 This method does not apply per-obsid corrections. The following
182 corrections are applied:
184 * On June 24th 2019 the detector was changed from ITL-3800C-098
185 to ITL-3800C-068. The header is intended to be correct in the
186 future.
187 * In late 2019 the DATE-OBS and MJD-OBS headers were reporting
188 1970 dates. To correct, the DATE/MJD headers are copied in to
189 replace them and the -END headers are cleared.
190 * Until November 2019 the IMGTYPE was set in the GROUPID header.
191 The value is moved to IMGTYPE.
192 * SHUTTIME is always forced to be `None`.
194 Corrections are reported as debug level log messages.
196 See `~astro_metadata_translator.fix_header` for details of the general
197 process.
198 """
199 modified = False
201 # Calculate the standard label to use for log messages
202 log_label = cls._construct_log_prefix(obsid, filename)
204 if "OBSID" not in header:
205 # Very old data used IMGNAME
206 header["OBSID"] = obsid
207 modified = True
208 # We are reporting the OBSID so no need to repeat it at start
209 # of log message. Use filename if we have it.
210 log_prefix = f"{filename}: " if filename else ""
211 log.debug("%sAssigning OBSID to a value of '%s'", log_prefix, header["OBSID"])
213 if "DAYOBS" not in header:
214 # OBS-NITE could have the value for DAYOBS but it is safer
215 # for older data to set it from the OBSID. Fall back to OBS-NITE
216 # if we have no alternative
217 dayObs = None
218 try:
219 dayObs = obsid.split("_", 3)[2]
220 except (AttributeError, ValueError):
221 # did not split as expected
222 pass
223 if dayObs is None or len(dayObs) != 8:
224 if "OBS-NITE" in header:
225 dayObs = header["OBS-NITE"]
226 log.debug("%s: Setting DAYOBS to '%s' from OBS-NITE header", log_label, dayObs)
227 else:
228 log.debug("%s: Unable to determine DAYOBS from header", log_label)
229 else:
230 log.debug("%s: Setting DAYOBS to '%s' from OBSID", log_label, dayObs)
231 if dayObs:
232 header["DAYOBS"] = dayObs
233 modified = True
235 if "SEQNUM" not in header:
236 try:
237 seqnum = obsid.split("_", 3)[3]
238 except (AttributeError, ValueError):
239 # did not split as expected
240 pass
241 else:
242 header["SEQNUM"] = int(seqnum)
243 modified = True
244 log.debug("%s: Extracting SEQNUM of '%s' from OBSID", log_label, header["SEQNUM"])
246 # The DATE-OBS / MJD-OBS keys can be 1970
247 if "DATE-OBS" in header and header["DATE-OBS"].startswith("1970"):
248 # Copy the headers from the DATE and MJD since we have no other
249 # choice.
250 header["DATE-OBS"] = header["DATE"]
251 header["DATE-BEG"] = header["DATE-OBS"]
252 header["MJD-OBS"] = header["MJD"]
253 header["MJD-BEG"] = header["MJD-OBS"]
255 # And clear the DATE-END and MJD-END -- the translator will use
256 # EXPTIME instead.
257 header["DATE-END"] = None
258 header["MJD-END"] = None
260 log.debug("%s: Forcing 1970 dates to '%s'", log_label, header["DATE"])
261 modified = True
263 # Create a translator since we need the date
264 translator = cls(header)
265 date = translator.to_datetime_begin()
266 if date > DETECTOR_068_DATE:
267 header["LSST_NUM"] = "ITL-3800C-068"
268 log.debug("%s: Forcing detector serial to %s", log_label, header["LSST_NUM"])
269 modified = True
271 if date < DATE_END_IS_BAD:
272 # DATE-END may or may not be in TAI and may or may not be
273 # before DATE-BEG. Simpler to clear it
274 if header.get("DATE-END"):
275 header["DATE-END"] = None
276 header["MJD-END"] = None
278 log.debug("%s: Clearing DATE-END as being untrustworthy", log_label)
279 modified = True
281 # Up until a certain date GROUPID was the IMGTYPE
282 if date < IMGTYPE_OKAY_DATE:
283 groupId = header.get("GROUPID")
284 if groupId and not groupId.startswith("test"):
285 imgType = header.get("IMGTYPE")
286 if not imgType:
287 if "_" in groupId:
288 # Sometimes have the form dark_0001_0002
289 # in this case we pull the IMGTYPE off the front and
290 # do not clear groupId (although groupId may now
291 # repeat on different days).
292 groupId, _ = groupId.split("_", 1)
293 elif groupId.upper() != "FOCUS" and groupId.upper().startswith("FOCUS"):
294 # If it is exactly FOCUS we want groupId cleared
295 groupId = "FOCUS"
296 else:
297 header["GROUPID"] = None
298 header["IMGTYPE"] = groupId
299 log.debug("%s: Setting IMGTYPE to '%s' from GROUPID", log_label, header["IMGTYPE"])
300 modified = True
301 else:
302 # Someone could be fixing headers in old data
303 # and we do not want GROUPID == IMGTYPE
304 if imgType == groupId:
305 # Clear the group so we default to original
306 header["GROUPID"] = None
308 # We were using OBJECT for engineering observations early on
309 if date < OBJECT_IS_ENGTEST:
310 imgType = header.get("IMGTYPE")
311 if imgType == "OBJECT":
312 header["IMGTYPE"] = "ENGTEST"
313 log.debug("%s: Changing OBJECT observation type to %s",
314 log_label, header["IMGTYPE"])
315 modified = True
317 # Early on the RA/DEC headers were stored in radians
318 if date < RADEC_IS_RADIANS:
319 if header.get("RA") is not None:
320 header["RA"] *= RAD2DEG
321 log.debug("%s: Changing RA header to degrees", log_label)
322 modified = True
323 if header.get("DEC") is not None:
324 header["DEC"] *= RAD2DEG
325 log.debug("%s: Changing DEC header to degrees", log_label)
326 modified = True
328 if header.get("SHUTTIME"):
329 log.debug("%s: Forcing SHUTTIME header to be None", log_label)
330 header["SHUTTIME"] = None
331 modified = True
333 if "OBJECT" not in header:
334 # Only patch OBJECT IMGTYPE
335 if "IMGTYPE" in header and header["IMGTYPE"] == "OBJECT":
336 log.debug("%s: Forcing OBJECT header to exist", log_label)
337 header["OBJECT"] = "NOTSET"
338 modified = True
340 if date < TARGET_STARTS_SPECCOLON:
341 if "OBJECT" in header:
342 header["OBJECT"] = header['OBJECT'].replace('spec:', '')
343 modified = True
345 if "RADESYS" in header:
346 if header["RADESYS"] == "":
347 # Default to ICRS
348 header["RADESYS"] = "ICRS"
349 log.debug("%s: Forcing blank RADESYS to '%s'", log_label, header["RADESYS"])
350 modified = True
352 if date < RASTART_IS_HOURS:
353 # Avoid two checks for case where RASTART is fine
354 if date < RASTART_IS_BAD:
355 # The wrong telescope position was used. Unsetting these will
356 # force the RA/DEC demand headers to be used instead.
357 for h in ("RASTART", "DECSTART", "RAEND", "DECEND"):
358 header[h] = None
359 log.debug("%s: Forcing derived RA/Dec headers to undefined", log_label)
360 modified = True
361 else:
362 # Correct hours to degrees
363 for h in ("RASTART", "RAEND"):
364 if header[h]:
365 header[h] *= 15.0
366 log.debug("%s: Correcting RASTART/END from hours to degrees", log_label)
367 modified = True
369 # RASTART/END headers have a TAI/UTC confusion causing an offset
370 # of 37 seconds for a period of time.
371 if RASTART_IS_BAD < date < RASTART_IS_OKAY:
372 modified = True
373 offset = (37.0 / 3600.0) * 15.0
374 for epoch in ("START", "END"):
375 h = "RA" + epoch
376 if header[h]:
377 header[h] += offset
379 if date < ROTPA_CONVENTION_180_SWITCH2 and date > ROTPA_CONVENTION_180_SWITCH1:
380 header['ROTPA'] = header['ROTPA'] - 180
381 modified = True
383 return modified
385 def _is_on_mountain(self):
386 date = self.to_datetime_begin()
387 if date > TSTART:
388 return True
389 return False
391 @staticmethod
392 def compute_detector_exposure_id(exposure_id, detector_num):
393 """Compute the detector exposure ID from detector number and
394 exposure ID.
396 This is a helper method to allow code working outside the translator
397 infrastructure to use the same algorithm.
399 Parameters
400 ----------
401 exposure_id : `int`
402 Unique exposure ID.
403 detector_num : `int`
404 Detector number.
406 Returns
407 -------
408 detector_exposure_id : `int`
409 The calculated ID.
410 """
411 if detector_num != 0:
412 log.warning("Unexpected non-zero detector number for LATISS")
413 return exposure_id
415 @cache_translation
416 def to_dark_time(self):
417 # Docstring will be inherited. Property defined in properties.py
419 # Always compare with exposure time
420 # We may revisit this later if there is a cutoff date where we
421 # can always trust the header.
422 exptime = self.to_exposure_time()
424 if self.is_key_ok("DARKTIME"):
425 darktime = self.quantity_from_card("DARKTIME", u.s)
426 if darktime >= exptime:
427 return darktime
428 reason = "Dark time less than exposure time."
429 else:
430 reason = "Dark time not defined."
432 log.warning("%s: %s Setting dark time to the exposure time.",
433 self._log_prefix, reason)
434 return exptime
436 @cache_translation
437 def to_exposure_time(self):
438 # Docstring will be inherited. Property defined in properties.py
439 # Some data is missing a value for EXPTIME.
440 # Have to be careful we do not have circular logic when trying to
441 # guess
442 if self.is_key_ok("EXPTIME"):
443 return self.quantity_from_card("EXPTIME", u.s)
445 # A missing or undefined EXPTIME is problematic. Set to -1
446 # to indicate that none was found.
447 log.warning("%s: Insufficient information to derive exposure time. Setting to -1.0s",
448 self._log_prefix)
449 return -1.0 * u.s
451 @cache_translation
452 def to_observation_type(self):
453 """Determine the observation type.
455 In the absence of an ``IMGTYPE`` header, assumes lab data is always a
456 dark if exposure time is non-zero, else bias.
458 Returns
459 -------
460 obstype : `str`
461 Observation type.
462 """
464 # LATISS observation type is documented to appear in OBSTYPE
465 # but for historical reasons prefers IMGTYPE.
466 # Test the keys in order until we find one that contains a
467 # defined value.
468 obstype_keys = ["OBSTYPE", "IMGTYPE"]
470 obstype = None
471 for k in obstype_keys:
472 if self.is_key_ok(k):
473 obstype = self._header[k]
474 self._used_these_cards(k)
475 obstype = obstype.lower()
476 break
478 if obstype is not None:
479 if obstype == "object" and not self._is_on_mountain():
480 # Do not map object to science in lab since most
481 # code assume science is on sky with RA/Dec.
482 obstype = "labobject"
483 elif obstype in ("skyexp", "object"):
484 obstype = "science"
486 return obstype
488 # In the absence of any observation type information, return
489 # unknown unless we think it might be a bias.
490 exptime = self.to_exposure_time()
491 if exptime == 0.0:
492 obstype = "bias"
493 else:
494 obstype = "unknown"
495 log.warning("%s: Unable to determine observation type. Guessing '%s'",
496 self._log_prefix, obstype)
497 return obstype
499 @cache_translation
500 def to_physical_filter(self):
501 """Calculate the physical filter name.
503 Returns
504 -------
505 filter : `str`
506 Name of filter. A combination of FILTER and GRATING
507 headers joined by a "~". The filter and grating are always
508 combined. The filter or grating part will be "none" if no value
509 is specified. Uses "empty" if any of the filters or gratings
510 indicate an "empty_N" name. "unknown" indicates that the filter is
511 not defined anywhere but we think it should be. "none" indicates
512 that the filter was not defined but the observation is a dark
513 or bias.
514 """
516 physical_filter = self._determine_primary_filter()
518 if self.is_key_ok("GRATING"):
519 grating = self._header["GRATING"]
520 self._used_these_cards("GRATING")
522 if not grating or grating.lower().startswith("empty"):
523 grating = "empty"
524 else:
525 # Be explicit about having no knowledge of the grating
526 grating = "unknown"
528 physical_filter = f"{physical_filter}{FILTER_DELIMITER}{grating}"
530 return physical_filter