Coverage for python/lsst/obs/lsst/translators/latiss.py : 13%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is currently part of obs_lsst but is written to allow it
2# to be migrated to the astro_metadata_translator package at a later date.
3#
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the LICENSE file in this directory for details of code ownership.
7#
8# Use of this source code is governed by a 3-clause BSD-style
9# license that can be found in the LICENSE file.
11"""Metadata translation code for LSST LATISS headers"""
13__all__ = ("LatissTranslator", )
15import logging
16import math
18import astropy.units as u
19from astropy.time import Time
20from astropy.coordinates import EarthLocation
22from astro_metadata_translator import cache_translation
23from astro_metadata_translator.translators.helpers import is_non_science
24from .lsst import LsstBaseTranslator, FILTER_DELIMITER
26log = logging.getLogger(__name__)
29# AuxTel is not the same place as LSST
30# These coordinates read from Apple Maps
31AUXTEL_LOCATION = EarthLocation.from_geodetic(-70.747698, -30.244728, 2663.0)
33# Date instrument is taking data at telescope
34# Prior to this date many parameters are automatically nulled out
35# since the headers have not historically been reliable
36TSTART = Time("2020-01-01T00:00", format="isot", scale="utc")
38# Define the sensor and group name for AuxTel globally so that it can be used
39# in multiple places. There is no raft but for consistency with other LSST
40# cameras we define one.
41_DETECTOR_GROUP_NAME = "RXX"
42_DETECTOR_NAME = "S00"
44# Date 068 detector was put in LATISS
45DETECTOR_068_DATE = Time("2019-06-24T00:00", format="isot", scale="utc")
47# IMGTYPE header is filled in after this date
48IMGTYPE_OKAY_DATE = Time("2019-11-07T00:00", format="isot", scale="utc")
50# OBJECT IMGTYPE really means ENGTEST until this date
51OBJECT_IS_ENGTEST = Time("2020-01-27T20:00", format="isot", scale="utc")
53# RA and DEC headers are in radians until this date
54RADEC_IS_RADIANS = Time("2020-01-28T22:00", format="isot", scale="utc")
56# RASTART/DECSTART/RAEND/DECEND used wrong telescope location before this
57RASTART_IS_BAD = Time("2020-02-01T00:00", format="isot", scale="utc")
59# DATE-END is not to be trusted before this date
60DATE_END_IS_BAD = Time("2020-02-01T00:00", format="isot", scale="utc")
62# Scaling factor radians to degrees. Keep it simple.
63RAD2DEG = 180.0 / math.pi
66def is_non_science_or_lab(self):
67 """Pseudo method to determine whether this is a lab or non-science
68 header.
70 Raises
71 ------
72 KeyError
73 If this is a science observation and on the mountain.
74 """
75 # Return without raising if this is not a science observation
76 # since the defaults are fine.
77 try:
78 # This will raise if it is a science observation
79 is_non_science(self)
80 return
81 except KeyError:
82 pass
84 # We are still in the lab, return and use the default
85 if not self._is_on_mountain():
86 return
88 # This is a science observation on the mountain so we should not
89 # use defaults
90 raise KeyError("Required key is missing and this is a mountain science observation")
93class LatissTranslator(LsstBaseTranslator):
94 """Metadata translator for LSST LATISS data from AuxTel.
96 For lab measurements many values are masked out.
97 """
99 name = "LSST_LATISS"
100 """Name of this translation class"""
102 supported_instrument = "LATISS"
103 """Supports the LATISS instrument."""
105 _const_map = {
106 # LATISS is not yet attached to a telescope so many translations
107 # are null.
108 "instrument": "LATISS",
109 "telescope": "LSSTAuxTel",
110 "detector_group": _DETECTOR_GROUP_NAME,
111 "detector_num": 0,
112 "detector_name": _DETECTOR_NAME, # Single sensor
113 "science_program": "unknown",
114 "relative_humidity": None,
115 "pressure": None,
116 "temperature": None,
117 }
119 _trivial_map = {
120 "observation_id": (["OBSID", "IMGNAME"], dict(default=None, checker=is_non_science)),
121 "detector_serial": ["LSST_NUM", "DETSER"],
122 "object": ("OBJECT", dict(checker=is_non_science_or_lab, default="UNKNOWN")),
123 "boresight_rotation_angle": (["ROTPA", "ROTANGLE"], dict(checker=is_non_science_or_lab,
124 default=float("nan"), unit=u.deg)),
125 }
127 DETECTOR_GROUP_NAME = _DETECTOR_GROUP_NAME
128 """Fixed name of detector group."""
130 DETECTOR_NAME = _DETECTOR_NAME
131 """Fixed name of single sensor."""
133 DETECTOR_MAX = 0
134 """Maximum number of detectors to use when calculating the
135 detector_exposure_id."""
137 _DEFAULT_LOCATION = AUXTEL_LOCATION
138 """Default telescope location in absence of relevant FITS headers."""
140 @classmethod
141 def can_translate(cls, header, filename=None):
142 """Indicate whether this translation class can translate the
143 supplied header.
145 Parameters
146 ----------
147 header : `dict`-like
148 Header to convert to standardized form.
149 filename : `str`, optional
150 Name of file being translated.
152 Returns
153 -------
154 can : `bool`
155 `True` if the header is recognized by this class. `False`
156 otherwise.
157 """
158 # INSTRUME keyword might be of two types
159 if "INSTRUME" in header:
160 instrume = header["INSTRUME"]
161 for v in ("LSST_ATISS", "LATISS"):
162 if instrume == v:
163 return True
164 # Calibration files strip important headers at the moment so guess
165 if "DETNAME" in header and header["DETNAME"] == "RXX_S00":
166 return True
167 return False
169 @classmethod
170 def fix_header(cls, header):
171 """Fix an incorrect LATISS header.
173 Parameters
174 ----------
175 header : `dict`
176 The header to update. Updates are in place.
178 Returns
179 -------
180 modified = `bool`
181 Returns `True` if the header was updated.
183 Notes
184 -----
185 This method does not apply per-obsid corrections. The following
186 corrections are applied:
188 * On June 24th 2019 the detector was changed from ITL-3800C-098
189 to ITL-3800C-068. The header is intended to be correct in the
190 future.
191 * In late 2019 the DATE-OBS and MJD-OBS headers were reporting
192 1970 dates. To correct, the DATE/MJD headers are copied in to
193 replace them and the -END headers are cleared.
194 * Until November 2019 the IMGTYPE was set in the GROUPID header.
195 The value is moved to IMGTYPE.
196 * SHUTTIME is always forced to be `None`.
198 Corrections are reported as debug level log messages.
199 """
200 modified = False
202 if "OBSID" not in header:
203 # Very old data used IMGNAME
204 header["OBSID"] = header.get("IMGNAME", "unknown")
205 modified = True
206 log.debug("Assigning OBSID to a value of '%s'", header["OBSID"])
208 obsid = header["OBSID"]
210 if "DAYOBS" not in header:
211 # OBS-NITE could have the value for DAYOBS but it is safer
212 # for older data to set it from the OBSID. Fall back to OBS-NITE
213 # if we have no alternative
214 dayObs = None
215 try:
216 dayObs = obsid.split("_", 3)[2]
217 except ValueError:
218 # did not split as expected
219 pass
220 if dayObs is None or len(dayObs) != 8:
221 dayObs = header["OBS-NITE"]
222 log.debug("%s: Setting DAYOBS to '%s' from OBS-NITE header", obsid, dayObs)
223 else:
224 log.debug("%s: Setting DAYOBS to '%s' from OBSID", obsid, dayObs)
225 header["DAYOBS"] = dayObs
226 modified = True
228 if "SEQNUM" not in header:
229 try:
230 seqnum = obsid.split("_", 3)[3]
231 except ValueError:
232 # did not split as expected
233 pass
234 else:
235 header["SEQNUM"] = int(seqnum)
236 modified = True
237 log.debug("%s: Extracting SEQNUM of '%s' from OBSID", obsid, header["SEQNUM"])
239 # The DATE-OBS / MJD-OBS keys can be 1970
240 if header["DATE-OBS"].startswith("1970"):
241 # Copy the headers from the DATE and MJD since we have no other
242 # choice.
243 header["DATE-OBS"] = header["DATE"]
244 header["DATE-BEG"] = header["DATE-OBS"]
245 header["MJD-OBS"] = header["MJD"]
246 header["MJD-BEG"] = header["MJD-OBS"]
248 # And clear the DATE-END and MJD-END -- the translator will use
249 # EXPTIME instead.
250 header["DATE-END"] = None
251 header["MJD-END"] = None
253 log.debug("%s: Forcing 1970 dates to '%s'", obsid, header["DATE"])
254 modified = True
256 # Create a translator since we need the date
257 translator = cls(header)
258 date = translator.to_datetime_begin()
259 if date > DETECTOR_068_DATE:
260 header["LSST_NUM"] = "ITL-3800C-068"
261 log.debug("%s: Forcing detector serial to %s", obsid, header["LSST_NUM"])
262 modified = True
264 if date < DATE_END_IS_BAD:
265 # DATE-END may or may not be in TAI and may or may not be
266 # before DATE-BEG. Simpler to clear it
267 if header.get("DATE-END"):
268 header["DATE-END"] = None
269 header["MJD-END"] = None
271 log.debug("%s: Clearing DATE-END as being untrustworthy", obsid)
272 modified = True
274 # Up until a certain date GROUPID was the IMGTYPE
275 if date < IMGTYPE_OKAY_DATE:
276 groupId = header.get("GROUPID")
277 if groupId and not groupId.startswith("test"):
278 imgType = header.get("IMGTYPE")
279 if not imgType:
280 if "_" in groupId:
281 # Sometimes have the form dark_0001_0002
282 # in this case we pull the IMGTYPE off the front and
283 # do not clear groupId (although groupId may now
284 # repeat on different days).
285 groupId, _ = groupId.split("_", 1)
286 elif groupId.upper() != "FOCUS" and groupId.upper().startswith("FOCUS"):
287 # If it is exactly FOCUS we want groupId cleared
288 groupId = "FOCUS"
289 else:
290 header["GROUPID"] = None
291 header["IMGTYPE"] = groupId
292 log.debug("%s: Setting IMGTYPE to '%s' from GROUPID", obsid, header["IMGTYPE"])
293 modified = True
294 else:
295 # Someone could be fixing headers in old data
296 # and we do not want GROUPID == IMGTYPE
297 if imgType == groupId:
298 # Clear the group so we default to original
299 header["GROUPID"] = None
301 # We were using OBJECT for engineering observations early on
302 if date < OBJECT_IS_ENGTEST:
303 imgType = header.get("IMGTYPE")
304 if imgType == "OBJECT":
305 header["IMGTYPE"] = "ENGTEST"
306 log.debug("%s: Changing OBJECT observation type to %s",
307 obsid, header["IMGTYPE"])
308 modified = True
310 # Early on the RA/DEC headers were stored in radians
311 if date < RADEC_IS_RADIANS:
312 if header.get("RA") is not None:
313 header["RA"] *= RAD2DEG
314 log.debug("%s: Changing RA header to degrees", obsid)
315 modified = True
316 if header.get("DEC") is not None:
317 header["DEC"] *= RAD2DEG
318 log.debug("%s: Changing DEC header to degrees", obsid)
319 modified = True
321 if header.get("SHUTTIME"):
322 log.debug("%s: Forcing SHUTTIME header to be None", obsid)
323 header["SHUTTIME"] = None
324 modified = True
326 if "OBJECT" not in header:
327 # Only patch OBJECT IMGTYPE
328 if "IMGTYPE" in header and header["IMGTYPE"] == "OBJECT":
329 log.debug("%s: Forcing OBJECT header to exist", obsid)
330 header["OBJECT"] = "NOTSET"
331 modified = True
333 if "RADESYS" in header:
334 if header["RADESYS"] == "":
335 # Default to ICRS
336 header["RADESYS"] = "ICRS"
337 log.debug("%s: Forcing blank RADESYS to '%s'", obsid, header["RADESYS"])
338 modified = True
340 if date < RASTART_IS_BAD:
341 # The wrong telescope position was used. Unsetting these will force
342 # the RA/DEC demand headers to be used instead.
343 for h in ("RASTART", "DECSTART", "RAEND", "DECEND"):
344 header[h] = None
345 log.debug("%s: Forcing derived RA/Dec headers to undefined", obsid)
347 return modified
349 def _is_on_mountain(self):
350 date = self.to_datetime_begin()
351 if date > TSTART:
352 return True
353 return False
355 @staticmethod
356 def compute_detector_exposure_id(exposure_id, detector_num):
357 """Compute the detector exposure ID from detector number and
358 exposure ID.
360 This is a helper method to allow code working outside the translator
361 infrastructure to use the same algorithm.
363 Parameters
364 ----------
365 exposure_id : `int`
366 Unique exposure ID.
367 detector_num : `int`
368 Detector number.
370 Returns
371 -------
372 detector_exposure_id : `int`
373 The calculated ID.
374 """
375 if detector_num != 0:
376 log.warning("Unexpected non-zero detector number for LATISS")
377 return exposure_id
379 @cache_translation
380 def to_dark_time(self):
381 # Docstring will be inherited. Property defined in properties.py
383 # Always compare with exposure time
384 # We may revisit this later if there is a cutoff date where we
385 # can always trust the header.
386 exptime = self.to_exposure_time()
388 if self.is_key_ok("DARKTIME"):
389 darktime = self.quantity_from_card("DARKTIME", u.s)
390 if darktime >= exptime:
391 return darktime
392 reason = "Dark time less than exposure time."
393 else:
394 reason = "Dark time not defined."
396 log.warning("%s: %s Setting dark time to the exposure time.",
397 self.to_observation_id(), reason)
398 return exptime
400 @cache_translation
401 def to_exposure_time(self):
402 # Docstring will be inherited. Property defined in properties.py
403 # Some data is missing a value for EXPTIME.
404 # Have to be careful we do not have circular logic when trying to
405 # guess
406 if self.is_key_ok("EXPTIME"):
407 return self.quantity_from_card("EXPTIME", u.s)
409 # A missing or undefined EXPTIME is problematic. Set to -1
410 # to indicate that none was found.
411 log.warning("%s: Insufficient information to derive exposure time. Setting to -1.0s",
412 self.to_observation_id())
413 return -1.0 * u.s
415 @cache_translation
416 def to_observation_type(self):
417 """Determine the observation type.
419 In the absence of an ``IMGTYPE`` header, assumes lab data is always a
420 dark if exposure time is non-zero, else bias.
422 Returns
423 -------
424 obstype : `str`
425 Observation type.
426 """
428 # LATISS observation type is documented to appear in OBSTYPE
429 # but for historical reasons prefers IMGTYPE.
430 # Test the keys in order until we find one that contains a
431 # defined value.
432 obstype_keys = ["OBSTYPE", "IMGTYPE"]
434 obstype = None
435 for k in obstype_keys:
436 if self.is_key_ok(k):
437 obstype = self._header[k]
438 self._used_these_cards(k)
439 obstype = obstype.lower()
440 break
442 if obstype is not None:
443 if obstype == "object" and not self._is_on_mountain():
444 # Do not map object to science in lab since most
445 # code assume science is on sky with RA/Dec.
446 obstype = "labobject"
447 elif obstype in ("skyexp", "object"):
448 obstype = "science"
450 return obstype
452 # In the absence of any observation type information, return
453 # unknown unless we think it might be a bias.
454 exptime = self.to_exposure_time()
455 if exptime == 0.0:
456 obstype = "bias"
457 else:
458 obstype = "unknown"
459 log.warning("%s: Unable to determine observation type. Guessing '%s'",
460 self.to_observation_id(), obstype)
461 return obstype
463 @cache_translation
464 def to_physical_filter(self):
465 """Calculate the physical filter name.
467 Returns
468 -------
469 filter : `str`
470 Name of filter. A combination of FILTER and GRATING
471 headers joined by a "~". The filter and grating are always
472 combined. The filter or grating part will be "NONE" if no value
473 is specified. Uses "EMPTY" if any of the filters or gratings
474 indicate an "empty_N" name. "????" indicates that the filter is
475 not defined anywhere but we think it should be. "NONE" indicates
476 that the filter was not defined but the observation is a dark
477 or bias.
478 """
480 # If there is no filter defined we want to report this as a special
481 # filter. ???? indicates that we think it should be set.
482 obstype = self.to_observation_type()
483 undefined_filter = "????"
484 log_undefined = True
485 if obstype in ("bias", "dark"):
486 undefined_filter = "NONE"
487 log_undefined = False
489 if self.is_key_ok("FILTER"):
490 physical_filter = self._header["FILTER"]
491 self._used_these_cards("FILTER")
493 if physical_filter.lower().startswith("empty"):
494 physical_filter = "EMPTY"
495 else:
496 # Be explicit about having no knowledge of the filter
497 physical_filter = undefined_filter
498 if log_undefined:
499 log.warning("%s: Unable to determine the filter",
500 self.to_observation_id())
502 if self.is_key_ok("GRATING"):
503 grating = self._header["GRATING"]
504 self._used_these_cards("GRATING")
506 if not grating or grating.lower().startswith("empty"):
507 grating = "EMPTY"
508 else:
509 # Be explicit about having no knowledge of the grating
510 grating = undefined_filter
512 physical_filter = f"{physical_filter}{FILTER_DELIMITER}{grating}"
514 return physical_filter
516 @cache_translation
517 def to_boresight_rotation_coord(self):
518 """Boresight rotation angle.
520 Only relevant for science observations.
521 """
522 unknown = "unknown"
523 if not self.is_on_sky():
524 return unknown
526 self._used_these_cards("ROTCOORD")
527 coord = self._header.get("ROTCOORD", unknown)
528 if coord is None:
529 coord = unknown
530 return coord
532 @cache_translation
533 def to_boresight_airmass(self):
534 """Calculate airmass at boresight at start of observation.
536 Notes
537 -----
538 Early data are missing AMSTART header so we fall back to calculating
539 it from ELSTART.
540 """
541 if not self.is_on_sky():
542 return None
544 # This observation should have AMSTART
545 amkey = "AMSTART"
546 if self.is_key_ok(amkey):
547 self._used_these_cards(amkey)
548 return self._header[amkey]
550 # Instead we need to look at azel
551 altaz = self.to_altaz_begin()
552 if altaz is not None:
553 return altaz.secz.to_value()
555 log.warning("%s: Unable to determine airmass of a science observation, returning 1.",
556 self.to_observation_id())
557 return 1.0