Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is currently part of obs_lsst but is written to allow it 

2# to be migrated to the astro_metadata_translator package at a later date. 

3# 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the LICENSE file in this directory for details of code ownership. 

7# 

8# Use of this source code is governed by a 3-clause BSD-style 

9# license that can be found in the LICENSE file. 

10 

11"""Metadata translation code for LSST LATISS headers""" 

12 

13__all__ = ("LatissTranslator", ) 

14 

15import logging 

16import math 

17 

18import astropy.units as u 

19from astropy.time import Time 

20from astropy.coordinates import EarthLocation 

21 

22from astro_metadata_translator import cache_translation 

23from astro_metadata_translator.translators.helpers import is_non_science 

24from .lsst import LsstBaseTranslator, FILTER_DELIMITER 

25 

26log = logging.getLogger(__name__) 

27 

28 

29# AuxTel is not the same place as LSST 

30# These coordinates read from Apple Maps 

31AUXTEL_LOCATION = EarthLocation.from_geodetic(-70.747698, -30.244728, 2663.0) 

32 

33# Date instrument is taking data at telescope 

34# Prior to this date many parameters are automatically nulled out 

35# since the headers have not historically been reliable 

36TSTART = Time("2020-01-01T00:00", format="isot", scale="utc") 

37 

38# Define the sensor and group name for AuxTel globally so that it can be used 

39# in multiple places. There is no raft but for consistency with other LSST 

40# cameras we define one. 

41_DETECTOR_GROUP_NAME = "RXX" 

42_DETECTOR_NAME = "S00" 

43 

44# Date 068 detector was put in LATISS 

45DETECTOR_068_DATE = Time("2019-06-24T00:00", format="isot", scale="utc") 

46 

47# IMGTYPE header is filled in after this date 

48IMGTYPE_OKAY_DATE = Time("2019-11-07T00:00", format="isot", scale="utc") 

49 

50# OBJECT IMGTYPE really means ENGTEST until this date 

51OBJECT_IS_ENGTEST = Time("2020-01-27T20:00", format="isot", scale="utc") 

52 

53# RA and DEC headers are in radians until this date 

54RADEC_IS_RADIANS = Time("2020-01-28T22:00", format="isot", scale="utc") 

55 

56# RASTART/DECSTART/RAEND/DECEND used wrong telescope location before this 

57# 2020-02-01T00:00 we fixed the telescope location, but RASTART is still 

58# in mount coordinates, so off by pointing model. 

59RASTART_IS_BAD = Time("2020-05-01T00:00", format="isot", scale="utc") 

60 

61# DATE-END is not to be trusted before this date 

62DATE_END_IS_BAD = Time("2020-02-01T00:00", format="isot", scale="utc") 

63 

64# Scaling factor radians to degrees. Keep it simple. 

65RAD2DEG = 180.0 / math.pi 

66 

67 

68def is_non_science_or_lab(self): 

69 """Pseudo method to determine whether this is a lab or non-science 

70 header. 

71 

72 Raises 

73 ------ 

74 KeyError 

75 If this is a science observation and on the mountain. 

76 """ 

77 # Return without raising if this is not a science observation 

78 # since the defaults are fine. 

79 try: 

80 # This will raise if it is a science observation 

81 is_non_science(self) 

82 return 

83 except KeyError: 

84 pass 

85 

86 # We are still in the lab, return and use the default 

87 if not self._is_on_mountain(): 

88 return 

89 

90 # This is a science observation on the mountain so we should not 

91 # use defaults 

92 raise KeyError(f"{self._log_prefix}: Required key is missing and this is a mountain science observation") 

93 

94 

95class LatissTranslator(LsstBaseTranslator): 

96 """Metadata translator for LSST LATISS data from AuxTel. 

97 

98 For lab measurements many values are masked out. 

99 """ 

100 

101 name = "LSST_LATISS" 

102 """Name of this translation class""" 

103 

104 supported_instrument = "LATISS" 

105 """Supports the LATISS instrument.""" 

106 

107 _const_map = { 

108 "instrument": "LATISS", 

109 "telescope": "Rubin Auxiliary Telescope", 

110 "detector_group": _DETECTOR_GROUP_NAME, 

111 "detector_num": 0, 

112 "detector_name": _DETECTOR_NAME, # Single sensor 

113 "science_program": "unknown", 

114 "relative_humidity": None, 

115 "pressure": None, 

116 "temperature": None, 

117 } 

118 

119 _trivial_map = { 

120 "observation_id": (["OBSID", "IMGNAME"], dict(default=None, checker=is_non_science)), 

121 "detector_serial": ["LSST_NUM", "DETSER"], 

122 "object": ("OBJECT", dict(checker=is_non_science_or_lab, default="UNKNOWN")), 

123 "boresight_rotation_angle": (["ROTPA", "ROTANGLE"], dict(checker=is_non_science_or_lab, 

124 default=float("nan"), unit=u.deg)), 

125 } 

126 

127 DETECTOR_GROUP_NAME = _DETECTOR_GROUP_NAME 

128 """Fixed name of detector group.""" 

129 

130 DETECTOR_NAME = _DETECTOR_NAME 

131 """Fixed name of single sensor.""" 

132 

133 DETECTOR_MAX = 0 

134 """Maximum number of detectors to use when calculating the 

135 detector_exposure_id.""" 

136 

137 _DEFAULT_LOCATION = AUXTEL_LOCATION 

138 """Default telescope location in absence of relevant FITS headers.""" 

139 

140 @classmethod 

141 def can_translate(cls, header, filename=None): 

142 """Indicate whether this translation class can translate the 

143 supplied header. 

144 

145 Parameters 

146 ---------- 

147 header : `dict`-like 

148 Header to convert to standardized form. 

149 filename : `str`, optional 

150 Name of file being translated. 

151 

152 Returns 

153 ------- 

154 can : `bool` 

155 `True` if the header is recognized by this class. `False` 

156 otherwise. 

157 """ 

158 # INSTRUME keyword might be of two types 

159 if "INSTRUME" in header: 

160 instrume = header["INSTRUME"] 

161 for v in ("LSST_ATISS", "LATISS"): 

162 if instrume == v: 

163 return True 

164 # Calibration files strip important headers at the moment so guess 

165 if "DETNAME" in header and header["DETNAME"] == "RXX_S00": 

166 return True 

167 return False 

168 

169 @classmethod 

170 def fix_header(cls, header, instrument, obsid, filename=None): 

171 """Fix an incorrect LATISS header. 

172 

173 Parameters 

174 ---------- 

175 header : `dict` 

176 The header to update. Updates are in place. 

177 instrument : `str` 

178 The name of the instrument. 

179 obsid : `str` 

180 Unique observation identifier associated with this header. 

181 Will always be provided. 

182 filename : `str`, optional 

183 Filename associated with this header. May not be set since headers 

184 can be fixed independently of any filename being known. 

185 

186 Returns 

187 ------- 

188 modified = `bool` 

189 Returns `True` if the header was updated. 

190 

191 Notes 

192 ----- 

193 This method does not apply per-obsid corrections. The following 

194 corrections are applied: 

195 

196 * On June 24th 2019 the detector was changed from ITL-3800C-098 

197 to ITL-3800C-068. The header is intended to be correct in the 

198 future. 

199 * In late 2019 the DATE-OBS and MJD-OBS headers were reporting 

200 1970 dates. To correct, the DATE/MJD headers are copied in to 

201 replace them and the -END headers are cleared. 

202 * Until November 2019 the IMGTYPE was set in the GROUPID header. 

203 The value is moved to IMGTYPE. 

204 * SHUTTIME is always forced to be `None`. 

205 

206 Corrections are reported as debug level log messages. 

207 

208 See `~astro_metadata_translator.fix_header` for details of the general 

209 process. 

210 """ 

211 modified = False 

212 

213 # Calculate the standard label to use for log messages 

214 log_label = cls._construct_log_prefix(obsid, filename) 

215 

216 if "OBSID" not in header: 

217 # Very old data used IMGNAME 

218 header["OBSID"] = obsid 

219 modified = True 

220 # We are reporting the OBSID so no need to repeat it at start 

221 # of log message. Use filename if we have it. 

222 log_prefix = f"{filename}: " if filename else "" 

223 log.debug("%sAssigning OBSID to a value of '%s'", log_prefix, header["OBSID"]) 

224 

225 if "DAYOBS" not in header: 

226 # OBS-NITE could have the value for DAYOBS but it is safer 

227 # for older data to set it from the OBSID. Fall back to OBS-NITE 

228 # if we have no alternative 

229 dayObs = None 

230 try: 

231 dayObs = obsid.split("_", 3)[2] 

232 except (AttributeError, ValueError): 

233 # did not split as expected 

234 pass 

235 if dayObs is None or len(dayObs) != 8: 

236 dayObs = header["OBS-NITE"] 

237 log.debug("%s: Setting DAYOBS to '%s' from OBS-NITE header", log_label, dayObs) 

238 else: 

239 log.debug("%s: Setting DAYOBS to '%s' from OBSID", log_label, dayObs) 

240 header["DAYOBS"] = dayObs 

241 modified = True 

242 

243 if "SEQNUM" not in header: 

244 try: 

245 seqnum = obsid.split("_", 3)[3] 

246 except (AttributeError, ValueError): 

247 # did not split as expected 

248 pass 

249 else: 

250 header["SEQNUM"] = int(seqnum) 

251 modified = True 

252 log.debug("%s: Extracting SEQNUM of '%s' from OBSID", log_label, header["SEQNUM"]) 

253 

254 # The DATE-OBS / MJD-OBS keys can be 1970 

255 if header["DATE-OBS"].startswith("1970"): 

256 # Copy the headers from the DATE and MJD since we have no other 

257 # choice. 

258 header["DATE-OBS"] = header["DATE"] 

259 header["DATE-BEG"] = header["DATE-OBS"] 

260 header["MJD-OBS"] = header["MJD"] 

261 header["MJD-BEG"] = header["MJD-OBS"] 

262 

263 # And clear the DATE-END and MJD-END -- the translator will use 

264 # EXPTIME instead. 

265 header["DATE-END"] = None 

266 header["MJD-END"] = None 

267 

268 log.debug("%s: Forcing 1970 dates to '%s'", log_label, header["DATE"]) 

269 modified = True 

270 

271 # Create a translator since we need the date 

272 translator = cls(header) 

273 date = translator.to_datetime_begin() 

274 if date > DETECTOR_068_DATE: 

275 header["LSST_NUM"] = "ITL-3800C-068" 

276 log.debug("%s: Forcing detector serial to %s", log_label, header["LSST_NUM"]) 

277 modified = True 

278 

279 if date < DATE_END_IS_BAD: 

280 # DATE-END may or may not be in TAI and may or may not be 

281 # before DATE-BEG. Simpler to clear it 

282 if header.get("DATE-END"): 

283 header["DATE-END"] = None 

284 header["MJD-END"] = None 

285 

286 log.debug("%s: Clearing DATE-END as being untrustworthy", log_label) 

287 modified = True 

288 

289 # Up until a certain date GROUPID was the IMGTYPE 

290 if date < IMGTYPE_OKAY_DATE: 

291 groupId = header.get("GROUPID") 

292 if groupId and not groupId.startswith("test"): 

293 imgType = header.get("IMGTYPE") 

294 if not imgType: 

295 if "_" in groupId: 

296 # Sometimes have the form dark_0001_0002 

297 # in this case we pull the IMGTYPE off the front and 

298 # do not clear groupId (although groupId may now 

299 # repeat on different days). 

300 groupId, _ = groupId.split("_", 1) 

301 elif groupId.upper() != "FOCUS" and groupId.upper().startswith("FOCUS"): 

302 # If it is exactly FOCUS we want groupId cleared 

303 groupId = "FOCUS" 

304 else: 

305 header["GROUPID"] = None 

306 header["IMGTYPE"] = groupId 

307 log.debug("%s: Setting IMGTYPE to '%s' from GROUPID", log_label, header["IMGTYPE"]) 

308 modified = True 

309 else: 

310 # Someone could be fixing headers in old data 

311 # and we do not want GROUPID == IMGTYPE 

312 if imgType == groupId: 

313 # Clear the group so we default to original 

314 header["GROUPID"] = None 

315 

316 # We were using OBJECT for engineering observations early on 

317 if date < OBJECT_IS_ENGTEST: 

318 imgType = header.get("IMGTYPE") 

319 if imgType == "OBJECT": 

320 header["IMGTYPE"] = "ENGTEST" 

321 log.debug("%s: Changing OBJECT observation type to %s", 

322 log_label, header["IMGTYPE"]) 

323 modified = True 

324 

325 # Early on the RA/DEC headers were stored in radians 

326 if date < RADEC_IS_RADIANS: 

327 if header.get("RA") is not None: 

328 header["RA"] *= RAD2DEG 

329 log.debug("%s: Changing RA header to degrees", log_label) 

330 modified = True 

331 if header.get("DEC") is not None: 

332 header["DEC"] *= RAD2DEG 

333 log.debug("%s: Changing DEC header to degrees", log_label) 

334 modified = True 

335 

336 if header.get("SHUTTIME"): 

337 log.debug("%s: Forcing SHUTTIME header to be None", log_label) 

338 header["SHUTTIME"] = None 

339 modified = True 

340 

341 if "OBJECT" not in header: 

342 # Only patch OBJECT IMGTYPE 

343 if "IMGTYPE" in header and header["IMGTYPE"] == "OBJECT": 

344 log.debug("%s: Forcing OBJECT header to exist", log_label) 

345 header["OBJECT"] = "NOTSET" 

346 modified = True 

347 

348 if "RADESYS" in header: 

349 if header["RADESYS"] == "": 

350 # Default to ICRS 

351 header["RADESYS"] = "ICRS" 

352 log.debug("%s: Forcing blank RADESYS to '%s'", log_label, header["RADESYS"]) 

353 modified = True 

354 

355 if date < RASTART_IS_BAD: 

356 # The wrong telescope position was used. Unsetting these will force 

357 # the RA/DEC demand headers to be used instead. 

358 for h in ("RASTART", "DECSTART", "RAEND", "DECEND"): 

359 header[h] = None 

360 log.debug("%s: Forcing derived RA/Dec headers to undefined", log_label) 

361 

362 return modified 

363 

364 def _is_on_mountain(self): 

365 date = self.to_datetime_begin() 

366 if date > TSTART: 

367 return True 

368 return False 

369 

370 @staticmethod 

371 def compute_detector_exposure_id(exposure_id, detector_num): 

372 """Compute the detector exposure ID from detector number and 

373 exposure ID. 

374 

375 This is a helper method to allow code working outside the translator 

376 infrastructure to use the same algorithm. 

377 

378 Parameters 

379 ---------- 

380 exposure_id : `int` 

381 Unique exposure ID. 

382 detector_num : `int` 

383 Detector number. 

384 

385 Returns 

386 ------- 

387 detector_exposure_id : `int` 

388 The calculated ID. 

389 """ 

390 if detector_num != 0: 

391 log.warning("Unexpected non-zero detector number for LATISS") 

392 return exposure_id 

393 

394 @cache_translation 

395 def to_dark_time(self): 

396 # Docstring will be inherited. Property defined in properties.py 

397 

398 # Always compare with exposure time 

399 # We may revisit this later if there is a cutoff date where we 

400 # can always trust the header. 

401 exptime = self.to_exposure_time() 

402 

403 if self.is_key_ok("DARKTIME"): 

404 darktime = self.quantity_from_card("DARKTIME", u.s) 

405 if darktime >= exptime: 

406 return darktime 

407 reason = "Dark time less than exposure time." 

408 else: 

409 reason = "Dark time not defined." 

410 

411 log.warning("%s: %s Setting dark time to the exposure time.", 

412 self._log_prefix, reason) 

413 return exptime 

414 

415 @cache_translation 

416 def to_exposure_time(self): 

417 # Docstring will be inherited. Property defined in properties.py 

418 # Some data is missing a value for EXPTIME. 

419 # Have to be careful we do not have circular logic when trying to 

420 # guess 

421 if self.is_key_ok("EXPTIME"): 

422 return self.quantity_from_card("EXPTIME", u.s) 

423 

424 # A missing or undefined EXPTIME is problematic. Set to -1 

425 # to indicate that none was found. 

426 log.warning("%s: Insufficient information to derive exposure time. Setting to -1.0s", 

427 self._log_prefix) 

428 return -1.0 * u.s 

429 

430 @cache_translation 

431 def to_observation_type(self): 

432 """Determine the observation type. 

433 

434 In the absence of an ``IMGTYPE`` header, assumes lab data is always a 

435 dark if exposure time is non-zero, else bias. 

436 

437 Returns 

438 ------- 

439 obstype : `str` 

440 Observation type. 

441 """ 

442 

443 # LATISS observation type is documented to appear in OBSTYPE 

444 # but for historical reasons prefers IMGTYPE. 

445 # Test the keys in order until we find one that contains a 

446 # defined value. 

447 obstype_keys = ["OBSTYPE", "IMGTYPE"] 

448 

449 obstype = None 

450 for k in obstype_keys: 

451 if self.is_key_ok(k): 

452 obstype = self._header[k] 

453 self._used_these_cards(k) 

454 obstype = obstype.lower() 

455 break 

456 

457 if obstype is not None: 

458 if obstype == "object" and not self._is_on_mountain(): 

459 # Do not map object to science in lab since most 

460 # code assume science is on sky with RA/Dec. 

461 obstype = "labobject" 

462 elif obstype in ("skyexp", "object"): 

463 obstype = "science" 

464 

465 return obstype 

466 

467 # In the absence of any observation type information, return 

468 # unknown unless we think it might be a bias. 

469 exptime = self.to_exposure_time() 

470 if exptime == 0.0: 

471 obstype = "bias" 

472 else: 

473 obstype = "unknown" 

474 log.warning("%s: Unable to determine observation type. Guessing '%s'", 

475 self._log_prefix, obstype) 

476 return obstype 

477 

478 @cache_translation 

479 def to_physical_filter(self): 

480 """Calculate the physical filter name. 

481 

482 Returns 

483 ------- 

484 filter : `str` 

485 Name of filter. A combination of FILTER and GRATING 

486 headers joined by a "~". The filter and grating are always 

487 combined. The filter or grating part will be "none" if no value 

488 is specified. Uses "empty" if any of the filters or gratings 

489 indicate an "empty_N" name. "unknown" indicates that the filter is 

490 not defined anywhere but we think it should be. "none" indicates 

491 that the filter was not defined but the observation is a dark 

492 or bias. 

493 """ 

494 

495 physical_filter = self._determine_primary_filter() 

496 

497 if self.is_key_ok("GRATING"): 

498 grating = self._header["GRATING"] 

499 self._used_these_cards("GRATING") 

500 

501 if not grating or grating.lower().startswith("empty"): 

502 grating = "empty" 

503 else: 

504 # Be explicit about having no knowledge of the grating 

505 grating = "unknown" 

506 

507 physical_filter = f"{physical_filter}{FILTER_DELIMITER}{grating}" 

508 

509 return physical_filter 

510 

511 @cache_translation 

512 def to_boresight_rotation_coord(self): 

513 """Boresight rotation angle. 

514 

515 Only relevant for science observations. 

516 """ 

517 unknown = "unknown" 

518 if not self.is_on_sky(): 

519 return unknown 

520 

521 self._used_these_cards("ROTCOORD") 

522 coord = self._header.get("ROTCOORD", unknown) 

523 if coord is None: 

524 coord = unknown 

525 return coord 

526 

527 @cache_translation 

528 def to_boresight_airmass(self): 

529 """Calculate airmass at boresight at start of observation. 

530 

531 Notes 

532 ----- 

533 Early data are missing AMSTART header so we fall back to calculating 

534 it from ELSTART. 

535 """ 

536 if not self.is_on_sky(): 

537 return None 

538 

539 # This observation should have AMSTART 

540 amkey = "AMSTART" 

541 if self.is_key_ok(amkey): 

542 self._used_these_cards(amkey) 

543 return self._header[amkey] 

544 

545 # Instead we need to look at azel 

546 altaz = self.to_altaz_begin() 

547 if altaz is not None: 

548 return altaz.secz.to_value() 

549 

550 log.warning("%s: Unable to determine airmass of a science observation, returning 1.", 

551 self._log_prefix) 

552 return 1.0