Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is currently part of obs_lsst but is written to allow it 

2# to be migrated to the astro_metadata_translator package at a later date. 

3# 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the LICENSE file in this directory for details of code ownership. 

7# 

8# Use of this source code is governed by a 3-clause BSD-style 

9# license that can be found in the LICENSE file. 

10 

11"""Metadata translation code for LSST LATISS headers""" 

12 

13__all__ = ("LatissTranslator", ) 

14 

15import logging 

16import math 

17 

18import astropy.units as u 

19from astropy.time import Time 

20from astropy.coordinates import EarthLocation 

21 

22from astro_metadata_translator import cache_translation 

23from astro_metadata_translator.translators.helpers import is_non_science 

24from .lsst import LsstBaseTranslator, FILTER_DELIMITER 

25 

26log = logging.getLogger(__name__) 

27 

28 

29# AuxTel is not the same place as LSST 

30# These coordinates read from Apple Maps 

31AUXTEL_LOCATION = EarthLocation.from_geodetic(-70.747698, -30.244728, 2663.0) 

32 

33# Date instrument is taking data at telescope 

34# Prior to this date many parameters are automatically nulled out 

35# since the headers have not historically been reliable 

36TSTART = Time("2020-01-01T00:00", format="isot", scale="utc") 

37 

38# Define the sensor and group name for AuxTel globally so that it can be used 

39# in multiple places. There is no raft but for consistency with other LSST 

40# cameras we define one. 

41_DETECTOR_GROUP_NAME = "RXX" 

42_DETECTOR_NAME = "S00" 

43 

44# Date 068 detector was put in LATISS 

45DETECTOR_068_DATE = Time("2019-06-24T00:00", format="isot", scale="utc") 

46 

47# IMGTYPE header is filled in after this date 

48IMGTYPE_OKAY_DATE = Time("2019-11-07T00:00", format="isot", scale="utc") 

49 

50# OBJECT IMGTYPE really means ENGTEST until this date 

51OBJECT_IS_ENGTEST = Time("2020-01-27T20:00", format="isot", scale="utc") 

52 

53# RA and DEC headers are in radians until this date 

54RADEC_IS_RADIANS = Time("2020-01-28T22:00", format="isot", scale="utc") 

55 

56# RASTART/DECSTART/RAEND/DECEND used wrong telescope location before this 

57RASTART_IS_BAD = Time("2020-02-01T00:00", format="isot", scale="utc") 

58 

59# DATE-END is not to be trusted before this date 

60DATE_END_IS_BAD = Time("2020-02-01T00:00", format="isot", scale="utc") 

61 

62# Scaling factor radians to degrees. Keep it simple. 

63RAD2DEG = 180.0 / math.pi 

64 

65 

66def is_non_science_or_lab(self): 

67 """Pseudo method to determine whether this is a lab or non-science 

68 header. 

69 

70 Raises 

71 ------ 

72 KeyError 

73 If this is a science observation and on the mountain. 

74 """ 

75 # Return without raising if this is not a science observation 

76 # since the defaults are fine. 

77 try: 

78 # This will raise if it is a science observation 

79 is_non_science(self) 

80 return 

81 except KeyError: 

82 pass 

83 

84 # We are still in the lab, return and use the default 

85 if not self._is_on_mountain(): 

86 return 

87 

88 # This is a science observation on the mountain so we should not 

89 # use defaults 

90 raise KeyError("Required key is missing and this is a mountain science observation") 

91 

92 

93class LatissTranslator(LsstBaseTranslator): 

94 """Metadata translator for LSST LATISS data from AuxTel. 

95 

96 For lab measurements many values are masked out. 

97 """ 

98 

99 name = "LSST_LATISS" 

100 """Name of this translation class""" 

101 

102 supported_instrument = "LATISS" 

103 """Supports the LATISS instrument.""" 

104 

105 _const_map = { 

106 "instrument": "LATISS", 

107 "telescope": "Rubin Auxiliary Telescope", 

108 "detector_group": _DETECTOR_GROUP_NAME, 

109 "detector_num": 0, 

110 "detector_name": _DETECTOR_NAME, # Single sensor 

111 "science_program": "unknown", 

112 "relative_humidity": None, 

113 "pressure": None, 

114 "temperature": None, 

115 } 

116 

117 _trivial_map = { 

118 "observation_id": (["OBSID", "IMGNAME"], dict(default=None, checker=is_non_science)), 

119 "detector_serial": ["LSST_NUM", "DETSER"], 

120 "object": ("OBJECT", dict(checker=is_non_science_or_lab, default="UNKNOWN")), 

121 "boresight_rotation_angle": (["ROTPA", "ROTANGLE"], dict(checker=is_non_science_or_lab, 

122 default=float("nan"), unit=u.deg)), 

123 } 

124 

125 DETECTOR_GROUP_NAME = _DETECTOR_GROUP_NAME 

126 """Fixed name of detector group.""" 

127 

128 DETECTOR_NAME = _DETECTOR_NAME 

129 """Fixed name of single sensor.""" 

130 

131 DETECTOR_MAX = 0 

132 """Maximum number of detectors to use when calculating the 

133 detector_exposure_id.""" 

134 

135 _DEFAULT_LOCATION = AUXTEL_LOCATION 

136 """Default telescope location in absence of relevant FITS headers.""" 

137 

138 @classmethod 

139 def can_translate(cls, header, filename=None): 

140 """Indicate whether this translation class can translate the 

141 supplied header. 

142 

143 Parameters 

144 ---------- 

145 header : `dict`-like 

146 Header to convert to standardized form. 

147 filename : `str`, optional 

148 Name of file being translated. 

149 

150 Returns 

151 ------- 

152 can : `bool` 

153 `True` if the header is recognized by this class. `False` 

154 otherwise. 

155 """ 

156 # INSTRUME keyword might be of two types 

157 if "INSTRUME" in header: 

158 instrume = header["INSTRUME"] 

159 for v in ("LSST_ATISS", "LATISS"): 

160 if instrume == v: 

161 return True 

162 # Calibration files strip important headers at the moment so guess 

163 if "DETNAME" in header and header["DETNAME"] == "RXX_S00": 

164 return True 

165 return False 

166 

167 @classmethod 

168 def fix_header(cls, header): 

169 """Fix an incorrect LATISS header. 

170 

171 Parameters 

172 ---------- 

173 header : `dict` 

174 The header to update. Updates are in place. 

175 

176 Returns 

177 ------- 

178 modified = `bool` 

179 Returns `True` if the header was updated. 

180 

181 Notes 

182 ----- 

183 This method does not apply per-obsid corrections. The following 

184 corrections are applied: 

185 

186 * On June 24th 2019 the detector was changed from ITL-3800C-098 

187 to ITL-3800C-068. The header is intended to be correct in the 

188 future. 

189 * In late 2019 the DATE-OBS and MJD-OBS headers were reporting 

190 1970 dates. To correct, the DATE/MJD headers are copied in to 

191 replace them and the -END headers are cleared. 

192 * Until November 2019 the IMGTYPE was set in the GROUPID header. 

193 The value is moved to IMGTYPE. 

194 * SHUTTIME is always forced to be `None`. 

195 

196 Corrections are reported as debug level log messages. 

197 """ 

198 modified = False 

199 

200 if "OBSID" not in header: 

201 # Very old data used IMGNAME 

202 header["OBSID"] = header.get("IMGNAME", "unknown") 

203 modified = True 

204 log.debug("Assigning OBSID to a value of '%s'", header["OBSID"]) 

205 

206 obsid = header["OBSID"] 

207 

208 if "DAYOBS" not in header: 

209 # OBS-NITE could have the value for DAYOBS but it is safer 

210 # for older data to set it from the OBSID. Fall back to OBS-NITE 

211 # if we have no alternative 

212 dayObs = None 

213 try: 

214 dayObs = obsid.split("_", 3)[2] 

215 except ValueError: 

216 # did not split as expected 

217 pass 

218 if dayObs is None or len(dayObs) != 8: 

219 dayObs = header["OBS-NITE"] 

220 log.debug("%s: Setting DAYOBS to '%s' from OBS-NITE header", obsid, dayObs) 

221 else: 

222 log.debug("%s: Setting DAYOBS to '%s' from OBSID", obsid, dayObs) 

223 header["DAYOBS"] = dayObs 

224 modified = True 

225 

226 if "SEQNUM" not in header: 

227 try: 

228 seqnum = obsid.split("_", 3)[3] 

229 except ValueError: 

230 # did not split as expected 

231 pass 

232 else: 

233 header["SEQNUM"] = int(seqnum) 

234 modified = True 

235 log.debug("%s: Extracting SEQNUM of '%s' from OBSID", obsid, header["SEQNUM"]) 

236 

237 # The DATE-OBS / MJD-OBS keys can be 1970 

238 if header["DATE-OBS"].startswith("1970"): 

239 # Copy the headers from the DATE and MJD since we have no other 

240 # choice. 

241 header["DATE-OBS"] = header["DATE"] 

242 header["DATE-BEG"] = header["DATE-OBS"] 

243 header["MJD-OBS"] = header["MJD"] 

244 header["MJD-BEG"] = header["MJD-OBS"] 

245 

246 # And clear the DATE-END and MJD-END -- the translator will use 

247 # EXPTIME instead. 

248 header["DATE-END"] = None 

249 header["MJD-END"] = None 

250 

251 log.debug("%s: Forcing 1970 dates to '%s'", obsid, header["DATE"]) 

252 modified = True 

253 

254 # Create a translator since we need the date 

255 translator = cls(header) 

256 date = translator.to_datetime_begin() 

257 if date > DETECTOR_068_DATE: 

258 header["LSST_NUM"] = "ITL-3800C-068" 

259 log.debug("%s: Forcing detector serial to %s", obsid, header["LSST_NUM"]) 

260 modified = True 

261 

262 if date < DATE_END_IS_BAD: 

263 # DATE-END may or may not be in TAI and may or may not be 

264 # before DATE-BEG. Simpler to clear it 

265 if header.get("DATE-END"): 

266 header["DATE-END"] = None 

267 header["MJD-END"] = None 

268 

269 log.debug("%s: Clearing DATE-END as being untrustworthy", obsid) 

270 modified = True 

271 

272 # Up until a certain date GROUPID was the IMGTYPE 

273 if date < IMGTYPE_OKAY_DATE: 

274 groupId = header.get("GROUPID") 

275 if groupId and not groupId.startswith("test"): 

276 imgType = header.get("IMGTYPE") 

277 if not imgType: 

278 if "_" in groupId: 

279 # Sometimes have the form dark_0001_0002 

280 # in this case we pull the IMGTYPE off the front and 

281 # do not clear groupId (although groupId may now 

282 # repeat on different days). 

283 groupId, _ = groupId.split("_", 1) 

284 elif groupId.upper() != "FOCUS" and groupId.upper().startswith("FOCUS"): 

285 # If it is exactly FOCUS we want groupId cleared 

286 groupId = "FOCUS" 

287 else: 

288 header["GROUPID"] = None 

289 header["IMGTYPE"] = groupId 

290 log.debug("%s: Setting IMGTYPE to '%s' from GROUPID", obsid, header["IMGTYPE"]) 

291 modified = True 

292 else: 

293 # Someone could be fixing headers in old data 

294 # and we do not want GROUPID == IMGTYPE 

295 if imgType == groupId: 

296 # Clear the group so we default to original 

297 header["GROUPID"] = None 

298 

299 # We were using OBJECT for engineering observations early on 

300 if date < OBJECT_IS_ENGTEST: 

301 imgType = header.get("IMGTYPE") 

302 if imgType == "OBJECT": 

303 header["IMGTYPE"] = "ENGTEST" 

304 log.debug("%s: Changing OBJECT observation type to %s", 

305 obsid, header["IMGTYPE"]) 

306 modified = True 

307 

308 # Early on the RA/DEC headers were stored in radians 

309 if date < RADEC_IS_RADIANS: 

310 if header.get("RA") is not None: 

311 header["RA"] *= RAD2DEG 

312 log.debug("%s: Changing RA header to degrees", obsid) 

313 modified = True 

314 if header.get("DEC") is not None: 

315 header["DEC"] *= RAD2DEG 

316 log.debug("%s: Changing DEC header to degrees", obsid) 

317 modified = True 

318 

319 if header.get("SHUTTIME"): 

320 log.debug("%s: Forcing SHUTTIME header to be None", obsid) 

321 header["SHUTTIME"] = None 

322 modified = True 

323 

324 if "OBJECT" not in header: 

325 # Only patch OBJECT IMGTYPE 

326 if "IMGTYPE" in header and header["IMGTYPE"] == "OBJECT": 

327 log.debug("%s: Forcing OBJECT header to exist", obsid) 

328 header["OBJECT"] = "NOTSET" 

329 modified = True 

330 

331 if "RADESYS" in header: 

332 if header["RADESYS"] == "": 

333 # Default to ICRS 

334 header["RADESYS"] = "ICRS" 

335 log.debug("%s: Forcing blank RADESYS to '%s'", obsid, header["RADESYS"]) 

336 modified = True 

337 

338 if date < RASTART_IS_BAD: 

339 # The wrong telescope position was used. Unsetting these will force 

340 # the RA/DEC demand headers to be used instead. 

341 for h in ("RASTART", "DECSTART", "RAEND", "DECEND"): 

342 header[h] = None 

343 log.debug("%s: Forcing derived RA/Dec headers to undefined", obsid) 

344 

345 return modified 

346 

347 def _is_on_mountain(self): 

348 date = self.to_datetime_begin() 

349 if date > TSTART: 

350 return True 

351 return False 

352 

353 @staticmethod 

354 def compute_detector_exposure_id(exposure_id, detector_num): 

355 """Compute the detector exposure ID from detector number and 

356 exposure ID. 

357 

358 This is a helper method to allow code working outside the translator 

359 infrastructure to use the same algorithm. 

360 

361 Parameters 

362 ---------- 

363 exposure_id : `int` 

364 Unique exposure ID. 

365 detector_num : `int` 

366 Detector number. 

367 

368 Returns 

369 ------- 

370 detector_exposure_id : `int` 

371 The calculated ID. 

372 """ 

373 if detector_num != 0: 

374 log.warning("Unexpected non-zero detector number for LATISS") 

375 return exposure_id 

376 

377 @cache_translation 

378 def to_dark_time(self): 

379 # Docstring will be inherited. Property defined in properties.py 

380 

381 # Always compare with exposure time 

382 # We may revisit this later if there is a cutoff date where we 

383 # can always trust the header. 

384 exptime = self.to_exposure_time() 

385 

386 if self.is_key_ok("DARKTIME"): 

387 darktime = self.quantity_from_card("DARKTIME", u.s) 

388 if darktime >= exptime: 

389 return darktime 

390 reason = "Dark time less than exposure time." 

391 else: 

392 reason = "Dark time not defined." 

393 

394 log.warning("%s: %s Setting dark time to the exposure time.", 

395 self.to_observation_id(), reason) 

396 return exptime 

397 

398 @cache_translation 

399 def to_exposure_time(self): 

400 # Docstring will be inherited. Property defined in properties.py 

401 # Some data is missing a value for EXPTIME. 

402 # Have to be careful we do not have circular logic when trying to 

403 # guess 

404 if self.is_key_ok("EXPTIME"): 

405 return self.quantity_from_card("EXPTIME", u.s) 

406 

407 # A missing or undefined EXPTIME is problematic. Set to -1 

408 # to indicate that none was found. 

409 log.warning("%s: Insufficient information to derive exposure time. Setting to -1.0s", 

410 self.to_observation_id()) 

411 return -1.0 * u.s 

412 

413 @cache_translation 

414 def to_observation_type(self): 

415 """Determine the observation type. 

416 

417 In the absence of an ``IMGTYPE`` header, assumes lab data is always a 

418 dark if exposure time is non-zero, else bias. 

419 

420 Returns 

421 ------- 

422 obstype : `str` 

423 Observation type. 

424 """ 

425 

426 # LATISS observation type is documented to appear in OBSTYPE 

427 # but for historical reasons prefers IMGTYPE. 

428 # Test the keys in order until we find one that contains a 

429 # defined value. 

430 obstype_keys = ["OBSTYPE", "IMGTYPE"] 

431 

432 obstype = None 

433 for k in obstype_keys: 

434 if self.is_key_ok(k): 

435 obstype = self._header[k] 

436 self._used_these_cards(k) 

437 obstype = obstype.lower() 

438 break 

439 

440 if obstype is not None: 

441 if obstype == "object" and not self._is_on_mountain(): 

442 # Do not map object to science in lab since most 

443 # code assume science is on sky with RA/Dec. 

444 obstype = "labobject" 

445 elif obstype in ("skyexp", "object"): 

446 obstype = "science" 

447 

448 return obstype 

449 

450 # In the absence of any observation type information, return 

451 # unknown unless we think it might be a bias. 

452 exptime = self.to_exposure_time() 

453 if exptime == 0.0: 

454 obstype = "bias" 

455 else: 

456 obstype = "unknown" 

457 log.warning("%s: Unable to determine observation type. Guessing '%s'", 

458 self.to_observation_id(), obstype) 

459 return obstype 

460 

461 @cache_translation 

462 def to_physical_filter(self): 

463 """Calculate the physical filter name. 

464 

465 Returns 

466 ------- 

467 filter : `str` 

468 Name of filter. A combination of FILTER and GRATING 

469 headers joined by a "~". The filter and grating are always 

470 combined. The filter or grating part will be "NONE" if no value 

471 is specified. Uses "EMPTY" if any of the filters or gratings 

472 indicate an "empty_N" name. "????" indicates that the filter is 

473 not defined anywhere but we think it should be. "NONE" indicates 

474 that the filter was not defined but the observation is a dark 

475 or bias. 

476 """ 

477 

478 # If there is no filter defined we want to report this as a special 

479 # filter. ???? indicates that we think it should be set. 

480 obstype = self.to_observation_type() 

481 undefined_filter = "????" 

482 log_undefined = True 

483 if obstype in ("bias", "dark"): 

484 undefined_filter = "NONE" 

485 log_undefined = False 

486 

487 if self.is_key_ok("FILTER"): 

488 physical_filter = self._header["FILTER"] 

489 self._used_these_cards("FILTER") 

490 

491 if physical_filter.lower().startswith("empty"): 

492 physical_filter = "EMPTY" 

493 else: 

494 # Be explicit about having no knowledge of the filter 

495 physical_filter = undefined_filter 

496 if log_undefined: 

497 log.warning("%s: Unable to determine the filter", 

498 self.to_observation_id()) 

499 

500 if self.is_key_ok("GRATING"): 

501 grating = self._header["GRATING"] 

502 self._used_these_cards("GRATING") 

503 

504 if not grating or grating.lower().startswith("empty"): 

505 grating = "EMPTY" 

506 else: 

507 # Be explicit about having no knowledge of the grating 

508 grating = undefined_filter 

509 

510 physical_filter = f"{physical_filter}{FILTER_DELIMITER}{grating}" 

511 

512 return physical_filter 

513 

514 @cache_translation 

515 def to_boresight_rotation_coord(self): 

516 """Boresight rotation angle. 

517 

518 Only relevant for science observations. 

519 """ 

520 unknown = "unknown" 

521 if not self.is_on_sky(): 

522 return unknown 

523 

524 self._used_these_cards("ROTCOORD") 

525 coord = self._header.get("ROTCOORD", unknown) 

526 if coord is None: 

527 coord = unknown 

528 return coord 

529 

530 @cache_translation 

531 def to_boresight_airmass(self): 

532 """Calculate airmass at boresight at start of observation. 

533 

534 Notes 

535 ----- 

536 Early data are missing AMSTART header so we fall back to calculating 

537 it from ELSTART. 

538 """ 

539 if not self.is_on_sky(): 

540 return None 

541 

542 # This observation should have AMSTART 

543 amkey = "AMSTART" 

544 if self.is_key_ok(amkey): 

545 self._used_these_cards(amkey) 

546 return self._header[amkey] 

547 

548 # Instead we need to look at azel 

549 altaz = self.to_altaz_begin() 

550 if altaz is not None: 

551 return altaz.secz.to_value() 

552 

553 log.warning("%s: Unable to determine airmass of a science observation, returning 1.", 

554 self.to_observation_id()) 

555 return 1.0