Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is currently part of obs_lsst but is written to allow it 

2# to be migrated to the astro_metadata_translator package at a later date. 

3# 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the LICENSE file in this directory for details of code ownership. 

7# 

8# Use of this source code is governed by a 3-clause BSD-style 

9# license that can be found in the LICENSE file. 

10 

11"""Metadata translation code for LSST LATISS headers""" 

12 

13__all__ = ("LatissTranslator", ) 

14 

15import logging 

16import math 

17 

18import astropy.units as u 

19from astropy.time import Time 

20from astropy.coordinates import EarthLocation 

21 

22from astro_metadata_translator import cache_translation 

23from astro_metadata_translator.translators.helpers import is_non_science 

24from .lsst import LsstBaseTranslator, FILTER_DELIMITER 

25 

26log = logging.getLogger(__name__) 

27 

28 

29# AuxTel is not the same place as LSST 

30# These coordinates read from Apple Maps 

31AUXTEL_LOCATION = EarthLocation.from_geodetic(-70.747698, -30.244728, 2663.0) 

32 

33# Date instrument is taking data at telescope 

34# Prior to this date many parameters are automatically nulled out 

35# since the headers have not historically been reliable 

36TSTART = Time("2020-01-01T00:00", format="isot", scale="utc") 

37 

38# Define the sensor and group name for AuxTel globally so that it can be used 

39# in multiple places. There is no raft but for consistency with other LSST 

40# cameras we define one. 

41_DETECTOR_GROUP_NAME = "RXX" 

42_DETECTOR_NAME = "S00" 

43 

44# Date 068 detector was put in LATISS 

45DETECTOR_068_DATE = Time("2019-06-24T00:00", format="isot", scale="utc") 

46 

47# IMGTYPE header is filled in after this date 

48IMGTYPE_OKAY_DATE = Time("2019-11-07T00:00", format="isot", scale="utc") 

49 

50# OBJECT IMGTYPE really means ENGTEST until this date 

51OBJECT_IS_ENGTEST = Time("2020-01-27T20:00", format="isot", scale="utc") 

52 

53# RA and DEC headers are in radians until this date 

54RADEC_IS_RADIANS = Time("2020-01-28T22:00", format="isot", scale="utc") 

55 

56# RASTART/DECSTART/RAEND/DECEND used wrong telescope location before this 

57# 2020-02-01T00:00 we fixed the telescope location, but RASTART is still 

58# in mount coordinates, so off by pointing model. 

59RASTART_IS_BAD = Time("2020-05-01T00:00", format="isot", scale="utc") 

60 

61# Between RASTART_IS_BAD and this time the RASTART header uses hours 

62# instead of degrees. 

63RASTART_IS_HOURS = Time("2021-02-11T18:45", format="isot", scale="utc") 

64 

65# From this date RASTART is correct as-is. 

66RASTART_IS_OKAY = Time("2021-02-12T00:00", format="isot", scale="utc") 

67 

68# DATE-END is not to be trusted before this date 

69DATE_END_IS_BAD = Time("2020-02-01T00:00", format="isot", scale="utc") 

70 

71# Scaling factor radians to degrees. Keep it simple. 

72RAD2DEG = 180.0 / math.pi 

73 

74 

75def is_non_science_or_lab(self): 

76 """Pseudo method to determine whether this is a lab or non-science 

77 header. 

78 

79 Raises 

80 ------ 

81 KeyError 

82 If this is a science observation and on the mountain. 

83 """ 

84 # Return without raising if this is not a science observation 

85 # since the defaults are fine. 

86 try: 

87 # This will raise if it is a science observation 

88 is_non_science(self) 

89 return 

90 except KeyError: 

91 pass 

92 

93 # We are still in the lab, return and use the default 

94 if not self._is_on_mountain(): 

95 return 

96 

97 # This is a science observation on the mountain so we should not 

98 # use defaults 

99 raise KeyError(f"{self._log_prefix}: Required key is missing and this is a mountain science observation") 

100 

101 

102class LatissTranslator(LsstBaseTranslator): 

103 """Metadata translator for LSST LATISS data from AuxTel. 

104 

105 For lab measurements many values are masked out. 

106 """ 

107 

108 name = "LSST_LATISS" 

109 """Name of this translation class""" 

110 

111 supported_instrument = "LATISS" 

112 """Supports the LATISS instrument.""" 

113 

114 _const_map = { 

115 "instrument": "LATISS", 

116 "telescope": "Rubin Auxiliary Telescope", 

117 "detector_group": _DETECTOR_GROUP_NAME, 

118 "detector_num": 0, 

119 "detector_name": _DETECTOR_NAME, # Single sensor 

120 "science_program": "unknown", 

121 "relative_humidity": None, 

122 "pressure": None, 

123 "temperature": None, 

124 } 

125 

126 _trivial_map = { 

127 "observation_id": (["OBSID", "IMGNAME"], dict(default=None, checker=is_non_science)), 

128 "detector_serial": ["LSST_NUM", "DETSER"], 

129 "object": ("OBJECT", dict(checker=is_non_science_or_lab, default="UNKNOWN")), 

130 "boresight_rotation_angle": (["ROTPA", "ROTANGLE"], dict(checker=is_non_science_or_lab, 

131 default=float("nan"), unit=u.deg)), 

132 } 

133 

134 DETECTOR_GROUP_NAME = _DETECTOR_GROUP_NAME 

135 """Fixed name of detector group.""" 

136 

137 DETECTOR_NAME = _DETECTOR_NAME 

138 """Fixed name of single sensor.""" 

139 

140 DETECTOR_MAX = 0 

141 """Maximum number of detectors to use when calculating the 

142 detector_exposure_id.""" 

143 

144 _DEFAULT_LOCATION = AUXTEL_LOCATION 

145 """Default telescope location in absence of relevant FITS headers.""" 

146 

147 @classmethod 

148 def can_translate(cls, header, filename=None): 

149 """Indicate whether this translation class can translate the 

150 supplied header. 

151 

152 Parameters 

153 ---------- 

154 header : `dict`-like 

155 Header to convert to standardized form. 

156 filename : `str`, optional 

157 Name of file being translated. 

158 

159 Returns 

160 ------- 

161 can : `bool` 

162 `True` if the header is recognized by this class. `False` 

163 otherwise. 

164 """ 

165 # INSTRUME keyword might be of two types 

166 if "INSTRUME" in header: 

167 instrume = header["INSTRUME"] 

168 for v in ("LSST_ATISS", "LATISS"): 

169 if instrume == v: 

170 return True 

171 # Calibration files strip important headers at the moment so guess 

172 if "DETNAME" in header and header["DETNAME"] == "RXX_S00": 

173 return True 

174 return False 

175 

176 @classmethod 

177 def fix_header(cls, header, instrument, obsid, filename=None): 

178 """Fix an incorrect LATISS header. 

179 

180 Parameters 

181 ---------- 

182 header : `dict` 

183 The header to update. Updates are in place. 

184 instrument : `str` 

185 The name of the instrument. 

186 obsid : `str` 

187 Unique observation identifier associated with this header. 

188 Will always be provided. 

189 filename : `str`, optional 

190 Filename associated with this header. May not be set since headers 

191 can be fixed independently of any filename being known. 

192 

193 Returns 

194 ------- 

195 modified = `bool` 

196 Returns `True` if the header was updated. 

197 

198 Notes 

199 ----- 

200 This method does not apply per-obsid corrections. The following 

201 corrections are applied: 

202 

203 * On June 24th 2019 the detector was changed from ITL-3800C-098 

204 to ITL-3800C-068. The header is intended to be correct in the 

205 future. 

206 * In late 2019 the DATE-OBS and MJD-OBS headers were reporting 

207 1970 dates. To correct, the DATE/MJD headers are copied in to 

208 replace them and the -END headers are cleared. 

209 * Until November 2019 the IMGTYPE was set in the GROUPID header. 

210 The value is moved to IMGTYPE. 

211 * SHUTTIME is always forced to be `None`. 

212 

213 Corrections are reported as debug level log messages. 

214 

215 See `~astro_metadata_translator.fix_header` for details of the general 

216 process. 

217 """ 

218 modified = False 

219 

220 # Calculate the standard label to use for log messages 

221 log_label = cls._construct_log_prefix(obsid, filename) 

222 

223 if "OBSID" not in header: 

224 # Very old data used IMGNAME 

225 header["OBSID"] = obsid 

226 modified = True 

227 # We are reporting the OBSID so no need to repeat it at start 

228 # of log message. Use filename if we have it. 

229 log_prefix = f"{filename}: " if filename else "" 

230 log.debug("%sAssigning OBSID to a value of '%s'", log_prefix, header["OBSID"]) 

231 

232 if "DAYOBS" not in header: 

233 # OBS-NITE could have the value for DAYOBS but it is safer 

234 # for older data to set it from the OBSID. Fall back to OBS-NITE 

235 # if we have no alternative 

236 dayObs = None 

237 try: 

238 dayObs = obsid.split("_", 3)[2] 

239 except (AttributeError, ValueError): 

240 # did not split as expected 

241 pass 

242 if dayObs is None or len(dayObs) != 8: 

243 if "OBS-NITE" in header: 

244 dayObs = header["OBS-NITE"] 

245 log.debug("%s: Setting DAYOBS to '%s' from OBS-NITE header", log_label, dayObs) 

246 else: 

247 log.debug("%s: Unable to determine DAYOBS from header", log_label) 

248 else: 

249 log.debug("%s: Setting DAYOBS to '%s' from OBSID", log_label, dayObs) 

250 if dayObs: 

251 header["DAYOBS"] = dayObs 

252 modified = True 

253 

254 if "SEQNUM" not in header: 

255 try: 

256 seqnum = obsid.split("_", 3)[3] 

257 except (AttributeError, ValueError): 

258 # did not split as expected 

259 pass 

260 else: 

261 header["SEQNUM"] = int(seqnum) 

262 modified = True 

263 log.debug("%s: Extracting SEQNUM of '%s' from OBSID", log_label, header["SEQNUM"]) 

264 

265 # The DATE-OBS / MJD-OBS keys can be 1970 

266 if "DATE-OBS" in header and header["DATE-OBS"].startswith("1970"): 

267 # Copy the headers from the DATE and MJD since we have no other 

268 # choice. 

269 header["DATE-OBS"] = header["DATE"] 

270 header["DATE-BEG"] = header["DATE-OBS"] 

271 header["MJD-OBS"] = header["MJD"] 

272 header["MJD-BEG"] = header["MJD-OBS"] 

273 

274 # And clear the DATE-END and MJD-END -- the translator will use 

275 # EXPTIME instead. 

276 header["DATE-END"] = None 

277 header["MJD-END"] = None 

278 

279 log.debug("%s: Forcing 1970 dates to '%s'", log_label, header["DATE"]) 

280 modified = True 

281 

282 # Create a translator since we need the date 

283 translator = cls(header) 

284 date = translator.to_datetime_begin() 

285 if date > DETECTOR_068_DATE: 

286 header["LSST_NUM"] = "ITL-3800C-068" 

287 log.debug("%s: Forcing detector serial to %s", log_label, header["LSST_NUM"]) 

288 modified = True 

289 

290 if date < DATE_END_IS_BAD: 

291 # DATE-END may or may not be in TAI and may or may not be 

292 # before DATE-BEG. Simpler to clear it 

293 if header.get("DATE-END"): 

294 header["DATE-END"] = None 

295 header["MJD-END"] = None 

296 

297 log.debug("%s: Clearing DATE-END as being untrustworthy", log_label) 

298 modified = True 

299 

300 # Up until a certain date GROUPID was the IMGTYPE 

301 if date < IMGTYPE_OKAY_DATE: 

302 groupId = header.get("GROUPID") 

303 if groupId and not groupId.startswith("test"): 

304 imgType = header.get("IMGTYPE") 

305 if not imgType: 

306 if "_" in groupId: 

307 # Sometimes have the form dark_0001_0002 

308 # in this case we pull the IMGTYPE off the front and 

309 # do not clear groupId (although groupId may now 

310 # repeat on different days). 

311 groupId, _ = groupId.split("_", 1) 

312 elif groupId.upper() != "FOCUS" and groupId.upper().startswith("FOCUS"): 

313 # If it is exactly FOCUS we want groupId cleared 

314 groupId = "FOCUS" 

315 else: 

316 header["GROUPID"] = None 

317 header["IMGTYPE"] = groupId 

318 log.debug("%s: Setting IMGTYPE to '%s' from GROUPID", log_label, header["IMGTYPE"]) 

319 modified = True 

320 else: 

321 # Someone could be fixing headers in old data 

322 # and we do not want GROUPID == IMGTYPE 

323 if imgType == groupId: 

324 # Clear the group so we default to original 

325 header["GROUPID"] = None 

326 

327 # We were using OBJECT for engineering observations early on 

328 if date < OBJECT_IS_ENGTEST: 

329 imgType = header.get("IMGTYPE") 

330 if imgType == "OBJECT": 

331 header["IMGTYPE"] = "ENGTEST" 

332 log.debug("%s: Changing OBJECT observation type to %s", 

333 log_label, header["IMGTYPE"]) 

334 modified = True 

335 

336 # Early on the RA/DEC headers were stored in radians 

337 if date < RADEC_IS_RADIANS: 

338 if header.get("RA") is not None: 

339 header["RA"] *= RAD2DEG 

340 log.debug("%s: Changing RA header to degrees", log_label) 

341 modified = True 

342 if header.get("DEC") is not None: 

343 header["DEC"] *= RAD2DEG 

344 log.debug("%s: Changing DEC header to degrees", log_label) 

345 modified = True 

346 

347 if header.get("SHUTTIME"): 

348 log.debug("%s: Forcing SHUTTIME header to be None", log_label) 

349 header["SHUTTIME"] = None 

350 modified = True 

351 

352 if "OBJECT" not in header: 

353 # Only patch OBJECT IMGTYPE 

354 if "IMGTYPE" in header and header["IMGTYPE"] == "OBJECT": 

355 log.debug("%s: Forcing OBJECT header to exist", log_label) 

356 header["OBJECT"] = "NOTSET" 

357 modified = True 

358 

359 if "RADESYS" in header: 

360 if header["RADESYS"] == "": 

361 # Default to ICRS 

362 header["RADESYS"] = "ICRS" 

363 log.debug("%s: Forcing blank RADESYS to '%s'", log_label, header["RADESYS"]) 

364 modified = True 

365 

366 if date < RASTART_IS_HOURS: 

367 # Avoid two checks for case where RASTART is fine 

368 if date < RASTART_IS_BAD: 

369 # The wrong telescope position was used. Unsetting these will 

370 # force the RA/DEC demand headers to be used instead. 

371 for h in ("RASTART", "DECSTART", "RAEND", "DECEND"): 

372 header[h] = None 

373 log.debug("%s: Forcing derived RA/Dec headers to undefined", log_label) 

374 modified = True 

375 else: 

376 # Correct hours to degrees 

377 for h in ("RASTART", "RAEND"): 

378 if header[h]: 

379 header[h] *= 15.0 

380 log.debug("%s: Correcting RASTART/END from hours to degrees", log_label) 

381 modified = True 

382 

383 # RASTART/END headers have a TAI/UTC confusion causing an offset 

384 # of 37 seconds for a period of time. 

385 if RASTART_IS_BAD < date < RASTART_IS_OKAY: 

386 modified = True 

387 offset = (37.0 / 3600.0) * 15.0 

388 for epoch in ("START", "END"): 

389 h = "RA" + epoch 

390 if header[h]: 

391 header[h] += offset 

392 

393 return modified 

394 

395 def _is_on_mountain(self): 

396 date = self.to_datetime_begin() 

397 if date > TSTART: 

398 return True 

399 return False 

400 

401 @staticmethod 

402 def compute_detector_exposure_id(exposure_id, detector_num): 

403 """Compute the detector exposure ID from detector number and 

404 exposure ID. 

405 

406 This is a helper method to allow code working outside the translator 

407 infrastructure to use the same algorithm. 

408 

409 Parameters 

410 ---------- 

411 exposure_id : `int` 

412 Unique exposure ID. 

413 detector_num : `int` 

414 Detector number. 

415 

416 Returns 

417 ------- 

418 detector_exposure_id : `int` 

419 The calculated ID. 

420 """ 

421 if detector_num != 0: 

422 log.warning("Unexpected non-zero detector number for LATISS") 

423 return exposure_id 

424 

425 @cache_translation 

426 def to_dark_time(self): 

427 # Docstring will be inherited. Property defined in properties.py 

428 

429 # Always compare with exposure time 

430 # We may revisit this later if there is a cutoff date where we 

431 # can always trust the header. 

432 exptime = self.to_exposure_time() 

433 

434 if self.is_key_ok("DARKTIME"): 

435 darktime = self.quantity_from_card("DARKTIME", u.s) 

436 if darktime >= exptime: 

437 return darktime 

438 reason = "Dark time less than exposure time." 

439 else: 

440 reason = "Dark time not defined." 

441 

442 log.warning("%s: %s Setting dark time to the exposure time.", 

443 self._log_prefix, reason) 

444 return exptime 

445 

446 @cache_translation 

447 def to_exposure_time(self): 

448 # Docstring will be inherited. Property defined in properties.py 

449 # Some data is missing a value for EXPTIME. 

450 # Have to be careful we do not have circular logic when trying to 

451 # guess 

452 if self.is_key_ok("EXPTIME"): 

453 return self.quantity_from_card("EXPTIME", u.s) 

454 

455 # A missing or undefined EXPTIME is problematic. Set to -1 

456 # to indicate that none was found. 

457 log.warning("%s: Insufficient information to derive exposure time. Setting to -1.0s", 

458 self._log_prefix) 

459 return -1.0 * u.s 

460 

461 @cache_translation 

462 def to_observation_type(self): 

463 """Determine the observation type. 

464 

465 In the absence of an ``IMGTYPE`` header, assumes lab data is always a 

466 dark if exposure time is non-zero, else bias. 

467 

468 Returns 

469 ------- 

470 obstype : `str` 

471 Observation type. 

472 """ 

473 

474 # LATISS observation type is documented to appear in OBSTYPE 

475 # but for historical reasons prefers IMGTYPE. 

476 # Test the keys in order until we find one that contains a 

477 # defined value. 

478 obstype_keys = ["OBSTYPE", "IMGTYPE"] 

479 

480 obstype = None 

481 for k in obstype_keys: 

482 if self.is_key_ok(k): 

483 obstype = self._header[k] 

484 self._used_these_cards(k) 

485 obstype = obstype.lower() 

486 break 

487 

488 if obstype is not None: 

489 if obstype == "object" and not self._is_on_mountain(): 

490 # Do not map object to science in lab since most 

491 # code assume science is on sky with RA/Dec. 

492 obstype = "labobject" 

493 elif obstype in ("skyexp", "object"): 

494 obstype = "science" 

495 

496 return obstype 

497 

498 # In the absence of any observation type information, return 

499 # unknown unless we think it might be a bias. 

500 exptime = self.to_exposure_time() 

501 if exptime == 0.0: 

502 obstype = "bias" 

503 else: 

504 obstype = "unknown" 

505 log.warning("%s: Unable to determine observation type. Guessing '%s'", 

506 self._log_prefix, obstype) 

507 return obstype 

508 

509 @cache_translation 

510 def to_physical_filter(self): 

511 """Calculate the physical filter name. 

512 

513 Returns 

514 ------- 

515 filter : `str` 

516 Name of filter. A combination of FILTER and GRATING 

517 headers joined by a "~". The filter and grating are always 

518 combined. The filter or grating part will be "none" if no value 

519 is specified. Uses "empty" if any of the filters or gratings 

520 indicate an "empty_N" name. "unknown" indicates that the filter is 

521 not defined anywhere but we think it should be. "none" indicates 

522 that the filter was not defined but the observation is a dark 

523 or bias. 

524 """ 

525 

526 physical_filter = self._determine_primary_filter() 

527 

528 if self.is_key_ok("GRATING"): 

529 grating = self._header["GRATING"] 

530 self._used_these_cards("GRATING") 

531 

532 if not grating or grating.lower().startswith("empty"): 

533 grating = "empty" 

534 else: 

535 # Be explicit about having no knowledge of the grating 

536 grating = "unknown" 

537 

538 physical_filter = f"{physical_filter}{FILTER_DELIMITER}{grating}" 

539 

540 return physical_filter