Coverage for python/lsst/obs/lsst/translators/latiss.py: 16%

233 statements  

« prev     ^ index     » next       coverage.py v6.4, created at 2022-05-24 03:42 -0700

1# This file is currently part of obs_lsst but is written to allow it 

2# to be migrated to the astro_metadata_translator package at a later date. 

3# 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the LICENSE file in this directory for details of code ownership. 

7# 

8# Use of this source code is governed by a 3-clause BSD-style 

9# license that can be found in the LICENSE file. 

10 

11"""Metadata translation code for LSST LATISS headers""" 

12 

13__all__ = ("LatissTranslator", ) 

14 

15import logging 

16import math 

17 

18import astropy.units as u 

19from astropy.time import Time 

20from astropy.coordinates import EarthLocation 

21 

22from astro_metadata_translator import cache_translation 

23from astro_metadata_translator.translators.helpers import is_non_science 

24from .lsst import LsstBaseTranslator, FILTER_DELIMITER 

25 

26log = logging.getLogger(__name__) 

27 

28 

29# AuxTel is not the same place as LSST 

30# These coordinates read from Apple Maps 

31AUXTEL_LOCATION = EarthLocation.from_geodetic(-70.747698, -30.244728, 2663.0) 

32 

33# Date instrument is taking data at telescope 

34# Prior to this date many parameters are automatically nulled out 

35# since the headers have not historically been reliable 

36TSTART = Time("2020-01-01T00:00", format="isot", scale="utc") 

37 

38# Define the sensor and group name for AuxTel globally so that it can be used 

39# in multiple places. There is no raft but for consistency with other LSST 

40# cameras we define one. 

41_DETECTOR_GROUP_NAME = "RXX" 

42_DETECTOR_NAME = "S00" 

43 

44# Date 068 detector was put in LATISS 

45DETECTOR_068_DATE = Time("2019-06-24T00:00", format="isot", scale="utc") 

46 

47# IMGTYPE header is filled in after this date 

48IMGTYPE_OKAY_DATE = Time("2019-11-07T00:00", format="isot", scale="utc") 

49 

50# OBJECT IMGTYPE really means ENGTEST until this date 

51OBJECT_IS_ENGTEST = Time("2020-01-27T20:00", format="isot", scale="utc") 

52 

53# RA and DEC headers are in radians until this date 

54RADEC_IS_RADIANS = Time("2020-01-28T22:00", format="isot", scale="utc") 

55 

56# RASTART/DECSTART/RAEND/DECEND used wrong telescope location before this 

57# 2020-02-01T00:00 we fixed the telescope location, but RASTART is still 

58# in mount coordinates, so off by pointing model. 

59RASTART_IS_BAD = Time("2020-05-01T00:00", format="isot", scale="utc") 

60 

61# Between RASTART_IS_BAD and this time the RASTART header uses hours 

62# instead of degrees. 

63RASTART_IS_HOURS = Time("2021-02-11T18:45", format="isot", scale="utc") 

64 

65# From this date RASTART is correct as-is. 

66RASTART_IS_OKAY = Time("2021-02-12T00:00", format="isot", scale="utc") 

67 

68# DATE-END is not to be trusted before this date 

69DATE_END_IS_BAD = Time("2020-02-01T00:00", format="isot", scale="utc") 

70 

71# The convention for the reporting of ROTPA changed by 180 here 

72ROTPA_CONVENTION_180_SWITCH1 = Time("2020-11-19T00:00", format="isot", scale="utc") 

73ROTPA_CONVENTION_180_SWITCH2 = Time("2021-10-29T00:00", format="isot", scale="utc") 

74 

75# Scaling factor radians to degrees. Keep it simple. 

76RAD2DEG = 180.0 / math.pi 

77 

78 

79def is_non_science_or_lab(self): 

80 """Pseudo method to determine whether this is a lab or non-science 

81 header. 

82 

83 Raises 

84 ------ 

85 KeyError 

86 If this is a science observation and on the mountain. 

87 """ 

88 # Return without raising if this is not a science observation 

89 # since the defaults are fine. 

90 try: 

91 # This will raise if it is a science observation 

92 is_non_science(self) 

93 return 

94 except KeyError: 

95 pass 

96 

97 # We are still in the lab, return and use the default 

98 if not self._is_on_mountain(): 

99 return 

100 

101 # This is a science observation on the mountain so we should not 

102 # use defaults 

103 raise KeyError(f"{self._log_prefix}: Required key is missing and this is a mountain science observation") 

104 

105 

106class LatissTranslator(LsstBaseTranslator): 

107 """Metadata translator for LSST LATISS data from AuxTel. 

108 

109 For lab measurements many values are masked out. 

110 """ 

111 

112 name = "LSST_LATISS" 

113 """Name of this translation class""" 

114 

115 supported_instrument = "LATISS" 

116 """Supports the LATISS instrument.""" 

117 

118 _const_map = { 

119 "instrument": "LATISS", 

120 "telescope": "Rubin Auxiliary Telescope", 

121 "detector_group": _DETECTOR_GROUP_NAME, 

122 "detector_num": 0, 

123 "detector_name": _DETECTOR_NAME, # Single sensor 

124 "relative_humidity": None, 

125 "pressure": None, 

126 "temperature": None, 

127 } 

128 

129 _trivial_map = { 

130 "observation_id": (["OBSID", "IMGNAME"], dict(default=None, checker=is_non_science)), 

131 "detector_serial": ["LSST_NUM", "DETSER"], 

132 "object": ("OBJECT", dict(checker=is_non_science_or_lab, default="UNKNOWN")), 

133 "boresight_rotation_angle": (["ROTPA", "ROTANGLE"], dict(checker=is_non_science_or_lab, 

134 default=float("nan"), unit=u.deg)), 

135 "science_program": ("PROGRAM", dict(default="unknown")), 

136 } 

137 

138 DETECTOR_GROUP_NAME = _DETECTOR_GROUP_NAME 

139 """Fixed name of detector group.""" 

140 

141 DETECTOR_NAME = _DETECTOR_NAME 

142 """Fixed name of single sensor.""" 

143 

144 DETECTOR_MAX = 0 

145 """Maximum number of detectors to use when calculating the 

146 detector_exposure_id.""" 

147 

148 _DEFAULT_LOCATION = AUXTEL_LOCATION 

149 """Default telescope location in absence of relevant FITS headers.""" 

150 

151 @classmethod 

152 def can_translate(cls, header, filename=None): 

153 """Indicate whether this translation class can translate the 

154 supplied header. 

155 

156 Parameters 

157 ---------- 

158 header : `dict`-like 

159 Header to convert to standardized form. 

160 filename : `str`, optional 

161 Name of file being translated. 

162 

163 Returns 

164 ------- 

165 can : `bool` 

166 `True` if the header is recognized by this class. `False` 

167 otherwise. 

168 """ 

169 # INSTRUME keyword might be of two types 

170 if "INSTRUME" in header: 

171 instrume = header["INSTRUME"] 

172 for v in ("LSST_ATISS", "LATISS"): 

173 if instrume == v: 

174 return True 

175 # Calibration files strip important headers at the moment so guess 

176 if "DETNAME" in header and header["DETNAME"] == "RXX_S00": 

177 return True 

178 return False 

179 

180 @classmethod 

181 def fix_header(cls, header, instrument, obsid, filename=None): 

182 """Fix an incorrect LATISS header. 

183 

184 Parameters 

185 ---------- 

186 header : `dict` 

187 The header to update. Updates are in place. 

188 instrument : `str` 

189 The name of the instrument. 

190 obsid : `str` 

191 Unique observation identifier associated with this header. 

192 Will always be provided. 

193 filename : `str`, optional 

194 Filename associated with this header. May not be set since headers 

195 can be fixed independently of any filename being known. 

196 

197 Returns 

198 ------- 

199 modified = `bool` 

200 Returns `True` if the header was updated. 

201 

202 Notes 

203 ----- 

204 This method does not apply per-obsid corrections. The following 

205 corrections are applied: 

206 

207 * On June 24th 2019 the detector was changed from ITL-3800C-098 

208 to ITL-3800C-068. The header is intended to be correct in the 

209 future. 

210 * In late 2019 the DATE-OBS and MJD-OBS headers were reporting 

211 1970 dates. To correct, the DATE/MJD headers are copied in to 

212 replace them and the -END headers are cleared. 

213 * Until November 2019 the IMGTYPE was set in the GROUPID header. 

214 The value is moved to IMGTYPE. 

215 * SHUTTIME is always forced to be `None`. 

216 

217 Corrections are reported as debug level log messages. 

218 

219 See `~astro_metadata_translator.fix_header` for details of the general 

220 process. 

221 """ 

222 modified = False 

223 

224 # Calculate the standard label to use for log messages 

225 log_label = cls._construct_log_prefix(obsid, filename) 

226 

227 if "OBSID" not in header: 

228 # Very old data used IMGNAME 

229 header["OBSID"] = obsid 

230 modified = True 

231 # We are reporting the OBSID so no need to repeat it at start 

232 # of log message. Use filename if we have it. 

233 log_prefix = f"{filename}: " if filename else "" 

234 log.debug("%sAssigning OBSID to a value of '%s'", log_prefix, header["OBSID"]) 

235 

236 if "DAYOBS" not in header: 

237 # OBS-NITE could have the value for DAYOBS but it is safer 

238 # for older data to set it from the OBSID. Fall back to OBS-NITE 

239 # if we have no alternative 

240 dayObs = None 

241 try: 

242 dayObs = obsid.split("_", 3)[2] 

243 except (AttributeError, ValueError): 

244 # did not split as expected 

245 pass 

246 if dayObs is None or len(dayObs) != 8: 

247 if "OBS-NITE" in header: 

248 dayObs = header["OBS-NITE"] 

249 log.debug("%s: Setting DAYOBS to '%s' from OBS-NITE header", log_label, dayObs) 

250 else: 

251 log.debug("%s: Unable to determine DAYOBS from header", log_label) 

252 else: 

253 log.debug("%s: Setting DAYOBS to '%s' from OBSID", log_label, dayObs) 

254 if dayObs: 

255 header["DAYOBS"] = dayObs 

256 modified = True 

257 

258 if "SEQNUM" not in header: 

259 try: 

260 seqnum = obsid.split("_", 3)[3] 

261 except (AttributeError, ValueError): 

262 # did not split as expected 

263 pass 

264 else: 

265 header["SEQNUM"] = int(seqnum) 

266 modified = True 

267 log.debug("%s: Extracting SEQNUM of '%s' from OBSID", log_label, header["SEQNUM"]) 

268 

269 # The DATE-OBS / MJD-OBS keys can be 1970 

270 if "DATE-OBS" in header and header["DATE-OBS"].startswith("1970"): 

271 # Copy the headers from the DATE and MJD since we have no other 

272 # choice. 

273 header["DATE-OBS"] = header["DATE"] 

274 header["DATE-BEG"] = header["DATE-OBS"] 

275 header["MJD-OBS"] = header["MJD"] 

276 header["MJD-BEG"] = header["MJD-OBS"] 

277 

278 # And clear the DATE-END and MJD-END -- the translator will use 

279 # EXPTIME instead. 

280 header["DATE-END"] = None 

281 header["MJD-END"] = None 

282 

283 log.debug("%s: Forcing 1970 dates to '%s'", log_label, header["DATE"]) 

284 modified = True 

285 

286 # Create a translator since we need the date 

287 translator = cls(header) 

288 date = translator.to_datetime_begin() 

289 if date > DETECTOR_068_DATE: 

290 header["LSST_NUM"] = "ITL-3800C-068" 

291 log.debug("%s: Forcing detector serial to %s", log_label, header["LSST_NUM"]) 

292 modified = True 

293 

294 if date < DATE_END_IS_BAD: 

295 # DATE-END may or may not be in TAI and may or may not be 

296 # before DATE-BEG. Simpler to clear it 

297 if header.get("DATE-END"): 

298 header["DATE-END"] = None 

299 header["MJD-END"] = None 

300 

301 log.debug("%s: Clearing DATE-END as being untrustworthy", log_label) 

302 modified = True 

303 

304 # Up until a certain date GROUPID was the IMGTYPE 

305 if date < IMGTYPE_OKAY_DATE: 

306 groupId = header.get("GROUPID") 

307 if groupId and not groupId.startswith("test"): 

308 imgType = header.get("IMGTYPE") 

309 if not imgType: 

310 if "_" in groupId: 

311 # Sometimes have the form dark_0001_0002 

312 # in this case we pull the IMGTYPE off the front and 

313 # do not clear groupId (although groupId may now 

314 # repeat on different days). 

315 groupId, _ = groupId.split("_", 1) 

316 elif groupId.upper() != "FOCUS" and groupId.upper().startswith("FOCUS"): 

317 # If it is exactly FOCUS we want groupId cleared 

318 groupId = "FOCUS" 

319 else: 

320 header["GROUPID"] = None 

321 header["IMGTYPE"] = groupId 

322 log.debug("%s: Setting IMGTYPE to '%s' from GROUPID", log_label, header["IMGTYPE"]) 

323 modified = True 

324 else: 

325 # Someone could be fixing headers in old data 

326 # and we do not want GROUPID == IMGTYPE 

327 if imgType == groupId: 

328 # Clear the group so we default to original 

329 header["GROUPID"] = None 

330 

331 # We were using OBJECT for engineering observations early on 

332 if date < OBJECT_IS_ENGTEST: 

333 imgType = header.get("IMGTYPE") 

334 if imgType == "OBJECT": 

335 header["IMGTYPE"] = "ENGTEST" 

336 log.debug("%s: Changing OBJECT observation type to %s", 

337 log_label, header["IMGTYPE"]) 

338 modified = True 

339 

340 # Early on the RA/DEC headers were stored in radians 

341 if date < RADEC_IS_RADIANS: 

342 if header.get("RA") is not None: 

343 header["RA"] *= RAD2DEG 

344 log.debug("%s: Changing RA header to degrees", log_label) 

345 modified = True 

346 if header.get("DEC") is not None: 

347 header["DEC"] *= RAD2DEG 

348 log.debug("%s: Changing DEC header to degrees", log_label) 

349 modified = True 

350 

351 if header.get("SHUTTIME"): 

352 log.debug("%s: Forcing SHUTTIME header to be None", log_label) 

353 header["SHUTTIME"] = None 

354 modified = True 

355 

356 if "OBJECT" not in header: 

357 # Only patch OBJECT IMGTYPE 

358 if "IMGTYPE" in header and header["IMGTYPE"] == "OBJECT": 

359 log.debug("%s: Forcing OBJECT header to exist", log_label) 

360 header["OBJECT"] = "NOTSET" 

361 modified = True 

362 

363 if "RADESYS" in header: 

364 if header["RADESYS"] == "": 

365 # Default to ICRS 

366 header["RADESYS"] = "ICRS" 

367 log.debug("%s: Forcing blank RADESYS to '%s'", log_label, header["RADESYS"]) 

368 modified = True 

369 

370 if date < RASTART_IS_HOURS: 

371 # Avoid two checks for case where RASTART is fine 

372 if date < RASTART_IS_BAD: 

373 # The wrong telescope position was used. Unsetting these will 

374 # force the RA/DEC demand headers to be used instead. 

375 for h in ("RASTART", "DECSTART", "RAEND", "DECEND"): 

376 header[h] = None 

377 log.debug("%s: Forcing derived RA/Dec headers to undefined", log_label) 

378 modified = True 

379 else: 

380 # Correct hours to degrees 

381 for h in ("RASTART", "RAEND"): 

382 if header[h]: 

383 header[h] *= 15.0 

384 log.debug("%s: Correcting RASTART/END from hours to degrees", log_label) 

385 modified = True 

386 

387 # RASTART/END headers have a TAI/UTC confusion causing an offset 

388 # of 37 seconds for a period of time. 

389 if RASTART_IS_BAD < date < RASTART_IS_OKAY: 

390 modified = True 

391 offset = (37.0 / 3600.0) * 15.0 

392 for epoch in ("START", "END"): 

393 h = "RA" + epoch 

394 if header[h]: 

395 header[h] += offset 

396 

397 if date < ROTPA_CONVENTION_180_SWITCH2 and date > ROTPA_CONVENTION_180_SWITCH1: 

398 header['ROTPA'] = header['ROTPA'] - 180 

399 modified = True 

400 

401 return modified 

402 

403 def _is_on_mountain(self): 

404 date = self.to_datetime_begin() 

405 if date > TSTART: 

406 return True 

407 return False 

408 

409 @staticmethod 

410 def compute_detector_exposure_id(exposure_id, detector_num): 

411 """Compute the detector exposure ID from detector number and 

412 exposure ID. 

413 

414 This is a helper method to allow code working outside the translator 

415 infrastructure to use the same algorithm. 

416 

417 Parameters 

418 ---------- 

419 exposure_id : `int` 

420 Unique exposure ID. 

421 detector_num : `int` 

422 Detector number. 

423 

424 Returns 

425 ------- 

426 detector_exposure_id : `int` 

427 The calculated ID. 

428 """ 

429 if detector_num != 0: 

430 log.warning("Unexpected non-zero detector number for LATISS") 

431 return exposure_id 

432 

433 @cache_translation 

434 def to_dark_time(self): 

435 # Docstring will be inherited. Property defined in properties.py 

436 

437 # Always compare with exposure time 

438 # We may revisit this later if there is a cutoff date where we 

439 # can always trust the header. 

440 exptime = self.to_exposure_time() 

441 

442 if self.is_key_ok("DARKTIME"): 

443 darktime = self.quantity_from_card("DARKTIME", u.s) 

444 if darktime >= exptime: 

445 return darktime 

446 reason = "Dark time less than exposure time." 

447 else: 

448 reason = "Dark time not defined." 

449 

450 log.warning("%s: %s Setting dark time to the exposure time.", 

451 self._log_prefix, reason) 

452 return exptime 

453 

454 @cache_translation 

455 def to_exposure_time(self): 

456 # Docstring will be inherited. Property defined in properties.py 

457 # Some data is missing a value for EXPTIME. 

458 # Have to be careful we do not have circular logic when trying to 

459 # guess 

460 if self.is_key_ok("EXPTIME"): 

461 return self.quantity_from_card("EXPTIME", u.s) 

462 

463 # A missing or undefined EXPTIME is problematic. Set to -1 

464 # to indicate that none was found. 

465 log.warning("%s: Insufficient information to derive exposure time. Setting to -1.0s", 

466 self._log_prefix) 

467 return -1.0 * u.s 

468 

469 @cache_translation 

470 def to_observation_type(self): 

471 """Determine the observation type. 

472 

473 In the absence of an ``IMGTYPE`` header, assumes lab data is always a 

474 dark if exposure time is non-zero, else bias. 

475 

476 Returns 

477 ------- 

478 obstype : `str` 

479 Observation type. 

480 """ 

481 

482 # LATISS observation type is documented to appear in OBSTYPE 

483 # but for historical reasons prefers IMGTYPE. 

484 # Test the keys in order until we find one that contains a 

485 # defined value. 

486 obstype_keys = ["OBSTYPE", "IMGTYPE"] 

487 

488 obstype = None 

489 for k in obstype_keys: 

490 if self.is_key_ok(k): 

491 obstype = self._header[k] 

492 self._used_these_cards(k) 

493 obstype = obstype.lower() 

494 break 

495 

496 if obstype is not None: 

497 if obstype == "object" and not self._is_on_mountain(): 

498 # Do not map object to science in lab since most 

499 # code assume science is on sky with RA/Dec. 

500 obstype = "labobject" 

501 elif obstype in ("skyexp", "object"): 

502 obstype = "science" 

503 

504 return obstype 

505 

506 # In the absence of any observation type information, return 

507 # unknown unless we think it might be a bias. 

508 exptime = self.to_exposure_time() 

509 if exptime == 0.0: 

510 obstype = "bias" 

511 else: 

512 obstype = "unknown" 

513 log.warning("%s: Unable to determine observation type. Guessing '%s'", 

514 self._log_prefix, obstype) 

515 return obstype 

516 

517 @cache_translation 

518 def to_physical_filter(self): 

519 """Calculate the physical filter name. 

520 

521 Returns 

522 ------- 

523 filter : `str` 

524 Name of filter. A combination of FILTER and GRATING 

525 headers joined by a "~". The filter and grating are always 

526 combined. The filter or grating part will be "none" if no value 

527 is specified. Uses "empty" if any of the filters or gratings 

528 indicate an "empty_N" name. "unknown" indicates that the filter is 

529 not defined anywhere but we think it should be. "none" indicates 

530 that the filter was not defined but the observation is a dark 

531 or bias. 

532 """ 

533 

534 physical_filter = self._determine_primary_filter() 

535 

536 if self.is_key_ok("GRATING"): 

537 grating = self._header["GRATING"] 

538 self._used_these_cards("GRATING") 

539 

540 if not grating or grating.lower().startswith("empty"): 

541 grating = "empty" 

542 else: 

543 # Be explicit about having no knowledge of the grating 

544 grating = "unknown" 

545 

546 physical_filter = f"{physical_filter}{FILTER_DELIMITER}{grating}" 

547 

548 return physical_filter