Coverage for python/lsst/obs/lsst/translators/latiss.py: 16%

244 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-12-08 15:41 -0800

1# This file is currently part of obs_lsst but is written to allow it 

2# to be migrated to the astro_metadata_translator package at a later date. 

3# 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the LICENSE file in this directory for details of code ownership. 

7# 

8# Use of this source code is governed by a 3-clause BSD-style 

9# license that can be found in the LICENSE file. 

10 

11"""Metadata translation code for LSST LATISS headers""" 

12 

13__all__ = ("LatissTranslator", ) 

14 

15import logging 

16import math 

17 

18import astropy.units as u 

19from astropy.time import Time 

20from astropy.coordinates import EarthLocation 

21 

22from astro_metadata_translator import cache_translation 

23from astro_metadata_translator.translators.helpers import is_non_science 

24from .lsst import LsstBaseTranslator, FILTER_DELIMITER 

25 

26log = logging.getLogger(__name__) 

27 

28 

29# AuxTel is not the same place as LSST 

30# These coordinates read from Apple Maps 

31AUXTEL_LOCATION = EarthLocation.from_geodetic(-70.747698, -30.244728, 2663.0) 

32 

33# Date instrument is taking data at telescope 

34# Prior to this date many parameters are automatically nulled out 

35# since the headers have not historically been reliable 

36TSTART = Time("2020-01-01T00:00", format="isot", scale="utc") 

37 

38# Define the sensor and group name for AuxTel globally so that it can be used 

39# in multiple places. There is no raft but for consistency with other LSST 

40# cameras we define one. 

41_DETECTOR_GROUP_NAME = "RXX" 

42_DETECTOR_NAME = "S00" 

43 

44# Date 068 detector was put in LATISS 

45DETECTOR_068_DATE = Time("2019-06-24T00:00", format="isot", scale="utc") 

46 

47# IMGTYPE header is filled in after this date 

48IMGTYPE_OKAY_DATE = Time("2019-11-07T00:00", format="isot", scale="utc") 

49 

50# OBJECT IMGTYPE really means ENGTEST until this date 

51OBJECT_IS_ENGTEST = Time("2020-01-27T20:00", format="isot", scale="utc") 

52 

53# RA and DEC headers are in radians until this date 

54RADEC_IS_RADIANS = Time("2020-01-28T22:00", format="isot", scale="utc") 

55 

56# RASTART/DECSTART/RAEND/DECEND used wrong telescope location before this 

57# 2020-02-01T00:00 we fixed the telescope location, but RASTART is still 

58# in mount coordinates, so off by pointing model. 

59RASTART_IS_BAD = Time("2020-05-01T00:00", format="isot", scale="utc") 

60 

61# Between RASTART_IS_BAD and this time the RASTART header uses hours 

62# instead of degrees. 

63RASTART_IS_HOURS = Time("2021-02-11T18:45", format="isot", scale="utc") 

64 

65# From this date RASTART is correct as-is. 

66RASTART_IS_OKAY = Time("2021-02-12T00:00", format="isot", scale="utc") 

67 

68# DATE-END is not to be trusted before this date 

69DATE_END_IS_BAD = Time("2020-02-01T00:00", format="isot", scale="utc") 

70 

71# The convention for the reporting of ROTPA changed by 180 here 

72ROTPA_CONVENTION_180_SWITCH1 = Time("2020-11-19T00:00", format="isot", scale="utc") 

73ROTPA_CONVENTION_180_SWITCH2 = Time("2021-10-29T00:00", format="isot", scale="utc") 

74 

75# TARGET is set to start with 'spec:' for dispsered images before this date 

76TARGET_STARTS_SPECCOLON = Time("2022-07-10T00:00", format="isot", scale="utc") 

77 

78# Scaling factor radians to degrees. Keep it simple. 

79RAD2DEG = 180.0 / math.pi 

80 

81 

82def is_non_science_or_lab(self): 

83 """Pseudo method to determine whether this is a lab or non-science 

84 header. 

85 

86 Raises 

87 ------ 

88 KeyError 

89 If this is a science observation and on the mountain. 

90 """ 

91 # Return without raising if this is not a science observation 

92 # since the defaults are fine. 

93 try: 

94 # This will raise if it is a science observation 

95 is_non_science(self) 

96 return 

97 except KeyError: 

98 pass 

99 

100 # We are still in the lab, return and use the default 

101 if not self._is_on_mountain(): 

102 return 

103 

104 # This is a science observation on the mountain so we should not 

105 # use defaults 

106 raise KeyError(f"{self._log_prefix}: Required key is missing and this is a mountain science observation") 

107 

108 

109class LatissTranslator(LsstBaseTranslator): 

110 """Metadata translator for LSST LATISS data from AuxTel. 

111 

112 For lab measurements many values are masked out. 

113 """ 

114 

115 name = "LSST_LATISS" 

116 """Name of this translation class""" 

117 

118 supported_instrument = "LATISS" 

119 """Supports the LATISS instrument.""" 

120 

121 _const_map = { 

122 "instrument": "LATISS", 

123 "telescope": "Rubin Auxiliary Telescope", 

124 "detector_group": _DETECTOR_GROUP_NAME, 

125 "detector_num": 0, 

126 "detector_name": _DETECTOR_NAME, # Single sensor 

127 "relative_humidity": None, 

128 "pressure": None, 

129 "temperature": None, 

130 } 

131 

132 _trivial_map = { 

133 "observation_id": (["OBSID", "IMGNAME"], dict(default=None, checker=is_non_science)), 

134 "detector_serial": ["LSST_NUM", "DETSER"], 

135 "object": ("OBJECT", dict(checker=is_non_science_or_lab, default="UNKNOWN")), 

136 "boresight_rotation_angle": (["ROTPA", "ROTANGLE"], dict(checker=is_non_science_or_lab, 

137 default=float("nan"), unit=u.deg)), 

138 "science_program": ("PROGRAM", dict(default="unknown")), 

139 } 

140 

141 DETECTOR_GROUP_NAME = _DETECTOR_GROUP_NAME 

142 """Fixed name of detector group.""" 

143 

144 DETECTOR_NAME = _DETECTOR_NAME 

145 """Fixed name of single sensor.""" 

146 

147 DETECTOR_MAX = 0 

148 """Maximum number of detectors to use when calculating the 

149 detector_exposure_id.""" 

150 

151 _DEFAULT_LOCATION = AUXTEL_LOCATION 

152 """Default telescope location in absence of relevant FITS headers.""" 

153 

154 @classmethod 

155 def can_translate(cls, header, filename=None): 

156 """Indicate whether this translation class can translate the 

157 supplied header. 

158 

159 Parameters 

160 ---------- 

161 header : `dict`-like 

162 Header to convert to standardized form. 

163 filename : `str`, optional 

164 Name of file being translated. 

165 

166 Returns 

167 ------- 

168 can : `bool` 

169 `True` if the header is recognized by this class. `False` 

170 otherwise. 

171 """ 

172 # INSTRUME keyword might be of two types 

173 if "INSTRUME" in header: 

174 instrume = header["INSTRUME"] 

175 for v in ("LSST_ATISS", "LATISS"): 

176 if instrume == v: 

177 return True 

178 # Calibration files strip important headers at the moment so guess 

179 if "DETNAME" in header and header["DETNAME"] == "RXX_S00": 

180 return True 

181 return False 

182 

183 @classmethod 

184 def fix_header(cls, header, instrument, obsid, filename=None): 

185 """Fix an incorrect LATISS header. 

186 

187 Parameters 

188 ---------- 

189 header : `dict` 

190 The header to update. Updates are in place. 

191 instrument : `str` 

192 The name of the instrument. 

193 obsid : `str` 

194 Unique observation identifier associated with this header. 

195 Will always be provided. 

196 filename : `str`, optional 

197 Filename associated with this header. May not be set since headers 

198 can be fixed independently of any filename being known. 

199 

200 Returns 

201 ------- 

202 modified = `bool` 

203 Returns `True` if the header was updated. 

204 

205 Notes 

206 ----- 

207 This method does not apply per-obsid corrections. The following 

208 corrections are applied: 

209 

210 * On June 24th 2019 the detector was changed from ITL-3800C-098 

211 to ITL-3800C-068. The header is intended to be correct in the 

212 future. 

213 * In late 2019 the DATE-OBS and MJD-OBS headers were reporting 

214 1970 dates. To correct, the DATE/MJD headers are copied in to 

215 replace them and the -END headers are cleared. 

216 * Until November 2019 the IMGTYPE was set in the GROUPID header. 

217 The value is moved to IMGTYPE. 

218 * SHUTTIME is always forced to be `None`. 

219 

220 Corrections are reported as debug level log messages. 

221 

222 See `~astro_metadata_translator.fix_header` for details of the general 

223 process. 

224 """ 

225 modified = False 

226 

227 # Calculate the standard label to use for log messages 

228 log_label = cls._construct_log_prefix(obsid, filename) 

229 

230 if "OBSID" not in header: 

231 # Very old data used IMGNAME 

232 header["OBSID"] = obsid 

233 modified = True 

234 # We are reporting the OBSID so no need to repeat it at start 

235 # of log message. Use filename if we have it. 

236 log_prefix = f"{filename}: " if filename else "" 

237 log.debug("%sAssigning OBSID to a value of '%s'", log_prefix, header["OBSID"]) 

238 

239 if "DAYOBS" not in header: 

240 # OBS-NITE could have the value for DAYOBS but it is safer 

241 # for older data to set it from the OBSID. Fall back to OBS-NITE 

242 # if we have no alternative 

243 dayObs = None 

244 try: 

245 dayObs = obsid.split("_", 3)[2] 

246 except (AttributeError, ValueError): 

247 # did not split as expected 

248 pass 

249 if dayObs is None or len(dayObs) != 8: 

250 if "OBS-NITE" in header: 

251 dayObs = header["OBS-NITE"] 

252 log.debug("%s: Setting DAYOBS to '%s' from OBS-NITE header", log_label, dayObs) 

253 else: 

254 log.debug("%s: Unable to determine DAYOBS from header", log_label) 

255 else: 

256 log.debug("%s: Setting DAYOBS to '%s' from OBSID", log_label, dayObs) 

257 if dayObs: 

258 header["DAYOBS"] = dayObs 

259 modified = True 

260 

261 if "SEQNUM" not in header: 

262 try: 

263 seqnum = obsid.split("_", 3)[3] 

264 except (AttributeError, ValueError): 

265 # did not split as expected 

266 pass 

267 else: 

268 header["SEQNUM"] = int(seqnum) 

269 modified = True 

270 log.debug("%s: Extracting SEQNUM of '%s' from OBSID", log_label, header["SEQNUM"]) 

271 

272 # The DATE-OBS / MJD-OBS keys can be 1970 

273 if "DATE-OBS" in header and header["DATE-OBS"].startswith("1970"): 

274 # Copy the headers from the DATE and MJD since we have no other 

275 # choice. 

276 header["DATE-OBS"] = header["DATE"] 

277 header["DATE-BEG"] = header["DATE-OBS"] 

278 header["MJD-OBS"] = header["MJD"] 

279 header["MJD-BEG"] = header["MJD-OBS"] 

280 

281 # And clear the DATE-END and MJD-END -- the translator will use 

282 # EXPTIME instead. 

283 header["DATE-END"] = None 

284 header["MJD-END"] = None 

285 

286 log.debug("%s: Forcing 1970 dates to '%s'", log_label, header["DATE"]) 

287 modified = True 

288 

289 # Create a translator since we need the date 

290 translator = cls(header) 

291 date = translator.to_datetime_begin() 

292 if date > DETECTOR_068_DATE: 

293 header["LSST_NUM"] = "ITL-3800C-068" 

294 log.debug("%s: Forcing detector serial to %s", log_label, header["LSST_NUM"]) 

295 modified = True 

296 

297 if date < DATE_END_IS_BAD: 

298 # DATE-END may or may not be in TAI and may or may not be 

299 # before DATE-BEG. Simpler to clear it 

300 if header.get("DATE-END"): 

301 header["DATE-END"] = None 

302 header["MJD-END"] = None 

303 

304 log.debug("%s: Clearing DATE-END as being untrustworthy", log_label) 

305 modified = True 

306 

307 # Up until a certain date GROUPID was the IMGTYPE 

308 if date < IMGTYPE_OKAY_DATE: 

309 groupId = header.get("GROUPID") 

310 if groupId and not groupId.startswith("test"): 

311 imgType = header.get("IMGTYPE") 

312 if not imgType: 

313 if "_" in groupId: 

314 # Sometimes have the form dark_0001_0002 

315 # in this case we pull the IMGTYPE off the front and 

316 # do not clear groupId (although groupId may now 

317 # repeat on different days). 

318 groupId, _ = groupId.split("_", 1) 

319 elif groupId.upper() != "FOCUS" and groupId.upper().startswith("FOCUS"): 

320 # If it is exactly FOCUS we want groupId cleared 

321 groupId = "FOCUS" 

322 else: 

323 header["GROUPID"] = None 

324 header["IMGTYPE"] = groupId 

325 log.debug("%s: Setting IMGTYPE to '%s' from GROUPID", log_label, header["IMGTYPE"]) 

326 modified = True 

327 else: 

328 # Someone could be fixing headers in old data 

329 # and we do not want GROUPID == IMGTYPE 

330 if imgType == groupId: 

331 # Clear the group so we default to original 

332 header["GROUPID"] = None 

333 

334 # We were using OBJECT for engineering observations early on 

335 if date < OBJECT_IS_ENGTEST: 

336 imgType = header.get("IMGTYPE") 

337 if imgType == "OBJECT": 

338 header["IMGTYPE"] = "ENGTEST" 

339 log.debug("%s: Changing OBJECT observation type to %s", 

340 log_label, header["IMGTYPE"]) 

341 modified = True 

342 

343 # Early on the RA/DEC headers were stored in radians 

344 if date < RADEC_IS_RADIANS: 

345 if header.get("RA") is not None: 

346 header["RA"] *= RAD2DEG 

347 log.debug("%s: Changing RA header to degrees", log_label) 

348 modified = True 

349 if header.get("DEC") is not None: 

350 header["DEC"] *= RAD2DEG 

351 log.debug("%s: Changing DEC header to degrees", log_label) 

352 modified = True 

353 

354 if header.get("SHUTTIME"): 

355 log.debug("%s: Forcing SHUTTIME header to be None", log_label) 

356 header["SHUTTIME"] = None 

357 modified = True 

358 

359 if "OBJECT" not in header: 

360 # Only patch OBJECT IMGTYPE 

361 if "IMGTYPE" in header and header["IMGTYPE"] == "OBJECT": 

362 log.debug("%s: Forcing OBJECT header to exist", log_label) 

363 header["OBJECT"] = "NOTSET" 

364 modified = True 

365 

366 if date < TARGET_STARTS_SPECCOLON: 

367 if "OBJECT" in header: 

368 header["OBJECT"] = header['OBJECT'].replace('spec:', '') 

369 modified = True 

370 

371 if "RADESYS" in header: 

372 if header["RADESYS"] == "": 

373 # Default to ICRS 

374 header["RADESYS"] = "ICRS" 

375 log.debug("%s: Forcing blank RADESYS to '%s'", log_label, header["RADESYS"]) 

376 modified = True 

377 

378 if date < RASTART_IS_HOURS: 

379 # Avoid two checks for case where RASTART is fine 

380 if date < RASTART_IS_BAD: 

381 # The wrong telescope position was used. Unsetting these will 

382 # force the RA/DEC demand headers to be used instead. 

383 for h in ("RASTART", "DECSTART", "RAEND", "DECEND"): 

384 header[h] = None 

385 log.debug("%s: Forcing derived RA/Dec headers to undefined", log_label) 

386 modified = True 

387 else: 

388 # Correct hours to degrees 

389 for h in ("RASTART", "RAEND"): 

390 if header[h]: 

391 header[h] *= 15.0 

392 log.debug("%s: Correcting RASTART/END from hours to degrees", log_label) 

393 modified = True 

394 

395 # RASTART/END headers have a TAI/UTC confusion causing an offset 

396 # of 37 seconds for a period of time. 

397 if RASTART_IS_BAD < date < RASTART_IS_OKAY: 

398 modified = True 

399 offset = (37.0 / 3600.0) * 15.0 

400 for epoch in ("START", "END"): 

401 h = "RA" + epoch 

402 if header[h]: 

403 header[h] += offset 

404 

405 if date < ROTPA_CONVENTION_180_SWITCH2 and date > ROTPA_CONVENTION_180_SWITCH1: 

406 header['ROTPA'] = header['ROTPA'] - 180 

407 modified = True 

408 

409 return modified 

410 

411 def _is_on_mountain(self): 

412 date = self.to_datetime_begin() 

413 if date > TSTART: 

414 return True 

415 return False 

416 

417 @staticmethod 

418 def compute_detector_exposure_id(exposure_id, detector_num): 

419 """Compute the detector exposure ID from detector number and 

420 exposure ID. 

421 

422 This is a helper method to allow code working outside the translator 

423 infrastructure to use the same algorithm. 

424 

425 Parameters 

426 ---------- 

427 exposure_id : `int` 

428 Unique exposure ID. 

429 detector_num : `int` 

430 Detector number. 

431 

432 Returns 

433 ------- 

434 detector_exposure_id : `int` 

435 The calculated ID. 

436 """ 

437 if detector_num != 0: 

438 log.warning("Unexpected non-zero detector number for LATISS") 

439 return exposure_id 

440 

441 @cache_translation 

442 def to_dark_time(self): 

443 # Docstring will be inherited. Property defined in properties.py 

444 

445 # Always compare with exposure time 

446 # We may revisit this later if there is a cutoff date where we 

447 # can always trust the header. 

448 exptime = self.to_exposure_time() 

449 

450 if self.is_key_ok("DARKTIME"): 

451 darktime = self.quantity_from_card("DARKTIME", u.s) 

452 if darktime >= exptime: 

453 return darktime 

454 reason = "Dark time less than exposure time." 

455 else: 

456 reason = "Dark time not defined." 

457 

458 log.warning("%s: %s Setting dark time to the exposure time.", 

459 self._log_prefix, reason) 

460 return exptime 

461 

462 @cache_translation 

463 def to_exposure_time(self): 

464 # Docstring will be inherited. Property defined in properties.py 

465 # Some data is missing a value for EXPTIME. 

466 # Have to be careful we do not have circular logic when trying to 

467 # guess 

468 if self.is_key_ok("EXPTIME"): 

469 return self.quantity_from_card("EXPTIME", u.s) 

470 

471 # A missing or undefined EXPTIME is problematic. Set to -1 

472 # to indicate that none was found. 

473 log.warning("%s: Insufficient information to derive exposure time. Setting to -1.0s", 

474 self._log_prefix) 

475 return -1.0 * u.s 

476 

477 @cache_translation 

478 def to_observation_type(self): 

479 """Determine the observation type. 

480 

481 In the absence of an ``IMGTYPE`` header, assumes lab data is always a 

482 dark if exposure time is non-zero, else bias. 

483 

484 Returns 

485 ------- 

486 obstype : `str` 

487 Observation type. 

488 """ 

489 

490 # LATISS observation type is documented to appear in OBSTYPE 

491 # but for historical reasons prefers IMGTYPE. 

492 # Test the keys in order until we find one that contains a 

493 # defined value. 

494 obstype_keys = ["OBSTYPE", "IMGTYPE"] 

495 

496 obstype = None 

497 for k in obstype_keys: 

498 if self.is_key_ok(k): 

499 obstype = self._header[k] 

500 self._used_these_cards(k) 

501 obstype = obstype.lower() 

502 break 

503 

504 if obstype is not None: 

505 if obstype == "object" and not self._is_on_mountain(): 

506 # Do not map object to science in lab since most 

507 # code assume science is on sky with RA/Dec. 

508 obstype = "labobject" 

509 elif obstype in ("skyexp", "object"): 

510 obstype = "science" 

511 

512 return obstype 

513 

514 # In the absence of any observation type information, return 

515 # unknown unless we think it might be a bias. 

516 exptime = self.to_exposure_time() 

517 if exptime == 0.0: 

518 obstype = "bias" 

519 else: 

520 obstype = "unknown" 

521 log.warning("%s: Unable to determine observation type. Guessing '%s'", 

522 self._log_prefix, obstype) 

523 return obstype 

524 

525 @cache_translation 

526 def to_physical_filter(self): 

527 """Calculate the physical filter name. 

528 

529 Returns 

530 ------- 

531 filter : `str` 

532 Name of filter. A combination of FILTER and GRATING 

533 headers joined by a "~". The filter and grating are always 

534 combined. The filter or grating part will be "none" if no value 

535 is specified. Uses "empty" if any of the filters or gratings 

536 indicate an "empty_N" name. "unknown" indicates that the filter is 

537 not defined anywhere but we think it should be. "none" indicates 

538 that the filter was not defined but the observation is a dark 

539 or bias. 

540 """ 

541 

542 physical_filter = self._determine_primary_filter() 

543 

544 if self.is_key_ok("GRATING"): 

545 grating = self._header["GRATING"] 

546 self._used_these_cards("GRATING") 

547 

548 if not grating or grating.lower().startswith("empty"): 

549 grating = "empty" 

550 else: 

551 # Be explicit about having no knowledge of the grating 

552 grating = "unknown" 

553 

554 physical_filter = f"{physical_filter}{FILTER_DELIMITER}{grating}" 

555 

556 return physical_filter