Coverage for python/lsst/obs/lsst/translators/latiss.py: 17%

236 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-12-09 02:58 -0800

1# This file is currently part of obs_lsst but is written to allow it 

2# to be migrated to the astro_metadata_translator package at a later date. 

3# 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the LICENSE file in this directory for details of code ownership. 

7# 

8# Use of this source code is governed by a 3-clause BSD-style 

9# license that can be found in the LICENSE file. 

10 

11"""Metadata translation code for LSST LATISS headers""" 

12 

13__all__ = ("LatissTranslator", ) 

14 

15import logging 

16import math 

17 

18import astropy.units as u 

19from astropy.time import Time 

20from astropy.coordinates import EarthLocation 

21 

22from astro_metadata_translator import cache_translation 

23from astro_metadata_translator.translators.helpers import is_non_science 

24from .lsst import LsstBaseTranslator, FILTER_DELIMITER 

25from .lsstCam import is_non_science_or_lab 

26 

27log = logging.getLogger(__name__) 

28 

29 

30# AuxTel is not the same place as LSST 

31# These coordinates read from Apple Maps 

32AUXTEL_LOCATION = EarthLocation.from_geodetic(-70.747698, -30.244728, 2663.0) 

33 

34# Date instrument is taking data at telescope 

35# Prior to this date many parameters are automatically nulled out 

36# since the headers have not historically been reliable 

37TSTART = Time("2020-01-01T00:00", format="isot", scale="utc") 

38 

39# Define the sensor and group name for AuxTel globally so that it can be used 

40# in multiple places. There is no raft but for consistency with other LSST 

41# cameras we define one. 

42_DETECTOR_GROUP_NAME = "RXX" 

43_DETECTOR_NAME = "S00" 

44 

45# Date 068 detector was put in LATISS 

46DETECTOR_068_DATE = Time("2019-06-24T00:00", format="isot", scale="utc") 

47 

48# IMGTYPE header is filled in after this date 

49IMGTYPE_OKAY_DATE = Time("2019-11-07T00:00", format="isot", scale="utc") 

50 

51# OBJECT IMGTYPE really means ENGTEST until this date 

52OBJECT_IS_ENGTEST = Time("2020-01-27T20:00", format="isot", scale="utc") 

53 

54# RA and DEC headers are in radians until this date 

55RADEC_IS_RADIANS = Time("2020-01-28T22:00", format="isot", scale="utc") 

56 

57# RASTART/DECSTART/RAEND/DECEND used wrong telescope location before this 

58# 2020-02-01T00:00 we fixed the telescope location, but RASTART is still 

59# in mount coordinates, so off by pointing model. 

60RASTART_IS_BAD = Time("2020-05-01T00:00", format="isot", scale="utc") 

61 

62# Between RASTART_IS_BAD and this time the RASTART header uses hours 

63# instead of degrees. 

64RASTART_IS_HOURS = Time("2021-02-11T18:45", format="isot", scale="utc") 

65 

66# From this date RASTART is correct as-is. 

67RASTART_IS_OKAY = Time("2021-02-12T00:00", format="isot", scale="utc") 

68 

69# DATE-END is not to be trusted before this date 

70DATE_END_IS_BAD = Time("2020-02-01T00:00", format="isot", scale="utc") 

71 

72# The convention for the reporting of ROTPA changed by 180 here 

73ROTPA_CONVENTION_180_SWITCH1 = Time("2020-11-19T00:00", format="isot", scale="utc") 

74ROTPA_CONVENTION_180_SWITCH2 = Time("2021-10-29T00:00", format="isot", scale="utc") 

75 

76# TARGET is set to start with 'spec:' for dispsered images before this date 

77TARGET_STARTS_SPECCOLON = Time("2022-07-10T00:00", format="isot", scale="utc") 

78 

79# Scaling factor radians to degrees. Keep it simple. 

80RAD2DEG = 180.0 / math.pi 

81 

82 

83class LatissTranslator(LsstBaseTranslator): 

84 """Metadata translator for LSST LATISS data from AuxTel. 

85 

86 For lab measurements many values are masked out. 

87 """ 

88 

89 name = "LSST_LATISS" 

90 """Name of this translation class""" 

91 

92 supported_instrument = "LATISS" 

93 """Supports the LATISS instrument.""" 

94 

95 _const_map = { 

96 "instrument": "LATISS", 

97 "telescope": "Rubin Auxiliary Telescope", 

98 "detector_group": _DETECTOR_GROUP_NAME, 

99 "detector_num": 0, 

100 "detector_name": _DETECTOR_NAME, # Single sensor 

101 "relative_humidity": None, 

102 "pressure": None, 

103 "temperature": None, 

104 } 

105 

106 _trivial_map = { 

107 "observation_id": (["OBSID", "IMGNAME"], dict(default=None, checker=is_non_science)), 

108 "detector_serial": ["LSST_NUM", "DETSER"], 

109 "object": ("OBJECT", dict(checker=is_non_science_or_lab, default="UNKNOWN")), 

110 "boresight_rotation_angle": (["ROTPA", "ROTANGLE"], dict(checker=is_non_science_or_lab, 

111 default=float("nan"), unit=u.deg)), 

112 "science_program": ("PROGRAM", dict(default="unknown")), 

113 } 

114 

115 DETECTOR_GROUP_NAME = _DETECTOR_GROUP_NAME 

116 """Fixed name of detector group.""" 

117 

118 DETECTOR_NAME = _DETECTOR_NAME 

119 """Fixed name of single sensor.""" 

120 

121 DETECTOR_MAX = 0 

122 """Maximum number of detectors to use when calculating the 

123 detector_exposure_id.""" 

124 

125 _DEFAULT_LOCATION = AUXTEL_LOCATION 

126 """Default telescope location in absence of relevant FITS headers.""" 

127 

128 @classmethod 

129 def can_translate(cls, header, filename=None): 

130 """Indicate whether this translation class can translate the 

131 supplied header. 

132 

133 Parameters 

134 ---------- 

135 header : `dict`-like 

136 Header to convert to standardized form. 

137 filename : `str`, optional 

138 Name of file being translated. 

139 

140 Returns 

141 ------- 

142 can : `bool` 

143 `True` if the header is recognized by this class. `False` 

144 otherwise. 

145 """ 

146 # INSTRUME keyword might be of two types 

147 if "INSTRUME" in header: 

148 instrume = header["INSTRUME"] 

149 for v in ("LSST_ATISS", "LATISS"): 

150 if instrume == v: 

151 return True 

152 # Calibration files strip important headers at the moment so guess 

153 if "DETNAME" in header and header["DETNAME"] == "RXX_S00": 

154 return True 

155 return False 

156 

157 @classmethod 

158 def fix_header(cls, header, instrument, obsid, filename=None): 

159 """Fix an incorrect LATISS header. 

160 

161 Parameters 

162 ---------- 

163 header : `dict` 

164 The header to update. Updates are in place. 

165 instrument : `str` 

166 The name of the instrument. 

167 obsid : `str` 

168 Unique observation identifier associated with this header. 

169 Will always be provided. 

170 filename : `str`, optional 

171 Filename associated with this header. May not be set since headers 

172 can be fixed independently of any filename being known. 

173 

174 Returns 

175 ------- 

176 modified = `bool` 

177 Returns `True` if the header was updated. 

178 

179 Notes 

180 ----- 

181 This method does not apply per-obsid corrections. The following 

182 corrections are applied: 

183 

184 * On June 24th 2019 the detector was changed from ITL-3800C-098 

185 to ITL-3800C-068. The header is intended to be correct in the 

186 future. 

187 * In late 2019 the DATE-OBS and MJD-OBS headers were reporting 

188 1970 dates. To correct, the DATE/MJD headers are copied in to 

189 replace them and the -END headers are cleared. 

190 * Until November 2019 the IMGTYPE was set in the GROUPID header. 

191 The value is moved to IMGTYPE. 

192 * SHUTTIME is always forced to be `None`. 

193 

194 Corrections are reported as debug level log messages. 

195 

196 See `~astro_metadata_translator.fix_header` for details of the general 

197 process. 

198 """ 

199 modified = False 

200 

201 # Calculate the standard label to use for log messages 

202 log_label = cls._construct_log_prefix(obsid, filename) 

203 

204 if "OBSID" not in header: 

205 # Very old data used IMGNAME 

206 header["OBSID"] = obsid 

207 modified = True 

208 # We are reporting the OBSID so no need to repeat it at start 

209 # of log message. Use filename if we have it. 

210 log_prefix = f"{filename}: " if filename else "" 

211 log.debug("%sAssigning OBSID to a value of '%s'", log_prefix, header["OBSID"]) 

212 

213 if "DAYOBS" not in header: 

214 # OBS-NITE could have the value for DAYOBS but it is safer 

215 # for older data to set it from the OBSID. Fall back to OBS-NITE 

216 # if we have no alternative 

217 dayObs = None 

218 try: 

219 dayObs = obsid.split("_", 3)[2] 

220 except (AttributeError, ValueError): 

221 # did not split as expected 

222 pass 

223 if dayObs is None or len(dayObs) != 8: 

224 if "OBS-NITE" in header: 

225 dayObs = header["OBS-NITE"] 

226 log.debug("%s: Setting DAYOBS to '%s' from OBS-NITE header", log_label, dayObs) 

227 else: 

228 log.debug("%s: Unable to determine DAYOBS from header", log_label) 

229 else: 

230 log.debug("%s: Setting DAYOBS to '%s' from OBSID", log_label, dayObs) 

231 if dayObs: 

232 header["DAYOBS"] = dayObs 

233 modified = True 

234 

235 if "SEQNUM" not in header: 

236 try: 

237 seqnum = obsid.split("_", 3)[3] 

238 except (AttributeError, ValueError): 

239 # did not split as expected 

240 pass 

241 else: 

242 header["SEQNUM"] = int(seqnum) 

243 modified = True 

244 log.debug("%s: Extracting SEQNUM of '%s' from OBSID", log_label, header["SEQNUM"]) 

245 

246 # The DATE-OBS / MJD-OBS keys can be 1970 

247 if "DATE-OBS" in header and header["DATE-OBS"].startswith("1970"): 

248 # Copy the headers from the DATE and MJD since we have no other 

249 # choice. 

250 header["DATE-OBS"] = header["DATE"] 

251 header["DATE-BEG"] = header["DATE-OBS"] 

252 header["MJD-OBS"] = header["MJD"] 

253 header["MJD-BEG"] = header["MJD-OBS"] 

254 

255 # And clear the DATE-END and MJD-END -- the translator will use 

256 # EXPTIME instead. 

257 header["DATE-END"] = None 

258 header["MJD-END"] = None 

259 

260 log.debug("%s: Forcing 1970 dates to '%s'", log_label, header["DATE"]) 

261 modified = True 

262 

263 # Create a translator since we need the date 

264 translator = cls(header) 

265 date = translator.to_datetime_begin() 

266 if date > DETECTOR_068_DATE: 

267 header["LSST_NUM"] = "ITL-3800C-068" 

268 log.debug("%s: Forcing detector serial to %s", log_label, header["LSST_NUM"]) 

269 modified = True 

270 

271 if date < DATE_END_IS_BAD: 

272 # DATE-END may or may not be in TAI and may or may not be 

273 # before DATE-BEG. Simpler to clear it 

274 if header.get("DATE-END"): 

275 header["DATE-END"] = None 

276 header["MJD-END"] = None 

277 

278 log.debug("%s: Clearing DATE-END as being untrustworthy", log_label) 

279 modified = True 

280 

281 # Up until a certain date GROUPID was the IMGTYPE 

282 if date < IMGTYPE_OKAY_DATE: 

283 groupId = header.get("GROUPID") 

284 if groupId and not groupId.startswith("test"): 

285 imgType = header.get("IMGTYPE") 

286 if not imgType: 

287 if "_" in groupId: 

288 # Sometimes have the form dark_0001_0002 

289 # in this case we pull the IMGTYPE off the front and 

290 # do not clear groupId (although groupId may now 

291 # repeat on different days). 

292 groupId, _ = groupId.split("_", 1) 

293 elif groupId.upper() != "FOCUS" and groupId.upper().startswith("FOCUS"): 

294 # If it is exactly FOCUS we want groupId cleared 

295 groupId = "FOCUS" 

296 else: 

297 header["GROUPID"] = None 

298 header["IMGTYPE"] = groupId 

299 log.debug("%s: Setting IMGTYPE to '%s' from GROUPID", log_label, header["IMGTYPE"]) 

300 modified = True 

301 else: 

302 # Someone could be fixing headers in old data 

303 # and we do not want GROUPID == IMGTYPE 

304 if imgType == groupId: 

305 # Clear the group so we default to original 

306 header["GROUPID"] = None 

307 

308 # We were using OBJECT for engineering observations early on 

309 if date < OBJECT_IS_ENGTEST: 

310 imgType = header.get("IMGTYPE") 

311 if imgType == "OBJECT": 

312 header["IMGTYPE"] = "ENGTEST" 

313 log.debug("%s: Changing OBJECT observation type to %s", 

314 log_label, header["IMGTYPE"]) 

315 modified = True 

316 

317 # Early on the RA/DEC headers were stored in radians 

318 if date < RADEC_IS_RADIANS: 

319 if header.get("RA") is not None: 

320 header["RA"] *= RAD2DEG 

321 log.debug("%s: Changing RA header to degrees", log_label) 

322 modified = True 

323 if header.get("DEC") is not None: 

324 header["DEC"] *= RAD2DEG 

325 log.debug("%s: Changing DEC header to degrees", log_label) 

326 modified = True 

327 

328 if header.get("SHUTTIME"): 

329 log.debug("%s: Forcing SHUTTIME header to be None", log_label) 

330 header["SHUTTIME"] = None 

331 modified = True 

332 

333 if "OBJECT" not in header: 

334 # Only patch OBJECT IMGTYPE 

335 if "IMGTYPE" in header and header["IMGTYPE"] == "OBJECT": 

336 log.debug("%s: Forcing OBJECT header to exist", log_label) 

337 header["OBJECT"] = "NOTSET" 

338 modified = True 

339 

340 if date < TARGET_STARTS_SPECCOLON: 

341 if "OBJECT" in header: 

342 header["OBJECT"] = header['OBJECT'].replace('spec:', '') 

343 modified = True 

344 

345 if "RADESYS" in header: 

346 if header["RADESYS"] == "": 

347 # Default to ICRS 

348 header["RADESYS"] = "ICRS" 

349 log.debug("%s: Forcing blank RADESYS to '%s'", log_label, header["RADESYS"]) 

350 modified = True 

351 

352 if date < RASTART_IS_HOURS: 

353 # Avoid two checks for case where RASTART is fine 

354 if date < RASTART_IS_BAD: 

355 # The wrong telescope position was used. Unsetting these will 

356 # force the RA/DEC demand headers to be used instead. 

357 for h in ("RASTART", "DECSTART", "RAEND", "DECEND"): 

358 header[h] = None 

359 log.debug("%s: Forcing derived RA/Dec headers to undefined", log_label) 

360 modified = True 

361 else: 

362 # Correct hours to degrees 

363 for h in ("RASTART", "RAEND"): 

364 if header[h]: 

365 header[h] *= 15.0 

366 log.debug("%s: Correcting RASTART/END from hours to degrees", log_label) 

367 modified = True 

368 

369 # RASTART/END headers have a TAI/UTC confusion causing an offset 

370 # of 37 seconds for a period of time. 

371 if RASTART_IS_BAD < date < RASTART_IS_OKAY: 

372 modified = True 

373 offset = (37.0 / 3600.0) * 15.0 

374 for epoch in ("START", "END"): 

375 h = "RA" + epoch 

376 if header[h]: 

377 header[h] += offset 

378 

379 if date < ROTPA_CONVENTION_180_SWITCH2 and date > ROTPA_CONVENTION_180_SWITCH1: 

380 header['ROTPA'] = header['ROTPA'] - 180 

381 modified = True 

382 

383 return modified 

384 

385 def _is_on_mountain(self): 

386 date = self.to_datetime_begin() 

387 if date > TSTART: 

388 return True 

389 return False 

390 

391 @staticmethod 

392 def compute_detector_exposure_id(exposure_id, detector_num): 

393 """Compute the detector exposure ID from detector number and 

394 exposure ID. 

395 

396 This is a helper method to allow code working outside the translator 

397 infrastructure to use the same algorithm. 

398 

399 Parameters 

400 ---------- 

401 exposure_id : `int` 

402 Unique exposure ID. 

403 detector_num : `int` 

404 Detector number. 

405 

406 Returns 

407 ------- 

408 detector_exposure_id : `int` 

409 The calculated ID. 

410 """ 

411 if detector_num != 0: 

412 log.warning("Unexpected non-zero detector number for LATISS") 

413 return exposure_id 

414 

415 @cache_translation 

416 def to_dark_time(self): 

417 # Docstring will be inherited. Property defined in properties.py 

418 

419 # Always compare with exposure time 

420 # We may revisit this later if there is a cutoff date where we 

421 # can always trust the header. 

422 exptime = self.to_exposure_time() 

423 

424 if self.is_key_ok("DARKTIME"): 

425 darktime = self.quantity_from_card("DARKTIME", u.s) 

426 if darktime >= exptime: 

427 return darktime 

428 reason = "Dark time less than exposure time." 

429 else: 

430 reason = "Dark time not defined." 

431 

432 log.warning("%s: %s Setting dark time to the exposure time.", 

433 self._log_prefix, reason) 

434 return exptime 

435 

436 @cache_translation 

437 def to_exposure_time(self): 

438 # Docstring will be inherited. Property defined in properties.py 

439 # Some data is missing a value for EXPTIME. 

440 # Have to be careful we do not have circular logic when trying to 

441 # guess 

442 if self.is_key_ok("EXPTIME"): 

443 return self.quantity_from_card("EXPTIME", u.s) 

444 

445 # A missing or undefined EXPTIME is problematic. Set to -1 

446 # to indicate that none was found. 

447 log.warning("%s: Insufficient information to derive exposure time. Setting to -1.0s", 

448 self._log_prefix) 

449 return -1.0 * u.s 

450 

451 @cache_translation 

452 def to_observation_type(self): 

453 """Determine the observation type. 

454 

455 In the absence of an ``IMGTYPE`` header, assumes lab data is always a 

456 dark if exposure time is non-zero, else bias. 

457 

458 Returns 

459 ------- 

460 obstype : `str` 

461 Observation type. 

462 """ 

463 

464 # LATISS observation type is documented to appear in OBSTYPE 

465 # but for historical reasons prefers IMGTYPE. 

466 # Test the keys in order until we find one that contains a 

467 # defined value. 

468 obstype_keys = ["OBSTYPE", "IMGTYPE"] 

469 

470 obstype = None 

471 for k in obstype_keys: 

472 if self.is_key_ok(k): 

473 obstype = self._header[k] 

474 self._used_these_cards(k) 

475 obstype = obstype.lower() 

476 break 

477 

478 if obstype is not None: 

479 if obstype == "object" and not self._is_on_mountain(): 

480 # Do not map object to science in lab since most 

481 # code assume science is on sky with RA/Dec. 

482 obstype = "labobject" 

483 elif obstype in ("skyexp", "object"): 

484 obstype = "science" 

485 

486 return obstype 

487 

488 # In the absence of any observation type information, return 

489 # unknown unless we think it might be a bias. 

490 exptime = self.to_exposure_time() 

491 if exptime == 0.0: 

492 obstype = "bias" 

493 else: 

494 obstype = "unknown" 

495 log.warning("%s: Unable to determine observation type. Guessing '%s'", 

496 self._log_prefix, obstype) 

497 return obstype 

498 

499 @cache_translation 

500 def to_physical_filter(self): 

501 """Calculate the physical filter name. 

502 

503 Returns 

504 ------- 

505 filter : `str` 

506 Name of filter. A combination of FILTER and GRATING 

507 headers joined by a "~". The filter and grating are always 

508 combined. The filter or grating part will be "none" if no value 

509 is specified. Uses "empty" if any of the filters or gratings 

510 indicate an "empty_N" name. "unknown" indicates that the filter is 

511 not defined anywhere but we think it should be. "none" indicates 

512 that the filter was not defined but the observation is a dark 

513 or bias. 

514 """ 

515 

516 physical_filter = self._determine_primary_filter() 

517 

518 if self.is_key_ok("GRATING"): 

519 grating = self._header["GRATING"] 

520 self._used_these_cards("GRATING") 

521 

522 if not grating or grating.lower().startswith("empty"): 

523 grating = "empty" 

524 else: 

525 # Be explicit about having no knowledge of the grating 

526 grating = "unknown" 

527 

528 physical_filter = f"{physical_filter}{FILTER_DELIMITER}{grating}" 

529 

530 return physical_filter