Coverage for python/lsst/obs/lsst/translators/latiss.py: 17%

236 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-01 04:00 -0700

1# This file is currently part of obs_lsst but is written to allow it 

2# to be migrated to the astro_metadata_translator package at a later date. 

3# 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the LICENSE file in this directory for details of code ownership. 

7# 

8# Use of this source code is governed by a 3-clause BSD-style 

9# license that can be found in the LICENSE file. 

10 

11"""Metadata translation code for LSST LATISS headers""" 

12 

13__all__ = ("LatissTranslator", ) 

14 

15import logging 

16import math 

17 

18import astropy.units as u 

19from astropy.time import Time 

20from astropy.coordinates import EarthLocation 

21 

22from astro_metadata_translator import cache_translation 

23from astro_metadata_translator.translators.helpers import is_non_science 

24from .lsst import LsstBaseTranslator, FILTER_DELIMITER 

25from .lsstCam import is_non_science_or_lab 

26 

27log = logging.getLogger(__name__) 

28 

29 

30# AuxTel is not the same place as LSST 

31# These coordinates read from Apple Maps 

32AUXTEL_LOCATION = EarthLocation.from_geodetic(-70.747698, -30.244728, 2663.0) 

33 

34# Date instrument is taking data at telescope 

35# Prior to this date many parameters are automatically nulled out 

36# since the headers have not historically been reliable 

37TSTART = Time("2020-01-01T00:00", format="isot", scale="utc") 

38 

39# Define the sensor and group name for AuxTel globally so that it can be used 

40# in multiple places. There is no raft but for consistency with other LSST 

41# cameras we define one. 

42_DETECTOR_GROUP_NAME = "RXX" 

43_DETECTOR_NAME = "S00" 

44 

45# Date 068 detector was put in LATISS 

46DETECTOR_068_DATE = Time("2019-06-24T00:00", format="isot", scale="utc") 

47 

48# IMGTYPE header is filled in after this date 

49IMGTYPE_OKAY_DATE = Time("2019-11-07T00:00", format="isot", scale="utc") 

50 

51# OBJECT IMGTYPE really means ENGTEST until this date 

52OBJECT_IS_ENGTEST = Time("2020-01-27T20:00", format="isot", scale="utc") 

53 

54# RA and DEC headers are in radians until this date 

55RADEC_IS_RADIANS = Time("2020-01-28T22:00", format="isot", scale="utc") 

56 

57# RASTART/DECSTART/RAEND/DECEND used wrong telescope location before this 

58# 2020-02-01T00:00 we fixed the telescope location, but RASTART is still 

59# in mount coordinates, so off by pointing model. 

60RASTART_IS_BAD = Time("2020-05-01T00:00", format="isot", scale="utc") 

61 

62# Between RASTART_IS_BAD and this time the RASTART header uses hours 

63# instead of degrees. 

64RASTART_IS_HOURS = Time("2021-02-11T18:45", format="isot", scale="utc") 

65 

66# From this date RASTART is correct as-is. 

67RASTART_IS_OKAY = Time("2021-02-12T00:00", format="isot", scale="utc") 

68 

69# DATE-END is not to be trusted before this date 

70DATE_END_IS_BAD = Time("2020-02-01T00:00", format="isot", scale="utc") 

71 

72# The convention for the reporting of ROTPA changed by 180 here 

73ROTPA_CONVENTION_180_SWITCH1 = Time("2020-11-19T00:00", format="isot", scale="utc") 

74ROTPA_CONVENTION_180_SWITCH2 = Time("2021-10-29T00:00", format="isot", scale="utc") 

75 

76# TARGET is set to start with 'spec:' for dispsered images before this date 

77TARGET_STARTS_SPECCOLON = Time("2022-07-10T00:00", format="isot", scale="utc") 

78 

79# Scaling factor radians to degrees. Keep it simple. 

80RAD2DEG = 180.0 / math.pi 

81 

82 

83class LatissTranslator(LsstBaseTranslator): 

84 """Metadata translator for LSST LATISS data from AuxTel. 

85 

86 For lab measurements many values are masked out. 

87 """ 

88 

89 name = "LSST_LATISS" 

90 """Name of this translation class""" 

91 

92 supported_instrument = "LATISS" 

93 """Supports the LATISS instrument.""" 

94 

95 _const_map = { 

96 "instrument": "LATISS", 

97 "telescope": "Rubin Auxiliary Telescope", 

98 "detector_group": _DETECTOR_GROUP_NAME, 

99 "detector_num": 0, 

100 "detector_name": _DETECTOR_NAME, # Single sensor 

101 "relative_humidity": None, 

102 "pressure": None, 

103 "temperature": None, 

104 } 

105 

106 _trivial_map = { 

107 "observation_id": (["OBSID", "IMGNAME"], dict(default=None, checker=is_non_science)), 

108 "detector_serial": ["LSST_NUM", "DETSER"], 

109 "object": ("OBJECT", dict(checker=is_non_science_or_lab, default="UNKNOWN")), 

110 "boresight_rotation_angle": (["ROTPA", "ROTANGLE"], dict(checker=is_non_science_or_lab, 

111 default=float("nan"), unit=u.deg)), 

112 "science_program": ("PROGRAM", dict(default="unknown")), 

113 } 

114 

115 DETECTOR_GROUP_NAME = _DETECTOR_GROUP_NAME 

116 """Fixed name of detector group.""" 

117 

118 DETECTOR_NAME = _DETECTOR_NAME 

119 """Fixed name of single sensor.""" 

120 

121 DETECTOR_MAX = 0 

122 """Maximum number of detectors to use when calculating the 

123 detector_exposure_id.""" 

124 

125 _DEFAULT_LOCATION = AUXTEL_LOCATION 

126 """Default telescope location in absence of relevant FITS headers.""" 

127 

128 @classmethod 

129 def can_translate(cls, header, filename=None): 

130 """Indicate whether this translation class can translate the 

131 supplied header. 

132 

133 Parameters 

134 ---------- 

135 header : `dict`-like 

136 Header to convert to standardized form. 

137 filename : `str`, optional 

138 Name of file being translated. 

139 

140 Returns 

141 ------- 

142 can : `bool` 

143 `True` if the header is recognized by this class. `False` 

144 otherwise. 

145 """ 

146 # INSTRUME keyword might be of two types 

147 if "INSTRUME" in header: 

148 instrume = header["INSTRUME"] 

149 for v in ("LSST_ATISS", "LATISS"): 

150 if instrume == v: 

151 return True 

152 # Calibration files strip important headers at the moment so guess 

153 if "DETNAME" in header and header["DETNAME"] == "RXX_S00": 

154 return True 

155 return False 

156 

157 @classmethod 

158 def fix_header(cls, header, instrument, obsid, filename=None): 

159 """Fix an incorrect LATISS header. 

160 

161 Parameters 

162 ---------- 

163 header : `dict` 

164 The header to update. Updates are in place. 

165 instrument : `str` 

166 The name of the instrument. 

167 obsid : `str` 

168 Unique observation identifier associated with this header. 

169 Will always be provided. 

170 filename : `str`, optional 

171 Filename associated with this header. May not be set since headers 

172 can be fixed independently of any filename being known. 

173 

174 Returns 

175 ------- 

176 modified = `bool` 

177 Returns `True` if the header was updated. 

178 

179 Notes 

180 ----- 

181 This method does not apply per-obsid corrections. The following 

182 corrections are applied: 

183 

184 * On June 24th 2019 the detector was changed from ITL-3800C-098 

185 to ITL-3800C-068. The header is intended to be correct in the 

186 future. 

187 * In late 2019 the DATE-OBS and MJD-OBS headers were reporting 

188 1970 dates. To correct, the DATE/MJD headers are copied in to 

189 replace them and the -END headers are cleared. 

190 * Until November 2019 the IMGTYPE was set in the GROUPID header. 

191 The value is moved to IMGTYPE. 

192 * SHUTTIME is always forced to be `None`. 

193 

194 Corrections are reported as debug level log messages. 

195 

196 See `~astro_metadata_translator.fix_header` for details of the general 

197 process. 

198 """ 

199 modified = False 

200 

201 # Calculate the standard label to use for log messages 

202 log_label = cls._construct_log_prefix(obsid, filename) 

203 

204 if "OBSID" not in header: 

205 # Very old data used IMGNAME 

206 header["OBSID"] = obsid 

207 modified = True 

208 # We are reporting the OBSID so no need to repeat it at start 

209 # of log message. Use filename if we have it. 

210 log_prefix = f"{filename}: " if filename else "" 

211 log.debug("%sAssigning OBSID to a value of '%s'", log_prefix, header["OBSID"]) 

212 

213 if "DAYOBS" not in header: 

214 # OBS-NITE could have the value for DAYOBS but it is safer 

215 # for older data to set it from the OBSID. Fall back to OBS-NITE 

216 # if we have no alternative 

217 dayObs = None 

218 try: 

219 dayObs = obsid.split("_", 3)[2] 

220 except (AttributeError, ValueError): 

221 # did not split as expected 

222 pass 

223 if dayObs is None or len(dayObs) != 8: 

224 if "OBS-NITE" in header: 

225 dayObs = header["OBS-NITE"] 

226 log.debug("%s: Setting DAYOBS to '%s' from OBS-NITE header", log_label, dayObs) 

227 else: 

228 log.debug("%s: Unable to determine DAYOBS from header", log_label) 

229 else: 

230 log.debug("%s: Setting DAYOBS to '%s' from OBSID", log_label, dayObs) 

231 if dayObs: 

232 header["DAYOBS"] = dayObs 

233 modified = True 

234 

235 if "SEQNUM" not in header: 

236 try: 

237 seqnum = obsid.split("_", 3)[3] 

238 except (AttributeError, ValueError): 

239 # did not split as expected 

240 pass 

241 else: 

242 header["SEQNUM"] = int(seqnum) 

243 modified = True 

244 log.debug("%s: Extracting SEQNUM of '%s' from OBSID", log_label, header["SEQNUM"]) 

245 

246 # The DATE-OBS / MJD-OBS keys can be 1970 

247 if "DATE-OBS" in header and header["DATE-OBS"].startswith("1970"): 

248 # Copy the headers from the DATE and MJD since we have no other 

249 # choice. 

250 header["DATE-OBS"] = header["DATE"] 

251 header["DATE-BEG"] = header["DATE-OBS"] 

252 header["MJD-OBS"] = header["MJD"] 

253 header["MJD-BEG"] = header["MJD-OBS"] 

254 

255 # And clear the DATE-END and MJD-END -- the translator will use 

256 # EXPTIME instead. 

257 header["DATE-END"] = None 

258 header["MJD-END"] = None 

259 

260 log.debug("%s: Forcing 1970 dates to '%s'", log_label, header["DATE"]) 

261 modified = True 

262 

263 # Create a translator since we need the date 

264 translator = cls(header) 

265 date = translator.to_datetime_begin() 

266 if date > DETECTOR_068_DATE: 

267 header["LSST_NUM"] = "ITL-3800C-068" 

268 log.debug("%s: Forcing detector serial to %s", log_label, header["LSST_NUM"]) 

269 modified = True 

270 

271 if date < DATE_END_IS_BAD: 

272 # DATE-END may or may not be in TAI and may or may not be 

273 # before DATE-BEG. Simpler to clear it 

274 if header.get("DATE-END"): 

275 header["DATE-END"] = None 

276 header["MJD-END"] = None 

277 

278 log.debug("%s: Clearing DATE-END as being untrustworthy", log_label) 

279 modified = True 

280 

281 # Up until a certain date GROUPID was the IMGTYPE 

282 if date < IMGTYPE_OKAY_DATE: 

283 groupId = header.get("GROUPID") 

284 if groupId and not groupId.startswith("test"): 

285 imgType = header.get("IMGTYPE") 

286 if not imgType: 

287 if "_" in groupId: 

288 # Sometimes have the form dark_0001_0002 

289 # in this case we pull the IMGTYPE off the front and 

290 # do not clear groupId (although groupId may now 

291 # repeat on different days). 

292 groupId, _ = groupId.split("_", 1) 

293 elif groupId.upper() != "FOCUS" and groupId.upper().startswith("FOCUS"): 

294 # If it is exactly FOCUS we want groupId cleared 

295 groupId = "FOCUS" 

296 else: 

297 header["GROUPID"] = None 

298 header["IMGTYPE"] = groupId 

299 log.debug("%s: Setting IMGTYPE to '%s' from GROUPID", log_label, header["IMGTYPE"]) 

300 modified = True 

301 else: 

302 # Someone could be fixing headers in old data 

303 # and we do not want GROUPID == IMGTYPE 

304 if imgType == groupId: 

305 # Clear the group so we default to original 

306 header["GROUPID"] = None 

307 

308 # We were using OBJECT for engineering observations early on 

309 if date < OBJECT_IS_ENGTEST: 

310 imgType = header.get("IMGTYPE") 

311 if imgType == "OBJECT": 

312 header["IMGTYPE"] = "ENGTEST" 

313 log.debug("%s: Changing OBJECT observation type to %s", 

314 log_label, header["IMGTYPE"]) 

315 modified = True 

316 

317 # Early on the RA/DEC headers were stored in radians 

318 if date < RADEC_IS_RADIANS: 

319 if header.get("RA") is not None: 

320 header["RA"] *= RAD2DEG 

321 log.debug("%s: Changing RA header to degrees", log_label) 

322 modified = True 

323 if header.get("DEC") is not None: 

324 header["DEC"] *= RAD2DEG 

325 log.debug("%s: Changing DEC header to degrees", log_label) 

326 modified = True 

327 

328 if header.get("SHUTTIME"): 

329 log.debug("%s: Forcing SHUTTIME header to be None", log_label) 

330 header["SHUTTIME"] = None 

331 modified = True 

332 

333 if "OBJECT" not in header: 

334 # Only patch OBJECT IMGTYPE 

335 if "IMGTYPE" in header and header["IMGTYPE"] == "OBJECT": 

336 log.debug("%s: Forcing OBJECT header to exist", log_label) 

337 header["OBJECT"] = "NOTSET" 

338 modified = True 

339 

340 if date < TARGET_STARTS_SPECCOLON: 

341 if "OBJECT" in header: 

342 header["OBJECT"] = header['OBJECT'].replace('spec:', '') 

343 modified = True 

344 

345 if "RADESYS" in header: 

346 if header["RADESYS"] == "": 

347 # Default to ICRS 

348 header["RADESYS"] = "ICRS" 

349 log.debug("%s: Forcing blank RADESYS to '%s'", log_label, header["RADESYS"]) 

350 modified = True 

351 

352 if date < RASTART_IS_HOURS: 

353 # Avoid two checks for case where RASTART is fine 

354 if date < RASTART_IS_BAD: 

355 # The wrong telescope position was used. Unsetting these will 

356 # force the RA/DEC demand headers to be used instead. 

357 for h in ("RASTART", "DECSTART", "RAEND", "DECEND"): 

358 header[h] = None 

359 log.debug("%s: Forcing derived RA/Dec headers to undefined", log_label) 

360 modified = True 

361 else: 

362 # Correct hours to degrees 

363 for h in ("RASTART", "RAEND"): 

364 if header[h]: 

365 header[h] *= 15.0 

366 log.debug("%s: Correcting RASTART/END from hours to degrees", log_label) 

367 modified = True 

368 

369 # RASTART/END headers have a TAI/UTC confusion causing an offset 

370 # of 37 seconds for a period of time. 

371 if RASTART_IS_BAD < date < RASTART_IS_OKAY: 

372 modified = True 

373 offset = (37.0 / 3600.0) * 15.0 

374 for epoch in ("START", "END"): 

375 h = "RA" + epoch 

376 if header[h]: 

377 header[h] += offset 

378 

379 if date < ROTPA_CONVENTION_180_SWITCH2 and date > ROTPA_CONVENTION_180_SWITCH1: 

380 header['ROTPA'] = header['ROTPA'] - 180 

381 modified = True 

382 

383 return modified 

384 

385 def _is_on_mountain(self): 

386 date = self.to_datetime_begin() 

387 if date > TSTART: 

388 return True 

389 return False 

390 

391 @staticmethod 

392 def compute_detector_exposure_id(exposure_id, detector_num): 

393 # Docstring inherited. 

394 if detector_num != 0: 

395 log.warning("Unexpected non-zero detector number for LATISS") 

396 return LsstBaseTranslator.compute_detector_exposure_id(exposure_id, detector_num) 

397 

398 @cache_translation 

399 def to_dark_time(self): 

400 # Docstring will be inherited. Property defined in properties.py 

401 

402 # Always compare with exposure time 

403 # We may revisit this later if there is a cutoff date where we 

404 # can always trust the header. 

405 exptime = self.to_exposure_time() 

406 

407 if self.is_key_ok("DARKTIME"): 

408 darktime = self.quantity_from_card("DARKTIME", u.s) 

409 if darktime >= exptime: 

410 return darktime 

411 reason = "Dark time less than exposure time." 

412 else: 

413 reason = "Dark time not defined." 

414 

415 log.warning("%s: %s Setting dark time to the exposure time.", 

416 self._log_prefix, reason) 

417 return exptime 

418 

419 @cache_translation 

420 def to_exposure_time(self): 

421 # Docstring will be inherited. Property defined in properties.py 

422 # Some data is missing a value for EXPTIME. 

423 # Have to be careful we do not have circular logic when trying to 

424 # guess 

425 if self.is_key_ok("EXPTIME"): 

426 return self.quantity_from_card("EXPTIME", u.s) 

427 

428 # A missing or undefined EXPTIME is problematic. Set to -1 

429 # to indicate that none was found. 

430 log.warning("%s: Insufficient information to derive exposure time. Setting to -1.0s", 

431 self._log_prefix) 

432 return -1.0 * u.s 

433 

434 @cache_translation 

435 def to_observation_type(self): 

436 """Determine the observation type. 

437 

438 In the absence of an ``IMGTYPE`` header, assumes lab data is always a 

439 dark if exposure time is non-zero, else bias. 

440 

441 Returns 

442 ------- 

443 obstype : `str` 

444 Observation type. 

445 """ 

446 

447 # LATISS observation type is documented to appear in OBSTYPE 

448 # but for historical reasons prefers IMGTYPE. 

449 # Test the keys in order until we find one that contains a 

450 # defined value. 

451 obstype_keys = ["OBSTYPE", "IMGTYPE"] 

452 

453 obstype = None 

454 for k in obstype_keys: 

455 if self.is_key_ok(k): 

456 obstype = self._header[k] 

457 self._used_these_cards(k) 

458 obstype = obstype.lower() 

459 break 

460 

461 if obstype is not None: 

462 if obstype == "object" and not self._is_on_mountain(): 

463 # Do not map object to science in lab since most 

464 # code assume science is on sky with RA/Dec. 

465 obstype = "labobject" 

466 elif obstype in ("skyexp", "object"): 

467 obstype = "science" 

468 

469 return obstype 

470 

471 # In the absence of any observation type information, return 

472 # unknown unless we think it might be a bias. 

473 exptime = self.to_exposure_time() 

474 if exptime == 0.0: 

475 obstype = "bias" 

476 else: 

477 obstype = "unknown" 

478 log.warning("%s: Unable to determine observation type. Guessing '%s'", 

479 self._log_prefix, obstype) 

480 return obstype 

481 

482 @cache_translation 

483 def to_physical_filter(self): 

484 """Calculate the physical filter name. 

485 

486 Returns 

487 ------- 

488 filter : `str` 

489 Name of filter. A combination of FILTER and GRATING 

490 headers joined by a "~". The filter and grating are always 

491 combined. The filter or grating part will be "none" if no value 

492 is specified. Uses "empty" if any of the filters or gratings 

493 indicate an "empty_N" name. "unknown" indicates that the filter is 

494 not defined anywhere but we think it should be. "none" indicates 

495 that the filter was not defined but the observation is a dark 

496 or bias. 

497 """ 

498 

499 physical_filter = self._determine_primary_filter() 

500 

501 if self.is_key_ok("GRATING"): 

502 grating = self._header["GRATING"] 

503 self._used_these_cards("GRATING") 

504 

505 if not grating or grating.lower().startswith("empty"): 

506 grating = "empty" 

507 else: 

508 # Be explicit about having no knowledge of the grating 

509 grating = "unknown" 

510 

511 physical_filter = f"{physical_filter}{FILTER_DELIMITER}{grating}" 

512 

513 return physical_filter