Coverage for python/lsst/obs/lsst/translators/latiss.py: 19%

244 statements  

« prev     ^ index     » next       coverage.py v7.4.3, created at 2024-02-25 09:36 +0000

1# This file is currently part of obs_lsst but is written to allow it 

2# to be migrated to the astro_metadata_translator package at a later date. 

3# 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the LICENSE file in this directory for details of code ownership. 

7# 

8# Use of this source code is governed by a 3-clause BSD-style 

9# license that can be found in the LICENSE file. 

10 

11"""Metadata translation code for LSST LATISS headers""" 

12 

13__all__ = ("LatissTranslator", ) 

14 

15import logging 

16import math 

17 

18import astropy.units as u 

19from astropy.time import Time 

20from astropy.coordinates import EarthLocation 

21 

22from astro_metadata_translator import cache_translation 

23from astro_metadata_translator.translators.helpers import is_non_science 

24from .lsst import LsstBaseTranslator, FILTER_DELIMITER 

25from .lsstCam import is_non_science_or_lab 

26 

27log = logging.getLogger(__name__) 

28 

29 

30# AuxTel is not the same place as LSST 

31# These coordinates read from Apple Maps 

32AUXTEL_LOCATION = EarthLocation.from_geodetic(-70.747698, -30.244728, 2663.0) 

33 

34# Date instrument is taking data at telescope 

35# Prior to this date many parameters are automatically nulled out 

36# since the headers have not historically been reliable 

37TSTART = Time("2020-01-01T00:00", format="isot", scale="utc") 

38 

39# Define the sensor and group name for AuxTel globally so that it can be used 

40# in multiple places. There is no raft but for consistency with other LSST 

41# cameras we define one. 

42_DETECTOR_GROUP_NAME = "RXX" 

43_DETECTOR_NAME = "S00" 

44 

45# Date 068 detector was put in LATISS 

46DETECTOR_068_DATE = Time("2019-06-24T00:00", format="isot", scale="utc") 

47 

48# IMGTYPE header is filled in after this date 

49IMGTYPE_OKAY_DATE = Time("2019-11-07T00:00", format="isot", scale="utc") 

50 

51# OBJECT IMGTYPE really means ENGTEST until this date 

52OBJECT_IS_ENGTEST = Time("2020-01-27T20:00", format="isot", scale="utc") 

53 

54# RA and DEC headers are in radians until this date 

55RADEC_IS_RADIANS = Time("2020-01-28T22:00", format="isot", scale="utc") 

56 

57# RASTART/DECSTART/RAEND/DECEND used wrong telescope location before this 

58# 2020-02-01T00:00 we fixed the telescope location, but RASTART is still 

59# in mount coordinates, so off by pointing model. 

60RASTART_IS_BAD = Time("2020-05-01T00:00", format="isot", scale="utc") 

61 

62# Between RASTART_IS_BAD and this time the RASTART header uses hours 

63# instead of degrees. 

64RASTART_IS_HOURS = Time("2021-02-11T18:45", format="isot", scale="utc") 

65 

66# From this date RASTART is correct as-is. 

67RASTART_IS_OKAY = Time("2021-02-12T00:00", format="isot", scale="utc") 

68 

69# DATE-END is not to be trusted before this date 

70DATE_END_IS_BAD = Time("2020-02-01T00:00", format="isot", scale="utc") 

71 

72# The convention for the reporting of ROTPA changed by 180 here 

73ROTPA_CONVENTION_180_SWITCH1 = Time("2020-11-19T00:00", format="isot", scale="utc") 

74ROTPA_CONVENTION_180_SWITCH2 = Time("2021-10-29T00:00", format="isot", scale="utc") 

75 

76# TARGET is set to start with 'spec:' for dispsered images before this date 

77TARGET_STARTS_SPECCOLON = Time("2022-07-10T00:00", format="isot", scale="utc") 

78 

79# Scaling factor radians to degrees. Keep it simple. 

80RAD2DEG = 180.0 / math.pi 

81 

82 

83class LatissTranslator(LsstBaseTranslator): 

84 """Metadata translator for LSST LATISS data from AuxTel. 

85 

86 For lab measurements many values are masked out. 

87 """ 

88 

89 name = "LSST_LATISS" 

90 """Name of this translation class""" 

91 

92 supported_instrument = "LATISS" 

93 """Supports the LATISS instrument.""" 

94 

95 _const_map = { 

96 "instrument": "LATISS", 

97 "telescope": "Rubin Auxiliary Telescope", 

98 "detector_group": _DETECTOR_GROUP_NAME, 

99 "detector_num": 0, 

100 "detector_name": _DETECTOR_NAME, # Single sensor 

101 } 

102 

103 _trivial_map = { 

104 "observation_id": (["OBSID", "IMGNAME"], dict(default=None, checker=is_non_science)), 

105 "detector_serial": ["LSST_NUM", "DETSER"], 

106 "object": ("OBJECT", dict(checker=is_non_science_or_lab, default="UNKNOWN")), 

107 "boresight_rotation_angle": (["ROTPA", "ROTANGLE"], dict(checker=is_non_science_or_lab, 

108 default=float("nan"), unit=u.deg)), 

109 "science_program": ("PROGRAM", dict(default="unknown")), 

110 } 

111 

112 DETECTOR_GROUP_NAME = _DETECTOR_GROUP_NAME 

113 """Fixed name of detector group.""" 

114 

115 DETECTOR_NAME = _DETECTOR_NAME 

116 """Fixed name of single sensor.""" 

117 

118 DETECTOR_MAX = 0 

119 """Maximum number of detectors to use when calculating the 

120 detector_exposure_id.""" 

121 

122 _DEFAULT_LOCATION = AUXTEL_LOCATION 

123 """Default telescope location in absence of relevant FITS headers.""" 

124 

125 @classmethod 

126 def can_translate(cls, header, filename=None): 

127 """Indicate whether this translation class can translate the 

128 supplied header. 

129 

130 Parameters 

131 ---------- 

132 header : `dict`-like 

133 Header to convert to standardized form. 

134 filename : `str`, optional 

135 Name of file being translated. 

136 

137 Returns 

138 ------- 

139 can : `bool` 

140 `True` if the header is recognized by this class. `False` 

141 otherwise. 

142 """ 

143 # INSTRUME keyword might be of two types 

144 if "INSTRUME" in header: 

145 instrume = header["INSTRUME"] 

146 for v in ("LSST_ATISS", "LATISS"): 

147 if instrume == v: 

148 return True 

149 # Calibration files strip important headers at the moment so guess 

150 if "DETNAME" in header and header["DETNAME"] == "RXX_S00": 

151 return True 

152 return False 

153 

154 @classmethod 

155 def fix_header(cls, header, instrument, obsid, filename=None): 

156 """Fix an incorrect LATISS header. 

157 

158 Parameters 

159 ---------- 

160 header : `dict` 

161 The header to update. Updates are in place. 

162 instrument : `str` 

163 The name of the instrument. 

164 obsid : `str` 

165 Unique observation identifier associated with this header. 

166 Will always be provided. 

167 filename : `str`, optional 

168 Filename associated with this header. May not be set since headers 

169 can be fixed independently of any filename being known. 

170 

171 Returns 

172 ------- 

173 modified = `bool` 

174 Returns `True` if the header was updated. 

175 

176 Notes 

177 ----- 

178 This method does not apply per-obsid corrections. The following 

179 corrections are applied: 

180 

181 * On June 24th 2019 the detector was changed from ITL-3800C-098 

182 to ITL-3800C-068. The header is intended to be correct in the 

183 future. 

184 * In late 2019 the DATE-OBS and MJD-OBS headers were reporting 

185 1970 dates. To correct, the DATE/MJD headers are copied in to 

186 replace them and the -END headers are cleared. 

187 * Until November 2019 the IMGTYPE was set in the GROUPID header. 

188 The value is moved to IMGTYPE. 

189 * SHUTTIME is always forced to be `None`. 

190 

191 Corrections are reported as debug level log messages. 

192 

193 See `~astro_metadata_translator.fix_header` for details of the general 

194 process. 

195 """ 

196 modified = False 

197 

198 # Calculate the standard label to use for log messages 

199 log_label = cls._construct_log_prefix(obsid, filename) 

200 

201 if "OBSID" not in header: 

202 # Very old data used IMGNAME 

203 header["OBSID"] = obsid 

204 modified = True 

205 # We are reporting the OBSID so no need to repeat it at start 

206 # of log message. Use filename if we have it. 

207 log_prefix = f"{filename}: " if filename else "" 

208 log.debug("%sAssigning OBSID to a value of '%s'", log_prefix, header["OBSID"]) 

209 

210 if "DAYOBS" not in header: 

211 # OBS-NITE could have the value for DAYOBS but it is safer 

212 # for older data to set it from the OBSID. Fall back to OBS-NITE 

213 # if we have no alternative 

214 dayObs = None 

215 try: 

216 dayObs = obsid.split("_", 3)[2] 

217 except (AttributeError, ValueError): 

218 # did not split as expected 

219 pass 

220 if dayObs is None or len(dayObs) != 8: 

221 if "OBS-NITE" in header: 

222 dayObs = header["OBS-NITE"] 

223 log.debug("%s: Setting DAYOBS to '%s' from OBS-NITE header", log_label, dayObs) 

224 else: 

225 log.debug("%s: Unable to determine DAYOBS from header", log_label) 

226 else: 

227 log.debug("%s: Setting DAYOBS to '%s' from OBSID", log_label, dayObs) 

228 if dayObs: 

229 header["DAYOBS"] = dayObs 

230 modified = True 

231 

232 if "SEQNUM" not in header: 

233 try: 

234 seqnum = obsid.split("_", 3)[3] 

235 except (AttributeError, ValueError): 

236 # did not split as expected 

237 pass 

238 else: 

239 header["SEQNUM"] = int(seqnum) 

240 modified = True 

241 log.debug("%s: Extracting SEQNUM of '%s' from OBSID", log_label, header["SEQNUM"]) 

242 

243 # The DATE-OBS / MJD-OBS keys can be 1970 

244 if "DATE-OBS" in header and header["DATE-OBS"].startswith("1970"): 

245 # Copy the headers from the DATE and MJD since we have no other 

246 # choice. 

247 header["DATE-OBS"] = header["DATE"] 

248 header["DATE-BEG"] = header["DATE-OBS"] 

249 header["MJD-OBS"] = header["MJD"] 

250 header["MJD-BEG"] = header["MJD-OBS"] 

251 

252 # And clear the DATE-END and MJD-END -- the translator will use 

253 # EXPTIME instead. 

254 header["DATE-END"] = None 

255 header["MJD-END"] = None 

256 

257 log.debug("%s: Forcing 1970 dates to '%s'", log_label, header["DATE"]) 

258 modified = True 

259 

260 # Create a translator since we need the date 

261 translator = cls(header) 

262 date = translator.to_datetime_begin() 

263 if date > DETECTOR_068_DATE: 

264 header["LSST_NUM"] = "ITL-3800C-068" 

265 log.debug("%s: Forcing detector serial to %s", log_label, header["LSST_NUM"]) 

266 modified = True 

267 

268 if date < DATE_END_IS_BAD: 

269 # DATE-END may or may not be in TAI and may or may not be 

270 # before DATE-BEG. Simpler to clear it 

271 if header.get("DATE-END"): 

272 header["DATE-END"] = None 

273 header["MJD-END"] = None 

274 

275 log.debug("%s: Clearing DATE-END as being untrustworthy", log_label) 

276 modified = True 

277 

278 # Up until a certain date GROUPID was the IMGTYPE 

279 if date < IMGTYPE_OKAY_DATE: 

280 groupId = header.get("GROUPID") 

281 if groupId and not groupId.startswith("test"): 

282 imgType = header.get("IMGTYPE") 

283 if not imgType: 

284 if "_" in groupId: 

285 # Sometimes have the form dark_0001_0002 

286 # in this case we pull the IMGTYPE off the front and 

287 # do not clear groupId (although groupId may now 

288 # repeat on different days). 

289 groupId, _ = groupId.split("_", 1) 

290 elif groupId.upper() != "FOCUS" and groupId.upper().startswith("FOCUS"): 

291 # If it is exactly FOCUS we want groupId cleared 

292 groupId = "FOCUS" 

293 else: 

294 header["GROUPID"] = None 

295 header["IMGTYPE"] = groupId 

296 log.debug("%s: Setting IMGTYPE to '%s' from GROUPID", log_label, header["IMGTYPE"]) 

297 modified = True 

298 else: 

299 # Someone could be fixing headers in old data 

300 # and we do not want GROUPID == IMGTYPE 

301 if imgType == groupId: 

302 # Clear the group so we default to original 

303 header["GROUPID"] = None 

304 

305 # We were using OBJECT for engineering observations early on 

306 if date < OBJECT_IS_ENGTEST: 

307 imgType = header.get("IMGTYPE") 

308 if imgType == "OBJECT": 

309 header["IMGTYPE"] = "ENGTEST" 

310 log.debug("%s: Changing OBJECT observation type to %s", 

311 log_label, header["IMGTYPE"]) 

312 modified = True 

313 

314 # Early on the RA/DEC headers were stored in radians 

315 if date < RADEC_IS_RADIANS: 

316 if header.get("RA") is not None: 

317 header["RA"] *= RAD2DEG 

318 log.debug("%s: Changing RA header to degrees", log_label) 

319 modified = True 

320 if header.get("DEC") is not None: 

321 header["DEC"] *= RAD2DEG 

322 log.debug("%s: Changing DEC header to degrees", log_label) 

323 modified = True 

324 

325 if header.get("SHUTTIME"): 

326 log.debug("%s: Forcing SHUTTIME header to be None", log_label) 

327 header["SHUTTIME"] = None 

328 modified = True 

329 

330 if "OBJECT" not in header: 

331 # Only patch OBJECT IMGTYPE 

332 if "IMGTYPE" in header and header["IMGTYPE"] == "OBJECT": 

333 log.debug("%s: Forcing OBJECT header to exist", log_label) 

334 header["OBJECT"] = "NOTSET" 

335 modified = True 

336 

337 if date < TARGET_STARTS_SPECCOLON: 

338 if "OBJECT" in header: 

339 header["OBJECT"] = header['OBJECT'].replace('spec:', '') 

340 modified = True 

341 

342 if "RADESYS" in header: 

343 if header["RADESYS"] == "": 

344 # Default to ICRS 

345 header["RADESYS"] = "ICRS" 

346 log.debug("%s: Forcing blank RADESYS to '%s'", log_label, header["RADESYS"]) 

347 modified = True 

348 

349 if date < RASTART_IS_HOURS: 

350 # Avoid two checks for case where RASTART is fine 

351 if date < RASTART_IS_BAD: 

352 # The wrong telescope position was used. Unsetting these will 

353 # force the RA/DEC demand headers to be used instead. 

354 for h in ("RASTART", "DECSTART", "RAEND", "DECEND"): 

355 header[h] = None 

356 log.debug("%s: Forcing derived RA/Dec headers to undefined", log_label) 

357 modified = True 

358 else: 

359 # Correct hours to degrees 

360 for h in ("RASTART", "RAEND"): 

361 if header[h]: 

362 header[h] *= 15.0 

363 log.debug("%s: Correcting RASTART/END from hours to degrees", log_label) 

364 modified = True 

365 

366 # RASTART/END headers have a TAI/UTC confusion causing an offset 

367 # of 37 seconds for a period of time. 

368 if RASTART_IS_BAD < date < RASTART_IS_OKAY: 

369 modified = True 

370 offset = (37.0 / 3600.0) * 15.0 

371 for epoch in ("START", "END"): 

372 h = "RA" + epoch 

373 if header[h]: 

374 header[h] += offset 

375 

376 if date < ROTPA_CONVENTION_180_SWITCH2 and date > ROTPA_CONVENTION_180_SWITCH1: 

377 header['ROTPA'] = header['ROTPA'] - 180 

378 modified = True 

379 

380 if obsgeo := header.get("OBSGEO-Z"): 

381 try: 

382 if obsgeo > 0.0: 

383 obsgeo *= -1.0 

384 header["OBSGEO-Z"] = obsgeo 

385 modified = True 

386 except TypeError: 

387 pass 

388 

389 return modified 

390 

391 def _is_on_mountain(self): 

392 date = self.to_datetime_begin() 

393 if date > TSTART: 

394 return True 

395 return False 

396 

397 @staticmethod 

398 def compute_detector_exposure_id(exposure_id, detector_num): 

399 # Docstring inherited. 

400 if detector_num != 0: 

401 log.warning("Unexpected non-zero detector number for LATISS") 

402 return LsstBaseTranslator.compute_detector_exposure_id(exposure_id, detector_num) 

403 

404 @cache_translation 

405 def to_dark_time(self): 

406 # Docstring will be inherited. Property defined in properties.py 

407 

408 # Always compare with exposure time 

409 # We may revisit this later if there is a cutoff date where we 

410 # can always trust the header. 

411 exptime = self.to_exposure_time() 

412 

413 if self.is_key_ok("DARKTIME"): 

414 darktime = self.quantity_from_card("DARKTIME", u.s) 

415 if darktime >= exptime: 

416 return darktime 

417 reason = "Dark time less than exposure time." 

418 else: 

419 reason = "Dark time not defined." 

420 

421 log.warning("%s: %s Setting dark time to the exposure time.", 

422 self._log_prefix, reason) 

423 return exptime 

424 

425 @cache_translation 

426 def to_exposure_time(self): 

427 # Docstring will be inherited. Property defined in properties.py 

428 # Some data is missing a value for EXPTIME. 

429 # Have to be careful we do not have circular logic when trying to 

430 # guess 

431 if self.is_key_ok("EXPTIME"): 

432 return self.quantity_from_card("EXPTIME", u.s) 

433 

434 # A missing or undefined EXPTIME is problematic. Set to -1 

435 # to indicate that none was found. 

436 log.warning("%s: Insufficient information to derive exposure time. Setting to -1.0s", 

437 self._log_prefix) 

438 return -1.0 * u.s 

439 

440 @cache_translation 

441 def to_observation_type(self): 

442 """Determine the observation type. 

443 

444 In the absence of an ``IMGTYPE`` header, assumes lab data is always a 

445 dark if exposure time is non-zero, else bias. 

446 

447 Returns 

448 ------- 

449 obstype : `str` 

450 Observation type. 

451 """ 

452 

453 # LATISS observation type is documented to appear in OBSTYPE 

454 # but for historical reasons prefers IMGTYPE. 

455 # Test the keys in order until we find one that contains a 

456 # defined value. 

457 obstype_keys = ["OBSTYPE", "IMGTYPE"] 

458 

459 obstype = None 

460 for k in obstype_keys: 

461 if self.is_key_ok(k): 

462 obstype = self._header[k] 

463 self._used_these_cards(k) 

464 obstype = obstype.lower() 

465 break 

466 

467 if obstype is not None: 

468 if obstype == "object" and not self._is_on_mountain(): 

469 # Do not map object to science in lab since most 

470 # code assume science is on sky with RA/Dec. 

471 obstype = "labobject" 

472 elif obstype in ("skyexp", "object"): 

473 obstype = "science" 

474 

475 return obstype 

476 

477 # In the absence of any observation type information, return 

478 # unknown unless we think it might be a bias. 

479 exptime = self.to_exposure_time() 

480 if exptime == 0.0: 

481 obstype = "bias" 

482 else: 

483 obstype = "unknown" 

484 log.warning("%s: Unable to determine observation type. Guessing '%s'", 

485 self._log_prefix, obstype) 

486 return obstype 

487 

488 @cache_translation 

489 def to_physical_filter(self): 

490 """Calculate the physical filter name. 

491 

492 Returns 

493 ------- 

494 filter : `str` 

495 Name of filter. A combination of FILTER and GRATING 

496 headers joined by a "~". The filter and grating are always 

497 combined. The filter or grating part will be "none" if no value 

498 is specified. Uses "empty" if any of the filters or gratings 

499 indicate an "empty_N" name. "unknown" indicates that the filter is 

500 not defined anywhere but we think it should be. "none" indicates 

501 that the filter was not defined but the observation is a dark 

502 or bias. 

503 """ 

504 

505 physical_filter = self._determine_primary_filter() 

506 

507 if self.is_key_ok("GRATING"): 

508 grating = self._header["GRATING"] 

509 self._used_these_cards("GRATING") 

510 

511 if not grating or grating.lower().startswith("empty"): 

512 grating = "empty" 

513 else: 

514 # Be explicit about having no knowledge of the grating 

515 grating = "unknown" 

516 

517 physical_filter = f"{physical_filter}{FILTER_DELIMITER}{grating}" 

518 

519 return physical_filter