Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is currently part of obs_lsst but is written to allow it 

2# to be migrated to the astro_metadata_translator package at a later date. 

3# 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the LICENSE file in this directory for details of code ownership. 

7# 

8# Use of this source code is governed by a 3-clause BSD-style 

9# license that can be found in the LICENSE file. 

10 

11"""Metadata translation support code for LSST headers""" 

12 

13__all__ = ("ROLLOVERTIME", "TZERO", "LSST_LOCATION", "read_detector_ids", 

14 "compute_detector_exposure_id_generic", "LsstBaseTranslator") 

15 

16import os.path 

17import yaml 

18import logging 

19import re 

20import datetime 

21import hashlib 

22 

23import astropy.coordinates 

24import astropy.units as u 

25from astropy.time import Time, TimeDelta 

26from astropy.coordinates import EarthLocation 

27 

28from lsst.utils import getPackageDir 

29 

30from astro_metadata_translator import cache_translation, FitsTranslator 

31from astro_metadata_translator.translators.helpers import tracking_from_degree_headers, \ 

32 altaz_from_degree_headers 

33 

34# LSST day clock starts at UTC+8 

35ROLLOVERTIME = TimeDelta(8*60*60, scale="tai", format="sec") 

36TZERO = Time("2015-01-01T00:00", format="isot", scale="utc") 

37TZERO_DATETIME = TZERO.to_datetime() 

38 

39# Delimiter to use for multiple filters/gratings 

40FILTER_DELIMITER = "~" 

41 

42# Regex to use for parsing a GROUPID string 

43GROUP_RE = re.compile(r"^(\d\d\d\d\-\d\d\-\d\dT\d\d:\d\d:\d\d)\.(\d\d\d)(?:[\+#](\d+))?$") 

44 

45# LSST Default location in the absence of headers 

46LSST_LOCATION = EarthLocation.from_geodetic(-70.749417, -30.244639, 2663.0) 

47 

48obs_lsst_packageDir = getPackageDir("obs_lsst") 

49 

50log = logging.getLogger(__name__) 

51 

52 

53def read_detector_ids(policyFile): 

54 """Read a camera policy file and retrieve the mapping from CCD name 

55 to ID. 

56 

57 Parameters 

58 ---------- 

59 policyFile : `str` 

60 Name of YAML policy file to read, relative to the obs_lsst 

61 package. 

62 

63 Returns 

64 ------- 

65 mapping : `dict` of `str` to (`int`, `str`) 

66 A `dict` with keys being the full names of the detectors, and the 

67 value is a `tuple` containing the integer detector number and the 

68 detector serial number. 

69 

70 Notes 

71 ----- 

72 Reads the camera YAML definition file directly and extracts just the 

73 IDs and serials. This routine does not use the standard 

74 `~lsst.obs.base.yamlCamera.YAMLCamera` infrastructure or 

75 `lsst.afw.cameraGeom`. This is because the translators are intended to 

76 have minimal dependencies on LSST infrastructure. 

77 """ 

78 

79 file = os.path.join(obs_lsst_packageDir, policyFile) 

80 try: 

81 with open(file) as fh: 

82 # Use the fast parser since these files are large 

83 camera = yaml.load(fh, Loader=yaml.CSafeLoader) 

84 except OSError as e: 

85 raise ValueError(f"Could not load camera policy file {file}") from e 

86 

87 mapping = {} 

88 for ccd, value in camera["CCDs"].items(): 

89 mapping[ccd] = (int(value["id"]), value["serial"]) 

90 

91 return mapping 

92 

93 

94def compute_detector_exposure_id_generic(exposure_id, detector_num, max_num=1000, mode="concat"): 

95 """Compute the detector_exposure_id from the exposure id and the 

96 detector number. 

97 

98 Parameters 

99 ---------- 

100 exposure_id : `int` 

101 The exposure ID. 

102 detector_num : `int` 

103 The detector number. 

104 max_num : `int`, optional 

105 Maximum number of detectors to make space for. Defaults to 1000. 

106 mode : `str`, optional 

107 Computation mode. Defaults to "concat". 

108 - concat : Concatenate the exposure ID and detector number, making 

109 sure that there is space for max_num and zero padding. 

110 - multiply : Multiply the exposure ID by the maximum detector 

111 number and add the detector number. 

112 

113 Returns 

114 ------- 

115 detector_exposure_id : `int` 

116 Computed ID. 

117 

118 Raises 

119 ------ 

120 ValueError 

121 The detector number is out of range. 

122 """ 

123 

124 if detector_num is None: 

125 raise ValueError("Detector number must be defined.") 

126 if detector_num > max_num or detector_num < 0: 

127 raise ValueError(f"Detector number out of range 0 <= {detector_num} <= {max_num}") 

128 

129 if mode == "concat": 

130 npad = len(str(max_num)) 

131 return int(f"{exposure_id}{detector_num:0{npad}d}") 

132 elif mode == "multiply": 

133 return max_num*exposure_id + detector_num 

134 else: 

135 raise ValueError(f"Computation mode of '{mode}' is not understood") 

136 

137 

138class LsstBaseTranslator(FitsTranslator): 

139 """Translation methods useful for all LSST-style headers.""" 

140 

141 _const_map = {} 

142 _trivial_map = {} 

143 

144 # Do not specify a name for this translator 

145 cameraPolicyFile = None 

146 """Path to policy file relative to obs_lsst root.""" 

147 

148 detectorMapping = None 

149 """Mapping of detector name to detector number and serial.""" 

150 

151 detectorSerials = None 

152 """Mapping of detector serial number to raft, number, and name.""" 

153 

154 DETECTOR_MAX = 999 

155 """Maximum number of detectors to use when calculating the 

156 detector_exposure_id.""" 

157 

158 _DEFAULT_LOCATION = LSST_LOCATION 

159 """Default telescope location in absence of relevant FITS headers.""" 

160 

161 @classmethod 

162 def __init_subclass__(cls, **kwargs): 

163 """Ensure that subclasses clear their own detector mapping entries 

164 such that subclasses of translators that use detector mappings 

165 do not pick up the incorrect values from a parent.""" 

166 

167 cls.detectorMapping = None 

168 cls.detectorSerials = None 

169 

170 super().__init_subclass__(**kwargs) 

171 

172 def search_paths(self): 

173 """Search paths to use for LSST data when looking for header correction 

174 files. 

175 

176 Returns 

177 ------- 

178 path : `list` 

179 List with a single element containing the full path to the 

180 ``corrections`` directory within the ``obs_lsst`` package. 

181 """ 

182 return [os.path.join(obs_lsst_packageDir, "corrections")] 

183 

184 @classmethod 

185 def compute_detector_exposure_id(cls, exposure_id, detector_num): 

186 """Compute the detector exposure ID from detector number and 

187 exposure ID. 

188 

189 This is a helper method to allow code working outside the translator 

190 infrastructure to use the same algorithm. 

191 

192 Parameters 

193 ---------- 

194 exposure_id : `int` 

195 Unique exposure ID. 

196 detector_num : `int` 

197 Detector number. 

198 

199 Returns 

200 ------- 

201 detector_exposure_id : `int` 

202 The calculated ID. 

203 """ 

204 return compute_detector_exposure_id_generic(exposure_id, detector_num, 

205 max_num=cls.DETECTOR_MAX, 

206 mode="concat") 

207 

208 @classmethod 

209 def max_detector_exposure_id(cls): 

210 """The maximum detector exposure ID expected to be generated by 

211 this instrument. 

212 

213 Returns 

214 ------- 

215 max_id : `int` 

216 The maximum value. 

217 """ 

218 max_exposure_id = cls.max_exposure_id() 

219 return cls.compute_detector_exposure_id(max_exposure_id, cls.DETECTOR_MAX) 

220 

221 @classmethod 

222 def max_exposure_id(cls): 

223 """The maximum exposure ID expected from this instrument. 

224 

225 Returns 

226 ------- 

227 max_exposure_id : `int` 

228 The maximum value. 

229 """ 

230 max_date = "2050-12-31T23:59.999" 

231 max_seqnum = 99_999 

232 max_controller = "C" # This controller triggers the largest numbers 

233 return cls.compute_exposure_id(max_date, max_seqnum, max_controller) 

234 

235 @classmethod 

236 def detector_mapping(cls): 

237 """Returns the mapping of full name to detector ID and serial. 

238 

239 Returns 

240 ------- 

241 mapping : `dict` of `str`:`tuple` 

242 Returns the mapping of full detector name (group+detector) 

243 to detector number and serial. 

244 

245 Raises 

246 ------ 

247 ValueError 

248 Raised if no camera policy file has been registered with this 

249 translation class. 

250 

251 Notes 

252 ----- 

253 Will construct the mapping if none has previously been constructed. 

254 """ 

255 if cls.cameraPolicyFile is not None: 

256 if cls.detectorMapping is None: 

257 cls.detectorMapping = read_detector_ids(cls.cameraPolicyFile) 

258 else: 

259 raise ValueError(f"Translation class '{cls.__name__}' has no registered camera policy file") 

260 

261 return cls.detectorMapping 

262 

263 @classmethod 

264 def detector_serials(cls): 

265 """Obtain the mapping of detector serial to detector group, name, 

266 and number. 

267 

268 Returns 

269 ------- 

270 info : `dict` of `tuple` of (`str`, `str`, `int`) 

271 A `dict` with the serial numbers as keys and values of detector 

272 group, name, and number. 

273 """ 

274 if cls.detectorSerials is None: 

275 detector_mapping = cls.detector_mapping() 

276 

277 if detector_mapping is not None: 

278 # Form mapping to go from serial number to names/numbers 

279 serials = {} 

280 for fullname, (id, serial) in cls.detectorMapping.items(): 

281 raft, detector_name = fullname.split("_") 

282 if serial in serials: 

283 raise RuntimeError(f"Serial {serial} is defined in multiple places") 

284 serials[serial] = (raft, detector_name, id) 

285 cls.detectorSerials = serials 

286 else: 

287 raise RuntimeError("Unable to obtain detector mapping information") 

288 

289 return cls.detectorSerials 

290 

291 @classmethod 

292 def compute_detector_num_from_name(cls, detector_group, detector_name): 

293 """Helper method to return the detector number from the name. 

294 

295 Parameters 

296 ---------- 

297 detector_group : `str` 

298 Name of the detector grouping. This is generally the raft name. 

299 detector_name : `str` 

300 Detector name. 

301 

302 Returns 

303 ------- 

304 num : `int` 

305 Detector number. 

306 """ 

307 fullname = f"{detector_group}_{detector_name}" 

308 

309 num = None 

310 detector_mapping = cls.detector_mapping() 

311 if detector_mapping is None: 

312 raise RuntimeError("Unable to obtain detector mapping information") 

313 

314 if fullname in detector_mapping: 

315 num = detector_mapping[fullname] 

316 else: 

317 log.warning(f"Unable to determine detector number from detector name {fullname}") 

318 return None 

319 

320 return num[0] 

321 

322 @classmethod 

323 def compute_detector_info_from_serial(cls, detector_serial): 

324 """Helper method to return the detector information from the serial. 

325 

326 Parameters 

327 ---------- 

328 detector_serial : `str` 

329 Detector serial ID. 

330 

331 Returns 

332 ------- 

333 info : `tuple` of (`str`, `str`, `int`) 

334 Detector group, name, and number. 

335 """ 

336 serial_mapping = cls.detector_serials() 

337 if serial_mapping is None: 

338 raise RuntimeError("Unable to obtain serial mapping information") 

339 

340 if detector_serial in serial_mapping: 

341 info = serial_mapping[detector_serial] 

342 else: 

343 raise RuntimeError("Unable to determine detector information from detector serial" 

344 f" {detector_serial}") 

345 

346 return info 

347 

348 @staticmethod 

349 def compute_exposure_id(dayobs, seqnum, controller=None): 

350 """Helper method to calculate the exposure_id. 

351 

352 Parameters 

353 ---------- 

354 dayobs : `str` 

355 Day of observation in either YYYYMMDD or YYYY-MM-DD format. 

356 If the string looks like ISO format it will be truncated before the 

357 ``T`` before being handled. 

358 seqnum : `int` or `str` 

359 Sequence number. 

360 controller : `str`, optional 

361 Controller to use. If this is "O", no change is made to the 

362 exposure ID. If it is "C" a 1000 is added to the year component 

363 of the exposure ID. 

364 `None` indicates that the controller is not relevant to the 

365 exposure ID calculation (generally this is the case for test 

366 stand data). 

367 

368 Returns 

369 ------- 

370 exposure_id : `int` 

371 Exposure ID in form YYYYMMDDnnnnn form. 

372 """ 

373 if "T" in dayobs: 

374 dayobs = dayobs[:dayobs.find("T")] 

375 

376 dayobs = dayobs.replace("-", "") 

377 

378 if len(dayobs) != 8: 

379 raise ValueError(f"Malformed dayobs: {dayobs}") 

380 

381 # Expect no more than 99,999 exposures in a day 

382 maxdigits = 5 

383 if seqnum >= 10**maxdigits: 

384 raise ValueError(f"Sequence number ({seqnum}) exceeds limit") 

385 

386 # Camera control changes the exposure ID 

387 if controller is not None: 

388 if controller == "O": 

389 pass 

390 elif controller == "C": 

391 # Add 1000 to the year component 

392 dayobs = int(dayobs) 

393 dayobs += 1000_00_00 

394 else: 

395 raise ValueError(f"Supplied controller, '{controller}' is neither 'O' nor 'C'") 

396 

397 # Form the number as a string zero padding the sequence number 

398 idstr = f"{dayobs}{seqnum:0{maxdigits}d}" 

399 

400 # Exposure ID has to be an integer 

401 return int(idstr) 

402 

403 def _is_on_mountain(self): 

404 """Indicate whether these data are coming from the instrument 

405 installed on the mountain. 

406 

407 Returns 

408 ------- 

409 is : `bool` 

410 `True` if instrument is on the mountain. 

411 """ 

412 if "TSTAND" in self._header: 

413 return False 

414 return True 

415 

416 def is_on_sky(self): 

417 """Determine if this is an on-sky observation. 

418 

419 Returns 

420 ------- 

421 is_on_sky : `bool` 

422 Returns True if this is a observation on sky on the 

423 summit. 

424 """ 

425 # For LSST we think on sky unless tracksys is local 

426 if self.is_key_ok("TRACKSYS"): 

427 if self._header["TRACKSYS"].lower() == "local": 

428 # not on sky 

429 return False 

430 

431 # These are obviously not on sky 

432 if self.to_observation_type() in ("bias", "dark", "flat"): 

433 return False 

434 

435 return self._is_on_mountain() 

436 

437 @cache_translation 

438 def to_location(self): 

439 # Docstring will be inherited. Property defined in properties.py 

440 if not self._is_on_mountain(): 

441 return None 

442 try: 

443 # Try standard FITS headers 

444 return super().to_location() 

445 except KeyError: 

446 return self._DEFAULT_LOCATION 

447 

448 @cache_translation 

449 def to_datetime_begin(self): 

450 # Docstring will be inherited. Property defined in properties.py 

451 self._used_these_cards("MJD-OBS") 

452 return Time(self._header["MJD-OBS"], scale="tai", format="mjd") 

453 

454 @cache_translation 

455 def to_datetime_end(self): 

456 # Docstring will be inherited. Property defined in properties.py 

457 if self.is_key_ok("DATE-END"): 

458 return super().to_datetime_end() 

459 

460 return self.to_datetime_begin() + self.to_exposure_time() 

461 

462 @cache_translation 

463 def to_detector_num(self): 

464 # Docstring will be inherited. Property defined in properties.py 

465 raft = self.to_detector_group() 

466 detector = self.to_detector_name() 

467 return self.compute_detector_num_from_name(raft, detector) 

468 

469 @cache_translation 

470 def to_detector_exposure_id(self): 

471 # Docstring will be inherited. Property defined in properties.py 

472 exposure_id = self.to_exposure_id() 

473 num = self.to_detector_num() 

474 return self.compute_detector_exposure_id(exposure_id, num) 

475 

476 @cache_translation 

477 def to_observation_type(self): 

478 # Docstring will be inherited. Property defined in properties.py 

479 obstype = self._header["IMGTYPE"] 

480 self._used_these_cards("IMGTYPE") 

481 obstype = obstype.lower() 

482 if obstype in ("skyexp", "object"): 

483 obstype = "science" 

484 return obstype 

485 

486 @cache_translation 

487 def to_dark_time(self): 

488 """Calculate the dark time. 

489 

490 If a DARKTIME header is not found, the value is assumed to be 

491 identical to the exposure time. 

492 

493 Returns 

494 ------- 

495 dark : `astropy.units.Quantity` 

496 The dark time in seconds. 

497 """ 

498 if self.is_key_ok("DARKTIME"): 

499 darktime = self._header["DARKTIME"]*u.s 

500 self._used_these_cards("DARKTIME") 

501 else: 

502 log.warning("%s: Unable to determine dark time. Setting from exposure time.", 

503 self.to_observation_id()) 

504 darktime = self.to_exposure_time() 

505 return darktime 

506 

507 @cache_translation 

508 def to_exposure_id(self): 

509 """Generate a unique exposure ID number 

510 

511 This is a combination of DAYOBS and SEQNUM, and optionally 

512 CONTRLLR. 

513 

514 Returns 

515 ------- 

516 exposure_id : `int` 

517 Unique exposure number. 

518 """ 

519 if "CALIB_ID" in self._header: 

520 self._used_these_cards("CALIB_ID") 

521 return None 

522 

523 dayobs = self._header["DAYOBS"] 

524 seqnum = self._header["SEQNUM"] 

525 self._used_these_cards("DAYOBS", "SEQNUM") 

526 

527 if self.is_key_ok("CONTRLLR"): 

528 controller = self._header["CONTRLLR"] 

529 self._used_these_cards("CONTRLLR") 

530 else: 

531 controller = None 

532 

533 return self.compute_exposure_id(dayobs, seqnum, controller=controller) 

534 

535 @cache_translation 

536 def to_visit_id(self): 

537 """Calculate the visit associated with this exposure. 

538 

539 Notes 

540 ----- 

541 For LATISS and LSSTCam the default visit is derived from the 

542 exposure group. For other instruments we return the exposure_id. 

543 """ 

544 

545 exposure_group = self.to_exposure_group() 

546 # If the group is an int we return it 

547 try: 

548 visit_id = int(exposure_group) 

549 return visit_id 

550 except ValueError: 

551 pass 

552 

553 # A Group is defined as ISO date with an extension 

554 # The integer must be the same for a given group so we can never 

555 # use datetime_begin. 

556 # Nominally a GROUPID looks like "ISODATE+N" where the +N is 

557 # optional. This can be converted to seconds since epoch with 

558 # an adjustment for N. 

559 # For early data lacking that form we hash the group and return 

560 # the int. 

561 matches_date = GROUP_RE.match(exposure_group) 

562 if matches_date: 

563 iso_str = matches_date.group(1) 

564 fraction = matches_date.group(2) 

565 n = matches_date.group(3) 

566 if n is not None: 

567 n = int(n) 

568 else: 

569 n = 0 

570 iso = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%S") 

571 

572 tdelta = iso - TZERO_DATETIME 

573 epoch = int(tdelta.total_seconds()) 

574 

575 # Form the integer from EPOCH + 3 DIGIT FRAC + 0-pad N 

576 visit_id = int(f"{epoch}{fraction}{n:04d}") 

577 else: 

578 # Non-standard string so convert to numbers 

579 # using a hash function. Use the first N hex digits 

580 group_bytes = exposure_group.encode("us-ascii") 

581 hasher = hashlib.blake2b(group_bytes) 

582 # Need to be big enough it does not possibly clash with the 

583 # date-based version above 

584 digest = hasher.hexdigest()[:14] 

585 visit_id = int(digest, base=16) 

586 

587 # To help with hash collision, append the string length 

588 visit_id = int(f"{visit_id}{len(exposure_group):02d}") 

589 

590 return visit_id 

591 

592 @cache_translation 

593 def to_physical_filter(self): 

594 """Calculate the physical filter name. 

595 

596 Returns 

597 ------- 

598 filter : `str` 

599 Name of filter. Can be a combination of FILTER, FILTER1 and FILTER2 

600 headers joined by a "~". Returns "NONE" if no filter is declared. 

601 """ 

602 joined = self._join_keyword_values(["FILTER", "FILTER1", "FILTER2"], delim=FILTER_DELIMITER) 

603 if not joined: 

604 joined = "NONE" 

605 

606 return joined 

607 

608 @cache_translation 

609 def to_tracking_radec(self): 

610 if not self.is_on_sky(): 

611 return None 

612 

613 # RA/DEC are *derived* headers and for the case where the DATE-BEG 

614 # is 1970 they are garbage and should not be used. 

615 if self._header["DATE-OBS"] == self._header["DATE"]: 

616 # A fixed up date -- use AZEL as source of truth 

617 altaz = self.to_altaz_begin() 

618 radec = astropy.coordinates.SkyCoord(altaz.transform_to(astropy.coordinates.ICRS), 

619 obstime=altaz.obstime, 

620 location=altaz.location) 

621 else: 

622 radecsys = ("RADESYS",) 

623 radecpairs = (("RASTART", "DECSTART"), ("RA", "DEC")) 

624 radec = tracking_from_degree_headers(self, radecsys, radecpairs) 

625 

626 return radec 

627 

628 @cache_translation 

629 def to_altaz_begin(self): 

630 if not self._is_on_mountain(): 

631 return None 

632 

633 # ALTAZ always relevant unless bias or dark 

634 if self.to_observation_type() in ("bias", "dark"): 

635 return None 

636 

637 return altaz_from_degree_headers(self, (("ELSTART", "AZSTART"),), 

638 self.to_datetime_begin(), is_zd=False) 

639 

640 @cache_translation 

641 def to_exposure_group(self): 

642 """Calculate the exposure group string. 

643 

644 For LSSTCam and LATISS this is read from the ``GROUPID`` header. 

645 If that header is missing the exposure_id is returned instead as 

646 a string. 

647 """ 

648 if self.is_key_ok("GROUPID"): 

649 exposure_group = self._header["GROUPID"] 

650 self._used_these_cards("GROUPID") 

651 return exposure_group 

652 return super().to_exposure_group()