Coverage for python/astro_metadata_translator/observationInfo.py: 15%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

256 statements  

1# This file is part of astro_metadata_translator. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the LICENSE file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12"""Represent standard metadata from instrument headers""" 

13 

14from __future__ import annotations 

15 

16__all__ = ("ObservationInfo", "makeObservationInfo") 

17 

18import copy 

19import itertools 

20import json 

21import logging 

22import math 

23from typing import ( 

24 TYPE_CHECKING, 

25 Any, 

26 Callable, 

27 Dict, 

28 FrozenSet, 

29 MutableMapping, 

30 Optional, 

31 Sequence, 

32 Set, 

33 Tuple, 

34 Type, 

35) 

36 

37import astropy.time 

38from astropy.coordinates import AltAz, SkyCoord 

39 

40from .headers import fix_header 

41from .properties import PROPERTIES, PropertyDefinition 

42from .translator import MetadataTranslator 

43 

44if TYPE_CHECKING: 44 ↛ 45line 44 didn't jump to line 45, because the condition on line 44 was never true

45 import astropy.coordinates 

46 import astropy.units 

47 

48log = logging.getLogger(__name__) 

49 

50 

51class ObservationInfo: 

52 """Standardized representation of an instrument header for a single 

53 exposure observation. 

54 

55 There is a core set of instrumental properties that are pre-defined. 

56 Additional properties may be defined, either through the 

57 ``makeObservationInfo`` factory function by providing the ``extensions`` 

58 definitions, or through the regular ``ObservationInfo`` constructor when 

59 the extensions have been defined in the ``MetadataTranslator`` for the 

60 instrument of interest (or in the provided ``translator_class``). 

61 

62 Parameters 

63 ---------- 

64 header : `dict`-like 

65 Representation of an instrument header accessible as a `dict`. 

66 May be updated with header corrections if corrections are found. 

67 filename : `str`, optional 

68 Name of the file whose header is being translated. For some 

69 datasets with missing header information this can sometimes 

70 allow for some fixups in translations. 

71 translator_class : `MetadataTranslator`-class, optional 

72 If not `None`, the class to use to translate the supplied headers 

73 into standard form. Otherwise each registered translator class will 

74 be asked in turn if it knows how to translate the supplied header. 

75 pedantic : `bool`, optional 

76 If True the translation must succeed for all properties. If False 

77 individual property translations must all be implemented but can fail 

78 and a warning will be issued. 

79 search_path : iterable, optional 

80 Override search paths to use during header fix up. 

81 required : `set`, optional 

82 This parameter can be used to confirm that all properties contained 

83 in the set must translate correctly and also be non-None. For the case 

84 where ``pedantic`` is `True` this will still check that the resulting 

85 value is not `None`. 

86 subset : `set`, optional 

87 If not `None`, controls the translations that will be performed 

88 during construction. This can be useful if the caller is only 

89 interested in a subset of the properties and knows that some of 

90 the others might be slow to compute (for example the airmass if it 

91 has to be derived). 

92 

93 Raises 

94 ------ 

95 ValueError 

96 Raised if the supplied header was not recognized by any of the 

97 registered translators. Also raised if the request property subset 

98 is not a subset of the known properties. 

99 TypeError 

100 Raised if the supplied translator class was not a MetadataTranslator. 

101 KeyError 

102 Raised if a required property cannot be calculated, or if pedantic 

103 mode is enabled and any translations fails. 

104 NotImplementedError 

105 Raised if the selected translator does not support a required 

106 property. 

107 

108 Notes 

109 ----- 

110 Headers will be corrected if correction files are located and this will 

111 modify the header provided to the constructor. 

112 

113 Values of the properties are read-only. 

114 """ 

115 

116 # Static typing requires that we define the standard dynamic properties 

117 # statically. 

118 if TYPE_CHECKING: 118 ↛ 119line 118 didn't jump to line 119, because the condition on line 118 was never true

119 telescope: int 

120 instrument: str 

121 location: astropy.coordinates.EarthLocation 

122 exposure_id: int 

123 visit_id: int 

124 physical_filter: str 

125 datetime_begin: astropy.time.Time 

126 datetime_end: astropy.time.Time 

127 exposure_time: astropy.units.Quantity 

128 dark_time: astropy.units.Quantity 

129 boresight_airmass: float 

130 boresight_rotation_angle: astropy.units.Quantity 

131 boresight_rotation_coord: str 

132 detector_num: int 

133 detector_name: str 

134 detector_serial: str 

135 detector_group: str 

136 detector_exposure_id: int 

137 object: str 

138 temperature: astropy.units.Quantity 

139 pressure: astropy.units.Quantity 

140 relative_humidity: float 

141 tracking_radec: astropy.coordinates.SkyCoord 

142 altaz_begin: astropy.coordinates.AltAz 

143 science_program: str 

144 observation_type: str 

145 observation_id: str 

146 

147 def __init__( 

148 self, 

149 header: Optional[MutableMapping[str, Any]], 

150 filename: Optional[str] = None, 

151 translator_class: Optional[Type[MetadataTranslator]] = None, 

152 pedantic: bool = False, 

153 search_path: Optional[Sequence[str]] = None, 

154 required: Optional[Set[str]] = None, 

155 subset: Optional[Set[str]] = None, 

156 ) -> None: 

157 

158 # Initialize the empty object 

159 self._header: MutableMapping[str, Any] = {} 

160 self.filename = filename 

161 self._translator = None 

162 self.translator_class_name = "<None>" 

163 

164 # To allow makeObservationInfo to work, we special case a None 

165 # header 

166 if header is None: 

167 return 

168 

169 # Fix up the header (if required) 

170 fix_header(header, translator_class=translator_class, filename=filename, search_path=search_path) 

171 

172 # Store the supplied header for later stripping 

173 self._header = header 

174 

175 if translator_class is None: 

176 translator_class = MetadataTranslator.determine_translator(header, filename=filename) 

177 elif not issubclass(translator_class, MetadataTranslator): 

178 raise TypeError(f"Translator class must be a MetadataTranslator, not {translator_class}") 

179 

180 self._declare_extensions(translator_class.extensions) 

181 

182 # Create an instance for this header 

183 translator = translator_class(header, filename=filename) 

184 

185 # Store the translator 

186 self._translator = translator 

187 self.translator_class_name = translator_class.__name__ 

188 

189 # Form file information string in case we need an error message 

190 if filename: 

191 file_info = f" and file {filename}" 

192 else: 

193 file_info = "" 

194 

195 # Determine the properties of interest 

196 full_set = set(self.all_properties) 

197 if subset is not None: 

198 if not subset: 

199 raise ValueError("Cannot request no properties be calculated.") 

200 if not subset.issubset(full_set): 

201 raise ValueError( 

202 "Requested subset is not a subset of known properties. " f"Got extra: {subset - full_set}" 

203 ) 

204 properties = subset 

205 else: 

206 properties = full_set 

207 

208 if required is None: 

209 required = set() 

210 else: 

211 if not required.issubset(full_set): 

212 raise ValueError("Requested required properties include unknowns: " f"{required - full_set}") 

213 

214 # Loop over each property and request the translated form 

215 for t in properties: 

216 # prototype code 

217 method = f"to_{t}" 

218 property = f"_{t}" if not t.startswith("ext_") else t 

219 

220 try: 

221 value = getattr(translator, method)() 

222 except NotImplementedError as e: 

223 raise NotImplementedError( 

224 f"No translation exists for property '{t}'" f" using translator {translator.__class__}" 

225 ) from e 

226 except KeyError as e: 

227 err_msg = ( 

228 f"Error calculating property '{t}' using translator {translator.__class__}" f"{file_info}" 

229 ) 

230 if pedantic or t in required: 

231 raise KeyError(err_msg) from e 

232 else: 

233 log.debug("Calculation of property '%s' failed with header: %s", t, header) 

234 log.warning(f"Ignoring {err_msg}: {e}") 

235 continue 

236 

237 definition = self.all_properties[t] 

238 if not self._is_property_ok(definition, value): 

239 err_msg = ( 

240 f"Value calculated for property '{t}' is wrong type " 

241 f"({type(value)} != {definition.str_type}) using translator {translator.__class__}" 

242 f"{file_info}" 

243 ) 

244 if pedantic or t in required: 

245 raise TypeError(err_msg) 

246 else: 

247 log.debug("Calcuation of property '%s' had unexpected type with header: %s", t, header) 

248 log.warning(f"Ignoring {err_msg}") 

249 

250 if value is None and t in required: 

251 raise KeyError(f"Calculation of required property {t} resulted in a value of None") 

252 

253 super().__setattr__(property, value) # allows setting even write-protected extensions 

254 

255 @staticmethod 

256 def _get_all_properties( 

257 extensions: Optional[Dict[str, PropertyDefinition]] = None 

258 ) -> Dict[str, PropertyDefinition]: 

259 """Return the definitions of all properties 

260 

261 Parameters 

262 ---------- 

263 extensions : `dict` [`str`: `PropertyDefinition`] 

264 List of extension property definitions, indexed by name (with no 

265 "ext_" prefix). 

266 

267 Returns 

268 ------- 

269 properties : `dict` [`str`: `PropertyDefinition`] 

270 Merged list of all property definitions, indexed by name. Extension 

271 properties will be listed with an ``ext_`` prefix. 

272 """ 

273 properties = dict(PROPERTIES) 

274 if extensions: 

275 properties.update({"ext_" + pp: dd for pp, dd in extensions.items()}) 

276 return properties 

277 

278 def _declare_extensions(self, extensions: Optional[Dict[str, PropertyDefinition]]) -> None: 

279 """Declare and set up extension properties 

280 

281 This should always be called internally as part of the creation of a 

282 new `ObservationInfo`. 

283 

284 The core set of properties each have a python ``property`` that makes 

285 them read-only, and serves as a useful place to hang the docstring. 

286 However, the core set are set up at compile time, whereas the extension 

287 properties have to be configured at run time (because we don't know 

288 what they will be until we look at the header and figure out what 

289 instrument we're dealing with) when we have an instance rather than a 

290 class (and python ``property`` doesn't work on instances; only on 

291 classes). We therefore use a separate scheme for the extension 

292 properties: we write them directly to their associated instance 

293 variable, and we use ``__setattr__`` to protect them as read-only. 

294 Unfortunately, with this scheme, we can't give extension properties a 

295 docstring; but we're setting them up at runtime, so maybe that's not 

296 terribly important. 

297 

298 Parameters 

299 ---------- 

300 extensions : `dict` [`str`: `PropertyDefinition`] 

301 List of extension property definitions, indexed by name (with no 

302 "ext_" prefix). 

303 """ 

304 if not extensions: 

305 extensions = {} 

306 for name in extensions: 

307 super().__setattr__("ext_" + name, None) 

308 self.extensions = extensions 

309 self.all_properties = self._get_all_properties(extensions) 

310 

311 def __setattr__(self, name: str, value: Any) -> Any: 

312 """Set attribute 

313 

314 This provides read-only protection for the extension properties. The 

315 core set of properties have read-only protection via the use of the 

316 python ``property``. 

317 """ 

318 if hasattr(self, "extensions") and name.startswith("ext_") and name[4:] in self.extensions: 

319 raise AttributeError(f"Attribute {name} is read-only") 

320 return super().__setattr__(name, value) 

321 

322 @classmethod 

323 def _is_property_ok(cls, definition: PropertyDefinition, value: Any) -> bool: 

324 """Compare the supplied value against the expected type as defined 

325 for the corresponding property. 

326 

327 Parameters 

328 ---------- 

329 definition : `PropertyDefinition` 

330 Property definition. 

331 value : `object` 

332 Value of the property to validate. 

333 

334 Returns 

335 ------- 

336 is_ok : `bool` 

337 `True` if the value is of an appropriate type. 

338 

339 Notes 

340 ----- 

341 Currently only the type of the property is validated. There is no 

342 attempt to check bounds or determine that a Quantity is compatible 

343 with the property. 

344 """ 

345 if value is None: 

346 return True 

347 

348 # For AltAz coordinates, they can either arrive as AltAz or 

349 # as SkyCoord(frame=AltAz) so try to find the frame inside 

350 # the SkyCoord. 

351 if issubclass(definition.py_type, AltAz) and isinstance(value, SkyCoord): 

352 value = value.frame 

353 

354 if not isinstance(value, definition.py_type): 

355 return False 

356 

357 return True 

358 

359 @property 

360 def cards_used(self) -> FrozenSet[str]: 

361 """Header cards used for the translation. 

362 

363 Returns 

364 ------- 

365 used : `frozenset` of `str` 

366 Set of card used. 

367 """ 

368 if not self._translator: 

369 return frozenset() 

370 return self._translator.cards_used() 

371 

372 def stripped_header(self) -> MutableMapping[str, Any]: 

373 """Return a copy of the supplied header with used keywords removed. 

374 

375 Returns 

376 ------- 

377 stripped : `dict`-like 

378 Same class as header supplied to constructor, but with the 

379 headers used to calculate the generic information removed. 

380 """ 

381 hdr = copy.copy(self._header) 

382 used = self.cards_used 

383 for c in used: 

384 del hdr[c] 

385 return hdr 

386 

387 def __str__(self) -> str: 

388 # Put more interesting answers at front of list 

389 # and then do remainder 

390 priority = ("instrument", "telescope", "datetime_begin") 

391 properties = sorted(set(self.all_properties) - set(priority)) 

392 

393 result = "" 

394 for p in itertools.chain(priority, properties): 

395 value = getattr(self, p) 

396 if isinstance(value, astropy.time.Time): 

397 value.format = "isot" 

398 value = str(value.value) 

399 result += f"{p}: {value}\n" 

400 

401 return result 

402 

403 def __eq__(self, other: Any) -> bool: 

404 """Compares equal if standard properties are equal""" 

405 if not isinstance(other, ObservationInfo): 

406 return NotImplemented 

407 

408 # Compare simplified forms. 

409 # Cannot compare directly because nan will not equate as equal 

410 # whereas they should be equal for our purposes 

411 self_simple = self.to_simple() 

412 other_simple = other.to_simple() 

413 

414 # We don't care about the translator internal detail 

415 self_simple.pop("_translator", None) 

416 other_simple.pop("_translator", None) 

417 

418 for k, self_value in self_simple.items(): 

419 other_value = other_simple[k] 

420 if self_value != other_value: 

421 if math.isnan(self_value) and math.isnan(other_value): 

422 # If both are nan this is fine 

423 continue 

424 return False 

425 return True 

426 

427 def __lt__(self, other: Any) -> bool: 

428 if not isinstance(other, ObservationInfo): 

429 return NotImplemented 

430 return self.datetime_begin < other.datetime_begin 

431 

432 def __gt__(self, other: Any) -> bool: 

433 if not isinstance(other, ObservationInfo): 

434 return NotImplemented 

435 return self.datetime_begin > other.datetime_begin 

436 

437 def __getstate__(self) -> Tuple[Any, ...]: 

438 """Get pickleable state 

439 

440 Returns the properties. Deliberately does not preserve the full 

441 current state; in particular, does not return the full header or 

442 translator. 

443 

444 Returns 

445 ------- 

446 state : `tuple` 

447 Pickled state. 

448 """ 

449 state = dict() 

450 for p in self.all_properties: 

451 state[p] = getattr(self, p) 

452 

453 return state, self.extensions 

454 

455 def __setstate__(self, state: Tuple[Any, ...]) -> None: 

456 """Set object state from pickle 

457 

458 Parameters 

459 ---------- 

460 state : `tuple` 

461 Pickled state. 

462 """ 

463 try: 

464 state, extensions = state 

465 except ValueError: 

466 # Backwards compatibility for pickles generated before DM-34175 

467 extensions = {} 

468 self._declare_extensions(extensions) 

469 for p in self.all_properties: 

470 if p.startswith("ext_"): 

471 # allows setting even write-protected extensions 

472 super().__setattr__(p, state[p]) # type: ignore 

473 else: 

474 property = f"_{p}" 

475 setattr(self, property, state[p]) # type: ignore 

476 

477 def to_simple(self) -> MutableMapping[str, Any]: 

478 """Convert the contents of this object to simple dict form. 

479 

480 The keys of the dict are the standard properties but the values 

481 can be simplified to support JSON serialization. For example a 

482 SkyCoord might be represented as an ICRS RA/Dec tuple rather than 

483 a full SkyCoord representation. 

484 

485 Any properties with `None` value will be skipped. 

486 

487 Can be converted back to an `ObservationInfo` using `from_simple()`. 

488 

489 Returns 

490 ------- 

491 simple : `dict` of [`str`, `Any`] 

492 Simple dict of all properties. 

493 

494 Notes 

495 ----- 

496 Round-tripping of extension properties requires that the 

497 `ObservationInfo` was created with the help of a registered 

498 `MetadataTranslator` (which contains the extension property 

499 definitions). 

500 """ 

501 simple = {} 

502 if hasattr(self, "_translator") and self._translator and self._translator.name: 

503 simple["_translator"] = self._translator.name 

504 

505 for p in self.all_properties: 

506 property = f"_{p}" if not p.startswith("ext_") else p 

507 value = getattr(self, property) 

508 if value is None: 

509 continue 

510 

511 # Access the function to simplify the property 

512 simplifier = self.all_properties[p].to_simple 

513 

514 if simplifier is None: 

515 simple[p] = value 

516 continue 

517 

518 simple[p] = simplifier(value) 

519 

520 return simple 

521 

522 def to_json(self) -> str: 

523 """Serialize the object to JSON string. 

524 

525 Returns 

526 ------- 

527 j : `str` 

528 The properties of the ObservationInfo in JSON string form. 

529 

530 Notes 

531 ----- 

532 Round-tripping of extension properties requires that the 

533 `ObservationInfo` was created with the help of a registered 

534 `MetadataTranslator` (which contains the extension property 

535 definitions). 

536 """ 

537 return json.dumps(self.to_simple()) 

538 

539 @classmethod 

540 def from_simple(cls, simple: MutableMapping[str, Any]) -> ObservationInfo: 

541 """Convert the entity returned by `to_simple` back into an 

542 `ObservationInfo`. 

543 

544 Parameters 

545 ---------- 

546 simple : `dict` [`str`, `Any`] 

547 The dict returned by `to_simple()` 

548 

549 Returns 

550 ------- 

551 obsinfo : `ObservationInfo` 

552 New object constructed from the dict. 

553 

554 Notes 

555 ----- 

556 Round-tripping of extension properties requires that the 

557 `ObservationInfo` was created with the help of a registered 

558 `MetadataTranslator` (which contains the extension property 

559 definitions). 

560 """ 

561 extensions = {} 

562 translator = simple.pop("_translator", None) 

563 if translator: 

564 if translator not in MetadataTranslator.translators: 

565 raise KeyError(f"Unrecognised translator: {translator}") 

566 extensions = MetadataTranslator.translators[translator].extensions 

567 

568 properties = cls._get_all_properties(extensions) 

569 

570 processed: Dict[str, Any] = {} 

571 for k, v in simple.items(): 

572 

573 if v is None: 

574 continue 

575 

576 # Access the function to convert from simple form 

577 complexifier = properties[k].from_simple 

578 

579 if complexifier is not None: 

580 v = complexifier(v, **processed) 

581 

582 processed[k] = v 

583 

584 return cls.makeObservationInfo(extensions=extensions, **processed) 

585 

586 @classmethod 

587 def from_json(cls, json_str: str) -> ObservationInfo: 

588 """Create `ObservationInfo` from JSON string. 

589 

590 Parameters 

591 ---------- 

592 json_str : `str` 

593 The JSON representation. 

594 

595 Returns 

596 ------- 

597 obsinfo : `ObservationInfo` 

598 Reconstructed object. 

599 

600 Notes 

601 ----- 

602 Round-tripping of extension properties requires that the 

603 `ObservationInfo` was created with the help of a registered 

604 `MetadataTranslator` (which contains the extension property 

605 definitions). 

606 """ 

607 simple = json.loads(json_str) 

608 return cls.from_simple(simple) 

609 

610 @classmethod 

611 def makeObservationInfo( # noqa: N802 

612 cls, *, extensions: Optional[Dict[str, PropertyDefinition]] = None, **kwargs: Any 

613 ) -> ObservationInfo: 

614 """Construct an `ObservationInfo` from the supplied parameters. 

615 

616 Parameters 

617 ---------- 

618 extensions : `dict` [`str`: `PropertyDefinition`], optional 

619 Optional extension definitions, indexed by extension name (without 

620 the ``ext_`` prefix, which will be added by `ObservationInfo`). 

621 **kwargs 

622 Name-value pairs for any properties to be set. In the case of 

623 extension properties, the names should include the ``ext_`` prefix. 

624 

625 Notes 

626 ----- 

627 The supplied parameters should use names matching the property. 

628 The type of the supplied value will be checked against the property. 

629 Any properties not supplied will be assigned a value of `None`. 

630 

631 Raises 

632 ------ 

633 KeyError 

634 Raised if a supplied parameter key is not a known property. 

635 TypeError 

636 Raised if a supplied value does not match the expected type 

637 of the property. 

638 """ 

639 

640 obsinfo = cls(None) 

641 obsinfo._declare_extensions(extensions) 

642 

643 unused = set(kwargs) 

644 

645 for p in obsinfo.all_properties: 

646 if p in kwargs: 

647 property = f"_{p}" if not p.startswith("ext_") else p 

648 value = kwargs[p] 

649 definition = obsinfo.all_properties[p] 

650 if not cls._is_property_ok(definition, value): 

651 raise TypeError( 

652 f"Supplied value {value} for property {p} " 

653 f"should be of class {definition.str_type} not {value.__class__}" 

654 ) 

655 super(cls, obsinfo).__setattr__(property, value) # allows setting write-protected extensions 

656 unused.remove(p) 

657 

658 if unused: 

659 n = len(unused) 

660 raise KeyError(f"Unrecognized propert{'y' if n == 1 else 'ies'} provided: {', '.join(unused)}") 

661 

662 return obsinfo 

663 

664 

665# Method to add the standard properties 

666def _make_property(property: str, doc: str, return_typedoc: str, return_type: Type) -> Callable: 

667 """Create a getter method with associated docstring. 

668 

669 Parameters 

670 ---------- 

671 property : `str` 

672 Name of the property getter to be created. 

673 doc : `str` 

674 Description of this property. 

675 return_typedoc : `str` 

676 Type string of this property (used in the doc string). 

677 return_type : `class` 

678 Type of this property. 

679 

680 Returns 

681 ------- 

682 p : `function` 

683 Getter method for this property. 

684 """ 

685 

686 def getter(self: ObservationInfo) -> Any: 

687 return getattr(self, f"_{property}") 

688 

689 getter.__doc__ = f"""{doc} 

690 

691 Returns 

692 ------- 

693 {property} : `{return_typedoc}` 

694 Access the property. 

695 """ 

696 return getter 

697 

698 

699# Set up the core set of properties 

700# In order to provide read-only protection, each attribute is hidden behind a 

701# python "property" wrapper. 

702for name, definition in PROPERTIES.items(): 

703 setattr(ObservationInfo, f"_{name}", None) 

704 setattr( 

705 ObservationInfo, 

706 name, 

707 property(_make_property(name, definition.doc, definition.str_type, definition.py_type)), 

708 ) 

709 

710 

711def makeObservationInfo( # noqa: N802 

712 *, extensions: Optional[Dict[str, PropertyDefinition]] = None, **kwargs: Any 

713) -> ObservationInfo: 

714 """Construct an `ObservationInfo` from the supplied parameters. 

715 

716 Parameters 

717 ---------- 

718 extensions : `dict` [`str`: `PropertyDefinition`], optional 

719 Optional extension definitions, indexed by extension name (without 

720 the ``ext_`` prefix, which will be added by `ObservationInfo`). 

721 **kwargs 

722 Name-value pairs for any properties to be set. In the case of 

723 extension properties, the names should include the ``ext_`` prefix. 

724 

725 Notes 

726 ----- 

727 The supplied parameters should use names matching the property. 

728 The type of the supplied value will be checked against the property. 

729 Any properties not supplied will be assigned a value of `None`. 

730 

731 Raises 

732 ------ 

733 KeyError 

734 Raised if a supplied parameter key is not a known property. 

735 TypeError 

736 Raised if a supplied value does not match the expected type 

737 of the property. 

738 """ 

739 return ObservationInfo.makeObservationInfo(extensions=extensions, **kwargs)