Coverage for python/astro_metadata_translator/translator.py: 40%

387 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-02-22 03:09 -0800

1# This file is part of astro_metadata_translator. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the LICENSE file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12"""Classes and support code for metadata translation""" 

13 

14from __future__ import annotations 

15 

16__all__ = ("MetadataTranslator", "StubTranslator", "cache_translation") 

17 

18import importlib 

19import inspect 

20import logging 

21import math 

22import warnings 

23from abc import abstractmethod 

24from typing import ( 

25 TYPE_CHECKING, 

26 Any, 

27 Callable, 

28 ClassVar, 

29 Dict, 

30 FrozenSet, 

31 Iterable, 

32 Iterator, 

33 List, 

34 Mapping, 

35 MutableMapping, 

36 Optional, 

37 Sequence, 

38 Set, 

39 Tuple, 

40 Type, 

41 Union, 

42) 

43 

44import astropy.io.fits.card 

45import astropy.units as u 

46from astropy.coordinates import Angle 

47 

48from .properties import PROPERTIES, PropertyDefinition 

49 

50if TYPE_CHECKING: 50 ↛ 51line 50 didn't jump to line 51, because the condition on line 50 was never true

51 import astropy.coordinates 

52 import astropy.time 

53 

54log = logging.getLogger(__name__) 

55 

56# Location of the root of the corrections resource files 

57CORRECTIONS_RESOURCE_ROOT = "corrections" 

58 

59"""Cache of version strings indexed by class.""" 

60_VERSION_CACHE: Dict[Type, str] = dict() 

61 

62 

63def cache_translation(func: Callable, method: Optional[str] = None) -> Callable: 

64 """Decorator to cache the result of a translation method. 

65 

66 Especially useful when a translation uses many other translation 

67 methods. Should be used only on ``to_x()`` methods. 

68 

69 Parameters 

70 ---------- 

71 func : `function` 

72 Translation method to cache. 

73 method : `str`, optional 

74 Name of the translation method to cache. Not needed if the decorator 

75 is used around a normal method, but necessary when the decorator is 

76 being used in a metaclass. 

77 

78 Returns 

79 ------- 

80 wrapped : `function` 

81 Method wrapped by the caching function. 

82 """ 

83 name = func.__name__ if method is None else method 

84 

85 def func_wrapper(self: MetadataTranslator) -> Any: 

86 if name not in self._translation_cache: 

87 self._translation_cache[name] = func(self) 

88 return self._translation_cache[name] 

89 

90 func_wrapper.__doc__ = func.__doc__ 

91 func_wrapper.__name__ = f"{name}_cached" 

92 return func_wrapper 

93 

94 

95class MetadataTranslator: 

96 """Per-instrument metadata translation support 

97 

98 Parameters 

99 ---------- 

100 header : `dict`-like 

101 Representation of an instrument header that can be manipulated 

102 as if it was a `dict`. 

103 filename : `str`, optional 

104 Name of the file whose header is being translated. For some 

105 datasets with missing header information this can sometimes 

106 allow for some fixups in translations. 

107 """ 

108 

109 # These are all deliberately empty in the base class. 

110 name: Optional[str] = None 

111 """The declared name of the translator.""" 

112 

113 default_search_path: Optional[Sequence[str]] = None 

114 """Default search path to use to locate header correction files.""" 

115 

116 default_resource_package = __name__.split(".")[0] 

117 """Module name to use to locate the correction resources.""" 

118 

119 default_resource_root: Optional[str] = None 

120 """Default package resource path root to use to locate header correction 

121 files within the ``default_resource_package`` package.""" 

122 

123 _trivial_map: Dict[str, Union[str, List[str], Tuple[Any, ...]]] = {} 

124 """Dict of one-to-one mappings for header translation from standard 

125 property to corresponding keyword.""" 

126 

127 _const_map: Dict[str, Any] = {} 

128 """Dict defining a constant for specified standard properties.""" 

129 

130 translators: Dict[str, Type] = dict() 

131 """All registered metadata translation classes.""" 

132 

133 supported_instrument: Optional[str] = None 

134 """Name of instrument understood by this translation class.""" 

135 

136 all_properties: Dict[str, PropertyDefinition] = {} 

137 """All the valid properties for this translator including extensions.""" 

138 

139 extensions: Dict[str, PropertyDefinition] = {} 

140 """Extension properties (`str`: `PropertyDefinition`) 

141 

142 Some instruments have important properties beyond the standard set; this is 

143 the place to declare that they exist, and they will be treated in the same 

144 way as the standard set, except that their names will everywhere be 

145 prefixed with ``ext_``. 

146 

147 Each property is indexed by name (`str`), with a corresponding 

148 `PropertyDefinition`. 

149 """ 

150 

151 # Static typing requires that we define the standard dynamic properties 

152 # statically. 

153 if TYPE_CHECKING: 153 ↛ 154line 153 didn't jump to line 154, because the condition on line 153 was never true

154 to_telescope: ClassVar[Callable[[MetadataTranslator], str]] 

155 to_instrument: ClassVar[Callable[[MetadataTranslator], str]] 

156 to_location: ClassVar[Callable[[MetadataTranslator], astropy.coordinates.EarthLocation]] 

157 to_exposure_id: ClassVar[Callable[[MetadataTranslator], int]] 

158 to_visit_id: ClassVar[Callable[[MetadataTranslator], int]] 

159 to_physical_filter: ClassVar[Callable[[MetadataTranslator], str]] 

160 to_datetime_begin: ClassVar[Callable[[MetadataTranslator], astropy.time.Time]] 

161 to_datetime_end: ClassVar[Callable[[MetadataTranslator], astropy.time.Time]] 

162 to_exposure_time: ClassVar[Callable[[MetadataTranslator], u.Quantity]] 

163 to_dark_time: ClassVar[Callable[[MetadataTranslator], u.Quantity]] 

164 to_boresight_airmass: ClassVar[Callable[[MetadataTranslator], float]] 

165 to_boresight_rotation_angle: ClassVar[Callable[[MetadataTranslator], u.Quantity]] 

166 to_boresight_rotation_coord: ClassVar[Callable[[MetadataTranslator], str]] 

167 to_detector_num: ClassVar[Callable[[MetadataTranslator], int]] 

168 to_detector_name: ClassVar[Callable[[MetadataTranslator], str]] 

169 to_detector_serial: ClassVar[Callable[[MetadataTranslator], str]] 

170 to_detector_group: ClassVar[Callable[[MetadataTranslator], Optional[str]]] 

171 to_detector_exposure_id: ClassVar[Callable[[MetadataTranslator], int]] 

172 to_object: ClassVar[Callable[[MetadataTranslator], str]] 

173 to_temperature: ClassVar[Callable[[MetadataTranslator], u.Quantity]] 

174 to_pressure: ClassVar[Callable[[MetadataTranslator], u.Quantity]] 

175 to_relative_humidity: ClassVar[Callable[[MetadataTranslator], float]] 

176 to_tracking_radec: ClassVar[Callable[[MetadataTranslator], astropy.coordinates.SkyCoord]] 

177 to_altaz_begin: ClassVar[Callable[[MetadataTranslator], astropy.coordinates.AltAz]] 

178 to_science_program: ClassVar[Callable[[MetadataTranslator], str]] 

179 to_observation_type: ClassVar[Callable[[MetadataTranslator], str]] 

180 to_observation_id: ClassVar[Callable[[MetadataTranslator], str]] 

181 

182 @classmethod 

183 def defined_in_this_class(cls, name: str) -> Optional[bool]: 

184 """Report if the specified class attribute is defined specifically in 

185 this class. 

186 

187 Parameters 

188 ---------- 

189 name : `str` 

190 Name of the attribute to test. 

191 

192 Returns 

193 ------- 

194 in_class : `bool` 

195 `True` if there is a attribute of that name defined in this 

196 specific subclass. 

197 `False` if the method is not defined in this specific subclass 

198 but is defined in a parent class. 

199 Returns `None` if the attribute is not defined anywhere 

200 in the class hierarchy (which can happen if translators have 

201 typos in their mapping tables). 

202 

203 Notes 

204 ----- 

205 Retrieves the attribute associated with the given name. 

206 Then looks in all the parent classes to determine whether that 

207 attribute comes from a parent class or from the current class. 

208 Attributes are compared using `id()`. 

209 """ 

210 # The attribute to compare. 

211 if not hasattr(cls, name): 

212 return None 

213 attr_id = id(getattr(cls, name)) 

214 

215 # Get all the classes in the hierarchy 

216 mro = list(inspect.getmro(cls)) 

217 

218 # Remove the first entry from the list since that will be the 

219 # current class 

220 mro.pop(0) 

221 

222 for parent in mro: 

223 # Some attributes may only exist in subclasses. Skip base classes 

224 # that are missing the attribute (such as object). 

225 if hasattr(parent, name): 

226 if id(getattr(parent, name)) == attr_id: 

227 return False 

228 return True 

229 

230 @classmethod 

231 def _make_const_mapping(cls, property_key: str, constant: Any) -> Callable: 

232 """Make a translator method that returns a constant value. 

233 

234 Parameters 

235 ---------- 

236 property_key : `str` 

237 Name of the property to be calculated (for the docstring). 

238 constant : `str` or `numbers.Number` 

239 Value to return for this translator. 

240 

241 Returns 

242 ------- 

243 f : `function` 

244 Function returning the constant. 

245 """ 

246 

247 def constant_translator(self: MetadataTranslator) -> Any: 

248 return constant 

249 

250 if property_key in cls.all_properties: 

251 property_doc = cls.all_properties[property_key].doc 

252 return_type = cls.all_properties[property_key].py_type 

253 else: 

254 return_type = type(constant) 

255 property_doc = f"Returns constant value for '{property_key}' property" 

256 

257 constant_translator.__doc__ = f"""{property_doc} 

258 

259 Returns 

260 ------- 

261 translation : `{return_type}` 

262 Translated property. 

263 """ 

264 return constant_translator 

265 

266 @classmethod 

267 def _make_trivial_mapping( 

268 cls, 

269 property_key: str, 

270 header_key: Union[str, Sequence[str]], 

271 default: Optional[Any] = None, 

272 minimum: Optional[Any] = None, 

273 maximum: Optional[Any] = None, 

274 unit: Optional[astropy.unit.Unit] = None, 

275 checker: Optional[Callable] = None, 

276 ) -> Callable: 

277 """Make a translator method returning a header value. 

278 

279 The header value can be converted to a `~astropy.units.Quantity` 

280 if desired, and can also have its value validated. 

281 

282 See `MetadataTranslator.validate_value()` for details on the use 

283 of default parameters. 

284 

285 Parameters 

286 ---------- 

287 property_key : `str` 

288 Name of the translator to be constructed (for the docstring). 

289 header_key : `str` or `list` of `str` 

290 Name of the key to look up in the header. If a `list` each 

291 key will be tested in turn until one matches. This can deal with 

292 header styles that evolve over time. 

293 default : `numbers.Number` or `astropy.units.Quantity`, `str`, optional 

294 If not `None`, default value to be used if the parameter read from 

295 the header is not defined or if the header is missing. 

296 minimum : `numbers.Number` or `astropy.units.Quantity`, optional 

297 If not `None`, and if ``default`` is not `None`, minimum value 

298 acceptable for this parameter. 

299 maximum : `numbers.Number` or `astropy.units.Quantity`, optional 

300 If not `None`, and if ``default`` is not `None`, maximum value 

301 acceptable for this parameter. 

302 unit : `astropy.units.Unit`, optional 

303 If not `None`, the value read from the header will be converted 

304 to a `~astropy.units.Quantity`. Only supported for numeric values. 

305 checker : `function`, optional 

306 Callback function to be used by the translator method in case the 

307 keyword is not present. Function will be executed as if it is 

308 a method of the translator class. Running without raising an 

309 exception will allow the default to be used. Should usually raise 

310 `KeyError`. 

311 

312 Returns 

313 ------- 

314 t : `function` 

315 Function implementing a translator with the specified 

316 parameters. 

317 """ 

318 if property_key in cls.all_properties: 318 ↛ 322line 318 didn't jump to line 322, because the condition on line 318 was never false

319 property_doc = cls.all_properties[property_key].doc 

320 return_type = cls.all_properties[property_key].str_type 

321 else: 

322 return_type = "str` or `numbers.Number" 

323 property_doc = f"Map '{header_key}' header keyword to '{property_key}' property" 

324 

325 def trivial_translator(self: MetadataTranslator) -> Any: 

326 if unit is not None: 

327 q = self.quantity_from_card( 

328 header_key, unit, default=default, minimum=minimum, maximum=maximum, checker=checker 

329 ) 

330 # Convert to Angle if this quantity is an angle 

331 if return_type == "astropy.coordinates.Angle": 

332 q = Angle(q) 

333 return q 

334 

335 keywords = header_key if isinstance(header_key, list) else [header_key] 

336 for key in keywords: 

337 if self.is_key_ok(key): 

338 value = self._header[key] 

339 if default is not None and not isinstance(value, str): 

340 value = self.validate_value(value, default, minimum=minimum, maximum=maximum) 

341 self._used_these_cards(key) 

342 break 

343 else: 

344 # No keywords found, use default, checking first, or raise 

345 # A None default is only allowed if a checker is provided. 

346 if checker is not None: 

347 try: 

348 checker(self) 

349 except Exception: 

350 raise KeyError(f"Could not find {keywords} in header") 

351 return default 

352 elif default is not None: 

353 value = default 

354 else: 

355 raise KeyError(f"Could not find {keywords} in header") 

356 

357 # If we know this is meant to be a string, force to a string. 

358 # Sometimes headers represent items as integers which generically 

359 # we want as strings (eg OBSID). Sometimes also floats are 

360 # written as "NaN" strings. 

361 casts = {"str": str, "float": float, "int": int} 

362 if return_type in casts and not isinstance(value, casts[return_type]) and value is not None: 

363 value = casts[return_type](value) 

364 

365 return value 

366 

367 # Docstring inheritance means it is confusing to specify here 

368 # exactly which header value is being used. 

369 trivial_translator.__doc__ = f"""{property_doc} 

370 

371 Returns 

372 ------- 

373 translation : `{return_type}` 

374 Translated value derived from the header. 

375 """ 

376 return trivial_translator 

377 

378 @classmethod 

379 def __init_subclass__(cls, **kwargs: Any) -> None: 

380 """Register all subclasses with the base class and create dynamic 

381 translator methods. 

382 

383 The method provides two facilities. Firstly, every subclass 

384 of `MetadataTranslator` that includes a ``name`` class property is 

385 registered as a translator class that could be selected when automatic 

386 header translation is attempted. Only name translator subclasses that 

387 correspond to a complete instrument. Translation classes providing 

388 generic translation support for multiple instrument translators should 

389 not be named. 

390 

391 The second feature of this method is to convert simple translations 

392 to full translator methods. Sometimes a translation is fixed (for 

393 example a specific instrument name should be used) and rather than 

394 provide a full ``to_property()`` translation method the mapping can be 

395 defined in a class variable named ``_constMap``. Similarly, for 

396 one-to-one trivial mappings from a header to a property, 

397 ``_trivialMap`` can be defined. Trivial mappings are a dict mapping a 

398 generic property to either a header keyword, or a tuple consisting of 

399 the header keyword and a dict containing key value pairs suitable for 

400 the `MetadataTranslator.quantity_from_card()` method. 

401 """ 

402 super().__init_subclass__(**kwargs) 

403 

404 # Only register classes with declared names 

405 if hasattr(cls, "name") and cls.name is not None: 

406 if cls.name in MetadataTranslator.translators: 406 ↛ 407line 406 didn't jump to line 407, because the condition on line 406 was never true

407 log.warning( 

408 "%s: Replacing %s translator with %s", 

409 cls.name, 

410 MetadataTranslator.translators[cls.name], 

411 cls, 

412 ) 

413 MetadataTranslator.translators[cls.name] = cls 

414 

415 # Check that we have not inherited constant/trivial mappings from 

416 # parent class that we have already applied. Empty maps are always 

417 # assumed okay 

418 const_map = cls._const_map if cls._const_map and cls.defined_in_this_class("_const_map") else {} 

419 trivial_map = ( 

420 cls._trivial_map if cls._trivial_map and cls.defined_in_this_class("_trivial_map") else {} 

421 ) 

422 

423 # Check for shadowing 

424 trivials = set(trivial_map.keys()) 

425 constants = set(const_map.keys()) 

426 both = trivials & constants 

427 if both: 427 ↛ 428line 427 didn't jump to line 428, because the condition on line 427 was never true

428 log.warning("%s: defined in both const_map and trivial_map: %s", cls.__name__, ", ".join(both)) 

429 

430 all = trivials | constants 

431 for name in all: 

432 if cls.defined_in_this_class(f"to_{name}"): 432 ↛ 435line 432 didn't jump to line 435, because the condition on line 432 was never true

433 # Must be one of trivial or constant. If in both then constant 

434 # overrides trivial. 

435 location = "by _trivial_map" 

436 if name in constants: 

437 location = "by _const_map" 

438 log.warning( 

439 "%s: %s is defined explicitly but will be replaced %s", cls.__name__, name, location 

440 ) 

441 

442 properties = set(PROPERTIES) | set(("ext_" + pp for pp in cls.extensions)) 

443 cls.all_properties = dict(PROPERTIES) 

444 cls.all_properties.update(cls.extensions) 

445 

446 # Go through the trival mappings for this class and create 

447 # corresponding translator methods 

448 for property_key, header_key in trivial_map.items(): 

449 kwargs = {} 

450 if type(header_key) == tuple: 

451 kwargs = header_key[1] 

452 header_key = header_key[0] 

453 translator = cls._make_trivial_mapping(property_key, header_key, **kwargs) 

454 method = f"to_{property_key}" 

455 translator.__name__ = f"{method}_trivial_in_{cls.__name__}" 

456 setattr(cls, method, cache_translation(translator, method=method)) 

457 if property_key not in properties: 457 ↛ 458line 457 didn't jump to line 458, because the condition on line 457 was never true

458 log.warning(f"Unexpected trivial translator for '{property_key}' defined in {cls}") 

459 

460 # Go through the constant mappings for this class and create 

461 # corresponding translator methods 

462 for property_key, constant in const_map.items(): 

463 translator = cls._make_const_mapping(property_key, constant) 

464 method = f"to_{property_key}" 

465 translator.__name__ = f"{method}_constant_in_{cls.__name__}" 

466 setattr(cls, method, translator) 

467 if property_key not in properties: 467 ↛ 468line 467 didn't jump to line 468, because the condition on line 467 was never true

468 log.warning(f"Unexpected constant translator for '{property_key}' defined in {cls}") 

469 

470 def __init__(self, header: Mapping[str, Any], filename: Optional[str] = None) -> None: 

471 self._header = header 

472 self.filename = filename 

473 self._used_cards: Set[str] = set() 

474 

475 # Prefix to use for warnings about failed translations 

476 self._log_prefix_cache: Optional[str] = None 

477 

478 # Cache assumes header is read-only once stored in object 

479 self._translation_cache: Dict[str, Any] = {} 

480 

481 @classmethod 

482 @abstractmethod 

483 def can_translate(cls, header: MutableMapping[str, Any], filename: Optional[str] = None) -> bool: 

484 """Indicate whether this translation class can translate the 

485 supplied header. 

486 

487 Parameters 

488 ---------- 

489 header : `dict`-like 

490 Header to convert to standardized form. 

491 filename : `str`, optional 

492 Name of file being translated. 

493 

494 Returns 

495 ------- 

496 can : `bool` 

497 `True` if the header is recognized by this class. `False` 

498 otherwise. 

499 """ 

500 raise NotImplementedError() 

501 

502 @classmethod 

503 def can_translate_with_options( 

504 cls, header: Mapping[str, Any], options: Dict[str, Any], filename: Optional[str] = None 

505 ) -> bool: 

506 """Helper method for `can_translate` allowing options. 

507 

508 Parameters 

509 ---------- 

510 header : `dict`-like 

511 Header to convert to standardized form. 

512 options : `dict` 

513 Headers to try to determine whether this header can 

514 be translated by this class. If a card is found it will 

515 be compared with the expected value and will return that 

516 comparison. Each card will be tried in turn until one is 

517 found. 

518 filename : `str`, optional 

519 Name of file being translated. 

520 

521 Returns 

522 ------- 

523 can : `bool` 

524 `True` if the header is recognized by this class. `False` 

525 otherwise. 

526 

527 Notes 

528 ----- 

529 Intended to be used from within `can_translate` implementations 

530 for specific translators. Is not intended to be called directly 

531 from `determine_translator`. 

532 """ 

533 for card, value in options.items(): 

534 if card in header: 

535 return header[card] == value 

536 return False 

537 

538 @classmethod 

539 def determine_translator( 

540 cls, header: Mapping[str, Any], filename: Optional[str] = None 

541 ) -> Type[MetadataTranslator]: 

542 """Determine a translation class by examining the header 

543 

544 Parameters 

545 ---------- 

546 header : `dict`-like 

547 Representation of a header. 

548 filename : `str`, optional 

549 Name of file being translated. 

550 

551 Returns 

552 ------- 

553 translator : `MetadataTranslator` 

554 Translation class that knows how to extract metadata from 

555 the supplied header. 

556 

557 Raises 

558 ------ 

559 ValueError 

560 None of the registered translation classes understood the supplied 

561 header. 

562 """ 

563 file_msg = "" 

564 if filename is not None: 

565 file_msg = f" from {filename}" 

566 for name, trans in cls.translators.items(): 

567 if trans.can_translate(header, filename=filename): 

568 log.debug("Using translation class %s%s", name, file_msg) 

569 return trans 

570 else: 

571 raise ValueError( 

572 f"None of the registered translation classes {list(cls.translators.keys())}" 

573 f" understood this header{file_msg}" 

574 ) 

575 

576 @classmethod 

577 def translator_version(cls) -> str: 

578 """Return the version string for this translator class. 

579 

580 Returns 

581 ------- 

582 version : `str` 

583 String identifying the version of this translator. 

584 

585 Notes 

586 ----- 

587 Assumes that the version is available from the ``__version__`` 

588 variable in the parent module. If this is not the case a translator 

589 should subclass this method. 

590 """ 

591 if cls in _VERSION_CACHE: 

592 return _VERSION_CACHE[cls] 

593 

594 version = "unknown" 

595 module_name = cls.__module__ 

596 components = module_name.split(".") 

597 while components: 

598 # This class has already been imported so importing it 

599 # should work. 

600 module = importlib.import_module(".".join(components)) 

601 if hasattr(module, v := "__version__"): 

602 version = getattr(module, v) 

603 if version == "unknown": 

604 # LSST software will have a fingerprint 

605 if hasattr(module, v := "__fingerprint__"): 

606 version = getattr(module, v) 

607 break 

608 else: 

609 # Remove last component from module name and try again 

610 components.pop() 

611 

612 _VERSION_CACHE[cls] = version 

613 return version 

614 

615 @classmethod 

616 def fix_header( 

617 cls, header: MutableMapping[str, Any], instrument: str, obsid: str, filename: Optional[str] = None 

618 ) -> bool: 

619 """Apply global fixes to a supplied header. 

620 

621 Parameters 

622 ---------- 

623 header : `dict` 

624 The header to correct. Correction is in place. 

625 instrument : `str` 

626 The name of the instrument. 

627 obsid : `str` 

628 Unique observation identifier associated with this header. 

629 Will always be provided. 

630 filename : `str`, optional 

631 Filename associated with this header. May not be set since headers 

632 can be fixed independently of any filename being known. 

633 

634 Returns 

635 ------- 

636 modified : `bool` 

637 `True` if a correction was applied. 

638 

639 Notes 

640 ----- 

641 This method is intended to support major discrepancies in headers 

642 such as: 

643 

644 * Periods of time where headers are known to be incorrect in some 

645 way that can be fixed either by deriving the correct value from 

646 the existing value or understanding the that correction is static 

647 for the given time. This requires that the date header is 

648 known. 

649 * The presence of a certain value is always wrong and should be 

650 corrected with a new static value regardless of date. 

651 

652 It is assumed that one off problems with headers have been applied 

653 before this method is called using the per-obsid correction system. 

654 

655 Usually called from `astro_metadata_translator.fix_header`. 

656 

657 For log messages, do not assume that the filename will be present. 

658 Always write log messages to fall back on using the ``obsid`` if 

659 ``filename`` is `None`. 

660 """ 

661 return False 

662 

663 @staticmethod 

664 def _construct_log_prefix(obsid: str, filename: Optional[str] = None) -> str: 

665 """Construct a log prefix string from the obsid and filename. 

666 

667 Parameters 

668 ---------- 

669 obsid : `str` 

670 The observation identifier. 

671 filename : `str`, optional 

672 The filename associated with the header being translated. 

673 Can be `None`. 

674 """ 

675 if filename: 

676 return f"{filename}({obsid})" 

677 return obsid 

678 

679 @property 

680 def _log_prefix(self) -> str: 

681 """Standard prefix that can be used for log messages to report 

682 useful context. 

683 

684 Will be either the filename and obsid, or just the obsid depending 

685 on whether a filename is known. 

686 

687 Returns 

688 ------- 

689 prefix : `str` 

690 The prefix to use. 

691 """ 

692 if self._log_prefix_cache is None: 

693 # Protect against the unfortunate event of the obsid failing to 

694 # be calculated. This should be rare but should not prevent a log 

695 # message from appearing. 

696 try: 

697 obsid = self.to_observation_id() 

698 except Exception: 

699 obsid = "unknown_obsid" 

700 self._log_prefix_cache = self._construct_log_prefix(obsid, self.filename) 

701 return self._log_prefix_cache 

702 

703 def _used_these_cards(self, *args: str) -> None: 

704 """Indicate that the supplied cards have been used for translation. 

705 

706 Parameters 

707 ---------- 

708 args : sequence of `str` 

709 Keywords used to process a translation. 

710 """ 

711 self._used_cards.update(set(args)) 

712 

713 def cards_used(self) -> FrozenSet[str]: 

714 """Cards used during metadata extraction. 

715 

716 Returns 

717 ------- 

718 used : `frozenset` of `str` 

719 Cards used when extracting metadata. 

720 """ 

721 return frozenset(self._used_cards) 

722 

723 @staticmethod 

724 def validate_value( 

725 value: float, default: float, minimum: Optional[float] = None, maximum: Optional[float] = None 

726 ) -> float: 

727 """Validate the supplied value, returning a new value if out of range 

728 

729 Parameters 

730 ---------- 

731 value : `float` 

732 Value to be validated. 

733 default : `float` 

734 Default value to use if supplied value is invalid or out of range. 

735 Assumed to be in the same units as the value expected in the 

736 header. 

737 minimum : `float` 

738 Minimum possible valid value, optional. If the calculated value 

739 is below this value, the default value will be used. 

740 maximum : `float` 

741 Maximum possible valid value, optional. If the calculated value 

742 is above this value, the default value will be used. 

743 

744 Returns 

745 ------- 

746 value : `float` 

747 Either the supplied value, or a default value. 

748 """ 

749 if value is None or math.isnan(value): 

750 value = default 

751 else: 

752 if minimum is not None and value < minimum: 

753 value = default 

754 elif maximum is not None and value > maximum: 

755 value = default 

756 return value 

757 

758 @staticmethod 

759 def is_keyword_defined(header: Mapping[str, Any], keyword: Optional[str]) -> bool: 

760 """Return `True` if the value associated with the named keyword is 

761 present in the supplied header and defined. 

762 

763 Parameters 

764 ---------- 

765 header : `dict`-lik 

766 Header to use as reference. 

767 keyword : `str` 

768 Keyword to check against header. 

769 

770 Returns 

771 ------- 

772 is_defined : `bool` 

773 `True` if the header is present and not-`None`. `False` otherwise. 

774 """ 

775 if keyword is None or keyword not in header: 

776 return False 

777 

778 if header[keyword] is None: 

779 return False 

780 

781 # Special case Astropy undefined value 

782 if isinstance(header[keyword], astropy.io.fits.card.Undefined): 

783 return False 

784 

785 return True 

786 

787 def resource_root(self) -> Tuple[Optional[str], Optional[str]]: 

788 """Package resource to use to locate correction resources within an 

789 installed package. 

790 

791 Returns 

792 ------- 

793 resource_package : `str` 

794 Package resource name. `None` if no package resource are to be 

795 used. 

796 resource_root : `str` 

797 The name of the resource root. `None` if no package resources 

798 are to be used. 

799 """ 

800 return (self.default_resource_package, self.default_resource_root) 

801 

802 def search_paths(self) -> List[str]: 

803 """Search paths to use when searching for header fix up correction 

804 files. 

805 

806 Returns 

807 ------- 

808 paths : `list` 

809 Directory paths to search. Can be an empty list if no special 

810 directories are defined. 

811 

812 Notes 

813 ----- 

814 Uses the classes ``default_search_path`` property if defined. 

815 """ 

816 if self.default_search_path is not None: 

817 return [p for p in self.default_search_path] 

818 return [] 

819 

820 def is_key_ok(self, keyword: Optional[str]) -> bool: 

821 """Return `True` if the value associated with the named keyword is 

822 present in this header and defined. 

823 

824 Parameters 

825 ---------- 

826 keyword : `str` 

827 Keyword to check against header. 

828 

829 Returns 

830 ------- 

831 is_ok : `bool` 

832 `True` if the header is present and not-`None`. `False` otherwise. 

833 """ 

834 return self.is_keyword_defined(self._header, keyword) 

835 

836 def are_keys_ok(self, keywords: Iterable[str]) -> bool: 

837 """Are the supplied keys all present and defined? 

838 

839 Parameters 

840 ---------- 

841 keywords : iterable of `str` 

842 Keywords to test. 

843 

844 Returns 

845 ------- 

846 all_ok : `bool` 

847 `True` if all supplied keys are present and defined. 

848 """ 

849 for k in keywords: 

850 if not self.is_key_ok(k): 

851 return False 

852 return True 

853 

854 def quantity_from_card( 

855 self, 

856 keywords: Union[str, Sequence[str]], 

857 unit: u.Unit, 

858 default: Optional[float] = None, 

859 minimum: Optional[float] = None, 

860 maximum: Optional[float] = None, 

861 checker: Optional[Callable] = None, 

862 ) -> u.Quantity: 

863 """Calculate a Astropy Quantity from a header card and a unit. 

864 

865 Parameters 

866 ---------- 

867 keywords : `str` or `list` of `str` 

868 Keyword to use from header. If a list each keyword will be tried 

869 in turn until one matches. 

870 unit : `astropy.units.UnitBase` 

871 Unit of the item in the header. 

872 default : `float`, optional 

873 Default value to use if the header value is invalid. Assumed 

874 to be in the same units as the value expected in the header. If 

875 None, no default value is used. 

876 minimum : `float`, optional 

877 Minimum possible valid value, optional. If the calculated value 

878 is below this value, the default value will be used. 

879 maximum : `float`, optional 

880 Maximum possible valid value, optional. If the calculated value 

881 is above this value, the default value will be used. 

882 checker : `function`, optional 

883 Callback function to be used by the translator method in case the 

884 keyword is not present. Function will be executed as if it is 

885 a method of the translator class. Running without raising an 

886 exception will allow the default to be used. Should usually raise 

887 `KeyError`. 

888 

889 Returns 

890 ------- 

891 q : `astropy.units.Quantity` 

892 Quantity representing the header value. 

893 

894 Raises 

895 ------ 

896 KeyError 

897 The supplied header key is not present. 

898 """ 

899 keyword_list = [keywords] if isinstance(keywords, str) else list(keywords) 

900 for k in keyword_list: 

901 if self.is_key_ok(k): 

902 value = self._header[k] 

903 keyword = k 

904 break 

905 else: 

906 if checker is not None: 

907 try: 

908 checker(self) 

909 value = default 

910 if value is not None: 

911 value = u.Quantity(value, unit=unit) 

912 return value 

913 except Exception: 

914 pass 

915 raise KeyError(f"Could not find {keywords} in header") 

916 if isinstance(value, str): 

917 # Sometimes the header has the wrong type in it but this must 

918 # be a number if we are creating a quantity. 

919 value = float(value) 

920 self._used_these_cards(keyword) 

921 if default is not None: 

922 value = self.validate_value(value, default, maximum=maximum, minimum=minimum) 

923 return u.Quantity(value, unit=unit) 

924 

925 def _join_keyword_values(self, keywords: Iterable[str], delim: str = "+") -> str: 

926 """Join values of all defined keywords with the specified delimiter. 

927 

928 Parameters 

929 ---------- 

930 keywords : iterable of `str` 

931 Keywords to look for in header. 

932 delim : `str`, optional 

933 Character to use to join the values together. 

934 

935 Returns 

936 ------- 

937 joined : `str` 

938 String formed from all the keywords found in the header with 

939 defined values joined by the delimiter. Empty string if no 

940 defined keywords found. 

941 """ 

942 values = [] 

943 for k in keywords: 

944 if self.is_key_ok(k): 

945 values.append(self._header[k]) 

946 self._used_these_cards(k) 

947 

948 if values: 

949 joined = delim.join(str(v) for v in values) 

950 else: 

951 joined = "" 

952 

953 return joined 

954 

955 @cache_translation 

956 def to_detector_unique_name(self) -> str: 

957 """Return a unique name for the detector. 

958 

959 Base class implementation attempts to combine ``detector_name`` with 

960 ``detector_group``. Group is only used if not `None`. 

961 

962 Can be over-ridden by specialist translator class. 

963 

964 Returns 

965 ------- 

966 name : `str` 

967 ``detector_group``_``detector_name`` if ``detector_group`` is 

968 defined, else the ``detector_name`` is assumed to be unique. 

969 If neither return a valid value an exception is raised. 

970 

971 Raises 

972 ------ 

973 NotImplementedError 

974 Raised if neither detector_name nor detector_group is defined. 

975 """ 

976 name = self.to_detector_name() 

977 group = self.to_detector_group() 

978 

979 if group is None and name is None: 

980 raise NotImplementedError("Can not determine unique name from detector_group and detector_name") 

981 

982 if group is not None: 

983 return f"{group}_{name}" 

984 

985 return name 

986 

987 @cache_translation 

988 def to_exposure_group(self) -> Optional[str]: 

989 """Return the group label associated with this exposure. 

990 

991 Base class implementation returns the ``exposure_id`` in string 

992 form. A subclass may do something different. 

993 

994 Returns 

995 ------- 

996 name : `str` 

997 The ``exposure_id`` converted to a string. 

998 """ 

999 exposure_id = self.to_exposure_id() 

1000 if exposure_id is None: 

1001 # mypy does not think this can ever happen but play it safe 

1002 # with subclasses. 

1003 return None # type: ignore 

1004 else: 

1005 return str(exposure_id) 

1006 

1007 @cache_translation 

1008 def to_observation_reason(self) -> str: 

1009 """Return the reason this observation was taken. 

1010 

1011 Base class implementation returns the ``science`` if the 

1012 ``observation_type`` is science, else ``unknown``. 

1013 A subclass may do something different. 

1014 

1015 Returns 

1016 ------- 

1017 name : `str` 

1018 The reason for this observation. 

1019 """ 

1020 obstype = self.to_observation_type() 

1021 if obstype == "science": 

1022 return "science" 

1023 return "unknown" 

1024 

1025 @cache_translation 

1026 def to_observing_day(self) -> int: 

1027 """Return the YYYYMMDD integer corresponding to the observing day. 

1028 

1029 Base class implementation uses the TAI date of the start of the 

1030 observation. 

1031 

1032 Returns 

1033 ------- 

1034 day : `int` 

1035 The observing day as an integer of form YYYYMMDD. If the header 

1036 is broken and is unable to obtain a date of observation, ``0`` 

1037 is returned and the assumption is made that the problem will 

1038 be caught elsewhere. 

1039 """ 

1040 datetime_begin = self.to_datetime_begin() 

1041 if datetime_begin is None: 

1042 return 0 

1043 return int(datetime_begin.tai.strftime("%Y%m%d")) 

1044 

1045 @cache_translation 

1046 def to_observation_counter(self) -> int: 

1047 """Return an integer corresponding to how this observation relates 

1048 to other observations. 

1049 

1050 Base class implementation returns ``0`` to indicate that it is not 

1051 known how an observatory will define a counter. Some observatories 

1052 may not use the concept, others may use a counter that increases 

1053 for every observation taken for that instrument, and others may 

1054 define it to be a counter within an observing day. 

1055 

1056 Returns 

1057 ------- 

1058 sequence : `int` 

1059 The observation counter. Always ``0`` for this implementation. 

1060 """ 

1061 return 0 

1062 

1063 @cache_translation 

1064 def to_group_counter_start(self) -> int: 

1065 """Return the observation counter of the observation that began 

1066 this group. 

1067 

1068 The definition of the relevant group is up to the metadata 

1069 translator. It can be the first observation in the exposure_group 

1070 or the first observation in the visit, but must be derivable 

1071 from the metadata of this observation. 

1072 

1073 Returns 

1074 ------- 

1075 counter : `int` 

1076 The observation counter for the start of the relevant group. 

1077 Default implementation always returns the observation counter 

1078 of this observation. 

1079 """ 

1080 return self.to_observation_counter() 

1081 

1082 @cache_translation 

1083 def to_group_counter_end(self) -> int: 

1084 """Return the observation counter of the observation that ends 

1085 this group. 

1086 

1087 The definition of the relevant group is up to the metadata 

1088 translator. It can be the last observation in the exposure_group 

1089 or the last observation in the visit, but must be derivable 

1090 from the metadata of this observation. It is of course possible 

1091 that the last observation in the group does not exist if a sequence 

1092 of observations was not completed. 

1093 

1094 Returns 

1095 ------- 

1096 counter : `int` 

1097 The observation counter for the end of the relevant group. 

1098 Default implementation always returns the observation counter 

1099 of this observation. 

1100 """ 

1101 return self.to_observation_counter() 

1102 

1103 @cache_translation 

1104 def to_has_simulated_content(self) -> bool: 

1105 """Return a boolean indicating whether any part of the observation 

1106 was simulated. 

1107 

1108 Returns 

1109 ------- 

1110 is_simulated : `bool` 

1111 `True` if this exposure has simulated content. This can be 

1112 if some parts of the metadata or data were simulated. Default 

1113 implementation always returns `False`. 

1114 """ 

1115 return False 

1116 

1117 @cache_translation 

1118 def to_focus_z(self) -> u.Quantity: 

1119 """Return a default defocal distance of 0.0 mm if there is no 

1120 keyword for defocal distance in the header. The default 

1121 keyword for defocal distance is ``FOCUSZ``. 

1122 

1123 Returns 

1124 ------- 

1125 focus_z: `astropy.units.Quantity` 

1126 The defocal distance from header or the 0.0mm default 

1127 """ 

1128 return 0.0 * u.mm 

1129 

1130 @classmethod 

1131 def determine_translatable_headers( 

1132 cls, filename: str, primary: Optional[MutableMapping[str, Any]] = None 

1133 ) -> Iterator[MutableMapping[str, Any]]: 

1134 """Given a file return all the headers usable for metadata translation. 

1135 

1136 This method can optionally be given a header from the file. This 

1137 header will generally be the primary header or a merge of the first 

1138 two headers. 

1139 

1140 In the base class implementation it is assumed that 

1141 this supplied header is the only useful header for metadata translation 

1142 and it will be returned unchanged if given. This can avoid 

1143 unnecesarily re-opening the file and re-reading the header when the 

1144 content is already known. 

1145 

1146 If no header is supplied, a header will be read from the supplied 

1147 file using `read_basic_metadata_from_file`, allowing it to merge 

1148 the primary and secondary header of a multi-extension FITS file. 

1149 Subclasses can read the header from the data file using whatever 

1150 technique is best for that instrument. 

1151 

1152 Subclasses can return multiple headers and ignore the externally 

1153 supplied header. They can also merge it with another header and return 

1154 a new derived header if that is required by the particular data file. 

1155 There is no requirement for the supplied header to be used. 

1156 

1157 Parameters 

1158 ---------- 

1159 filename : `str` 

1160 Path to a file in a format understood by this translator. 

1161 primary : `dict`-like, optional 

1162 The primary header obtained by the caller. This is sometimes 

1163 already known, for example if a system is trying to bootstrap 

1164 without already knowing what data is in the file. For many 

1165 instruments where the primary header is the only relevant 

1166 header, the primary header will be returned with no further 

1167 action. 

1168 

1169 Yields 

1170 ------ 

1171 headers : iterator of `dict`-like 

1172 A header usable for metadata translation. For this base 

1173 implementation it will be either the supplied primary header 

1174 or a header read from the file. This implementation will only 

1175 ever yield a single header. 

1176 

1177 Notes 

1178 ----- 

1179 Each translator class can have code specifically tailored to its 

1180 own file format. It is important not to call this method with 

1181 an incorrect translator class. The normal paradigm is for the 

1182 caller to have read the first header and then called 

1183 `determine_translator()` on the result to work out which translator 

1184 class to then call to obtain the real headers to be used for 

1185 translation. 

1186 """ 

1187 if primary is not None: 

1188 yield primary 

1189 else: 

1190 # Prevent circular import by deferring 

1191 from .file_helpers import read_basic_metadata_from_file 

1192 

1193 # Merge primary and secondary header if they exist. 

1194 header = read_basic_metadata_from_file(filename, -1) 

1195 assert header is not None # for mypy since can_raise=True 

1196 yield header 

1197 

1198 

1199def _make_abstract_translator_method( 

1200 property: str, doc: str, return_typedoc: str, return_type: Type 

1201) -> Callable: 

1202 """Create a an abstract translation method for this property. 

1203 

1204 Parameters 

1205 ---------- 

1206 property : `str` 

1207 Name of the translator for property to be created. 

1208 doc : `str` 

1209 Description of the property. 

1210 return_typedoc : `str` 

1211 Type string of this property (used in the doc string). 

1212 return_type : `class` 

1213 Type of this property. 

1214 

1215 Returns 

1216 ------- 

1217 m : `function` 

1218 Translator method for this property. 

1219 """ 

1220 

1221 def to_property(self: MetadataTranslator) -> None: 

1222 raise NotImplementedError(f"Translator for '{property}' undefined.") 

1223 

1224 to_property.__doc__ = f"""Return value of {property} from headers. 

1225 

1226 {doc} 

1227 

1228 Returns 

1229 ------- 

1230 {property} : `{return_typedoc}` 

1231 The translated property. 

1232 """ 

1233 return to_property 

1234 

1235 

1236# Make abstract methods for all the translators methods. 

1237# Unfortunately registering them as abstractmethods does not work 

1238# as these assignments come after the class has been created. 

1239# Assigning to __abstractmethods__ directly does work but interacts 

1240# poorly with the metaclass automatically generating methods from 

1241# _trivialMap and _constMap. 

1242# Note that subclasses that provide extension properties are assumed to not 

1243# need abstract methods created for them. 

1244 

1245# Allow for concrete translator methods to exist in the base class 

1246# These translator methods can be defined in terms of other properties 

1247CONCRETE = set() 

1248 

1249for name, definition in PROPERTIES.items(): 

1250 method = f"to_{name}" 

1251 if not MetadataTranslator.defined_in_this_class(method): 

1252 setattr( 

1253 MetadataTranslator, 

1254 f"to_{name}", 

1255 abstractmethod( 

1256 _make_abstract_translator_method( 

1257 name, definition.doc, definition.str_type, definition.py_type 

1258 ) 

1259 ), 

1260 ) 

1261 else: 

1262 CONCRETE.add(method) 

1263 

1264 

1265class StubTranslator(MetadataTranslator): 

1266 """Translator where all the translations are stubbed out and issue 

1267 warnings. 

1268 

1269 This translator can be used as a base class whilst developing a new 

1270 translator. It allows testing to proceed without being required to fully 

1271 define all translation methods. Once complete the class should be 

1272 removed from the inheritance tree. 

1273 

1274 """ 

1275 

1276 pass 

1277 

1278 

1279def _make_forwarded_stub_translator_method( 

1280 cls: Type[MetadataTranslator], property: str, doc: str, return_typedoc: str, return_type: Type 

1281) -> Callable: 

1282 """Create a stub translation method for this property that calls the 

1283 base method and catches `NotImplementedError`. 

1284 

1285 Parameters 

1286 ---------- 

1287 cls : `class` 

1288 Class to use when referencing `super()`. This would usually be 

1289 `StubTranslator`. 

1290 property : `str` 

1291 Name of the translator for property to be created. 

1292 doc : `str` 

1293 Description of the property. 

1294 return_typedoc : `str` 

1295 Type string of this property (used in the doc string). 

1296 return_type : `class` 

1297 Type of this property. 

1298 

1299 Returns 

1300 ------- 

1301 m : `function` 

1302 Stub translator method for this property. 

1303 """ 

1304 method = f"to_{property}" 

1305 

1306 def to_stub(self: MetadataTranslator) -> Any: 

1307 parent = getattr(super(cls, self), method, None) 

1308 try: 

1309 if parent is not None: 

1310 return parent() 

1311 except NotImplementedError: 

1312 pass 

1313 

1314 warnings.warn( 

1315 f"Please implement translator for property '{property}' for translator {self}", stacklevel=3 

1316 ) 

1317 return None 

1318 

1319 to_stub.__doc__ = f"""Unimplemented forwarding translator for {property}. 

1320 

1321 {doc} 

1322 

1323 Calls the base class translation method and if that fails with 

1324 `NotImplementedError` issues a warning reminding the implementer to 

1325 override this method. 

1326 

1327 Returns 

1328 ------- 

1329 {property} : `None` or `{return_typedoc}` 

1330 Always returns `None`. 

1331 """ 

1332 return to_stub 

1333 

1334 

1335# Create stub translation methods for each property. These stubs warn 

1336# rather than fail and should be overridden by translators. 

1337for name, description in PROPERTIES.items(): 

1338 setattr( 

1339 StubTranslator, 

1340 f"to_{name}", 

1341 _make_forwarded_stub_translator_method( 

1342 StubTranslator, name, definition.doc, definition.str_type, definition.py_type # type: ignore 

1343 ), 

1344 )