Coverage for python/astro_metadata_translator/translator.py: 46%

391 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-08-05 01:30 +0000

1# This file is part of astro_metadata_translator. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the LICENSE file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12"""Classes and support code for metadata translation.""" 

13 

14from __future__ import annotations 

15 

16__all__ = ("MetadataTranslator", "StubTranslator", "cache_translation") 

17 

18import importlib 

19import inspect 

20import logging 

21import math 

22import warnings 

23from abc import abstractmethod 

24from collections.abc import Callable, Iterable, Iterator, Mapping, MutableMapping, Sequence 

25from typing import TYPE_CHECKING, Any, ClassVar 

26 

27import astropy.io.fits.card 

28import astropy.units as u 

29from astropy.coordinates import Angle 

30 

31from .properties import PROPERTIES, PropertyDefinition 

32 

33if TYPE_CHECKING: 33 ↛ 34line 33 didn't jump to line 34, because the condition on line 33 was never true

34 import astropy.coordinates 

35 import astropy.time 

36 

37log = logging.getLogger(__name__) 

38 

39# Location of the root of the corrections resource files 

40CORRECTIONS_RESOURCE_ROOT = "corrections" 

41 

42"""Cache of version strings indexed by class.""" 

43_VERSION_CACHE: dict[type, str] = dict() 

44 

45 

46def cache_translation(func: Callable, method: str | None = None) -> Callable: 

47 """Cache the result of a translation method. 

48 

49 Parameters 

50 ---------- 

51 func : `~collections.abc.Callable` 

52 Translation method to cache. 

53 method : `str`, optional 

54 Name of the translation method to cache. Not needed if the decorator 

55 is used around a normal method, but necessary when the decorator is 

56 being used in a metaclass. 

57 

58 Returns 

59 ------- 

60 wrapped : `~collections.abc.Callable` 

61 Method wrapped by the caching function. 

62 

63 Notes 

64 ----- 

65 Especially useful when a translation uses many other translation 

66 methods or involves significant computation. 

67 Should be used only on ``to_x()`` methods. 

68 

69 .. code-block:: python 

70 

71 @cache_translation 

72 def to_detector_num(self): 

73 .... 

74 """ 

75 name = func.__name__ if method is None else method 

76 

77 def func_wrapper(self: MetadataTranslator) -> Any: 

78 if name not in self._translation_cache: 

79 self._translation_cache[name] = func(self) 

80 return self._translation_cache[name] 

81 

82 func_wrapper.__doc__ = func.__doc__ 

83 func_wrapper.__name__ = f"{name}_cached" 

84 return func_wrapper 

85 

86 

87class MetadataTranslator: 

88 """Per-instrument metadata translation support. 

89 

90 Parameters 

91 ---------- 

92 header : `dict`-like 

93 Representation of an instrument header that can be manipulated 

94 as if it was a `dict`. 

95 filename : `str`, optional 

96 Name of the file whose header is being translated. For some 

97 datasets with missing header information this can sometimes 

98 allow for some fixups in translations. 

99 """ 

100 

101 # These are all deliberately empty in the base class. 

102 name: str | None = None 

103 """The declared name of the translator.""" 

104 

105 default_search_path: Sequence[str] | None = None 

106 """Default search path to use to locate header correction files.""" 

107 

108 default_resource_package = __name__.split(".")[0] 

109 """Module name to use to locate the correction resources.""" 

110 

111 default_resource_root: str | None = None 

112 """Default package resource path root to use to locate header correction 

113 files within the ``default_resource_package`` package.""" 

114 

115 _trivial_map: dict[str, str | list[str] | tuple[Any, ...]] = {} 

116 """Dict of one-to-one mappings for header translation from standard 

117 property to corresponding keyword.""" 

118 

119 _const_map: dict[str, Any] = {} 

120 """Dict defining a constant for specified standard properties.""" 

121 

122 translators: dict[str, type[MetadataTranslator]] = dict() 

123 """All registered metadata translation classes.""" 

124 

125 supported_instrument: str | None = None 

126 """Name of instrument understood by this translation class.""" 

127 

128 all_properties: dict[str, PropertyDefinition] = {} 

129 """All the valid properties for this translator including extensions.""" 

130 

131 extensions: dict[str, PropertyDefinition] = {} 

132 """Extension properties (`str`: `PropertyDefinition`) 

133 

134 Some instruments have important properties beyond the standard set; this is 

135 the place to declare that they exist, and they will be treated in the same 

136 way as the standard set, except that their names will everywhere be 

137 prefixed with ``ext_``. 

138 

139 Each property is indexed by name (`str`), with a corresponding 

140 `PropertyDefinition`. 

141 """ 

142 

143 # Static typing requires that we define the standard dynamic properties 

144 # statically. 

145 if TYPE_CHECKING: 145 ↛ 146line 145 didn't jump to line 146, because the condition on line 145 was never true

146 to_telescope: ClassVar[Callable[[MetadataTranslator], str]] 

147 to_instrument: ClassVar[Callable[[MetadataTranslator], str]] 

148 to_location: ClassVar[Callable[[MetadataTranslator], astropy.coordinates.EarthLocation]] 

149 to_exposure_id: ClassVar[Callable[[MetadataTranslator], int]] 

150 to_visit_id: ClassVar[Callable[[MetadataTranslator], int]] 

151 to_physical_filter: ClassVar[Callable[[MetadataTranslator], str]] 

152 to_datetime_begin: ClassVar[Callable[[MetadataTranslator], astropy.time.Time]] 

153 to_datetime_end: ClassVar[Callable[[MetadataTranslator], astropy.time.Time]] 

154 to_exposure_time: ClassVar[Callable[[MetadataTranslator], u.Quantity]] 

155 to_dark_time: ClassVar[Callable[[MetadataTranslator], u.Quantity]] 

156 to_boresight_airmass: ClassVar[Callable[[MetadataTranslator], float]] 

157 to_boresight_rotation_angle: ClassVar[Callable[[MetadataTranslator], u.Quantity]] 

158 to_boresight_rotation_coord: ClassVar[Callable[[MetadataTranslator], str]] 

159 to_detector_num: ClassVar[Callable[[MetadataTranslator], int]] 

160 to_detector_name: ClassVar[Callable[[MetadataTranslator], str]] 

161 to_detector_serial: ClassVar[Callable[[MetadataTranslator], str]] 

162 to_detector_group: ClassVar[Callable[[MetadataTranslator], str | None]] 

163 to_detector_exposure_id: ClassVar[Callable[[MetadataTranslator], int]] 

164 to_object: ClassVar[Callable[[MetadataTranslator], str]] 

165 to_temperature: ClassVar[Callable[[MetadataTranslator], u.Quantity]] 

166 to_pressure: ClassVar[Callable[[MetadataTranslator], u.Quantity]] 

167 to_relative_humidity: ClassVar[Callable[[MetadataTranslator], float]] 

168 to_tracking_radec: ClassVar[Callable[[MetadataTranslator], astropy.coordinates.SkyCoord]] 

169 to_altaz_begin: ClassVar[Callable[[MetadataTranslator], astropy.coordinates.AltAz]] 

170 to_science_program: ClassVar[Callable[[MetadataTranslator], str]] 

171 to_observation_type: ClassVar[Callable[[MetadataTranslator], str]] 

172 to_observation_id: ClassVar[Callable[[MetadataTranslator], str]] 

173 

174 @classmethod 

175 def defined_in_this_class(cls, name: str) -> bool | None: 

176 """Report if the specified class attribute is defined specifically in 

177 this class. 

178 

179 Parameters 

180 ---------- 

181 name : `str` 

182 Name of the attribute to test. 

183 

184 Returns 

185 ------- 

186 in_class : `bool` 

187 `True` if there is a attribute of that name defined in this 

188 specific subclass. 

189 `False` if the method is not defined in this specific subclass 

190 but is defined in a parent class. 

191 Returns `None` if the attribute is not defined anywhere 

192 in the class hierarchy (which can happen if translators have 

193 typos in their mapping tables). 

194 

195 Notes 

196 ----- 

197 Retrieves the attribute associated with the given name. 

198 Then looks in all the parent classes to determine whether that 

199 attribute comes from a parent class or from the current class. 

200 Attributes are compared using :py:func:`id`. 

201 """ 

202 # The attribute to compare. 

203 if not hasattr(cls, name): 

204 return None 

205 attr_id = id(getattr(cls, name)) 

206 

207 # Get all the classes in the hierarchy 

208 mro = list(inspect.getmro(cls)) 

209 

210 # Remove the first entry from the list since that will be the 

211 # current class 

212 mro.pop(0) 

213 

214 for parent in mro: 

215 # Some attributes may only exist in subclasses. Skip base classes 

216 # that are missing the attribute (such as object). 

217 if hasattr(parent, name): 

218 if id(getattr(parent, name)) == attr_id: 

219 return False 

220 return True 

221 

222 @classmethod 

223 def _make_const_mapping(cls, property_key: str, constant: Any) -> Callable: 

224 """Make a translator method that returns a constant value. 

225 

226 Parameters 

227 ---------- 

228 property_key : `str` 

229 Name of the property to be calculated (for the docstring). 

230 constant : `str` or `numbers.Number` 

231 Value to return for this translator. 

232 

233 Returns 

234 ------- 

235 f : `~collections.abc.Callable` 

236 Function returning the constant. 

237 """ 

238 

239 def constant_translator(self: MetadataTranslator) -> Any: 

240 return constant 

241 

242 if property_key in cls.all_properties: 

243 property_doc = cls.all_properties[property_key].doc 

244 return_type = cls.all_properties[property_key].py_type 

245 else: 

246 return_type = type(constant) 

247 property_doc = f"Returns constant value for '{property_key}' property" 

248 

249 if return_type.__module__ == "builtins": 

250 full_name = return_type.__name__ 

251 else: 

252 full_name = f"{return_type.__module__}.{return_type.__qualname__}" 

253 

254 constant_translator.__doc__ = f"""{property_doc} 

255 

256 Returns 

257 ------- 

258 translation : `{full_name}` 

259 Translated property. 

260 """ 

261 return constant_translator 

262 

263 @classmethod 

264 def _make_trivial_mapping( 

265 cls, 

266 property_key: str, 

267 header_key: str | Sequence[str], 

268 default: Any | None = None, 

269 minimum: Any | None = None, 

270 maximum: Any | None = None, 

271 unit: astropy.unit.Unit | None = None, 

272 checker: Callable | None = None, 

273 ) -> Callable: 

274 """Make a translator method returning a header value. 

275 

276 The header value can be converted to a `~astropy.units.Quantity` 

277 if desired, and can also have its value validated. 

278 

279 See `MetadataTranslator.validate_value()` for details on the use 

280 of default parameters. 

281 

282 Parameters 

283 ---------- 

284 property_key : `str` 

285 Name of the translator to be constructed (for the docstring). 

286 header_key : `str` or `list` of `str` 

287 Name of the key to look up in the header. If a `list` each 

288 key will be tested in turn until one matches. This can deal with 

289 header styles that evolve over time. 

290 default : `numbers.Number` or `astropy.units.Quantity`, `str`, optional 

291 If not `None`, default value to be used if the parameter read from 

292 the header is not defined or if the header is missing. 

293 minimum : `numbers.Number` or `astropy.units.Quantity`, optional 

294 If not `None`, and if ``default`` is not `None`, minimum value 

295 acceptable for this parameter. 

296 maximum : `numbers.Number` or `astropy.units.Quantity`, optional 

297 If not `None`, and if ``default`` is not `None`, maximum value 

298 acceptable for this parameter. 

299 unit : `astropy.units.Unit`, optional 

300 If not `None`, the value read from the header will be converted 

301 to a `~astropy.units.Quantity`. Only supported for numeric values. 

302 checker : `~collections.abc.Callable`, optional 

303 Callback function to be used by the translator method in case the 

304 keyword is not present. Function will be executed as if it is 

305 a method of the translator class. Running without raising an 

306 exception will allow the default to be used. Should usually raise 

307 `KeyError`. 

308 

309 Returns 

310 ------- 

311 t : `~collections.abc.Callable` 

312 Function implementing a translator with the specified 

313 parameters. 

314 """ 

315 if property_key in cls.all_properties: 315 ↛ 319line 315 didn't jump to line 319, because the condition on line 315 was never false

316 property_doc = cls.all_properties[property_key].doc 

317 return_type = cls.all_properties[property_key].str_type 

318 else: 

319 return_type = "str` or `numbers.Number" 

320 property_doc = f"Map '{header_key}' header keyword to '{property_key}' property" 

321 

322 def trivial_translator(self: MetadataTranslator) -> Any: 

323 if unit is not None: 

324 q = self.quantity_from_card( 

325 header_key, unit, default=default, minimum=minimum, maximum=maximum, checker=checker 

326 ) 

327 # Convert to Angle if this quantity is an angle 

328 if return_type == "astropy.coordinates.Angle": 

329 q = Angle(q) 

330 return q 

331 

332 keywords = header_key if isinstance(header_key, list) else [header_key] 

333 for key in keywords: 

334 if self.is_key_ok(key): 

335 value = self._header[key] 

336 if default is not None and not isinstance(value, str): 

337 value = self.validate_value(value, default, minimum=minimum, maximum=maximum) 

338 self._used_these_cards(key) 

339 break 

340 else: 

341 # No keywords found, use default, checking first, or raise 

342 # A None default is only allowed if a checker is provided. 

343 if checker is not None: 

344 try: 

345 checker(self) 

346 except Exception: 

347 raise KeyError(f"Could not find {keywords} in header") 

348 return default 

349 elif default is not None: 

350 value = default 

351 else: 

352 raise KeyError(f"Could not find {keywords} in header") 

353 

354 # If we know this is meant to be a string, force to a string. 

355 # Sometimes headers represent items as integers which generically 

356 # we want as strings (eg OBSID). Sometimes also floats are 

357 # written as "NaN" strings. 

358 casts = {"str": str, "float": float, "int": int} 

359 if return_type in casts and not isinstance(value, casts[return_type]) and value is not None: 

360 value = casts[return_type](value) 

361 

362 return value 

363 

364 # Docstring inheritance means it is confusing to specify here 

365 # exactly which header value is being used. 

366 trivial_translator.__doc__ = f"""{property_doc} 

367 

368 Returns 

369 ------- 

370 translation : `{return_type}` 

371 Translated value derived from the header. 

372 """ 

373 return trivial_translator 

374 

375 @classmethod 

376 def __init_subclass__(cls, **kwargs: Any) -> None: 

377 """Register all subclasses with the base class and create dynamic 

378 translator methods. 

379 

380 The method provides two facilities. Firstly, every subclass 

381 of `MetadataTranslator` that includes a ``name`` class property is 

382 registered as a translator class that could be selected when automatic 

383 header translation is attempted. Only name translator subclasses that 

384 correspond to a complete instrument. Translation classes providing 

385 generic translation support for multiple instrument translators should 

386 not be named. 

387 

388 The second feature of this method is to convert simple translations 

389 to full translator methods. Sometimes a translation is fixed (for 

390 example a specific instrument name should be used) and rather than 

391 provide a full ``to_property()`` translation method the mapping can be 

392 defined in a class variable named ``_constMap``. Similarly, for 

393 one-to-one trivial mappings from a header to a property, 

394 ``_trivialMap`` can be defined. Trivial mappings are a dict mapping a 

395 generic property to either a header keyword, or a tuple consisting of 

396 the header keyword and a dict containing key value pairs suitable for 

397 the `MetadataTranslator.quantity_from_card` method. 

398 """ 

399 super().__init_subclass__(**kwargs) 

400 

401 # Only register classes with declared names 

402 if hasattr(cls, "name") and cls.name is not None: 

403 if cls.name in MetadataTranslator.translators: 403 ↛ 404line 403 didn't jump to line 404, because the condition on line 403 was never true

404 log.warning( 

405 "%s: Replacing %s translator with %s", 

406 cls.name, 

407 MetadataTranslator.translators[cls.name], 

408 cls, 

409 ) 

410 MetadataTranslator.translators[cls.name] = cls 

411 

412 # Check that we have not inherited constant/trivial mappings from 

413 # parent class that we have already applied. Empty maps are always 

414 # assumed okay 

415 const_map = cls._const_map if cls._const_map and cls.defined_in_this_class("_const_map") else {} 

416 trivial_map = ( 

417 cls._trivial_map if cls._trivial_map and cls.defined_in_this_class("_trivial_map") else {} 

418 ) 

419 

420 # Check for shadowing 

421 trivials = set(trivial_map.keys()) 

422 constants = set(const_map.keys()) 

423 both = trivials & constants 

424 if both: 424 ↛ 425line 424 didn't jump to line 425, because the condition on line 424 was never true

425 log.warning("%s: defined in both const_map and trivial_map: %s", cls.__name__, ", ".join(both)) 

426 

427 all = trivials | constants 

428 for name in all: 

429 if cls.defined_in_this_class(f"to_{name}"): 429 ↛ 432line 429 didn't jump to line 432, because the condition on line 429 was never true

430 # Must be one of trivial or constant. If in both then constant 

431 # overrides trivial. 

432 location = "by _trivial_map" 

433 if name in constants: 

434 location = "by _const_map" 

435 log.warning( 

436 "%s: %s is defined explicitly but will be replaced %s", cls.__name__, name, location 

437 ) 

438 

439 properties = set(PROPERTIES) | {"ext_" + pp for pp in cls.extensions} 

440 cls.all_properties = dict(PROPERTIES) 

441 cls.all_properties.update(cls.extensions) 

442 

443 # Go through the trival mappings for this class and create 

444 # corresponding translator methods 

445 for property_key, header_key in trivial_map.items(): 

446 kwargs = {} 

447 if type(header_key) == tuple: 

448 kwargs = header_key[1] 

449 header_key = header_key[0] 

450 translator = cls._make_trivial_mapping(property_key, header_key, **kwargs) 

451 method = f"to_{property_key}" 

452 translator.__name__ = f"{method}_trivial_in_{cls.__name__}" 

453 setattr(cls, method, cache_translation(translator, method=method)) 

454 if property_key not in properties: 454 ↛ 455line 454 didn't jump to line 455, because the condition on line 454 was never true

455 log.warning(f"Unexpected trivial translator for '{property_key}' defined in {cls}") 

456 

457 # Go through the constant mappings for this class and create 

458 # corresponding translator methods 

459 for property_key, constant in const_map.items(): 

460 translator = cls._make_const_mapping(property_key, constant) 

461 method = f"to_{property_key}" 

462 translator.__name__ = f"{method}_constant_in_{cls.__name__}" 

463 setattr(cls, method, translator) 

464 if property_key not in properties: 464 ↛ 465line 464 didn't jump to line 465, because the condition on line 464 was never true

465 log.warning(f"Unexpected constant translator for '{property_key}' defined in {cls}") 

466 

467 def __init__(self, header: Mapping[str, Any], filename: str | None = None) -> None: 

468 self._header = header 

469 self.filename = filename 

470 self._used_cards: set[str] = set() 

471 

472 # Prefix to use for warnings about failed translations 

473 self._log_prefix_cache: str | None = None 

474 

475 # Cache assumes header is read-only once stored in object 

476 self._translation_cache: dict[str, Any] = {} 

477 

478 @classmethod 

479 @abstractmethod 

480 def can_translate(cls, header: Mapping[str, Any], filename: str | None = None) -> bool: 

481 """Indicate whether this translation class can translate the 

482 supplied header. 

483 

484 Parameters 

485 ---------- 

486 header : `dict`-like 

487 Header to convert to standardized form. 

488 filename : `str`, optional 

489 Name of file being translated. 

490 

491 Returns 

492 ------- 

493 can : `bool` 

494 `True` if the header is recognized by this class. `False` 

495 otherwise. 

496 """ 

497 raise NotImplementedError() 

498 

499 @classmethod 

500 def can_translate_with_options( 

501 cls, header: Mapping[str, Any], options: dict[str, Any], filename: str | None = None 

502 ) -> bool: 

503 """Determine if a header can be translated with different criteria. 

504 

505 Parameters 

506 ---------- 

507 header : `dict`-like 

508 Header to convert to standardized form. 

509 options : `dict` 

510 Headers to try to determine whether this header can 

511 be translated by this class. If a card is found it will 

512 be compared with the expected value and will return that 

513 comparison. Each card will be tried in turn until one is 

514 found. 

515 filename : `str`, optional 

516 Name of file being translated. 

517 

518 Returns 

519 ------- 

520 can : `bool` 

521 `True` if the header is recognized by this class. `False` 

522 otherwise. 

523 

524 Notes 

525 ----- 

526 Intended to be used from within `can_translate` implementations 

527 for specific translators. Is not intended to be called directly 

528 from `determine_translator`. 

529 """ 

530 for card, value in options.items(): 

531 if card in header: 

532 return header[card] == value 

533 return False 

534 

535 @classmethod 

536 def determine_translator( 

537 cls, header: Mapping[str, Any], filename: str | None = None 

538 ) -> type[MetadataTranslator]: 

539 """Determine a translation class by examining the header. 

540 

541 Parameters 

542 ---------- 

543 header : `dict`-like 

544 Representation of a header. 

545 filename : `str`, optional 

546 Name of file being translated. 

547 

548 Returns 

549 ------- 

550 translator : `MetadataTranslator` 

551 Translation class that knows how to extract metadata from 

552 the supplied header. 

553 

554 Raises 

555 ------ 

556 ValueError 

557 None of the registered translation classes understood the supplied 

558 header. 

559 """ 

560 file_msg = "" 

561 if filename is not None: 

562 file_msg = f" from {filename}" 

563 for name, trans in cls.translators.items(): 

564 if trans.can_translate(header, filename=filename): 

565 log.debug("Using translation class %s%s", name, file_msg) 

566 return trans 

567 else: 

568 raise ValueError( 

569 f"None of the registered translation classes {list(cls.translators.keys())}" 

570 f" understood this header{file_msg}" 

571 ) 

572 

573 @classmethod 

574 def translator_version(cls) -> str: 

575 """Return the version string for this translator class. 

576 

577 Returns 

578 ------- 

579 version : `str` 

580 String identifying the version of this translator. 

581 

582 Notes 

583 ----- 

584 Assumes that the version is available from the ``__version__`` 

585 variable in the parent module. If this is not the case a translator 

586 should subclass this method. 

587 """ 

588 if cls in _VERSION_CACHE: 

589 return _VERSION_CACHE[cls] 

590 

591 version = "unknown" 

592 module_name = cls.__module__ 

593 components = module_name.split(".") 

594 while components: 

595 # This class has already been imported so importing it 

596 # should work. 

597 module = importlib.import_module(".".join(components)) 

598 if hasattr(module, v := "__version__"): 

599 version = getattr(module, v) 

600 if version == "unknown": 

601 # LSST software will have a fingerprint 

602 if hasattr(module, v := "__fingerprint__"): 

603 version = getattr(module, v) 

604 break 

605 else: 

606 # Remove last component from module name and try again 

607 components.pop() 

608 

609 _VERSION_CACHE[cls] = version 

610 return version 

611 

612 @classmethod 

613 def fix_header( 

614 cls, header: MutableMapping[str, Any], instrument: str, obsid: str, filename: str | None = None 

615 ) -> bool: 

616 """Apply global fixes to a supplied header. 

617 

618 Parameters 

619 ---------- 

620 header : `dict` 

621 The header to correct. Correction is in place. 

622 instrument : `str` 

623 The name of the instrument. 

624 obsid : `str` 

625 Unique observation identifier associated with this header. 

626 Will always be provided. 

627 filename : `str`, optional 

628 Filename associated with this header. May not be set since headers 

629 can be fixed independently of any filename being known. 

630 

631 Returns 

632 ------- 

633 modified : `bool` 

634 `True` if a correction was applied. 

635 

636 Notes 

637 ----- 

638 This method is intended to support major discrepancies in headers 

639 such as: 

640 

641 * Periods of time where headers are known to be incorrect in some 

642 way that can be fixed either by deriving the correct value from 

643 the existing value or understanding the that correction is static 

644 for the given time. This requires that the date header is 

645 known. 

646 * The presence of a certain value is always wrong and should be 

647 corrected with a new static value regardless of date. 

648 

649 It is assumed that one off problems with headers have been applied 

650 before this method is called using the per-obsid correction system. 

651 

652 Usually called from `astro_metadata_translator.fix_header`. 

653 

654 For log messages, do not assume that the filename will be present. 

655 Always write log messages to fall back on using the ``obsid`` if 

656 ``filename`` is `None`. 

657 """ 

658 return False 

659 

660 @staticmethod 

661 def _construct_log_prefix(obsid: str, filename: str | None = None) -> str: 

662 """Construct a log prefix string from the obsid and filename. 

663 

664 Parameters 

665 ---------- 

666 obsid : `str` 

667 The observation identifier. 

668 filename : `str`, optional 

669 The filename associated with the header being translated. 

670 Can be `None`. 

671 """ 

672 if filename: 

673 return f"{filename}({obsid})" 

674 return obsid 

675 

676 @property 

677 def _log_prefix(self) -> str: 

678 """Return standard prefix that can be used for log messages to report 

679 useful context. 

680 

681 Will be either the filename and obsid, or just the obsid depending 

682 on whether a filename is known. 

683 

684 Returns 

685 ------- 

686 prefix : `str` 

687 The prefix to use. 

688 """ 

689 if self._log_prefix_cache is None: 

690 # Protect against the unfortunate event of the obsid failing to 

691 # be calculated. This should be rare but should not prevent a log 

692 # message from appearing. 

693 try: 

694 obsid = self.to_observation_id() 

695 except Exception: 

696 obsid = "unknown_obsid" 

697 self._log_prefix_cache = self._construct_log_prefix(obsid, self.filename) 

698 return self._log_prefix_cache 

699 

700 def _used_these_cards(self, *args: str) -> None: 

701 """Indicate that the supplied cards have been used for translation. 

702 

703 Parameters 

704 ---------- 

705 args : sequence of `str` 

706 Keywords used to process a translation. 

707 """ 

708 self._used_cards.update(set(args)) 

709 

710 def cards_used(self) -> frozenset[str]: 

711 """Cards used during metadata extraction. 

712 

713 Returns 

714 ------- 

715 used : `frozenset` of `str` 

716 Cards used when extracting metadata. 

717 """ 

718 return frozenset(self._used_cards) 

719 

720 @staticmethod 

721 def validate_value( 

722 value: float, default: float, minimum: float | None = None, maximum: float | None = None 

723 ) -> float: 

724 """Validate the supplied value, returning a new value if out of range. 

725 

726 Parameters 

727 ---------- 

728 value : `float` 

729 Value to be validated. 

730 default : `float` 

731 Default value to use if supplied value is invalid or out of range. 

732 Assumed to be in the same units as the value expected in the 

733 header. 

734 minimum : `float` 

735 Minimum possible valid value, optional. If the calculated value 

736 is below this value, the default value will be used. 

737 maximum : `float` 

738 Maximum possible valid value, optional. If the calculated value 

739 is above this value, the default value will be used. 

740 

741 Returns 

742 ------- 

743 value : `float` 

744 Either the supplied value, or a default value. 

745 """ 

746 if value is None or math.isnan(value): 

747 value = default 

748 else: 

749 if minimum is not None and value < minimum: 

750 value = default 

751 elif maximum is not None and value > maximum: 

752 value = default 

753 return value 

754 

755 @staticmethod 

756 def is_keyword_defined(header: Mapping[str, Any], keyword: str | None) -> bool: 

757 """Return `True` if the value associated with the named keyword is 

758 present in the supplied header and defined. 

759 

760 Parameters 

761 ---------- 

762 header : `dict`-lik 

763 Header to use as reference. 

764 keyword : `str` 

765 Keyword to check against header. 

766 

767 Returns 

768 ------- 

769 is_defined : `bool` 

770 `True` if the header is present and not-`None`. `False` otherwise. 

771 """ 

772 if keyword is None or keyword not in header: 

773 return False 

774 

775 if header[keyword] is None: 

776 return False 

777 

778 # Special case Astropy undefined value 

779 if isinstance(header[keyword], astropy.io.fits.card.Undefined): 

780 return False 

781 

782 return True 

783 

784 def resource_root(self) -> tuple[str | None, str | None]: 

785 """Return package resource to use to locate correction resources within 

786 an installed package. 

787 

788 Returns 

789 ------- 

790 resource_package : `str` 

791 Package resource name. `None` if no package resource are to be 

792 used. 

793 resource_root : `str` 

794 The name of the resource root. `None` if no package resources 

795 are to be used. 

796 """ 

797 return (self.default_resource_package, self.default_resource_root) 

798 

799 def search_paths(self) -> list[str]: 

800 """Search paths to use when searching for header fix up correction 

801 files. 

802 

803 Returns 

804 ------- 

805 paths : `list` 

806 Directory paths to search. Can be an empty list if no special 

807 directories are defined. 

808 

809 Notes 

810 ----- 

811 Uses the classes ``default_search_path`` property if defined. 

812 """ 

813 if self.default_search_path is not None: 

814 return [p for p in self.default_search_path] 

815 return [] 

816 

817 def is_key_ok(self, keyword: str | None) -> bool: 

818 """Return `True` if the value associated with the named keyword is 

819 present in this header and defined. 

820 

821 Parameters 

822 ---------- 

823 keyword : `str` 

824 Keyword to check against header. 

825 

826 Returns 

827 ------- 

828 is_ok : `bool` 

829 `True` if the header is present and not-`None`. `False` otherwise. 

830 """ 

831 return self.is_keyword_defined(self._header, keyword) 

832 

833 def are_keys_ok(self, keywords: Iterable[str]) -> bool: 

834 """Are the supplied keys all present and defined?. 

835 

836 Parameters 

837 ---------- 

838 keywords : iterable of `str` 

839 Keywords to test. 

840 

841 Returns 

842 ------- 

843 all_ok : `bool` 

844 `True` if all supplied keys are present and defined. 

845 """ 

846 for k in keywords: 

847 if not self.is_key_ok(k): 

848 return False 

849 return True 

850 

851 def quantity_from_card( 

852 self, 

853 keywords: str | Sequence[str], 

854 unit: u.Unit, 

855 default: float | None = None, 

856 minimum: float | None = None, 

857 maximum: float | None = None, 

858 checker: Callable | None = None, 

859 ) -> u.Quantity: 

860 """Calculate a Astropy Quantity from a header card and a unit. 

861 

862 Parameters 

863 ---------- 

864 keywords : `str` or `list` of `str` 

865 Keyword to use from header. If a list each keyword will be tried 

866 in turn until one matches. 

867 unit : `astropy.units.UnitBase` 

868 Unit of the item in the header. 

869 default : `float`, optional 

870 Default value to use if the header value is invalid. Assumed 

871 to be in the same units as the value expected in the header. If 

872 None, no default value is used. 

873 minimum : `float`, optional 

874 Minimum possible valid value, optional. If the calculated value 

875 is below this value, the default value will be used. 

876 maximum : `float`, optional 

877 Maximum possible valid value, optional. If the calculated value 

878 is above this value, the default value will be used. 

879 checker : `~collections.abc.Callable`, optional 

880 Callback function to be used by the translator method in case the 

881 keyword is not present. Function will be executed as if it is 

882 a method of the translator class. Running without raising an 

883 exception will allow the default to be used. Should usually raise 

884 `KeyError`. 

885 

886 Returns 

887 ------- 

888 q : `astropy.units.Quantity` 

889 Quantity representing the header value. 

890 

891 Raises 

892 ------ 

893 KeyError 

894 The supplied header key is not present. 

895 """ 

896 keyword_list = [keywords] if isinstance(keywords, str) else list(keywords) 

897 for k in keyword_list: 

898 if self.is_key_ok(k): 

899 value = self._header[k] 

900 keyword = k 

901 break 

902 else: 

903 if checker is not None: 

904 try: 

905 checker(self) 

906 value = default 

907 if value is not None: 

908 value = u.Quantity(value, unit=unit) 

909 return value 

910 except Exception: 

911 pass 

912 raise KeyError(f"Could not find {keywords} in header") 

913 if isinstance(value, str): 

914 # Sometimes the header has the wrong type in it but this must 

915 # be a number if we are creating a quantity. 

916 value = float(value) 

917 self._used_these_cards(keyword) 

918 if default is not None: 

919 value = self.validate_value(value, default, maximum=maximum, minimum=minimum) 

920 return u.Quantity(value, unit=unit) 

921 

922 def _join_keyword_values(self, keywords: Iterable[str], delim: str = "+") -> str: 

923 """Join values of all defined keywords with the specified delimiter. 

924 

925 Parameters 

926 ---------- 

927 keywords : iterable of `str` 

928 Keywords to look for in header. 

929 delim : `str`, optional 

930 Character to use to join the values together. 

931 

932 Returns 

933 ------- 

934 joined : `str` 

935 String formed from all the keywords found in the header with 

936 defined values joined by the delimiter. Empty string if no 

937 defined keywords found. 

938 """ 

939 values = [] 

940 for k in keywords: 

941 if self.is_key_ok(k): 

942 values.append(self._header[k]) 

943 self._used_these_cards(k) 

944 

945 if values: 

946 joined = delim.join(str(v) for v in values) 

947 else: 

948 joined = "" 

949 

950 return joined 

951 

952 @cache_translation 

953 def to_detector_unique_name(self) -> str: 

954 """Return a unique name for the detector. 

955 

956 Base class implementation attempts to combine ``detector_name`` with 

957 ``detector_group``. Group is only used if not `None`. 

958 

959 Can be over-ridden by specialist translator class. 

960 

961 Returns 

962 ------- 

963 name : `str` 

964 ``detector_group``_``detector_name`` if ``detector_group`` is 

965 defined, else the ``detector_name`` is assumed to be unique. 

966 If neither return a valid value an exception is raised. 

967 

968 Raises 

969 ------ 

970 NotImplementedError 

971 Raised if neither detector_name nor detector_group is defined. 

972 """ 

973 name = self.to_detector_name() 

974 group = self.to_detector_group() 

975 

976 if group is None and name is None: 

977 raise NotImplementedError("Can not determine unique name from detector_group and detector_name") 

978 

979 if group is not None: 

980 return f"{group}_{name}" 

981 

982 return name 

983 

984 @cache_translation 

985 def to_exposure_group(self) -> str | None: 

986 """Return the group label associated with this exposure. 

987 

988 Base class implementation returns the ``exposure_id`` in string 

989 form. A subclass may do something different. 

990 

991 Returns 

992 ------- 

993 name : `str` 

994 The ``exposure_id`` converted to a string. 

995 """ 

996 exposure_id = self.to_exposure_id() 

997 if exposure_id is None: 

998 # mypy does not think this can ever happen but play it safe 

999 # with subclasses. 

1000 return None # type: ignore 

1001 else: 

1002 return str(exposure_id) 

1003 

1004 @cache_translation 

1005 def to_observation_reason(self) -> str: 

1006 """Return the reason this observation was taken. 

1007 

1008 Base class implementation returns the ``science`` if the 

1009 ``observation_type`` is science, else ``unknown``. 

1010 A subclass may do something different. 

1011 

1012 Returns 

1013 ------- 

1014 name : `str` 

1015 The reason for this observation. 

1016 """ 

1017 obstype = self.to_observation_type() 

1018 if obstype == "science": 

1019 return "science" 

1020 return "unknown" 

1021 

1022 @cache_translation 

1023 def to_observing_day(self) -> int: 

1024 """Return the YYYYMMDD integer corresponding to the observing day. 

1025 

1026 Base class implementation uses the TAI date of the start of the 

1027 observation. 

1028 

1029 Returns 

1030 ------- 

1031 day : `int` 

1032 The observing day as an integer of form YYYYMMDD. If the header 

1033 is broken and is unable to obtain a date of observation, ``0`` 

1034 is returned and the assumption is made that the problem will 

1035 be caught elsewhere. 

1036 """ 

1037 datetime_begin = self.to_datetime_begin() 

1038 if datetime_begin is None: 

1039 return 0 

1040 return int(datetime_begin.tai.strftime("%Y%m%d")) 

1041 

1042 @cache_translation 

1043 def to_observation_counter(self) -> int: 

1044 """Return an integer corresponding to how this observation relates 

1045 to other observations. 

1046 

1047 Base class implementation returns ``0`` to indicate that it is not 

1048 known how an observatory will define a counter. Some observatories 

1049 may not use the concept, others may use a counter that increases 

1050 for every observation taken for that instrument, and others may 

1051 define it to be a counter within an observing day. 

1052 

1053 Returns 

1054 ------- 

1055 sequence : `int` 

1056 The observation counter. Always ``0`` for this implementation. 

1057 """ 

1058 return 0 

1059 

1060 @cache_translation 

1061 def to_group_counter_start(self) -> int: 

1062 """Return the observation counter of the observation that began 

1063 this group. 

1064 

1065 The definition of the relevant group is up to the metadata 

1066 translator. It can be the first observation in the exposure_group 

1067 or the first observation in the visit, but must be derivable 

1068 from the metadata of this observation. 

1069 

1070 Returns 

1071 ------- 

1072 counter : `int` 

1073 The observation counter for the start of the relevant group. 

1074 Default implementation always returns the observation counter 

1075 of this observation. 

1076 """ 

1077 return self.to_observation_counter() 

1078 

1079 @cache_translation 

1080 def to_group_counter_end(self) -> int: 

1081 """Return the observation counter of the observation that ends 

1082 this group. 

1083 

1084 The definition of the relevant group is up to the metadata 

1085 translator. It can be the last observation in the exposure_group 

1086 or the last observation in the visit, but must be derivable 

1087 from the metadata of this observation. It is of course possible 

1088 that the last observation in the group does not exist if a sequence 

1089 of observations was not completed. 

1090 

1091 Returns 

1092 ------- 

1093 counter : `int` 

1094 The observation counter for the end of the relevant group. 

1095 Default implementation always returns the observation counter 

1096 of this observation. 

1097 """ 

1098 return self.to_observation_counter() 

1099 

1100 @cache_translation 

1101 def to_has_simulated_content(self) -> bool: 

1102 """Return a boolean indicating whether any part of the observation 

1103 was simulated. 

1104 

1105 Returns 

1106 ------- 

1107 is_simulated : `bool` 

1108 `True` if this exposure has simulated content. This can be 

1109 if some parts of the metadata or data were simulated. Default 

1110 implementation always returns `False`. 

1111 """ 

1112 return False 

1113 

1114 @cache_translation 

1115 def to_focus_z(self) -> u.Quantity: 

1116 """Return a default defocal distance of 0.0 mm if there is no 

1117 keyword for defocal distance in the header. The default 

1118 keyword for defocal distance is ``FOCUSZ``. 

1119 

1120 Returns 

1121 ------- 

1122 focus_z: `astropy.units.Quantity` 

1123 The defocal distance from header or the 0.0mm default 

1124 """ 

1125 return 0.0 * u.mm 

1126 

1127 @classmethod 

1128 def determine_translatable_headers( 

1129 cls, filename: str, primary: MutableMapping[str, Any] | None = None 

1130 ) -> Iterator[MutableMapping[str, Any]]: 

1131 """Given a file return all the headers usable for metadata translation. 

1132 

1133 This method can optionally be given a header from the file. This 

1134 header will generally be the primary header or a merge of the first 

1135 two headers. 

1136 

1137 In the base class implementation it is assumed that 

1138 this supplied header is the only useful header for metadata translation 

1139 and it will be returned unchanged if given. This can avoid 

1140 unnecessarily re-opening the file and re-reading the header when the 

1141 content is already known. 

1142 

1143 If no header is supplied, a header will be read from the supplied 

1144 file using `~.file_helpers.read_basic_metadata_from_file`, allowing it 

1145 to merge the primary and secondary header of a multi-extension FITS 

1146 file. Subclasses can read the header from the data file using whatever 

1147 technique is best for that instrument. 

1148 

1149 Subclasses can return multiple headers and ignore the externally 

1150 supplied header. They can also merge it with another header and return 

1151 a new derived header if that is required by the particular data file. 

1152 There is no requirement for the supplied header to be used. 

1153 

1154 Parameters 

1155 ---------- 

1156 filename : `str` 

1157 Path to a file in a format understood by this translator. 

1158 primary : `dict`-like, optional 

1159 The primary header obtained by the caller. This is sometimes 

1160 already known, for example if a system is trying to bootstrap 

1161 without already knowing what data is in the file. For many 

1162 instruments where the primary header is the only relevant 

1163 header, the primary header will be returned with no further 

1164 action. 

1165 

1166 Yields 

1167 ------ 

1168 headers : iterator of `dict`-like 

1169 A header usable for metadata translation. For this base 

1170 implementation it will be either the supplied primary header 

1171 or a header read from the file. This implementation will only 

1172 ever yield a single header. 

1173 

1174 Notes 

1175 ----- 

1176 Each translator class can have code specifically tailored to its 

1177 own file format. It is important not to call this method with 

1178 an incorrect translator class. The normal paradigm is for the 

1179 caller to have read the first header and then called 

1180 `determine_translator()` on the result to work out which translator 

1181 class to then call to obtain the real headers to be used for 

1182 translation. 

1183 """ 

1184 if primary is not None: 

1185 yield primary 

1186 else: 

1187 # Prevent circular import by deferring 

1188 from .file_helpers import read_basic_metadata_from_file 

1189 

1190 # Merge primary and secondary header if they exist. 

1191 header = read_basic_metadata_from_file(filename, -1) 

1192 assert header is not None # for mypy since can_raise=True 

1193 yield header 

1194 

1195 

1196def _make_abstract_translator_method( 

1197 property: str, doc: str, return_typedoc: str, return_type: type 

1198) -> Callable: 

1199 """Create a an abstract translation method for this property. 

1200 

1201 Parameters 

1202 ---------- 

1203 property : `str` 

1204 Name of the translator for property to be created. 

1205 doc : `str` 

1206 Description of the property. 

1207 return_typedoc : `str` 

1208 Type string of this property (used in the doc string). 

1209 return_type : `type` 

1210 Type of this property. 

1211 

1212 Returns 

1213 ------- 

1214 m : `~collections.abc.Callable` 

1215 Translator method for this property. 

1216 """ 

1217 

1218 def to_property(self: MetadataTranslator) -> None: 

1219 raise NotImplementedError(f"Translator for '{property}' undefined.") 

1220 

1221 to_property.__doc__ = f"""Return value of {property} from headers. 

1222 

1223 {doc} 

1224 

1225 Returns 

1226 ------- 

1227 {property} : `{return_typedoc}` 

1228 The translated property. 

1229 """ 

1230 return to_property 

1231 

1232 

1233# Make abstract methods for all the translators methods. 

1234# Unfortunately registering them as abstractmethods does not work 

1235# as these assignments come after the class has been created. 

1236# Assigning to __abstractmethods__ directly does work but interacts 

1237# poorly with the metaclass automatically generating methods from 

1238# _trivialMap and _constMap. 

1239# Note that subclasses that provide extension properties are assumed to not 

1240# need abstract methods created for them. 

1241 

1242# Allow for concrete translator methods to exist in the base class 

1243# These translator methods can be defined in terms of other properties 

1244CONCRETE = set() 

1245 

1246for name, definition in PROPERTIES.items(): 

1247 method = f"to_{name}" 

1248 if not MetadataTranslator.defined_in_this_class(method): 

1249 setattr( 

1250 MetadataTranslator, 

1251 f"to_{name}", 

1252 abstractmethod( 

1253 _make_abstract_translator_method( 

1254 name, definition.doc, definition.str_type, definition.py_type 

1255 ) 

1256 ), 

1257 ) 

1258 else: 

1259 CONCRETE.add(method) 

1260 

1261 

1262class StubTranslator(MetadataTranslator): 

1263 """Translator where all the translations are stubbed out and issue 

1264 warnings. 

1265 

1266 This translator can be used as a base class whilst developing a new 

1267 translator. It allows testing to proceed without being required to fully 

1268 define all translation methods. Once complete the class should be 

1269 removed from the inheritance tree. 

1270 

1271 """ 

1272 

1273 pass 

1274 

1275 

1276def _make_forwarded_stub_translator_method( 

1277 cls: type[MetadataTranslator], property: str, doc: str, return_typedoc: str, return_type: type 

1278) -> Callable: 

1279 """Create a stub translation method for this property that calls the 

1280 base method and catches `NotImplementedError`. 

1281 

1282 Parameters 

1283 ---------- 

1284 cls : `type` 

1285 Class to use when referencing `super()`. This would usually be 

1286 `StubTranslator`. 

1287 property : `str` 

1288 Name of the translator for property to be created. 

1289 doc : `str` 

1290 Description of the property. 

1291 return_typedoc : `str` 

1292 Type string of this property (used in the doc string). 

1293 return_type : `type` 

1294 Type of this property. 

1295 

1296 Returns 

1297 ------- 

1298 m : `~collections.abc.Callable` 

1299 Stub translator method for this property. 

1300 """ 

1301 method = f"to_{property}" 

1302 

1303 def to_stub(self: MetadataTranslator) -> Any: 

1304 parent = getattr(super(cls, self), method, None) 

1305 try: 

1306 if parent is not None: 

1307 return parent() 

1308 except NotImplementedError: 

1309 pass 

1310 

1311 warnings.warn( 

1312 f"Please implement translator for property '{property}' for translator {self}", stacklevel=3 

1313 ) 

1314 return None 

1315 

1316 to_stub.__doc__ = f"""Unimplemented forwarding translator for {property}. 

1317 

1318 {doc} 

1319 

1320 Calls the base class translation method and if that fails with 

1321 `NotImplementedError` issues a warning reminding the implementer to 

1322 override this method. 

1323 

1324 Returns 

1325 ------- 

1326 {property} : `None` or `{return_typedoc}` 

1327 Always returns `None`. 

1328 """ 

1329 return to_stub 

1330 

1331 

1332# Create stub translation methods for each property. These stubs warn 

1333# rather than fail and should be overridden by translators. 

1334for name, description in PROPERTIES.items(): 

1335 setattr( 

1336 StubTranslator, 

1337 f"to_{name}", 

1338 _make_forwarded_stub_translator_method( 

1339 StubTranslator, name, definition.doc, definition.str_type, definition.py_type # type: ignore 

1340 ), 

1341 )