Coverage for python/astro_metadata_translator/translator.py: 41%

388 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-27 02:38 -0700

1# This file is part of astro_metadata_translator. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the LICENSE file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12"""Classes and support code for metadata translation""" 

13 

14from __future__ import annotations 

15 

16__all__ = ("MetadataTranslator", "StubTranslator", "cache_translation") 

17 

18import importlib 

19import inspect 

20import logging 

21import math 

22import warnings 

23from abc import abstractmethod 

24from collections.abc import Callable, Iterable, Iterator, Mapping, MutableMapping, Sequence 

25from typing import TYPE_CHECKING, Any, ClassVar 

26 

27import astropy.io.fits.card 

28import astropy.units as u 

29from astropy.coordinates import Angle 

30 

31from .properties import PROPERTIES, PropertyDefinition 

32 

33if TYPE_CHECKING: 33 ↛ 34line 33 didn't jump to line 34, because the condition on line 33 was never true

34 import astropy.coordinates 

35 import astropy.time 

36 

37log = logging.getLogger(__name__) 

38 

39# Location of the root of the corrections resource files 

40CORRECTIONS_RESOURCE_ROOT = "corrections" 

41 

42"""Cache of version strings indexed by class.""" 

43_VERSION_CACHE: dict[type, str] = dict() 

44 

45 

46def cache_translation(func: Callable, method: str | None = None) -> Callable: 

47 """Decorator to cache the result of a translation method. 

48 

49 Especially useful when a translation uses many other translation 

50 methods. Should be used only on ``to_x()`` methods. 

51 

52 Parameters 

53 ---------- 

54 func : `function` 

55 Translation method to cache. 

56 method : `str`, optional 

57 Name of the translation method to cache. Not needed if the decorator 

58 is used around a normal method, but necessary when the decorator is 

59 being used in a metaclass. 

60 

61 Returns 

62 ------- 

63 wrapped : `function` 

64 Method wrapped by the caching function. 

65 """ 

66 name = func.__name__ if method is None else method 

67 

68 def func_wrapper(self: MetadataTranslator) -> Any: 

69 if name not in self._translation_cache: 

70 self._translation_cache[name] = func(self) 

71 return self._translation_cache[name] 

72 

73 func_wrapper.__doc__ = func.__doc__ 

74 func_wrapper.__name__ = f"{name}_cached" 

75 return func_wrapper 

76 

77 

78class MetadataTranslator: 

79 """Per-instrument metadata translation support 

80 

81 Parameters 

82 ---------- 

83 header : `dict`-like 

84 Representation of an instrument header that can be manipulated 

85 as if it was a `dict`. 

86 filename : `str`, optional 

87 Name of the file whose header is being translated. For some 

88 datasets with missing header information this can sometimes 

89 allow for some fixups in translations. 

90 """ 

91 

92 # These are all deliberately empty in the base class. 

93 name: str | None = None 

94 """The declared name of the translator.""" 

95 

96 default_search_path: Sequence[str] | None = None 

97 """Default search path to use to locate header correction files.""" 

98 

99 default_resource_package = __name__.split(".")[0] 

100 """Module name to use to locate the correction resources.""" 

101 

102 default_resource_root: str | None = None 

103 """Default package resource path root to use to locate header correction 

104 files within the ``default_resource_package`` package.""" 

105 

106 _trivial_map: dict[str, str | list[str] | tuple[Any, ...]] = {} 

107 """Dict of one-to-one mappings for header translation from standard 

108 property to corresponding keyword.""" 

109 

110 _const_map: dict[str, Any] = {} 

111 """Dict defining a constant for specified standard properties.""" 

112 

113 translators: dict[str, type[MetadataTranslator]] = dict() 

114 """All registered metadata translation classes.""" 

115 

116 supported_instrument: str | None = None 

117 """Name of instrument understood by this translation class.""" 

118 

119 all_properties: dict[str, PropertyDefinition] = {} 

120 """All the valid properties for this translator including extensions.""" 

121 

122 extensions: dict[str, PropertyDefinition] = {} 

123 """Extension properties (`str`: `PropertyDefinition`) 

124 

125 Some instruments have important properties beyond the standard set; this is 

126 the place to declare that they exist, and they will be treated in the same 

127 way as the standard set, except that their names will everywhere be 

128 prefixed with ``ext_``. 

129 

130 Each property is indexed by name (`str`), with a corresponding 

131 `PropertyDefinition`. 

132 """ 

133 

134 # Static typing requires that we define the standard dynamic properties 

135 # statically. 

136 if TYPE_CHECKING: 136 ↛ 137line 136 didn't jump to line 137, because the condition on line 136 was never true

137 to_telescope: ClassVar[Callable[[MetadataTranslator], str]] 

138 to_instrument: ClassVar[Callable[[MetadataTranslator], str]] 

139 to_location: ClassVar[Callable[[MetadataTranslator], astropy.coordinates.EarthLocation]] 

140 to_exposure_id: ClassVar[Callable[[MetadataTranslator], int]] 

141 to_visit_id: ClassVar[Callable[[MetadataTranslator], int]] 

142 to_physical_filter: ClassVar[Callable[[MetadataTranslator], str]] 

143 to_datetime_begin: ClassVar[Callable[[MetadataTranslator], astropy.time.Time]] 

144 to_datetime_end: ClassVar[Callable[[MetadataTranslator], astropy.time.Time]] 

145 to_exposure_time: ClassVar[Callable[[MetadataTranslator], u.Quantity]] 

146 to_dark_time: ClassVar[Callable[[MetadataTranslator], u.Quantity]] 

147 to_boresight_airmass: ClassVar[Callable[[MetadataTranslator], float]] 

148 to_boresight_rotation_angle: ClassVar[Callable[[MetadataTranslator], u.Quantity]] 

149 to_boresight_rotation_coord: ClassVar[Callable[[MetadataTranslator], str]] 

150 to_detector_num: ClassVar[Callable[[MetadataTranslator], int]] 

151 to_detector_name: ClassVar[Callable[[MetadataTranslator], str]] 

152 to_detector_serial: ClassVar[Callable[[MetadataTranslator], str]] 

153 to_detector_group: ClassVar[Callable[[MetadataTranslator], str | None]] 

154 to_detector_exposure_id: ClassVar[Callable[[MetadataTranslator], int]] 

155 to_object: ClassVar[Callable[[MetadataTranslator], str]] 

156 to_temperature: ClassVar[Callable[[MetadataTranslator], u.Quantity]] 

157 to_pressure: ClassVar[Callable[[MetadataTranslator], u.Quantity]] 

158 to_relative_humidity: ClassVar[Callable[[MetadataTranslator], float]] 

159 to_tracking_radec: ClassVar[Callable[[MetadataTranslator], astropy.coordinates.SkyCoord]] 

160 to_altaz_begin: ClassVar[Callable[[MetadataTranslator], astropy.coordinates.AltAz]] 

161 to_science_program: ClassVar[Callable[[MetadataTranslator], str]] 

162 to_observation_type: ClassVar[Callable[[MetadataTranslator], str]] 

163 to_observation_id: ClassVar[Callable[[MetadataTranslator], str]] 

164 

165 @classmethod 

166 def defined_in_this_class(cls, name: str) -> bool | None: 

167 """Report if the specified class attribute is defined specifically in 

168 this class. 

169 

170 Parameters 

171 ---------- 

172 name : `str` 

173 Name of the attribute to test. 

174 

175 Returns 

176 ------- 

177 in_class : `bool` 

178 `True` if there is a attribute of that name defined in this 

179 specific subclass. 

180 `False` if the method is not defined in this specific subclass 

181 but is defined in a parent class. 

182 Returns `None` if the attribute is not defined anywhere 

183 in the class hierarchy (which can happen if translators have 

184 typos in their mapping tables). 

185 

186 Notes 

187 ----- 

188 Retrieves the attribute associated with the given name. 

189 Then looks in all the parent classes to determine whether that 

190 attribute comes from a parent class or from the current class. 

191 Attributes are compared using `id()`. 

192 """ 

193 # The attribute to compare. 

194 if not hasattr(cls, name): 

195 return None 

196 attr_id = id(getattr(cls, name)) 

197 

198 # Get all the classes in the hierarchy 

199 mro = list(inspect.getmro(cls)) 

200 

201 # Remove the first entry from the list since that will be the 

202 # current class 

203 mro.pop(0) 

204 

205 for parent in mro: 

206 # Some attributes may only exist in subclasses. Skip base classes 

207 # that are missing the attribute (such as object). 

208 if hasattr(parent, name): 

209 if id(getattr(parent, name)) == attr_id: 

210 return False 

211 return True 

212 

213 @classmethod 

214 def _make_const_mapping(cls, property_key: str, constant: Any) -> Callable: 

215 """Make a translator method that returns a constant value. 

216 

217 Parameters 

218 ---------- 

219 property_key : `str` 

220 Name of the property to be calculated (for the docstring). 

221 constant : `str` or `numbers.Number` 

222 Value to return for this translator. 

223 

224 Returns 

225 ------- 

226 f : `function` 

227 Function returning the constant. 

228 """ 

229 

230 def constant_translator(self: MetadataTranslator) -> Any: 

231 return constant 

232 

233 if property_key in cls.all_properties: 

234 property_doc = cls.all_properties[property_key].doc 

235 return_type = cls.all_properties[property_key].py_type 

236 else: 

237 return_type = type(constant) 

238 property_doc = f"Returns constant value for '{property_key}' property" 

239 

240 constant_translator.__doc__ = f"""{property_doc} 

241 

242 Returns 

243 ------- 

244 translation : `{return_type}` 

245 Translated property. 

246 """ 

247 return constant_translator 

248 

249 @classmethod 

250 def _make_trivial_mapping( 

251 cls, 

252 property_key: str, 

253 header_key: str | Sequence[str], 

254 default: Any | None = None, 

255 minimum: Any | None = None, 

256 maximum: Any | None = None, 

257 unit: astropy.unit.Unit | None = None, 

258 checker: Callable | None = None, 

259 ) -> Callable: 

260 """Make a translator method returning a header value. 

261 

262 The header value can be converted to a `~astropy.units.Quantity` 

263 if desired, and can also have its value validated. 

264 

265 See `MetadataTranslator.validate_value()` for details on the use 

266 of default parameters. 

267 

268 Parameters 

269 ---------- 

270 property_key : `str` 

271 Name of the translator to be constructed (for the docstring). 

272 header_key : `str` or `list` of `str` 

273 Name of the key to look up in the header. If a `list` each 

274 key will be tested in turn until one matches. This can deal with 

275 header styles that evolve over time. 

276 default : `numbers.Number` or `astropy.units.Quantity`, `str`, optional 

277 If not `None`, default value to be used if the parameter read from 

278 the header is not defined or if the header is missing. 

279 minimum : `numbers.Number` or `astropy.units.Quantity`, optional 

280 If not `None`, and if ``default`` is not `None`, minimum value 

281 acceptable for this parameter. 

282 maximum : `numbers.Number` or `astropy.units.Quantity`, optional 

283 If not `None`, and if ``default`` is not `None`, maximum value 

284 acceptable for this parameter. 

285 unit : `astropy.units.Unit`, optional 

286 If not `None`, the value read from the header will be converted 

287 to a `~astropy.units.Quantity`. Only supported for numeric values. 

288 checker : `function`, optional 

289 Callback function to be used by the translator method in case the 

290 keyword is not present. Function will be executed as if it is 

291 a method of the translator class. Running without raising an 

292 exception will allow the default to be used. Should usually raise 

293 `KeyError`. 

294 

295 Returns 

296 ------- 

297 t : `function` 

298 Function implementing a translator with the specified 

299 parameters. 

300 """ 

301 if property_key in cls.all_properties: 301 ↛ 305line 301 didn't jump to line 305, because the condition on line 301 was never false

302 property_doc = cls.all_properties[property_key].doc 

303 return_type = cls.all_properties[property_key].str_type 

304 else: 

305 return_type = "str` or `numbers.Number" 

306 property_doc = f"Map '{header_key}' header keyword to '{property_key}' property" 

307 

308 def trivial_translator(self: MetadataTranslator) -> Any: 

309 if unit is not None: 

310 q = self.quantity_from_card( 

311 header_key, unit, default=default, minimum=minimum, maximum=maximum, checker=checker 

312 ) 

313 # Convert to Angle if this quantity is an angle 

314 if return_type == "astropy.coordinates.Angle": 

315 q = Angle(q) 

316 return q 

317 

318 keywords = header_key if isinstance(header_key, list) else [header_key] 

319 for key in keywords: 

320 if self.is_key_ok(key): 

321 value = self._header[key] 

322 if default is not None and not isinstance(value, str): 

323 value = self.validate_value(value, default, minimum=minimum, maximum=maximum) 

324 self._used_these_cards(key) 

325 break 

326 else: 

327 # No keywords found, use default, checking first, or raise 

328 # A None default is only allowed if a checker is provided. 

329 if checker is not None: 

330 try: 

331 checker(self) 

332 except Exception: 

333 raise KeyError(f"Could not find {keywords} in header") 

334 return default 

335 elif default is not None: 

336 value = default 

337 else: 

338 raise KeyError(f"Could not find {keywords} in header") 

339 

340 # If we know this is meant to be a string, force to a string. 

341 # Sometimes headers represent items as integers which generically 

342 # we want as strings (eg OBSID). Sometimes also floats are 

343 # written as "NaN" strings. 

344 casts = {"str": str, "float": float, "int": int} 

345 if return_type in casts and not isinstance(value, casts[return_type]) and value is not None: 

346 value = casts[return_type](value) 

347 

348 return value 

349 

350 # Docstring inheritance means it is confusing to specify here 

351 # exactly which header value is being used. 

352 trivial_translator.__doc__ = f"""{property_doc} 

353 

354 Returns 

355 ------- 

356 translation : `{return_type}` 

357 Translated value derived from the header. 

358 """ 

359 return trivial_translator 

360 

361 @classmethod 

362 def __init_subclass__(cls, **kwargs: Any) -> None: 

363 """Register all subclasses with the base class and create dynamic 

364 translator methods. 

365 

366 The method provides two facilities. Firstly, every subclass 

367 of `MetadataTranslator` that includes a ``name`` class property is 

368 registered as a translator class that could be selected when automatic 

369 header translation is attempted. Only name translator subclasses that 

370 correspond to a complete instrument. Translation classes providing 

371 generic translation support for multiple instrument translators should 

372 not be named. 

373 

374 The second feature of this method is to convert simple translations 

375 to full translator methods. Sometimes a translation is fixed (for 

376 example a specific instrument name should be used) and rather than 

377 provide a full ``to_property()`` translation method the mapping can be 

378 defined in a class variable named ``_constMap``. Similarly, for 

379 one-to-one trivial mappings from a header to a property, 

380 ``_trivialMap`` can be defined. Trivial mappings are a dict mapping a 

381 generic property to either a header keyword, or a tuple consisting of 

382 the header keyword and a dict containing key value pairs suitable for 

383 the `MetadataTranslator.quantity_from_card()` method. 

384 """ 

385 super().__init_subclass__(**kwargs) 

386 

387 # Only register classes with declared names 

388 if hasattr(cls, "name") and cls.name is not None: 

389 if cls.name in MetadataTranslator.translators: 389 ↛ 390line 389 didn't jump to line 390, because the condition on line 389 was never true

390 log.warning( 

391 "%s: Replacing %s translator with %s", 

392 cls.name, 

393 MetadataTranslator.translators[cls.name], 

394 cls, 

395 ) 

396 MetadataTranslator.translators[cls.name] = cls 

397 

398 # Check that we have not inherited constant/trivial mappings from 

399 # parent class that we have already applied. Empty maps are always 

400 # assumed okay 

401 const_map = cls._const_map if cls._const_map and cls.defined_in_this_class("_const_map") else {} 

402 trivial_map = ( 

403 cls._trivial_map if cls._trivial_map and cls.defined_in_this_class("_trivial_map") else {} 

404 ) 

405 

406 # Check for shadowing 

407 trivials = set(trivial_map.keys()) 

408 constants = set(const_map.keys()) 

409 both = trivials & constants 

410 if both: 410 ↛ 411line 410 didn't jump to line 411, because the condition on line 410 was never true

411 log.warning("%s: defined in both const_map and trivial_map: %s", cls.__name__, ", ".join(both)) 

412 

413 all = trivials | constants 

414 for name in all: 

415 if cls.defined_in_this_class(f"to_{name}"): 415 ↛ 418line 415 didn't jump to line 418, because the condition on line 415 was never true

416 # Must be one of trivial or constant. If in both then constant 

417 # overrides trivial. 

418 location = "by _trivial_map" 

419 if name in constants: 

420 location = "by _const_map" 

421 log.warning( 

422 "%s: %s is defined explicitly but will be replaced %s", cls.__name__, name, location 

423 ) 

424 

425 properties = set(PROPERTIES) | {"ext_" + pp for pp in cls.extensions} 

426 cls.all_properties = dict(PROPERTIES) 

427 cls.all_properties.update(cls.extensions) 

428 

429 # Go through the trival mappings for this class and create 

430 # corresponding translator methods 

431 for property_key, header_key in trivial_map.items(): 

432 kwargs = {} 

433 if type(header_key) == tuple: 

434 kwargs = header_key[1] 

435 header_key = header_key[0] 

436 translator = cls._make_trivial_mapping(property_key, header_key, **kwargs) 

437 method = f"to_{property_key}" 

438 translator.__name__ = f"{method}_trivial_in_{cls.__name__}" 

439 setattr(cls, method, cache_translation(translator, method=method)) 

440 if property_key not in properties: 440 ↛ 441line 440 didn't jump to line 441, because the condition on line 440 was never true

441 log.warning(f"Unexpected trivial translator for '{property_key}' defined in {cls}") 

442 

443 # Go through the constant mappings for this class and create 

444 # corresponding translator methods 

445 for property_key, constant in const_map.items(): 

446 translator = cls._make_const_mapping(property_key, constant) 

447 method = f"to_{property_key}" 

448 translator.__name__ = f"{method}_constant_in_{cls.__name__}" 

449 setattr(cls, method, translator) 

450 if property_key not in properties: 450 ↛ 451line 450 didn't jump to line 451, because the condition on line 450 was never true

451 log.warning(f"Unexpected constant translator for '{property_key}' defined in {cls}") 

452 

453 def __init__(self, header: Mapping[str, Any], filename: str | None = None) -> None: 

454 self._header = header 

455 self.filename = filename 

456 self._used_cards: set[str] = set() 

457 

458 # Prefix to use for warnings about failed translations 

459 self._log_prefix_cache: str | None = None 

460 

461 # Cache assumes header is read-only once stored in object 

462 self._translation_cache: dict[str, Any] = {} 

463 

464 @classmethod 

465 @abstractmethod 

466 def can_translate(cls, header: Mapping[str, Any], filename: str | None = None) -> bool: 

467 """Indicate whether this translation class can translate the 

468 supplied header. 

469 

470 Parameters 

471 ---------- 

472 header : `dict`-like 

473 Header to convert to standardized form. 

474 filename : `str`, optional 

475 Name of file being translated. 

476 

477 Returns 

478 ------- 

479 can : `bool` 

480 `True` if the header is recognized by this class. `False` 

481 otherwise. 

482 """ 

483 raise NotImplementedError() 

484 

485 @classmethod 

486 def can_translate_with_options( 

487 cls, header: Mapping[str, Any], options: dict[str, Any], filename: str | None = None 

488 ) -> bool: 

489 """Helper method for `can_translate` allowing options. 

490 

491 Parameters 

492 ---------- 

493 header : `dict`-like 

494 Header to convert to standardized form. 

495 options : `dict` 

496 Headers to try to determine whether this header can 

497 be translated by this class. If a card is found it will 

498 be compared with the expected value and will return that 

499 comparison. Each card will be tried in turn until one is 

500 found. 

501 filename : `str`, optional 

502 Name of file being translated. 

503 

504 Returns 

505 ------- 

506 can : `bool` 

507 `True` if the header is recognized by this class. `False` 

508 otherwise. 

509 

510 Notes 

511 ----- 

512 Intended to be used from within `can_translate` implementations 

513 for specific translators. Is not intended to be called directly 

514 from `determine_translator`. 

515 """ 

516 for card, value in options.items(): 

517 if card in header: 

518 return header[card] == value 

519 return False 

520 

521 @classmethod 

522 def determine_translator( 

523 cls, header: Mapping[str, Any], filename: str | None = None 

524 ) -> type[MetadataTranslator]: 

525 """Determine a translation class by examining the header 

526 

527 Parameters 

528 ---------- 

529 header : `dict`-like 

530 Representation of a header. 

531 filename : `str`, optional 

532 Name of file being translated. 

533 

534 Returns 

535 ------- 

536 translator : `MetadataTranslator` 

537 Translation class that knows how to extract metadata from 

538 the supplied header. 

539 

540 Raises 

541 ------ 

542 ValueError 

543 None of the registered translation classes understood the supplied 

544 header. 

545 """ 

546 file_msg = "" 

547 if filename is not None: 

548 file_msg = f" from {filename}" 

549 for name, trans in cls.translators.items(): 

550 if trans.can_translate(header, filename=filename): 

551 log.debug("Using translation class %s%s", name, file_msg) 

552 return trans 

553 else: 

554 raise ValueError( 

555 f"None of the registered translation classes {list(cls.translators.keys())}" 

556 f" understood this header{file_msg}" 

557 ) 

558 

559 @classmethod 

560 def translator_version(cls) -> str: 

561 """Return the version string for this translator class. 

562 

563 Returns 

564 ------- 

565 version : `str` 

566 String identifying the version of this translator. 

567 

568 Notes 

569 ----- 

570 Assumes that the version is available from the ``__version__`` 

571 variable in the parent module. If this is not the case a translator 

572 should subclass this method. 

573 """ 

574 if cls in _VERSION_CACHE: 

575 return _VERSION_CACHE[cls] 

576 

577 version = "unknown" 

578 module_name = cls.__module__ 

579 components = module_name.split(".") 

580 while components: 

581 # This class has already been imported so importing it 

582 # should work. 

583 module = importlib.import_module(".".join(components)) 

584 if hasattr(module, v := "__version__"): 

585 version = getattr(module, v) 

586 if version == "unknown": 

587 # LSST software will have a fingerprint 

588 if hasattr(module, v := "__fingerprint__"): 

589 version = getattr(module, v) 

590 break 

591 else: 

592 # Remove last component from module name and try again 

593 components.pop() 

594 

595 _VERSION_CACHE[cls] = version 

596 return version 

597 

598 @classmethod 

599 def fix_header( 

600 cls, header: MutableMapping[str, Any], instrument: str, obsid: str, filename: str | None = None 

601 ) -> bool: 

602 """Apply global fixes to a supplied header. 

603 

604 Parameters 

605 ---------- 

606 header : `dict` 

607 The header to correct. Correction is in place. 

608 instrument : `str` 

609 The name of the instrument. 

610 obsid : `str` 

611 Unique observation identifier associated with this header. 

612 Will always be provided. 

613 filename : `str`, optional 

614 Filename associated with this header. May not be set since headers 

615 can be fixed independently of any filename being known. 

616 

617 Returns 

618 ------- 

619 modified : `bool` 

620 `True` if a correction was applied. 

621 

622 Notes 

623 ----- 

624 This method is intended to support major discrepancies in headers 

625 such as: 

626 

627 * Periods of time where headers are known to be incorrect in some 

628 way that can be fixed either by deriving the correct value from 

629 the existing value or understanding the that correction is static 

630 for the given time. This requires that the date header is 

631 known. 

632 * The presence of a certain value is always wrong and should be 

633 corrected with a new static value regardless of date. 

634 

635 It is assumed that one off problems with headers have been applied 

636 before this method is called using the per-obsid correction system. 

637 

638 Usually called from `astro_metadata_translator.fix_header`. 

639 

640 For log messages, do not assume that the filename will be present. 

641 Always write log messages to fall back on using the ``obsid`` if 

642 ``filename`` is `None`. 

643 """ 

644 return False 

645 

646 @staticmethod 

647 def _construct_log_prefix(obsid: str, filename: str | None = None) -> str: 

648 """Construct a log prefix string from the obsid and filename. 

649 

650 Parameters 

651 ---------- 

652 obsid : `str` 

653 The observation identifier. 

654 filename : `str`, optional 

655 The filename associated with the header being translated. 

656 Can be `None`. 

657 """ 

658 if filename: 

659 return f"{filename}({obsid})" 

660 return obsid 

661 

662 @property 

663 def _log_prefix(self) -> str: 

664 """Standard prefix that can be used for log messages to report 

665 useful context. 

666 

667 Will be either the filename and obsid, or just the obsid depending 

668 on whether a filename is known. 

669 

670 Returns 

671 ------- 

672 prefix : `str` 

673 The prefix to use. 

674 """ 

675 if self._log_prefix_cache is None: 

676 # Protect against the unfortunate event of the obsid failing to 

677 # be calculated. This should be rare but should not prevent a log 

678 # message from appearing. 

679 try: 

680 obsid = self.to_observation_id() 

681 except Exception: 

682 obsid = "unknown_obsid" 

683 self._log_prefix_cache = self._construct_log_prefix(obsid, self.filename) 

684 return self._log_prefix_cache 

685 

686 def _used_these_cards(self, *args: str) -> None: 

687 """Indicate that the supplied cards have been used for translation. 

688 

689 Parameters 

690 ---------- 

691 args : sequence of `str` 

692 Keywords used to process a translation. 

693 """ 

694 self._used_cards.update(set(args)) 

695 

696 def cards_used(self) -> frozenset[str]: 

697 """Cards used during metadata extraction. 

698 

699 Returns 

700 ------- 

701 used : `frozenset` of `str` 

702 Cards used when extracting metadata. 

703 """ 

704 return frozenset(self._used_cards) 

705 

706 @staticmethod 

707 def validate_value( 

708 value: float, default: float, minimum: float | None = None, maximum: float | None = None 

709 ) -> float: 

710 """Validate the supplied value, returning a new value if out of range 

711 

712 Parameters 

713 ---------- 

714 value : `float` 

715 Value to be validated. 

716 default : `float` 

717 Default value to use if supplied value is invalid or out of range. 

718 Assumed to be in the same units as the value expected in the 

719 header. 

720 minimum : `float` 

721 Minimum possible valid value, optional. If the calculated value 

722 is below this value, the default value will be used. 

723 maximum : `float` 

724 Maximum possible valid value, optional. If the calculated value 

725 is above this value, the default value will be used. 

726 

727 Returns 

728 ------- 

729 value : `float` 

730 Either the supplied value, or a default value. 

731 """ 

732 if value is None or math.isnan(value): 

733 value = default 

734 else: 

735 if minimum is not None and value < minimum: 

736 value = default 

737 elif maximum is not None and value > maximum: 

738 value = default 

739 return value 

740 

741 @staticmethod 

742 def is_keyword_defined(header: Mapping[str, Any], keyword: str | None) -> bool: 

743 """Return `True` if the value associated with the named keyword is 

744 present in the supplied header and defined. 

745 

746 Parameters 

747 ---------- 

748 header : `dict`-lik 

749 Header to use as reference. 

750 keyword : `str` 

751 Keyword to check against header. 

752 

753 Returns 

754 ------- 

755 is_defined : `bool` 

756 `True` if the header is present and not-`None`. `False` otherwise. 

757 """ 

758 if keyword is None or keyword not in header: 

759 return False 

760 

761 if header[keyword] is None: 

762 return False 

763 

764 # Special case Astropy undefined value 

765 if isinstance(header[keyword], astropy.io.fits.card.Undefined): 

766 return False 

767 

768 return True 

769 

770 def resource_root(self) -> tuple[str | None, str | None]: 

771 """Package resource to use to locate correction resources within an 

772 installed package. 

773 

774 Returns 

775 ------- 

776 resource_package : `str` 

777 Package resource name. `None` if no package resource are to be 

778 used. 

779 resource_root : `str` 

780 The name of the resource root. `None` if no package resources 

781 are to be used. 

782 """ 

783 return (self.default_resource_package, self.default_resource_root) 

784 

785 def search_paths(self) -> list[str]: 

786 """Search paths to use when searching for header fix up correction 

787 files. 

788 

789 Returns 

790 ------- 

791 paths : `list` 

792 Directory paths to search. Can be an empty list if no special 

793 directories are defined. 

794 

795 Notes 

796 ----- 

797 Uses the classes ``default_search_path`` property if defined. 

798 """ 

799 if self.default_search_path is not None: 

800 return [p for p in self.default_search_path] 

801 return [] 

802 

803 def is_key_ok(self, keyword: str | None) -> bool: 

804 """Return `True` if the value associated with the named keyword is 

805 present in this header and defined. 

806 

807 Parameters 

808 ---------- 

809 keyword : `str` 

810 Keyword to check against header. 

811 

812 Returns 

813 ------- 

814 is_ok : `bool` 

815 `True` if the header is present and not-`None`. `False` otherwise. 

816 """ 

817 return self.is_keyword_defined(self._header, keyword) 

818 

819 def are_keys_ok(self, keywords: Iterable[str]) -> bool: 

820 """Are the supplied keys all present and defined? 

821 

822 Parameters 

823 ---------- 

824 keywords : iterable of `str` 

825 Keywords to test. 

826 

827 Returns 

828 ------- 

829 all_ok : `bool` 

830 `True` if all supplied keys are present and defined. 

831 """ 

832 for k in keywords: 

833 if not self.is_key_ok(k): 

834 return False 

835 return True 

836 

837 def quantity_from_card( 

838 self, 

839 keywords: str | Sequence[str], 

840 unit: u.Unit, 

841 default: float | None = None, 

842 minimum: float | None = None, 

843 maximum: float | None = None, 

844 checker: Callable | None = None, 

845 ) -> u.Quantity: 

846 """Calculate a Astropy Quantity from a header card and a unit. 

847 

848 Parameters 

849 ---------- 

850 keywords : `str` or `list` of `str` 

851 Keyword to use from header. If a list each keyword will be tried 

852 in turn until one matches. 

853 unit : `astropy.units.UnitBase` 

854 Unit of the item in the header. 

855 default : `float`, optional 

856 Default value to use if the header value is invalid. Assumed 

857 to be in the same units as the value expected in the header. If 

858 None, no default value is used. 

859 minimum : `float`, optional 

860 Minimum possible valid value, optional. If the calculated value 

861 is below this value, the default value will be used. 

862 maximum : `float`, optional 

863 Maximum possible valid value, optional. If the calculated value 

864 is above this value, the default value will be used. 

865 checker : `function`, optional 

866 Callback function to be used by the translator method in case the 

867 keyword is not present. Function will be executed as if it is 

868 a method of the translator class. Running without raising an 

869 exception will allow the default to be used. Should usually raise 

870 `KeyError`. 

871 

872 Returns 

873 ------- 

874 q : `astropy.units.Quantity` 

875 Quantity representing the header value. 

876 

877 Raises 

878 ------ 

879 KeyError 

880 The supplied header key is not present. 

881 """ 

882 keyword_list = [keywords] if isinstance(keywords, str) else list(keywords) 

883 for k in keyword_list: 

884 if self.is_key_ok(k): 

885 value = self._header[k] 

886 keyword = k 

887 break 

888 else: 

889 if checker is not None: 

890 try: 

891 checker(self) 

892 value = default 

893 if value is not None: 

894 value = u.Quantity(value, unit=unit) 

895 return value 

896 except Exception: 

897 pass 

898 raise KeyError(f"Could not find {keywords} in header") 

899 if isinstance(value, str): 

900 # Sometimes the header has the wrong type in it but this must 

901 # be a number if we are creating a quantity. 

902 value = float(value) 

903 self._used_these_cards(keyword) 

904 if default is not None: 

905 value = self.validate_value(value, default, maximum=maximum, minimum=minimum) 

906 return u.Quantity(value, unit=unit) 

907 

908 def _join_keyword_values(self, keywords: Iterable[str], delim: str = "+") -> str: 

909 """Join values of all defined keywords with the specified delimiter. 

910 

911 Parameters 

912 ---------- 

913 keywords : iterable of `str` 

914 Keywords to look for in header. 

915 delim : `str`, optional 

916 Character to use to join the values together. 

917 

918 Returns 

919 ------- 

920 joined : `str` 

921 String formed from all the keywords found in the header with 

922 defined values joined by the delimiter. Empty string if no 

923 defined keywords found. 

924 """ 

925 values = [] 

926 for k in keywords: 

927 if self.is_key_ok(k): 

928 values.append(self._header[k]) 

929 self._used_these_cards(k) 

930 

931 if values: 

932 joined = delim.join(str(v) for v in values) 

933 else: 

934 joined = "" 

935 

936 return joined 

937 

938 @cache_translation 

939 def to_detector_unique_name(self) -> str: 

940 """Return a unique name for the detector. 

941 

942 Base class implementation attempts to combine ``detector_name`` with 

943 ``detector_group``. Group is only used if not `None`. 

944 

945 Can be over-ridden by specialist translator class. 

946 

947 Returns 

948 ------- 

949 name : `str` 

950 ``detector_group``_``detector_name`` if ``detector_group`` is 

951 defined, else the ``detector_name`` is assumed to be unique. 

952 If neither return a valid value an exception is raised. 

953 

954 Raises 

955 ------ 

956 NotImplementedError 

957 Raised if neither detector_name nor detector_group is defined. 

958 """ 

959 name = self.to_detector_name() 

960 group = self.to_detector_group() 

961 

962 if group is None and name is None: 

963 raise NotImplementedError("Can not determine unique name from detector_group and detector_name") 

964 

965 if group is not None: 

966 return f"{group}_{name}" 

967 

968 return name 

969 

970 @cache_translation 

971 def to_exposure_group(self) -> str | None: 

972 """Return the group label associated with this exposure. 

973 

974 Base class implementation returns the ``exposure_id`` in string 

975 form. A subclass may do something different. 

976 

977 Returns 

978 ------- 

979 name : `str` 

980 The ``exposure_id`` converted to a string. 

981 """ 

982 exposure_id = self.to_exposure_id() 

983 if exposure_id is None: 

984 # mypy does not think this can ever happen but play it safe 

985 # with subclasses. 

986 return None # type: ignore 

987 else: 

988 return str(exposure_id) 

989 

990 @cache_translation 

991 def to_observation_reason(self) -> str: 

992 """Return the reason this observation was taken. 

993 

994 Base class implementation returns the ``science`` if the 

995 ``observation_type`` is science, else ``unknown``. 

996 A subclass may do something different. 

997 

998 Returns 

999 ------- 

1000 name : `str` 

1001 The reason for this observation. 

1002 """ 

1003 obstype = self.to_observation_type() 

1004 if obstype == "science": 

1005 return "science" 

1006 return "unknown" 

1007 

1008 @cache_translation 

1009 def to_observing_day(self) -> int: 

1010 """Return the YYYYMMDD integer corresponding to the observing day. 

1011 

1012 Base class implementation uses the TAI date of the start of the 

1013 observation. 

1014 

1015 Returns 

1016 ------- 

1017 day : `int` 

1018 The observing day as an integer of form YYYYMMDD. If the header 

1019 is broken and is unable to obtain a date of observation, ``0`` 

1020 is returned and the assumption is made that the problem will 

1021 be caught elsewhere. 

1022 """ 

1023 datetime_begin = self.to_datetime_begin() 

1024 if datetime_begin is None: 

1025 return 0 

1026 return int(datetime_begin.tai.strftime("%Y%m%d")) 

1027 

1028 @cache_translation 

1029 def to_observation_counter(self) -> int: 

1030 """Return an integer corresponding to how this observation relates 

1031 to other observations. 

1032 

1033 Base class implementation returns ``0`` to indicate that it is not 

1034 known how an observatory will define a counter. Some observatories 

1035 may not use the concept, others may use a counter that increases 

1036 for every observation taken for that instrument, and others may 

1037 define it to be a counter within an observing day. 

1038 

1039 Returns 

1040 ------- 

1041 sequence : `int` 

1042 The observation counter. Always ``0`` for this implementation. 

1043 """ 

1044 return 0 

1045 

1046 @cache_translation 

1047 def to_group_counter_start(self) -> int: 

1048 """Return the observation counter of the observation that began 

1049 this group. 

1050 

1051 The definition of the relevant group is up to the metadata 

1052 translator. It can be the first observation in the exposure_group 

1053 or the first observation in the visit, but must be derivable 

1054 from the metadata of this observation. 

1055 

1056 Returns 

1057 ------- 

1058 counter : `int` 

1059 The observation counter for the start of the relevant group. 

1060 Default implementation always returns the observation counter 

1061 of this observation. 

1062 """ 

1063 return self.to_observation_counter() 

1064 

1065 @cache_translation 

1066 def to_group_counter_end(self) -> int: 

1067 """Return the observation counter of the observation that ends 

1068 this group. 

1069 

1070 The definition of the relevant group is up to the metadata 

1071 translator. It can be the last observation in the exposure_group 

1072 or the last observation in the visit, but must be derivable 

1073 from the metadata of this observation. It is of course possible 

1074 that the last observation in the group does not exist if a sequence 

1075 of observations was not completed. 

1076 

1077 Returns 

1078 ------- 

1079 counter : `int` 

1080 The observation counter for the end of the relevant group. 

1081 Default implementation always returns the observation counter 

1082 of this observation. 

1083 """ 

1084 return self.to_observation_counter() 

1085 

1086 @cache_translation 

1087 def to_has_simulated_content(self) -> bool: 

1088 """Return a boolean indicating whether any part of the observation 

1089 was simulated. 

1090 

1091 Returns 

1092 ------- 

1093 is_simulated : `bool` 

1094 `True` if this exposure has simulated content. This can be 

1095 if some parts of the metadata or data were simulated. Default 

1096 implementation always returns `False`. 

1097 """ 

1098 return False 

1099 

1100 @cache_translation 

1101 def to_focus_z(self) -> u.Quantity: 

1102 """Return a default defocal distance of 0.0 mm if there is no 

1103 keyword for defocal distance in the header. The default 

1104 keyword for defocal distance is ``FOCUSZ``. 

1105 

1106 Returns 

1107 ------- 

1108 focus_z: `astropy.units.Quantity` 

1109 The defocal distance from header or the 0.0mm default 

1110 """ 

1111 return 0.0 * u.mm 

1112 

1113 @classmethod 

1114 def determine_translatable_headers( 

1115 cls, filename: str, primary: MutableMapping[str, Any] | None = None 

1116 ) -> Iterator[MutableMapping[str, Any]]: 

1117 """Given a file return all the headers usable for metadata translation. 

1118 

1119 This method can optionally be given a header from the file. This 

1120 header will generally be the primary header or a merge of the first 

1121 two headers. 

1122 

1123 In the base class implementation it is assumed that 

1124 this supplied header is the only useful header for metadata translation 

1125 and it will be returned unchanged if given. This can avoid 

1126 unnecesarily re-opening the file and re-reading the header when the 

1127 content is already known. 

1128 

1129 If no header is supplied, a header will be read from the supplied 

1130 file using `read_basic_metadata_from_file`, allowing it to merge 

1131 the primary and secondary header of a multi-extension FITS file. 

1132 Subclasses can read the header from the data file using whatever 

1133 technique is best for that instrument. 

1134 

1135 Subclasses can return multiple headers and ignore the externally 

1136 supplied header. They can also merge it with another header and return 

1137 a new derived header if that is required by the particular data file. 

1138 There is no requirement for the supplied header to be used. 

1139 

1140 Parameters 

1141 ---------- 

1142 filename : `str` 

1143 Path to a file in a format understood by this translator. 

1144 primary : `dict`-like, optional 

1145 The primary header obtained by the caller. This is sometimes 

1146 already known, for example if a system is trying to bootstrap 

1147 without already knowing what data is in the file. For many 

1148 instruments where the primary header is the only relevant 

1149 header, the primary header will be returned with no further 

1150 action. 

1151 

1152 Yields 

1153 ------ 

1154 headers : iterator of `dict`-like 

1155 A header usable for metadata translation. For this base 

1156 implementation it will be either the supplied primary header 

1157 or a header read from the file. This implementation will only 

1158 ever yield a single header. 

1159 

1160 Notes 

1161 ----- 

1162 Each translator class can have code specifically tailored to its 

1163 own file format. It is important not to call this method with 

1164 an incorrect translator class. The normal paradigm is for the 

1165 caller to have read the first header and then called 

1166 `determine_translator()` on the result to work out which translator 

1167 class to then call to obtain the real headers to be used for 

1168 translation. 

1169 """ 

1170 if primary is not None: 

1171 yield primary 

1172 else: 

1173 # Prevent circular import by deferring 

1174 from .file_helpers import read_basic_metadata_from_file 

1175 

1176 # Merge primary and secondary header if they exist. 

1177 header = read_basic_metadata_from_file(filename, -1) 

1178 assert header is not None # for mypy since can_raise=True 

1179 yield header 

1180 

1181 

1182def _make_abstract_translator_method( 

1183 property: str, doc: str, return_typedoc: str, return_type: type 

1184) -> Callable: 

1185 """Create a an abstract translation method for this property. 

1186 

1187 Parameters 

1188 ---------- 

1189 property : `str` 

1190 Name of the translator for property to be created. 

1191 doc : `str` 

1192 Description of the property. 

1193 return_typedoc : `str` 

1194 Type string of this property (used in the doc string). 

1195 return_type : `class` 

1196 Type of this property. 

1197 

1198 Returns 

1199 ------- 

1200 m : `function` 

1201 Translator method for this property. 

1202 """ 

1203 

1204 def to_property(self: MetadataTranslator) -> None: 

1205 raise NotImplementedError(f"Translator for '{property}' undefined.") 

1206 

1207 to_property.__doc__ = f"""Return value of {property} from headers. 

1208 

1209 {doc} 

1210 

1211 Returns 

1212 ------- 

1213 {property} : `{return_typedoc}` 

1214 The translated property. 

1215 """ 

1216 return to_property 

1217 

1218 

1219# Make abstract methods for all the translators methods. 

1220# Unfortunately registering them as abstractmethods does not work 

1221# as these assignments come after the class has been created. 

1222# Assigning to __abstractmethods__ directly does work but interacts 

1223# poorly with the metaclass automatically generating methods from 

1224# _trivialMap and _constMap. 

1225# Note that subclasses that provide extension properties are assumed to not 

1226# need abstract methods created for them. 

1227 

1228# Allow for concrete translator methods to exist in the base class 

1229# These translator methods can be defined in terms of other properties 

1230CONCRETE = set() 

1231 

1232for name, definition in PROPERTIES.items(): 

1233 method = f"to_{name}" 

1234 if not MetadataTranslator.defined_in_this_class(method): 

1235 setattr( 

1236 MetadataTranslator, 

1237 f"to_{name}", 

1238 abstractmethod( 

1239 _make_abstract_translator_method( 

1240 name, definition.doc, definition.str_type, definition.py_type 

1241 ) 

1242 ), 

1243 ) 

1244 else: 

1245 CONCRETE.add(method) 

1246 

1247 

1248class StubTranslator(MetadataTranslator): 

1249 """Translator where all the translations are stubbed out and issue 

1250 warnings. 

1251 

1252 This translator can be used as a base class whilst developing a new 

1253 translator. It allows testing to proceed without being required to fully 

1254 define all translation methods. Once complete the class should be 

1255 removed from the inheritance tree. 

1256 

1257 """ 

1258 

1259 pass 

1260 

1261 

1262def _make_forwarded_stub_translator_method( 

1263 cls: type[MetadataTranslator], property: str, doc: str, return_typedoc: str, return_type: type 

1264) -> Callable: 

1265 """Create a stub translation method for this property that calls the 

1266 base method and catches `NotImplementedError`. 

1267 

1268 Parameters 

1269 ---------- 

1270 cls : `class` 

1271 Class to use when referencing `super()`. This would usually be 

1272 `StubTranslator`. 

1273 property : `str` 

1274 Name of the translator for property to be created. 

1275 doc : `str` 

1276 Description of the property. 

1277 return_typedoc : `str` 

1278 Type string of this property (used in the doc string). 

1279 return_type : `class` 

1280 Type of this property. 

1281 

1282 Returns 

1283 ------- 

1284 m : `function` 

1285 Stub translator method for this property. 

1286 """ 

1287 method = f"to_{property}" 

1288 

1289 def to_stub(self: MetadataTranslator) -> Any: 

1290 parent = getattr(super(cls, self), method, None) 

1291 try: 

1292 if parent is not None: 

1293 return parent() 

1294 except NotImplementedError: 

1295 pass 

1296 

1297 warnings.warn( 

1298 f"Please implement translator for property '{property}' for translator {self}", stacklevel=3 

1299 ) 

1300 return None 

1301 

1302 to_stub.__doc__ = f"""Unimplemented forwarding translator for {property}. 

1303 

1304 {doc} 

1305 

1306 Calls the base class translation method and if that fails with 

1307 `NotImplementedError` issues a warning reminding the implementer to 

1308 override this method. 

1309 

1310 Returns 

1311 ------- 

1312 {property} : `None` or `{return_typedoc}` 

1313 Always returns `None`. 

1314 """ 

1315 return to_stub 

1316 

1317 

1318# Create stub translation methods for each property. These stubs warn 

1319# rather than fail and should be overridden by translators. 

1320for name, description in PROPERTIES.items(): 

1321 setattr( 

1322 StubTranslator, 

1323 f"to_{name}", 

1324 _make_forwarded_stub_translator_method( 

1325 StubTranslator, name, definition.doc, definition.str_type, definition.py_type # type: ignore 

1326 ), 

1327 )