Coverage for python/astro_metadata_translator/translator.py: 41%

387 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-08-18 18:35 +0000

1# This file is part of astro_metadata_translator. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the LICENSE file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12"""Classes and support code for metadata translation""" 

13 

14from __future__ import annotations 

15 

16__all__ = ("MetadataTranslator", "StubTranslator", "cache_translation") 

17 

18import importlib 

19import inspect 

20import logging 

21import math 

22import warnings 

23from abc import abstractmethod 

24from typing import ( 

25 TYPE_CHECKING, 

26 Any, 

27 Callable, 

28 Dict, 

29 FrozenSet, 

30 Iterable, 

31 Iterator, 

32 List, 

33 Mapping, 

34 MutableMapping, 

35 Optional, 

36 Sequence, 

37 Set, 

38 Tuple, 

39 Type, 

40 Union, 

41) 

42 

43import astropy.io.fits.card 

44import astropy.units as u 

45from astropy.coordinates import Angle 

46 

47from .properties import PROPERTIES, PropertyDefinition 

48 

49if TYPE_CHECKING: 49 ↛ 50line 49 didn't jump to line 50, because the condition on line 49 was never true

50 import astropy.coordinates 

51 import astropy.time 

52 

53log = logging.getLogger(__name__) 

54 

55# Location of the root of the corrections resource files 

56CORRECTIONS_RESOURCE_ROOT = "corrections" 

57 

58"""Cache of version strings indexed by class.""" 

59_VERSION_CACHE: Dict[Type, str] = dict() 

60 

61 

62def cache_translation(func: Callable, method: Optional[str] = None) -> Callable: 

63 """Decorator to cache the result of a translation method. 

64 

65 Especially useful when a translation uses many other translation 

66 methods. Should be used only on ``to_x()`` methods. 

67 

68 Parameters 

69 ---------- 

70 func : `function` 

71 Translation method to cache. 

72 method : `str`, optional 

73 Name of the translation method to cache. Not needed if the decorator 

74 is used around a normal method, but necessary when the decorator is 

75 being used in a metaclass. 

76 

77 Returns 

78 ------- 

79 wrapped : `function` 

80 Method wrapped by the caching function. 

81 """ 

82 name = func.__name__ if method is None else method 

83 

84 def func_wrapper(self: MetadataTranslator) -> Any: 

85 if name not in self._translation_cache: 

86 self._translation_cache[name] = func(self) 

87 return self._translation_cache[name] 

88 

89 func_wrapper.__doc__ = func.__doc__ 

90 func_wrapper.__name__ = f"{name}_cached" 

91 return func_wrapper 

92 

93 

94class MetadataTranslator: 

95 """Per-instrument metadata translation support 

96 

97 Parameters 

98 ---------- 

99 header : `dict`-like 

100 Representation of an instrument header that can be manipulated 

101 as if it was a `dict`. 

102 filename : `str`, optional 

103 Name of the file whose header is being translated. For some 

104 datasets with missing header information this can sometimes 

105 allow for some fixups in translations. 

106 """ 

107 

108 # These are all deliberately empty in the base class. 

109 name: Optional[str] = None 

110 """The declared name of the translator.""" 

111 

112 default_search_path: Optional[Sequence[str]] = None 

113 """Default search path to use to locate header correction files.""" 

114 

115 default_resource_package = __name__.split(".")[0] 

116 """Module name to use to locate the correction resources.""" 

117 

118 default_resource_root: Optional[str] = None 

119 """Default package resource path root to use to locate header correction 

120 files within the ``default_resource_package`` package.""" 

121 

122 _trivial_map: Dict[str, Union[str, List[str], Tuple[Any, ...]]] = {} 

123 """Dict of one-to-one mappings for header translation from standard 

124 property to corresponding keyword.""" 

125 

126 _const_map: Dict[str, Any] = {} 

127 """Dict defining a constant for specified standard properties.""" 

128 

129 translators: Dict[str, Type] = dict() 

130 """All registered metadata translation classes.""" 

131 

132 supported_instrument: Optional[str] = None 

133 """Name of instrument understood by this translation class.""" 

134 

135 all_properties: Dict[str, PropertyDefinition] = {} 

136 """All the valid properties for this translator including extensions.""" 

137 

138 extensions: Dict[str, PropertyDefinition] = {} 

139 """Extension properties (`str`: `PropertyDefinition`) 

140 

141 Some instruments have important properties beyond the standard set; this is 

142 the place to declare that they exist, and they will be treated in the same 

143 way as the standard set, except that their names will everywhere be 

144 prefixed with ``ext_``. 

145 

146 Each property is indexed by name (`str`), with a corresponding 

147 `PropertyDefinition`. 

148 """ 

149 

150 # Static typing requires that we define the standard dynamic properties 

151 # statically. 

152 if TYPE_CHECKING: 152 ↛ 153line 152 didn't jump to line 153, because the condition on line 152 was never true

153 to_telescope: Callable[[MetadataTranslator], str] 

154 to_instrument: Callable[[MetadataTranslator], str] 

155 to_location: Callable[[MetadataTranslator], astropy.coordinates.EarthLocation] 

156 to_exposure_id: Callable[[MetadataTranslator], int] 

157 to_visit_id: Callable[[MetadataTranslator], int] 

158 to_physical_filter: Callable[[MetadataTranslator], str] 

159 to_datetime_begin: Callable[[MetadataTranslator], astropy.time.Time] 

160 to_datetime_end: Callable[[MetadataTranslator], astropy.time.Time] 

161 to_exposure_time: Callable[[MetadataTranslator], u.Quantity] 

162 to_dark_time: Callable[[MetadataTranslator], u.Quantity] 

163 to_boresight_airmass: Callable[[MetadataTranslator], float] 

164 to_boresight_rotation_angle: Callable[[MetadataTranslator], u.Quantity] 

165 to_boresight_rotation_coord: Callable[[MetadataTranslator], str] 

166 to_detector_num: Callable[[MetadataTranslator], int] 

167 to_detector_name: Callable[[MetadataTranslator], str] 

168 to_detector_serial: Callable[[MetadataTranslator], str] 

169 to_detector_group: Callable[[MetadataTranslator], Optional[str]] 

170 to_detector_exposure_id: Callable[[MetadataTranslator], int] 

171 to_object: Callable[[MetadataTranslator], str] 

172 to_temperature: Callable[[MetadataTranslator], u.Quantity] 

173 to_pressure: Callable[[MetadataTranslator], u.Quantity] 

174 to_relative_humidity: Callable[[MetadataTranslator], float] 

175 to_tracking_radec: Callable[[MetadataTranslator], astropy.coordinates.SkyCoord] 

176 to_altaz_begin: Callable[[MetadataTranslator], astropy.coordinates.AltAz] 

177 to_science_program: Callable[[MetadataTranslator], str] 

178 to_observation_type: Callable[[MetadataTranslator], str] 

179 to_observation_id: Callable[[MetadataTranslator], str] 

180 

181 @classmethod 

182 def defined_in_this_class(cls, name: str) -> Optional[bool]: 

183 """Report if the specified class attribute is defined specifically in 

184 this class. 

185 

186 Parameters 

187 ---------- 

188 name : `str` 

189 Name of the attribute to test. 

190 

191 Returns 

192 ------- 

193 in_class : `bool` 

194 `True` if there is a attribute of that name defined in this 

195 specific subclass. 

196 `False` if the method is not defined in this specific subclass 

197 but is defined in a parent class. 

198 Returns `None` if the attribute is not defined anywhere 

199 in the class hierarchy (which can happen if translators have 

200 typos in their mapping tables). 

201 

202 Notes 

203 ----- 

204 Retrieves the attribute associated with the given name. 

205 Then looks in all the parent classes to determine whether that 

206 attribute comes from a parent class or from the current class. 

207 Attributes are compared using `id()`. 

208 """ 

209 # The attribute to compare. 

210 if not hasattr(cls, name): 

211 return None 

212 attr_id = id(getattr(cls, name)) 

213 

214 # Get all the classes in the hierarchy 

215 mro = list(inspect.getmro(cls)) 

216 

217 # Remove the first entry from the list since that will be the 

218 # current class 

219 mro.pop(0) 

220 

221 for parent in mro: 

222 # Some attributes may only exist in subclasses. Skip base classes 

223 # that are missing the attribute (such as object). 

224 if hasattr(parent, name): 

225 if id(getattr(parent, name)) == attr_id: 

226 return False 

227 return True 

228 

229 @classmethod 

230 def _make_const_mapping(cls, property_key: str, constant: Any) -> Callable: 

231 """Make a translator method that returns a constant value. 

232 

233 Parameters 

234 ---------- 

235 property_key : `str` 

236 Name of the property to be calculated (for the docstring). 

237 constant : `str` or `numbers.Number` 

238 Value to return for this translator. 

239 

240 Returns 

241 ------- 

242 f : `function` 

243 Function returning the constant. 

244 """ 

245 

246 def constant_translator(self: MetadataTranslator) -> Any: 

247 return constant 

248 

249 if property_key in cls.all_properties: 

250 property_doc = cls.all_properties[property_key].doc 

251 return_type = cls.all_properties[property_key].py_type 

252 else: 

253 return_type = type(constant) 

254 property_doc = f"Returns constant value for '{property_key}' property" 

255 

256 constant_translator.__doc__ = f"""{property_doc} 

257 

258 Returns 

259 ------- 

260 translation : `{return_type}` 

261 Translated property. 

262 """ 

263 return constant_translator 

264 

265 @classmethod 

266 def _make_trivial_mapping( 

267 cls, 

268 property_key: str, 

269 header_key: Union[str, Sequence[str]], 

270 default: Optional[Any] = None, 

271 minimum: Optional[Any] = None, 

272 maximum: Optional[Any] = None, 

273 unit: Optional[astropy.unit.Unit] = None, 

274 checker: Optional[Callable] = None, 

275 ) -> Callable: 

276 """Make a translator method returning a header value. 

277 

278 The header value can be converted to a `~astropy.units.Quantity` 

279 if desired, and can also have its value validated. 

280 

281 See `MetadataTranslator.validate_value()` for details on the use 

282 of default parameters. 

283 

284 Parameters 

285 ---------- 

286 property_key : `str` 

287 Name of the translator to be constructed (for the docstring). 

288 header_key : `str` or `list` of `str` 

289 Name of the key to look up in the header. If a `list` each 

290 key will be tested in turn until one matches. This can deal with 

291 header styles that evolve over time. 

292 default : `numbers.Number` or `astropy.units.Quantity`, `str`, optional 

293 If not `None`, default value to be used if the parameter read from 

294 the header is not defined or if the header is missing. 

295 minimum : `numbers.Number` or `astropy.units.Quantity`, optional 

296 If not `None`, and if ``default`` is not `None`, minimum value 

297 acceptable for this parameter. 

298 maximum : `numbers.Number` or `astropy.units.Quantity`, optional 

299 If not `None`, and if ``default`` is not `None`, maximum value 

300 acceptable for this parameter. 

301 unit : `astropy.units.Unit`, optional 

302 If not `None`, the value read from the header will be converted 

303 to a `~astropy.units.Quantity`. Only supported for numeric values. 

304 checker : `function`, optional 

305 Callback function to be used by the translator method in case the 

306 keyword is not present. Function will be executed as if it is 

307 a method of the translator class. Running without raising an 

308 exception will allow the default to be used. Should usually raise 

309 `KeyError`. 

310 

311 Returns 

312 ------- 

313 t : `function` 

314 Function implementing a translator with the specified 

315 parameters. 

316 """ 

317 if property_key in cls.all_properties: 317 ↛ 321line 317 didn't jump to line 321, because the condition on line 317 was never false

318 property_doc = cls.all_properties[property_key].doc 

319 return_type = cls.all_properties[property_key].str_type 

320 else: 

321 return_type = "str` or `numbers.Number" 

322 property_doc = f"Map '{header_key}' header keyword to '{property_key}' property" 

323 

324 def trivial_translator(self: MetadataTranslator) -> Any: 

325 if unit is not None: 

326 q = self.quantity_from_card( 

327 header_key, unit, default=default, minimum=minimum, maximum=maximum, checker=checker 

328 ) 

329 # Convert to Angle if this quantity is an angle 

330 if return_type == "astropy.coordinates.Angle": 

331 q = Angle(q) 

332 return q 

333 

334 keywords = header_key if isinstance(header_key, list) else [header_key] 

335 for key in keywords: 

336 if self.is_key_ok(key): 

337 value = self._header[key] 

338 if default is not None and not isinstance(value, str): 

339 value = self.validate_value(value, default, minimum=minimum, maximum=maximum) 

340 self._used_these_cards(key) 

341 break 

342 else: 

343 # No keywords found, use default, checking first, or raise 

344 # A None default is only allowed if a checker is provided. 

345 if checker is not None: 

346 try: 

347 checker(self) 

348 except Exception: 

349 raise KeyError(f"Could not find {keywords} in header") 

350 return default 

351 elif default is not None: 

352 value = default 

353 else: 

354 raise KeyError(f"Could not find {keywords} in header") 

355 

356 # If we know this is meant to be a string, force to a string. 

357 # Sometimes headers represent items as integers which generically 

358 # we want as strings (eg OBSID). Sometimes also floats are 

359 # written as "NaN" strings. 

360 casts = {"str": str, "float": float, "int": int} 

361 if return_type in casts and not isinstance(value, casts[return_type]) and value is not None: 

362 value = casts[return_type](value) 

363 

364 return value 

365 

366 # Docstring inheritance means it is confusing to specify here 

367 # exactly which header value is being used. 

368 trivial_translator.__doc__ = f"""{property_doc} 

369 

370 Returns 

371 ------- 

372 translation : `{return_type}` 

373 Translated value derived from the header. 

374 """ 

375 return trivial_translator 

376 

377 @classmethod 

378 def __init_subclass__(cls, **kwargs: Any) -> None: 

379 """Register all subclasses with the base class and create dynamic 

380 translator methods. 

381 

382 The method provides two facilities. Firstly, every subclass 

383 of `MetadataTranslator` that includes a ``name`` class property is 

384 registered as a translator class that could be selected when automatic 

385 header translation is attempted. Only name translator subclasses that 

386 correspond to a complete instrument. Translation classes providing 

387 generic translation support for multiple instrument translators should 

388 not be named. 

389 

390 The second feature of this method is to convert simple translations 

391 to full translator methods. Sometimes a translation is fixed (for 

392 example a specific instrument name should be used) and rather than 

393 provide a full ``to_property()`` translation method the mapping can be 

394 defined in a class variable named ``_constMap``. Similarly, for 

395 one-to-one trivial mappings from a header to a property, 

396 ``_trivialMap`` can be defined. Trivial mappings are a dict mapping a 

397 generic property to either a header keyword, or a tuple consisting of 

398 the header keyword and a dict containing key value pairs suitable for 

399 the `MetadataTranslator.quantity_from_card()` method. 

400 """ 

401 super().__init_subclass__(**kwargs) 

402 

403 # Only register classes with declared names 

404 if hasattr(cls, "name") and cls.name is not None: 

405 if cls.name in MetadataTranslator.translators: 405 ↛ 406line 405 didn't jump to line 406, because the condition on line 405 was never true

406 log.warning( 

407 "%s: Replacing %s translator with %s", 

408 cls.name, 

409 MetadataTranslator.translators[cls.name], 

410 cls, 

411 ) 

412 MetadataTranslator.translators[cls.name] = cls 

413 

414 # Check that we have not inherited constant/trivial mappings from 

415 # parent class that we have already applied. Empty maps are always 

416 # assumed okay 

417 const_map = cls._const_map if cls._const_map and cls.defined_in_this_class("_const_map") else {} 

418 trivial_map = ( 

419 cls._trivial_map if cls._trivial_map and cls.defined_in_this_class("_trivial_map") else {} 

420 ) 

421 

422 # Check for shadowing 

423 trivials = set(trivial_map.keys()) 

424 constants = set(const_map.keys()) 

425 both = trivials & constants 

426 if both: 426 ↛ 427line 426 didn't jump to line 427, because the condition on line 426 was never true

427 log.warning("%s: defined in both const_map and trivial_map: %s", cls.__name__, ", ".join(both)) 

428 

429 all = trivials | constants 

430 for name in all: 

431 if cls.defined_in_this_class(f"to_{name}"): 431 ↛ 434line 431 didn't jump to line 434, because the condition on line 431 was never true

432 # Must be one of trivial or constant. If in both then constant 

433 # overrides trivial. 

434 location = "by _trivial_map" 

435 if name in constants: 

436 location = "by _const_map" 

437 log.warning( 

438 "%s: %s is defined explicitly but will be replaced %s", cls.__name__, name, location 

439 ) 

440 

441 properties = set(PROPERTIES) | set(("ext_" + pp for pp in cls.extensions)) 

442 cls.all_properties = dict(PROPERTIES) 

443 cls.all_properties.update(cls.extensions) 

444 

445 # Go through the trival mappings for this class and create 

446 # corresponding translator methods 

447 for property_key, header_key in trivial_map.items(): 

448 kwargs = {} 

449 if type(header_key) == tuple: 

450 kwargs = header_key[1] 

451 header_key = header_key[0] 

452 translator = cls._make_trivial_mapping(property_key, header_key, **kwargs) 

453 method = f"to_{property_key}" 

454 translator.__name__ = f"{method}_trivial_in_{cls.__name__}" 

455 setattr(cls, method, cache_translation(translator, method=method)) 

456 if property_key not in properties: 456 ↛ 457line 456 didn't jump to line 457, because the condition on line 456 was never true

457 log.warning(f"Unexpected trivial translator for '{property_key}' defined in {cls}") 

458 

459 # Go through the constant mappings for this class and create 

460 # corresponding translator methods 

461 for property_key, constant in const_map.items(): 

462 translator = cls._make_const_mapping(property_key, constant) 

463 method = f"to_{property_key}" 

464 translator.__name__ = f"{method}_constant_in_{cls.__name__}" 

465 setattr(cls, method, translator) 

466 if property_key not in properties: 466 ↛ 467line 466 didn't jump to line 467, because the condition on line 466 was never true

467 log.warning(f"Unexpected constant translator for '{property_key}' defined in {cls}") 

468 

469 def __init__(self, header: Mapping[str, Any], filename: Optional[str] = None) -> None: 

470 self._header = header 

471 self.filename = filename 

472 self._used_cards: Set[str] = set() 

473 

474 # Prefix to use for warnings about failed translations 

475 self._log_prefix_cache: Optional[str] = None 

476 

477 # Cache assumes header is read-only once stored in object 

478 self._translation_cache: Dict[str, Any] = {} 

479 

480 @classmethod 

481 @abstractmethod 

482 def can_translate(cls, header: MutableMapping[str, Any], filename: Optional[str] = None) -> bool: 

483 """Indicate whether this translation class can translate the 

484 supplied header. 

485 

486 Parameters 

487 ---------- 

488 header : `dict`-like 

489 Header to convert to standardized form. 

490 filename : `str`, optional 

491 Name of file being translated. 

492 

493 Returns 

494 ------- 

495 can : `bool` 

496 `True` if the header is recognized by this class. `False` 

497 otherwise. 

498 """ 

499 raise NotImplementedError() 

500 

501 @classmethod 

502 def can_translate_with_options( 

503 cls, header: Mapping[str, Any], options: Dict[str, Any], filename: Optional[str] = None 

504 ) -> bool: 

505 """Helper method for `can_translate` allowing options. 

506 

507 Parameters 

508 ---------- 

509 header : `dict`-like 

510 Header to convert to standardized form. 

511 options : `dict` 

512 Headers to try to determine whether this header can 

513 be translated by this class. If a card is found it will 

514 be compared with the expected value and will return that 

515 comparison. Each card will be tried in turn until one is 

516 found. 

517 filename : `str`, optional 

518 Name of file being translated. 

519 

520 Returns 

521 ------- 

522 can : `bool` 

523 `True` if the header is recognized by this class. `False` 

524 otherwise. 

525 

526 Notes 

527 ----- 

528 Intended to be used from within `can_translate` implementations 

529 for specific translators. Is not intended to be called directly 

530 from `determine_translator`. 

531 """ 

532 for card, value in options.items(): 

533 if card in header: 

534 return header[card] == value 

535 return False 

536 

537 @classmethod 

538 def determine_translator( 

539 cls, header: Mapping[str, Any], filename: Optional[str] = None 

540 ) -> Type[MetadataTranslator]: 

541 """Determine a translation class by examining the header 

542 

543 Parameters 

544 ---------- 

545 header : `dict`-like 

546 Representation of a header. 

547 filename : `str`, optional 

548 Name of file being translated. 

549 

550 Returns 

551 ------- 

552 translator : `MetadataTranslator` 

553 Translation class that knows how to extract metadata from 

554 the supplied header. 

555 

556 Raises 

557 ------ 

558 ValueError 

559 None of the registered translation classes understood the supplied 

560 header. 

561 """ 

562 file_msg = "" 

563 if filename is not None: 

564 file_msg = f" from {filename}" 

565 for name, trans in cls.translators.items(): 

566 if trans.can_translate(header, filename=filename): 

567 log.debug("Using translation class %s%s", name, file_msg) 

568 return trans 

569 else: 

570 raise ValueError( 

571 f"None of the registered translation classes {list(cls.translators.keys())}" 

572 f" understood this header{file_msg}" 

573 ) 

574 

575 @classmethod 

576 def translator_version(cls) -> str: 

577 """Return the version string for this translator class. 

578 

579 Returns 

580 ------- 

581 version : `str` 

582 String identifying the version of this translator. 

583 

584 Notes 

585 ----- 

586 Assumes that the version is available from the ``__version__`` 

587 variable in the parent module. If this is not the case a translator 

588 should subclass this method. 

589 """ 

590 if cls in _VERSION_CACHE: 

591 return _VERSION_CACHE[cls] 

592 

593 version = "unknown" 

594 module_name = cls.__module__ 

595 components = module_name.split(".") 

596 while components: 

597 # This class has already been imported so importing it 

598 # should work. 

599 module = importlib.import_module(".".join(components)) 

600 if hasattr(module, v := "__version__"): 

601 version = getattr(module, v) 

602 if version == "unknown": 

603 # LSST software will have a fingerprint 

604 if hasattr(module, v := "__fingerprint__"): 

605 version = getattr(module, v) 

606 break 

607 else: 

608 # Remove last component from module name and try again 

609 components.pop() 

610 

611 _VERSION_CACHE[cls] = version 

612 return version 

613 

614 @classmethod 

615 def fix_header( 

616 cls, header: MutableMapping[str, Any], instrument: str, obsid: str, filename: Optional[str] = None 

617 ) -> bool: 

618 """Apply global fixes to a supplied header. 

619 

620 Parameters 

621 ---------- 

622 header : `dict` 

623 The header to correct. Correction is in place. 

624 instrument : `str` 

625 The name of the instrument. 

626 obsid : `str` 

627 Unique observation identifier associated with this header. 

628 Will always be provided. 

629 filename : `str`, optional 

630 Filename associated with this header. May not be set since headers 

631 can be fixed independently of any filename being known. 

632 

633 Returns 

634 ------- 

635 modified : `bool` 

636 `True` if a correction was applied. 

637 

638 Notes 

639 ----- 

640 This method is intended to support major discrepancies in headers 

641 such as: 

642 

643 * Periods of time where headers are known to be incorrect in some 

644 way that can be fixed either by deriving the correct value from 

645 the existing value or understanding the that correction is static 

646 for the given time. This requires that the date header is 

647 known. 

648 * The presence of a certain value is always wrong and should be 

649 corrected with a new static value regardless of date. 

650 

651 It is assumed that one off problems with headers have been applied 

652 before this method is called using the per-obsid correction system. 

653 

654 Usually called from `astro_metadata_translator.fix_header`. 

655 

656 For log messages, do not assume that the filename will be present. 

657 Always write log messages to fall back on using the ``obsid`` if 

658 ``filename`` is `None`. 

659 """ 

660 return False 

661 

662 @staticmethod 

663 def _construct_log_prefix(obsid: str, filename: Optional[str] = None) -> str: 

664 """Construct a log prefix string from the obsid and filename. 

665 

666 Parameters 

667 ---------- 

668 obsid : `str` 

669 The observation identifier. 

670 filename : `str`, optional 

671 The filename associated with the header being translated. 

672 Can be `None`. 

673 """ 

674 if filename: 

675 return f"{filename}({obsid})" 

676 return obsid 

677 

678 @property 

679 def _log_prefix(self) -> str: 

680 """Standard prefix that can be used for log messages to report 

681 useful context. 

682 

683 Will be either the filename and obsid, or just the obsid depending 

684 on whether a filename is known. 

685 

686 Returns 

687 ------- 

688 prefix : `str` 

689 The prefix to use. 

690 """ 

691 if self._log_prefix_cache is None: 

692 # Protect against the unfortunate event of the obsid failing to 

693 # be calculated. This should be rare but should not prevent a log 

694 # message from appearing. 

695 try: 

696 obsid = self.to_observation_id() 

697 except Exception: 

698 obsid = "unknown_obsid" 

699 self._log_prefix_cache = self._construct_log_prefix(obsid, self.filename) 

700 return self._log_prefix_cache 

701 

702 def _used_these_cards(self, *args: str) -> None: 

703 """Indicate that the supplied cards have been used for translation. 

704 

705 Parameters 

706 ---------- 

707 args : sequence of `str` 

708 Keywords used to process a translation. 

709 """ 

710 self._used_cards.update(set(args)) 

711 

712 def cards_used(self) -> FrozenSet[str]: 

713 """Cards used during metadata extraction. 

714 

715 Returns 

716 ------- 

717 used : `frozenset` of `str` 

718 Cards used when extracting metadata. 

719 """ 

720 return frozenset(self._used_cards) 

721 

722 @staticmethod 

723 def validate_value( 

724 value: float, default: float, minimum: Optional[float] = None, maximum: Optional[float] = None 

725 ) -> float: 

726 """Validate the supplied value, returning a new value if out of range 

727 

728 Parameters 

729 ---------- 

730 value : `float` 

731 Value to be validated. 

732 default : `float` 

733 Default value to use if supplied value is invalid or out of range. 

734 Assumed to be in the same units as the value expected in the 

735 header. 

736 minimum : `float` 

737 Minimum possible valid value, optional. If the calculated value 

738 is below this value, the default value will be used. 

739 maximum : `float` 

740 Maximum possible valid value, optional. If the calculated value 

741 is above this value, the default value will be used. 

742 

743 Returns 

744 ------- 

745 value : `float` 

746 Either the supplied value, or a default value. 

747 """ 

748 if value is None or math.isnan(value): 

749 value = default 

750 else: 

751 if minimum is not None and value < minimum: 

752 value = default 

753 elif maximum is not None and value > maximum: 

754 value = default 

755 return value 

756 

757 @staticmethod 

758 def is_keyword_defined(header: Mapping[str, Any], keyword: Optional[str]) -> bool: 

759 """Return `True` if the value associated with the named keyword is 

760 present in the supplied header and defined. 

761 

762 Parameters 

763 ---------- 

764 header : `dict`-lik 

765 Header to use as reference. 

766 keyword : `str` 

767 Keyword to check against header. 

768 

769 Returns 

770 ------- 

771 is_defined : `bool` 

772 `True` if the header is present and not-`None`. `False` otherwise. 

773 """ 

774 if keyword is None or keyword not in header: 

775 return False 

776 

777 if header[keyword] is None: 

778 return False 

779 

780 # Special case Astropy undefined value 

781 if isinstance(header[keyword], astropy.io.fits.card.Undefined): 

782 return False 

783 

784 return True 

785 

786 def resource_root(self) -> Tuple[Optional[str], Optional[str]]: 

787 """Package resource to use to locate correction resources within an 

788 installed package. 

789 

790 Returns 

791 ------- 

792 resource_package : `str` 

793 Package resource name. `None` if no package resource are to be 

794 used. 

795 resource_root : `str` 

796 The name of the resource root. `None` if no package resources 

797 are to be used. 

798 """ 

799 return (self.default_resource_package, self.default_resource_root) 

800 

801 def search_paths(self) -> List[str]: 

802 """Search paths to use when searching for header fix up correction 

803 files. 

804 

805 Returns 

806 ------- 

807 paths : `list` 

808 Directory paths to search. Can be an empty list if no special 

809 directories are defined. 

810 

811 Notes 

812 ----- 

813 Uses the classes ``default_search_path`` property if defined. 

814 """ 

815 if self.default_search_path is not None: 

816 return [p for p in self.default_search_path] 

817 return [] 

818 

819 def is_key_ok(self, keyword: Optional[str]) -> bool: 

820 """Return `True` if the value associated with the named keyword is 

821 present in this header and defined. 

822 

823 Parameters 

824 ---------- 

825 keyword : `str` 

826 Keyword to check against header. 

827 

828 Returns 

829 ------- 

830 is_ok : `bool` 

831 `True` if the header is present and not-`None`. `False` otherwise. 

832 """ 

833 return self.is_keyword_defined(self._header, keyword) 

834 

835 def are_keys_ok(self, keywords: Iterable[str]) -> bool: 

836 """Are the supplied keys all present and defined? 

837 

838 Parameters 

839 ---------- 

840 keywords : iterable of `str` 

841 Keywords to test. 

842 

843 Returns 

844 ------- 

845 all_ok : `bool` 

846 `True` if all supplied keys are present and defined. 

847 """ 

848 for k in keywords: 

849 if not self.is_key_ok(k): 

850 return False 

851 return True 

852 

853 def quantity_from_card( 

854 self, 

855 keywords: Union[str, Sequence[str]], 

856 unit: u.Unit, 

857 default: Optional[float] = None, 

858 minimum: Optional[float] = None, 

859 maximum: Optional[float] = None, 

860 checker: Optional[Callable] = None, 

861 ) -> u.Quantity: 

862 """Calculate a Astropy Quantity from a header card and a unit. 

863 

864 Parameters 

865 ---------- 

866 keywords : `str` or `list` of `str` 

867 Keyword to use from header. If a list each keyword will be tried 

868 in turn until one matches. 

869 unit : `astropy.units.UnitBase` 

870 Unit of the item in the header. 

871 default : `float`, optional 

872 Default value to use if the header value is invalid. Assumed 

873 to be in the same units as the value expected in the header. If 

874 None, no default value is used. 

875 minimum : `float`, optional 

876 Minimum possible valid value, optional. If the calculated value 

877 is below this value, the default value will be used. 

878 maximum : `float`, optional 

879 Maximum possible valid value, optional. If the calculated value 

880 is above this value, the default value will be used. 

881 checker : `function`, optional 

882 Callback function to be used by the translator method in case the 

883 keyword is not present. Function will be executed as if it is 

884 a method of the translator class. Running without raising an 

885 exception will allow the default to be used. Should usually raise 

886 `KeyError`. 

887 

888 Returns 

889 ------- 

890 q : `astropy.units.Quantity` 

891 Quantity representing the header value. 

892 

893 Raises 

894 ------ 

895 KeyError 

896 The supplied header key is not present. 

897 """ 

898 keyword_list = [keywords] if isinstance(keywords, str) else list(keywords) 

899 for k in keyword_list: 

900 if self.is_key_ok(k): 

901 value = self._header[k] 

902 keyword = k 

903 break 

904 else: 

905 if checker is not None: 

906 try: 

907 checker(self) 

908 value = default 

909 if value is not None: 

910 value = u.Quantity(value, unit=unit) 

911 return value 

912 except Exception: 

913 pass 

914 raise KeyError(f"Could not find {keywords} in header") 

915 if isinstance(value, str): 

916 # Sometimes the header has the wrong type in it but this must 

917 # be a number if we are creating a quantity. 

918 value = float(value) 

919 self._used_these_cards(keyword) 

920 if default is not None: 

921 value = self.validate_value(value, default, maximum=maximum, minimum=minimum) 

922 return u.Quantity(value, unit=unit) 

923 

924 def _join_keyword_values(self, keywords: Iterable[str], delim: str = "+") -> str: 

925 """Join values of all defined keywords with the specified delimiter. 

926 

927 Parameters 

928 ---------- 

929 keywords : iterable of `str` 

930 Keywords to look for in header. 

931 delim : `str`, optional 

932 Character to use to join the values together. 

933 

934 Returns 

935 ------- 

936 joined : `str` 

937 String formed from all the keywords found in the header with 

938 defined values joined by the delimiter. Empty string if no 

939 defined keywords found. 

940 """ 

941 values = [] 

942 for k in keywords: 

943 if self.is_key_ok(k): 

944 values.append(self._header[k]) 

945 self._used_these_cards(k) 

946 

947 if values: 

948 joined = delim.join(str(v) for v in values) 

949 else: 

950 joined = "" 

951 

952 return joined 

953 

954 @cache_translation 

955 def to_detector_unique_name(self) -> str: 

956 """Return a unique name for the detector. 

957 

958 Base class implementation attempts to combine ``detector_name`` with 

959 ``detector_group``. Group is only used if not `None`. 

960 

961 Can be over-ridden by specialist translator class. 

962 

963 Returns 

964 ------- 

965 name : `str` 

966 ``detector_group``_``detector_name`` if ``detector_group`` is 

967 defined, else the ``detector_name`` is assumed to be unique. 

968 If neither return a valid value an exception is raised. 

969 

970 Raises 

971 ------ 

972 NotImplementedError 

973 Raised if neither detector_name nor detector_group is defined. 

974 """ 

975 name = self.to_detector_name() 

976 group = self.to_detector_group() 

977 

978 if group is None and name is None: 

979 raise NotImplementedError("Can not determine unique name from detector_group and detector_name") 

980 

981 if group is not None: 

982 return f"{group}_{name}" 

983 

984 return name 

985 

986 @cache_translation 

987 def to_exposure_group(self) -> Optional[str]: 

988 """Return the group label associated with this exposure. 

989 

990 Base class implementation returns the ``exposure_id`` in string 

991 form. A subclass may do something different. 

992 

993 Returns 

994 ------- 

995 name : `str` 

996 The ``exposure_id`` converted to a string. 

997 """ 

998 exposure_id = self.to_exposure_id() 

999 if exposure_id is None: 

1000 # mypy does not think this can ever happen but play it safe 

1001 # with subclasses. 

1002 return None # type: ignore 

1003 else: 

1004 return str(exposure_id) 

1005 

1006 @cache_translation 

1007 def to_observation_reason(self) -> str: 

1008 """Return the reason this observation was taken. 

1009 

1010 Base class implementation returns the ``science`` if the 

1011 ``observation_type`` is science, else ``unknown``. 

1012 A subclass may do something different. 

1013 

1014 Returns 

1015 ------- 

1016 name : `str` 

1017 The reason for this observation. 

1018 """ 

1019 obstype = self.to_observation_type() 

1020 if obstype == "science": 

1021 return "science" 

1022 return "unknown" 

1023 

1024 @cache_translation 

1025 def to_observing_day(self) -> int: 

1026 """Return the YYYYMMDD integer corresponding to the observing day. 

1027 

1028 Base class implementation uses the TAI date of the start of the 

1029 observation. 

1030 

1031 Returns 

1032 ------- 

1033 day : `int` 

1034 The observing day as an integer of form YYYYMMDD. If the header 

1035 is broken and is unable to obtain a date of observation, ``0`` 

1036 is returned and the assumption is made that the problem will 

1037 be caught elsewhere. 

1038 """ 

1039 datetime_begin = self.to_datetime_begin() 

1040 if datetime_begin is None: 

1041 return 0 

1042 return int(datetime_begin.tai.strftime("%Y%m%d")) 

1043 

1044 @cache_translation 

1045 def to_observation_counter(self) -> int: 

1046 """Return an integer corresponding to how this observation relates 

1047 to other observations. 

1048 

1049 Base class implementation returns ``0`` to indicate that it is not 

1050 known how an observatory will define a counter. Some observatories 

1051 may not use the concept, others may use a counter that increases 

1052 for every observation taken for that instrument, and others may 

1053 define it to be a counter within an observing day. 

1054 

1055 Returns 

1056 ------- 

1057 sequence : `int` 

1058 The observation counter. Always ``0`` for this implementation. 

1059 """ 

1060 return 0 

1061 

1062 @cache_translation 

1063 def to_group_counter_start(self) -> int: 

1064 """Return the observation counter of the observation that began 

1065 this group. 

1066 

1067 The definition of the relevant group is up to the metadata 

1068 translator. It can be the first observation in the exposure_group 

1069 or the first observation in the visit, but must be derivable 

1070 from the metadata of this observation. 

1071 

1072 Returns 

1073 ------- 

1074 counter : `int` 

1075 The observation counter for the start of the relevant group. 

1076 Default implementation always returns the observation counter 

1077 of this observation. 

1078 """ 

1079 return self.to_observation_counter() 

1080 

1081 @cache_translation 

1082 def to_group_counter_end(self) -> int: 

1083 """Return the observation counter of the observation that ends 

1084 this group. 

1085 

1086 The definition of the relevant group is up to the metadata 

1087 translator. It can be the last observation in the exposure_group 

1088 or the last observation in the visit, but must be derivable 

1089 from the metadata of this observation. It is of course possible 

1090 that the last observation in the group does not exist if a sequence 

1091 of observations was not completed. 

1092 

1093 Returns 

1094 ------- 

1095 counter : `int` 

1096 The observation counter for the end of the relevant group. 

1097 Default implementation always returns the observation counter 

1098 of this observation. 

1099 """ 

1100 return self.to_observation_counter() 

1101 

1102 @cache_translation 

1103 def to_has_simulated_content(self) -> bool: 

1104 """Return a boolean indicating whether any part of the observation 

1105 was simulated. 

1106 

1107 Returns 

1108 ------- 

1109 is_simulated : `bool` 

1110 `True` if this exposure has simulated content. This can be 

1111 if some parts of the metadata or data were simulated. Default 

1112 implementation always returns `False`. 

1113 """ 

1114 return False 

1115 

1116 @cache_translation 

1117 def to_focus_z(self) -> u.Quantity: 

1118 """Return a default defocal distance of 0.0 mm if there is no 

1119 keyword for defocal distance in the header. The default 

1120 keyword for defocal distance is ``FOCUSZ``. 

1121 

1122 Returns 

1123 ------- 

1124 focus_z: `astropy.units.Quantity` 

1125 The defocal distance from header or the 0.0mm default 

1126 """ 

1127 return 0.0 * u.mm 

1128 

1129 @classmethod 

1130 def determine_translatable_headers( 

1131 cls, filename: str, primary: Optional[MutableMapping[str, Any]] = None 

1132 ) -> Iterator[MutableMapping[str, Any]]: 

1133 """Given a file return all the headers usable for metadata translation. 

1134 

1135 This method can optionally be given a header from the file. This 

1136 header will generally be the primary header or a merge of the first 

1137 two headers. 

1138 

1139 In the base class implementation it is assumed that 

1140 this supplied header is the only useful header for metadata translation 

1141 and it will be returned unchanged if given. This can avoid 

1142 unnecesarily re-opening the file and re-reading the header when the 

1143 content is already known. 

1144 

1145 If no header is supplied, a header will be read from the supplied 

1146 file using `read_basic_metadata_from_file`, allowing it to merge 

1147 the primary and secondary header of a multi-extension FITS file. 

1148 Subclasses can read the header from the data file using whatever 

1149 technique is best for that instrument. 

1150 

1151 Subclasses can return multiple headers and ignore the externally 

1152 supplied header. They can also merge it with another header and return 

1153 a new derived header if that is required by the particular data file. 

1154 There is no requirement for the supplied header to be used. 

1155 

1156 Parameters 

1157 ---------- 

1158 filename : `str` 

1159 Path to a file in a format understood by this translator. 

1160 primary : `dict`-like, optional 

1161 The primary header obtained by the caller. This is sometimes 

1162 already known, for example if a system is trying to bootstrap 

1163 without already knowing what data is in the file. For many 

1164 instruments where the primary header is the only relevant 

1165 header, the primary header will be returned with no further 

1166 action. 

1167 

1168 Yields 

1169 ------ 

1170 headers : iterator of `dict`-like 

1171 A header usable for metadata translation. For this base 

1172 implementation it will be either the supplied primary header 

1173 or a header read from the file. This implementation will only 

1174 ever yield a single header. 

1175 

1176 Notes 

1177 ----- 

1178 Each translator class can have code specifically tailored to its 

1179 own file format. It is important not to call this method with 

1180 an incorrect translator class. The normal paradigm is for the 

1181 caller to have read the first header and then called 

1182 `determine_translator()` on the result to work out which translator 

1183 class to then call to obtain the real headers to be used for 

1184 translation. 

1185 """ 

1186 if primary is not None: 

1187 yield primary 

1188 else: 

1189 # Prevent circular import by deferring 

1190 from .file_helpers import read_basic_metadata_from_file 

1191 

1192 # Merge primary and secondary header if they exist. 

1193 header = read_basic_metadata_from_file(filename, -1) 

1194 assert header is not None # for mypy since can_raise=True 

1195 yield header 

1196 

1197 

1198def _make_abstract_translator_method( 

1199 property: str, doc: str, return_typedoc: str, return_type: Type 

1200) -> Callable: 

1201 """Create a an abstract translation method for this property. 

1202 

1203 Parameters 

1204 ---------- 

1205 property : `str` 

1206 Name of the translator for property to be created. 

1207 doc : `str` 

1208 Description of the property. 

1209 return_typedoc : `str` 

1210 Type string of this property (used in the doc string). 

1211 return_type : `class` 

1212 Type of this property. 

1213 

1214 Returns 

1215 ------- 

1216 m : `function` 

1217 Translator method for this property. 

1218 """ 

1219 

1220 def to_property(self: MetadataTranslator) -> None: 

1221 raise NotImplementedError(f"Translator for '{property}' undefined.") 

1222 

1223 to_property.__doc__ = f"""Return value of {property} from headers. 

1224 

1225 {doc} 

1226 

1227 Returns 

1228 ------- 

1229 {property} : `{return_typedoc}` 

1230 The translated property. 

1231 """ 

1232 return to_property 

1233 

1234 

1235# Make abstract methods for all the translators methods. 

1236# Unfortunately registering them as abstractmethods does not work 

1237# as these assignments come after the class has been created. 

1238# Assigning to __abstractmethods__ directly does work but interacts 

1239# poorly with the metaclass automatically generating methods from 

1240# _trivialMap and _constMap. 

1241# Note that subclasses that provide extension properties are assumed to not 

1242# need abstract methods created for them. 

1243 

1244# Allow for concrete translator methods to exist in the base class 

1245# These translator methods can be defined in terms of other properties 

1246CONCRETE = set() 

1247 

1248for name, definition in PROPERTIES.items(): 

1249 method = f"to_{name}" 

1250 if not MetadataTranslator.defined_in_this_class(method): 

1251 setattr( 

1252 MetadataTranslator, 

1253 f"to_{name}", 

1254 abstractmethod( 

1255 _make_abstract_translator_method( 

1256 name, definition.doc, definition.str_type, definition.py_type 

1257 ) 

1258 ), 

1259 ) 

1260 else: 

1261 CONCRETE.add(method) 

1262 

1263 

1264class StubTranslator(MetadataTranslator): 

1265 """Translator where all the translations are stubbed out and issue 

1266 warnings. 

1267 

1268 This translator can be used as a base class whilst developing a new 

1269 translator. It allows testing to proceed without being required to fully 

1270 define all translation methods. Once complete the class should be 

1271 removed from the inheritance tree. 

1272 

1273 """ 

1274 

1275 pass 

1276 

1277 

1278def _make_forwarded_stub_translator_method( 

1279 cls: Type[MetadataTranslator], property: str, doc: str, return_typedoc: str, return_type: Type 

1280) -> Callable: 

1281 """Create a stub translation method for this property that calls the 

1282 base method and catches `NotImplementedError`. 

1283 

1284 Parameters 

1285 ---------- 

1286 cls : `class` 

1287 Class to use when referencing `super()`. This would usually be 

1288 `StubTranslator`. 

1289 property : `str` 

1290 Name of the translator for property to be created. 

1291 doc : `str` 

1292 Description of the property. 

1293 return_typedoc : `str` 

1294 Type string of this property (used in the doc string). 

1295 return_type : `class` 

1296 Type of this property. 

1297 

1298 Returns 

1299 ------- 

1300 m : `function` 

1301 Stub translator method for this property. 

1302 """ 

1303 method = f"to_{property}" 

1304 

1305 def to_stub(self: MetadataTranslator) -> Any: 

1306 parent = getattr(super(cls, self), method, None) 

1307 try: 

1308 if parent is not None: 

1309 return parent() 

1310 except NotImplementedError: 

1311 pass 

1312 

1313 warnings.warn( 

1314 f"Please implement translator for property '{property}' for translator {self}", stacklevel=3 

1315 ) 

1316 return None 

1317 

1318 to_stub.__doc__ = f"""Unimplemented forwarding translator for {property}. 

1319 

1320 {doc} 

1321 

1322 Calls the base class translation method and if that fails with 

1323 `NotImplementedError` issues a warning reminding the implementer to 

1324 override this method. 

1325 

1326 Returns 

1327 ------- 

1328 {property} : `None` or `{return_typedoc}` 

1329 Always returns `None`. 

1330 """ 

1331 return to_stub 

1332 

1333 

1334# Create stub translation methods for each property. These stubs warn 

1335# rather than fail and should be overridden by translators. 

1336for name, description in PROPERTIES.items(): 

1337 setattr( 

1338 StubTranslator, 

1339 f"to_{name}", 

1340 _make_forwarded_stub_translator_method( 

1341 StubTranslator, name, definition.doc, definition.str_type, definition.py_type # type: ignore 

1342 ), 

1343 )