Coverage for python/lsst/ip/isr/calibType.py: 16%

336 statements  

« prev     ^ index     » next       coverage.py v7.4.3, created at 2024-03-07 11:51 +0000

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21__all__ = ["IsrCalib", "IsrProvenance"] 

22 

23import abc 

24import datetime 

25import logging 

26import os.path 

27import warnings 

28import yaml 

29import numpy as np 

30 

31from astropy.table import Table 

32from astropy.io import fits 

33 

34from lsst.daf.base import PropertyList 

35from lsst.utils.introspection import get_full_type_name 

36from lsst.utils import doImport 

37 

38 

39class IsrCalib(abc.ABC): 

40 """Generic calibration type. 

41 

42 Subclasses must implement the toDict, fromDict, toTable, fromTable 

43 methods that allow the calibration information to be converted 

44 from dictionaries and afw tables. This will allow the calibration 

45 to be persisted using the base class read/write methods. 

46 

47 The validate method is intended to provide a common way to check 

48 that the calibration is valid (internally consistent) and 

49 appropriate (usable with the intended data). The apply method is 

50 intended to allow the calibration to be applied in a consistent 

51 manner. 

52 

53 Parameters 

54 ---------- 

55 camera : `lsst.afw.cameraGeom.Camera`, optional 

56 Camera to extract metadata from. 

57 detector : `lsst.afw.cameraGeom.Detector`, optional 

58 Detector to extract metadata from. 

59 log : `logging.Logger`, optional 

60 Log for messages. 

61 """ 

62 _OBSTYPE = "generic" 

63 _SCHEMA = "NO SCHEMA" 

64 _VERSION = 0 

65 

66 def __init__(self, camera=None, detector=None, log=None, **kwargs): 

67 self._instrument = None 

68 self._raftName = None 

69 self._slotName = None 

70 self._detectorName = None 

71 self._detectorSerial = None 

72 self._detectorId = None 

73 self._filter = None 

74 self._calibId = None 

75 self._seqfile = None 

76 self._seqname = None 

77 self._seqcksum = None 

78 self._metadata = PropertyList() 

79 self.setMetadata(PropertyList()) 

80 self.calibInfoFromDict(kwargs) 

81 

82 # Define the required attributes for this calibration. These 

83 # are entries that are automatically filled and propagated 

84 # from the metadata. The existence of the attribute is 

85 # required (they're checked for equivalence), but they do not 

86 # necessarily need to have a value (None == None in this 

87 # case). 

88 self.requiredAttributes = set(["_OBSTYPE", "_SCHEMA", "_VERSION"]) 

89 self.requiredAttributes.update(["_instrument", "_raftName", "_slotName", 

90 "_detectorName", "_detectorSerial", "_detectorId", 

91 "_filter", "_calibId", "_seqfile", "_seqname", "_seqcksum", 

92 "_metadata"]) 

93 

94 self.log = log if log else logging.getLogger(__name__) 

95 

96 if detector: 

97 self.fromDetector(detector) 

98 self.updateMetadata(camera=camera, detector=detector) 

99 

100 def __str__(self): 

101 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )" 

102 

103 def __eq__(self, other): 

104 """Calibration equivalence. 

105 

106 Running ``calib.log.setLevel(0)`` enables debug statements to 

107 identify problematic fields. 

108 """ 

109 if not isinstance(other, self.__class__): 

110 self.log.debug("Incorrect class type: %s %s", self.__class__, other.__class__) 

111 return False 

112 

113 for attr in self._requiredAttributes: 

114 attrSelf = getattr(self, attr) 

115 attrOther = getattr(other, attr) 

116 

117 if isinstance(attrSelf, dict): 

118 # Dictionary of arrays. 

119 if attrSelf.keys() != attrOther.keys(): 

120 self.log.debug("Dict Key Failure: %s %s %s", attr, attrSelf.keys(), attrOther.keys()) 

121 return False 

122 for key in attrSelf: 

123 try: 

124 if not np.allclose(attrSelf[key], attrOther[key], equal_nan=True): 

125 self.log.debug("Array Failure: %s %s %s", key, attrSelf[key], attrOther[key]) 

126 return False 

127 except TypeError: 

128 # If it is not something numpy can handle 

129 # (it's not a number or array of numbers), 

130 # then it needs to have its own equivalence 

131 # operator. 

132 if attrSelf[key] != attrOther[key]: 

133 return False 

134 elif isinstance(attrSelf, np.ndarray): 

135 # Bare array. 

136 if not np.allclose(attrSelf, attrOther, equal_nan=True): 

137 self.log.debug("Array Failure: %s %s %s", attr, attrSelf, attrOther) 

138 return False 

139 elif type(attrSelf) != type(attrOther): 

140 if set([attrSelf, attrOther]) == set([None, ""]): 

141 # Fits converts None to "", but None is not "". 

142 continue 

143 self.log.debug("Type Failure: %s %s %s %s %s", attr, type(attrSelf), type(attrOther), 

144 attrSelf, attrOther) 

145 return False 

146 else: 

147 if attrSelf != attrOther: 

148 self.log.debug("Value Failure: %s %s %s", attr, attrSelf, attrOther) 

149 return False 

150 

151 return True 

152 

153 @property 

154 def requiredAttributes(self): 

155 return self._requiredAttributes 

156 

157 @requiredAttributes.setter 

158 def requiredAttributes(self, value): 

159 self._requiredAttributes = value 

160 

161 def getMetadata(self): 

162 """Retrieve metadata associated with this calibration. 

163 

164 Returns 

165 ------- 

166 meta : `lsst.daf.base.PropertyList` 

167 Metadata. The returned `~lsst.daf.base.PropertyList` can be 

168 modified by the caller and the changes will be written to 

169 external files. 

170 """ 

171 return self._metadata 

172 

173 def setMetadata(self, metadata): 

174 """Store a copy of the supplied metadata with this calibration. 

175 

176 Parameters 

177 ---------- 

178 metadata : `lsst.daf.base.PropertyList` 

179 Metadata to associate with the calibration. Will be copied and 

180 overwrite existing metadata. 

181 """ 

182 if metadata is not None: 

183 self._metadata.update(metadata) 

184 

185 # Ensure that we have the obs type required by calibration ingest 

186 self._metadata["OBSTYPE"] = self._OBSTYPE 

187 self._metadata[self._OBSTYPE + "_SCHEMA"] = self._SCHEMA 

188 self._metadata[self._OBSTYPE + "_VERSION"] = self._VERSION 

189 

190 if isinstance(metadata, dict): 

191 self.calibInfoFromDict(metadata) 

192 elif isinstance(metadata, PropertyList): 

193 self.calibInfoFromDict(metadata.toDict()) 

194 

195 def updateMetadata(self, camera=None, detector=None, filterName=None, 

196 setCalibId=False, setCalibInfo=False, setDate=False, 

197 **kwargs): 

198 """Update metadata keywords with new values. 

199 

200 Parameters 

201 ---------- 

202 camera : `lsst.afw.cameraGeom.Camera`, optional 

203 Reference camera to use to set ``_instrument`` field. 

204 detector : `lsst.afw.cameraGeom.Detector`, optional 

205 Reference detector to use to set ``_detector*`` fields. 

206 filterName : `str`, optional 

207 Filter name to assign to this calibration. 

208 setCalibId : `bool`, optional 

209 Construct the ``_calibId`` field from other fields. 

210 setCalibInfo : `bool`, optional 

211 Set calibration parameters from metadata. 

212 setDate : `bool`, optional 

213 Ensure the metadata ``CALIBDATE`` fields are set to the current 

214 datetime. 

215 kwargs : `dict` or `collections.abc.Mapping`, optional 

216 Set of ``key=value`` pairs to assign to the metadata. 

217 """ 

218 mdOriginal = self.getMetadata() 

219 mdSupplemental = dict() 

220 

221 for k, v in kwargs.items(): 

222 if isinstance(v, fits.card.Undefined): 

223 kwargs[k] = None 

224 

225 if setCalibInfo: 

226 self.calibInfoFromDict(kwargs) 

227 

228 if camera: 

229 self._instrument = camera.getName() 

230 

231 if detector: 

232 self._detectorName = detector.getName() 

233 self._detectorSerial = detector.getSerial() 

234 self._detectorId = detector.getId() 

235 if "_" in self._detectorName: 

236 (self._raftName, self._slotName) = self._detectorName.split("_") 

237 

238 if filterName: 

239 # TOD0 DM-28093: I think this whole comment can go away, if we 

240 # always use physicalLabel everywhere in ip_isr. 

241 # If set via: 

242 # exposure.getInfo().getFilter().getName() 

243 # then this will hold the abstract filter. 

244 self._filter = filterName 

245 

246 if setDate: 

247 date = datetime.datetime.now() 

248 mdSupplemental["CALIBDATE"] = date.isoformat() 

249 mdSupplemental["CALIB_CREATION_DATE"] = date.date().isoformat() 

250 mdSupplemental["CALIB_CREATION_TIME"] = date.time().isoformat() 

251 

252 if setCalibId: 

253 values = [] 

254 values.append(f"instrument={self._instrument}") if self._instrument else None 

255 values.append(f"raftName={self._raftName}") if self._raftName else None 

256 values.append(f"detectorName={self._detectorName}") if self._detectorName else None 

257 values.append(f"detector={self._detectorId}") if self._detectorId else None 

258 values.append(f"filter={self._filter}") if self._filter else None 

259 

260 calibDate = mdOriginal.get("CALIBDATE", mdSupplemental.get("CALIBDATE", None)) 

261 values.append(f"calibDate={calibDate}") if calibDate else None 

262 

263 self._calibId = " ".join(values) 

264 

265 self._metadata["INSTRUME"] = self._instrument if self._instrument else None 

266 self._metadata["RAFTNAME"] = self._raftName if self._raftName else None 

267 self._metadata["SLOTNAME"] = self._slotName if self._slotName else None 

268 self._metadata["DETECTOR"] = self._detectorId 

269 self._metadata["DET_NAME"] = self._detectorName if self._detectorName else None 

270 self._metadata["DET_SER"] = self._detectorSerial if self._detectorSerial else None 

271 self._metadata["FILTER"] = self._filter if self._filter else None 

272 self._metadata["CALIB_ID"] = self._calibId if self._calibId else None 

273 self._metadata["SEQFILE"] = self._seqfile if self._seqfile else None 

274 self._metadata["SEQNAME"] = self._seqname if self._seqname else None 

275 self._metadata["SEQCKSUM"] = self._seqcksum if self._seqcksum else None 

276 self._metadata["CALIBCLS"] = get_full_type_name(self) 

277 

278 mdSupplemental.update(kwargs) 

279 mdOriginal.update(mdSupplemental) 

280 

281 def updateMetadataFromExposures(self, exposures): 

282 """Extract and unify metadata information. 

283 

284 Parameters 

285 ---------- 

286 exposures : `list` 

287 Exposures or other calibrations to scan. 

288 """ 

289 # This list of keywords is the set of header entries that 

290 # should be checked and propagated. Not having an entry is 

291 # not a failure, as they may not be defined for the exposures 

292 # being used. 

293 keywords = ["SEQNAME", "SEQFILE", "SEQCKSUM", "ODP", "AP0_RC"] 

294 metadata = {} 

295 

296 for exp in exposures: 

297 try: 

298 expMeta = exp.getMetadata() 

299 except AttributeError: 

300 continue 

301 for key in keywords: 

302 if key in expMeta: 

303 if key in metadata: 

304 if metadata[key] != expMeta[key]: 

305 self.log.warning("Metadata mismatch! Have: %s Found %s", 

306 metadata[key], expMeta[key]) 

307 else: 

308 metadata[key] = expMeta[key] 

309 self.updateMetadata(**metadata) 

310 

311 def calibInfoFromDict(self, dictionary): 

312 """Handle common keywords. 

313 

314 This isn't an ideal solution, but until all calibrations 

315 expect to find everything in the metadata, they still need to 

316 search through dictionaries. 

317 

318 Parameters 

319 ---------- 

320 dictionary : `dict` or `lsst.daf.base.PropertyList` 

321 Source for the common keywords. 

322 

323 Raises 

324 ------ 

325 RuntimeError 

326 Raised if the dictionary does not match the expected OBSTYPE. 

327 """ 

328 

329 def search(haystack, needles): 

330 """Search dictionary 'haystack' for an entry in 'needles' 

331 """ 

332 test = [haystack.get(x) for x in needles] 

333 test = set([x for x in test if x is not None]) 

334 if len(test) == 0: 

335 if "metadata" in haystack: 

336 return search(haystack["metadata"], needles) 

337 else: 

338 return None 

339 elif len(test) == 1: 

340 value = list(test)[0] 

341 if value == "": 

342 return None 

343 else: 

344 return value 

345 else: 

346 raise ValueError(f"Too many values found: {len(test)} {test} {needles}") 

347 

348 if "metadata" in dictionary: 

349 metadata = dictionary["metadata"] 

350 

351 if self._OBSTYPE != metadata["OBSTYPE"]: 

352 raise RuntimeError(f"Incorrect calibration supplied. Expected {self._OBSTYPE}, " 

353 f"found {metadata['OBSTYPE']}") 

354 

355 self._instrument = search(dictionary, ["INSTRUME", "instrument"]) 

356 self._raftName = search(dictionary, ["RAFTNAME"]) 

357 self._slotName = search(dictionary, ["SLOTNAME"]) 

358 self._detectorId = search(dictionary, ["DETECTOR", "detectorId"]) 

359 self._detectorName = search(dictionary, ["DET_NAME", "DETECTOR_NAME", "detectorName"]) 

360 self._detectorSerial = search(dictionary, ["DET_SER", "DETECTOR_SERIAL", "detectorSerial"]) 

361 self._filter = search(dictionary, ["FILTER", "filterName"]) 

362 self._calibId = search(dictionary, ["CALIB_ID"]) 

363 self._seqfile = search(dictionary, ["SEQFILE"]) 

364 self._seqname = search(dictionary, ["SEQNAME"]) 

365 self._seqcksum = search(dictionary, ["SEQCKSUM"]) 

366 

367 @classmethod 

368 def determineCalibClass(cls, metadata, message): 

369 """Attempt to find calibration class in metadata. 

370 

371 Parameters 

372 ---------- 

373 metadata : `dict` or `lsst.daf.base.PropertyList` 

374 Metadata possibly containing a calibration class entry. 

375 message : `str` 

376 Message to include in any errors. 

377 

378 Returns 

379 ------- 

380 calibClass : `object` 

381 The class to use to read the file contents. Should be an 

382 `lsst.ip.isr.IsrCalib` subclass. 

383 

384 Raises 

385 ------ 

386 ValueError 

387 Raised if the resulting calibClass is the base 

388 `lsst.ip.isr.IsrClass` (which does not implement the 

389 content methods). 

390 """ 

391 calibClassName = metadata.get("CALIBCLS") 

392 calibClass = doImport(calibClassName) if calibClassName is not None else cls 

393 if calibClass is IsrCalib: 

394 raise ValueError(f"Cannot use base class to read calibration data: {message}") 

395 return calibClass 

396 

397 @classmethod 

398 def readText(cls, filename, **kwargs): 

399 """Read calibration representation from a yaml/ecsv file. 

400 

401 Parameters 

402 ---------- 

403 filename : `str` 

404 Name of the file containing the calibration definition. 

405 kwargs : `dict` or collections.abc.Mapping`, optional 

406 Set of key=value pairs to pass to the ``fromDict`` or 

407 ``fromTable`` methods. 

408 

409 Returns 

410 ------- 

411 calib : `~lsst.ip.isr.IsrCalibType` 

412 Calibration class. 

413 

414 Raises 

415 ------ 

416 RuntimeError 

417 Raised if the filename does not end in ".ecsv" or ".yaml". 

418 """ 

419 if filename.endswith((".ecsv", ".ECSV")): 

420 data = Table.read(filename, format="ascii.ecsv") 

421 calibClass = cls.determineCalibClass(data.meta, "readText/ECSV") 

422 return calibClass.fromTable([data], **kwargs) 

423 elif filename.endswith((".yaml", ".YAML")): 

424 with open(filename, "r") as f: 

425 data = yaml.load(f, Loader=yaml.CLoader) 

426 calibClass = cls.determineCalibClass(data["metadata"], "readText/YAML") 

427 return calibClass.fromDict(data, **kwargs) 

428 else: 

429 raise RuntimeError(f"Unknown filename extension: {filename}") 

430 

431 def writeText(self, filename, format="auto"): 

432 """Write the calibration data to a text file. 

433 

434 Parameters 

435 ---------- 

436 filename : `str` 

437 Name of the file to write. 

438 format : `str` 

439 Format to write the file as. Supported values are: 

440 ``"auto"`` : Determine filetype from filename. 

441 ``"yaml"`` : Write as yaml. 

442 ``"ecsv"`` : Write as ecsv. 

443 

444 Returns 

445 ------- 

446 used : `str` 

447 The name of the file used to write the data. This may 

448 differ from the input if the format is explicitly chosen. 

449 

450 Raises 

451 ------ 

452 RuntimeError 

453 Raised if filename does not end in a known extension, or 

454 if all information cannot be written. 

455 

456 Notes 

457 ----- 

458 The file is written to YAML/ECSV format and will include any 

459 associated metadata. 

460 """ 

461 if format == "yaml" or (format == "auto" and filename.lower().endswith((".yaml", ".YAML"))): 

462 outDict = self.toDict() 

463 path, ext = os.path.splitext(filename) 

464 filename = path + ".yaml" 

465 with open(filename, "w") as f: 

466 yaml.dump(outDict, f) 

467 elif format == "ecsv" or (format == "auto" and filename.lower().endswith((".ecsv", ".ECSV"))): 

468 tableList = self.toTable() 

469 if len(tableList) > 1: 

470 # ECSV doesn't support multiple tables per file, so we 

471 # can only write the first table. 

472 raise RuntimeError(f"Unable to persist {len(tableList)}tables in ECSV format.") 

473 

474 table = tableList[0] 

475 path, ext = os.path.splitext(filename) 

476 filename = path + ".ecsv" 

477 table.write(filename, format="ascii.ecsv") 

478 else: 

479 raise RuntimeError(f"Attempt to write to a file {filename} " 

480 "that does not end in '.yaml' or '.ecsv'") 

481 

482 return filename 

483 

484 @classmethod 

485 def readFits(cls, filename, **kwargs): 

486 """Read calibration data from a FITS file. 

487 

488 Parameters 

489 ---------- 

490 filename : `str` 

491 Filename to read data from. 

492 kwargs : `dict` or collections.abc.Mapping`, optional 

493 Set of key=value pairs to pass to the ``fromTable`` 

494 method. 

495 

496 Returns 

497 ------- 

498 calib : `lsst.ip.isr.IsrCalib` 

499 Calibration contained within the file. 

500 """ 

501 tableList = [] 

502 tableList.append(Table.read(filename, hdu=1, mask_invalid=False)) 

503 extNum = 2 # Fits indices start at 1, we've read one already. 

504 keepTrying = True 

505 

506 while keepTrying: 

507 with warnings.catch_warnings(): 

508 warnings.simplefilter("error") 

509 try: 

510 newTable = Table.read(filename, hdu=extNum, mask_invalid=False) 

511 tableList.append(newTable) 

512 extNum += 1 

513 except Exception: 

514 keepTrying = False 

515 

516 for table in tableList: 

517 for k, v in table.meta.items(): 

518 if isinstance(v, fits.card.Undefined): 

519 table.meta[k] = None 

520 

521 calibClass = cls.determineCalibClass(tableList[0].meta, "readFits") 

522 return calibClass.fromTable(tableList, **kwargs) 

523 

524 def writeFits(self, filename): 

525 """Write calibration data to a FITS file. 

526 

527 Parameters 

528 ---------- 

529 filename : `str` 

530 Filename to write data to. 

531 

532 Returns 

533 ------- 

534 used : `str` 

535 The name of the file used to write the data. 

536 """ 

537 tableList = self.toTable() 

538 with warnings.catch_warnings(): 

539 warnings.filterwarnings("ignore", category=Warning, module="astropy.io") 

540 astropyList = [fits.table_to_hdu(table) for table in tableList] 

541 astropyList.insert(0, fits.PrimaryHDU()) 

542 

543 writer = fits.HDUList(astropyList) 

544 writer.writeto(filename, overwrite=True) 

545 return filename 

546 

547 def fromDetector(self, detector): 

548 """Modify the calibration parameters to match the supplied detector. 

549 

550 Parameters 

551 ---------- 

552 detector : `lsst.afw.cameraGeom.Detector` 

553 Detector to use to set parameters from. 

554 

555 Raises 

556 ------ 

557 NotImplementedError 

558 Raised if not implemented by a subclass. 

559 This needs to be implemented by subclasses for each 

560 calibration type. 

561 """ 

562 raise NotImplementedError("Must be implemented by subclass.") 

563 

564 @classmethod 

565 def fromDict(cls, dictionary, **kwargs): 

566 """Construct a calibration from a dictionary of properties. 

567 

568 Must be implemented by the specific calibration subclasses. 

569 

570 Parameters 

571 ---------- 

572 dictionary : `dict` 

573 Dictionary of properties. 

574 kwargs : `dict` or collections.abc.Mapping`, optional 

575 Set of key=value options. 

576 

577 Returns 

578 ------- 

579 calib : `lsst.ip.isr.CalibType` 

580 Constructed calibration. 

581 

582 Raises 

583 ------ 

584 NotImplementedError 

585 Raised if not implemented. 

586 """ 

587 raise NotImplementedError("Must be implemented by subclass.") 

588 

589 def toDict(self): 

590 """Return a dictionary containing the calibration properties. 

591 

592 The dictionary should be able to be round-tripped through 

593 `fromDict`. 

594 

595 Returns 

596 ------- 

597 dictionary : `dict` 

598 Dictionary of properties. 

599 

600 Raises 

601 ------ 

602 NotImplementedError 

603 Raised if not implemented. 

604 """ 

605 raise NotImplementedError("Must be implemented by subclass.") 

606 

607 @classmethod 

608 def fromTable(cls, tableList, **kwargs): 

609 """Construct a calibration from a dictionary of properties. 

610 

611 Must be implemented by the specific calibration subclasses. 

612 

613 Parameters 

614 ---------- 

615 tableList : `list` [`lsst.afw.table.Table`] 

616 List of tables of properties. 

617 kwargs : `dict` or collections.abc.Mapping`, optional 

618 Set of key=value options. 

619 

620 Returns 

621 ------- 

622 calib : `lsst.ip.isr.CalibType` 

623 Constructed calibration. 

624 

625 Raises 

626 ------ 

627 NotImplementedError 

628 Raised if not implemented. 

629 """ 

630 raise NotImplementedError("Must be implemented by subclass.") 

631 

632 def toTable(self): 

633 """Return a list of tables containing the calibration properties. 

634 

635 The table list should be able to be round-tripped through 

636 `fromDict`. 

637 

638 Returns 

639 ------- 

640 tableList : `list` [`lsst.afw.table.Table`] 

641 List of tables of properties. 

642 

643 Raises 

644 ------ 

645 NotImplementedError 

646 Raised if not implemented. 

647 """ 

648 raise NotImplementedError("Must be implemented by subclass.") 

649 

650 def validate(self, other=None): 

651 """Validate that this calibration is defined and can be used. 

652 

653 Parameters 

654 ---------- 

655 other : `object`, optional 

656 Thing to validate against. 

657 

658 Returns 

659 ------- 

660 valid : `bool` 

661 Returns true if the calibration is valid and appropriate. 

662 """ 

663 return False 

664 

665 def apply(self, target): 

666 """Method to apply the calibration to the target object. 

667 

668 Parameters 

669 ---------- 

670 target : `object` 

671 Thing to validate against. 

672 

673 Returns 

674 ------- 

675 valid : `bool` 

676 Returns true if the calibration was applied correctly. 

677 

678 Raises 

679 ------ 

680 NotImplementedError 

681 Raised if not implemented. 

682 """ 

683 raise NotImplementedError("Must be implemented by subclass.") 

684 

685 

686class IsrProvenance(IsrCalib): 

687 """Class for the provenance of data used to construct calibration. 

688 

689 Provenance is not really a calibration, but we would like to 

690 record this when constructing the calibration, and it provides an 

691 example of the base calibration class. 

692 

693 Parameters 

694 ---------- 

695 instrument : `str`, optional 

696 Name of the instrument the data was taken with. 

697 calibType : `str`, optional 

698 Type of calibration this provenance was generated for. 

699 detectorName : `str`, optional 

700 Name of the detector this calibration is for. 

701 detectorSerial : `str`, optional 

702 Identifier for the detector. 

703 

704 """ 

705 _OBSTYPE = "IsrProvenance" 

706 

707 def __init__(self, calibType="unknown", 

708 **kwargs): 

709 self.calibType = calibType 

710 self.dimensions = set() 

711 self.dataIdList = list() 

712 

713 super().__init__(**kwargs) 

714 

715 self.requiredAttributes.update(["calibType", "dimensions", "dataIdList"]) 

716 

717 def __str__(self): 

718 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )" 

719 

720 def __eq__(self, other): 

721 return super().__eq__(other) 

722 

723 def updateMetadata(self, setDate=False, **kwargs): 

724 """Update calibration metadata. 

725 

726 Parameters 

727 ---------- 

728 setDate : `bool`, optional 

729 Update the ``CALIBDATE`` fields in the metadata to the current 

730 time. Defaults to False. 

731 kwargs : `dict` or `collections.abc.Mapping`, optional 

732 Other keyword parameters to set in the metadata. 

733 """ 

734 kwargs["calibType"] = self.calibType 

735 super().updateMetadata(setDate=setDate, **kwargs) 

736 

737 def fromDataIds(self, dataIdList): 

738 """Update provenance from dataId List. 

739 

740 Parameters 

741 ---------- 

742 dataIdList : `list` [`lsst.daf.butler.DataId`] 

743 List of dataIds used in generating this calibration. 

744 """ 

745 for dataId in dataIdList: 

746 for key in dataId: 

747 if key not in self.dimensions: 

748 self.dimensions.add(key) 

749 self.dataIdList.append(dataId) 

750 

751 @classmethod 

752 def fromTable(cls, tableList): 

753 """Construct provenance from table list. 

754 

755 Parameters 

756 ---------- 

757 tableList : `list` [`lsst.afw.table.Table`] 

758 List of tables to construct the provenance from. 

759 

760 Returns 

761 ------- 

762 provenance : `lsst.ip.isr.IsrProvenance` 

763 The provenance defined in the tables. 

764 """ 

765 table = tableList[0] 

766 metadata = table.meta 

767 inDict = dict() 

768 inDict["metadata"] = metadata 

769 inDict["calibType"] = metadata["calibType"] 

770 inDict["dimensions"] = set() 

771 inDict["dataIdList"] = list() 

772 

773 schema = dict() 

774 for colName in table.columns: 

775 schema[colName.lower()] = colName 

776 inDict["dimensions"].add(colName.lower()) 

777 inDict["dimensions"] = sorted(inDict["dimensions"]) 

778 

779 for row in table: 

780 entry = dict() 

781 for dim in sorted(inDict["dimensions"]): 

782 entry[dim] = row[schema[dim]] 

783 inDict["dataIdList"].append(entry) 

784 

785 return cls.fromDict(inDict) 

786 

787 @classmethod 

788 def fromDict(cls, dictionary): 

789 """Construct provenance from a dictionary. 

790 

791 Parameters 

792 ---------- 

793 dictionary : `dict` 

794 Dictionary of provenance parameters. 

795 

796 Returns 

797 ------- 

798 provenance : `lsst.ip.isr.IsrProvenance` 

799 The provenance defined in the tables. 

800 """ 

801 calib = cls() 

802 if calib._OBSTYPE != dictionary["metadata"]["OBSTYPE"]: 

803 raise RuntimeError(f"Incorrect calibration supplied. Expected {calib._OBSTYPE}, " 

804 f"found {dictionary['metadata']['OBSTYPE']}") 

805 calib.updateMetadata(setDate=False, setCalibInfo=True, **dictionary["metadata"]) 

806 

807 # These properties should be in the metadata, but occasionally 

808 # are found in the dictionary itself. Check both places, 

809 # ending with `None` if neither contains the information. 

810 calib.calibType = dictionary["calibType"] 

811 calib.dimensions = set(dictionary["dimensions"]) 

812 calib.dataIdList = dictionary["dataIdList"] 

813 

814 calib.updateMetadata() 

815 return calib 

816 

817 def toDict(self): 

818 """Return a dictionary containing the provenance information. 

819 

820 Returns 

821 ------- 

822 dictionary : `dict` 

823 Dictionary of provenance. 

824 """ 

825 self.updateMetadata() 

826 

827 outDict = {} 

828 

829 metadata = self.getMetadata() 

830 outDict["metadata"] = metadata 

831 outDict["detectorName"] = self._detectorName 

832 outDict["detectorSerial"] = self._detectorSerial 

833 outDict["detectorId"] = self._detectorId 

834 outDict["instrument"] = self._instrument 

835 outDict["calibType"] = self.calibType 

836 outDict["dimensions"] = list(self.dimensions) 

837 outDict["dataIdList"] = self.dataIdList 

838 

839 return outDict 

840 

841 def toTable(self): 

842 """Return a list of tables containing the provenance. 

843 

844 This seems inefficient and slow, so this may not be the best 

845 way to store the data. 

846 

847 Returns 

848 ------- 

849 tableList : `list` [`lsst.afw.table.Table`] 

850 List of tables containing the provenance information 

851 """ 

852 tableList = [] 

853 self.updateMetadata(setDate=True, setCalibInfo=True) 

854 

855 catalog = Table(rows=self.dataIdList, 

856 names=self.dimensions) 

857 filteredMetadata = {k: v for k, v in self.getMetadata().toDict().items() if v is not None} 

858 catalog.meta = filteredMetadata 

859 tableList.append(catalog) 

860 return tableList