Coverage for python/lsst/ip/isr/calibType.py: 14%

312 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-11-18 02:49 -0800

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21__all__ = ["IsrCalib", "IsrProvenance"] 

22 

23import abc 

24import datetime 

25import logging 

26import os.path 

27import warnings 

28import yaml 

29import numpy as np 

30 

31from astropy.table import Table 

32from astropy.io import fits 

33 

34from lsst.daf.base import PropertyList 

35from lsst.utils.introspection import get_full_type_name 

36from lsst.utils import doImport 

37 

38 

39class IsrCalib(abc.ABC): 

40 """Generic calibration type. 

41 

42 Subclasses must implement the toDict, fromDict, toTable, fromTable 

43 methods that allow the calibration information to be converted 

44 from dictionaries and afw tables. This will allow the calibration 

45 to be persisted using the base class read/write methods. 

46 

47 The validate method is intended to provide a common way to check 

48 that the calibration is valid (internally consistent) and 

49 appropriate (usable with the intended data). The apply method is 

50 intended to allow the calibration to be applied in a consistent 

51 manner. 

52 

53 Parameters 

54 ---------- 

55 camera : `lsst.afw.cameraGeom.Camera`, optional 

56 Camera to extract metadata from. 

57 detector : `lsst.afw.cameraGeom.Detector`, optional 

58 Detector to extract metadata from. 

59 log : `logging.Logger`, optional 

60 Log for messages. 

61 """ 

62 _OBSTYPE = "generic" 

63 _SCHEMA = "NO SCHEMA" 

64 _VERSION = 0 

65 

66 def __init__(self, camera=None, detector=None, log=None, **kwargs): 

67 self._instrument = None 

68 self._raftName = None 

69 self._slotName = None 

70 self._detectorName = None 

71 self._detectorSerial = None 

72 self._detectorId = None 

73 self._filter = None 

74 self._calibId = None 

75 self._metadata = PropertyList() 

76 self.setMetadata(PropertyList()) 

77 self.calibInfoFromDict(kwargs) 

78 

79 # Define the required attributes for this calibration. 

80 self.requiredAttributes = set(["_OBSTYPE", "_SCHEMA", "_VERSION"]) 

81 self.requiredAttributes.update(["_instrument", "_raftName", "_slotName", 

82 "_detectorName", "_detectorSerial", "_detectorId", 

83 "_filter", "_calibId", "_metadata"]) 

84 

85 self.log = log if log else logging.getLogger(__name__) 

86 

87 if detector: 

88 self.fromDetector(detector) 

89 self.updateMetadata(camera=camera, detector=detector) 

90 

91 def __str__(self): 

92 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )" 

93 

94 def __eq__(self, other): 

95 """Calibration equivalence. 

96 

97 Running ``calib.log.setLevel(0)`` enables debug statements to 

98 identify problematic fields. 

99 """ 

100 if not isinstance(other, self.__class__): 

101 self.log.debug("Incorrect class type: %s %s", self.__class__, other.__class__) 

102 return False 

103 

104 for attr in self._requiredAttributes: 

105 attrSelf = getattr(self, attr) 

106 attrOther = getattr(other, attr) 

107 

108 if isinstance(attrSelf, dict): 

109 # Dictionary of arrays. 

110 if attrSelf.keys() != attrOther.keys(): 

111 self.log.debug("Dict Key Failure: %s %s %s", attr, attrSelf.keys(), attrOther.keys()) 

112 return False 

113 for key in attrSelf: 

114 try: 

115 if not np.allclose(attrSelf[key], attrOther[key], equal_nan=True): 

116 self.log.debug("Array Failure: %s %s %s", key, attrSelf[key], attrOther[key]) 

117 return False 

118 except TypeError: 

119 # If it is not something numpy can handle 

120 # (it's not a number or array of numbers), 

121 # then it needs to have its own equivalence 

122 # operator. 

123 if attrSelf[key] != attrOther[key]: 

124 return False 

125 elif isinstance(attrSelf, np.ndarray): 

126 # Bare array. 

127 if not np.allclose(attrSelf, attrOther, equal_nan=True): 

128 self.log.debug("Array Failure: %s %s %s", attr, attrSelf, attrOther) 

129 return False 

130 elif type(attrSelf) != type(attrOther): 

131 if set([attrSelf, attrOther]) == set([None, ""]): 

132 # Fits converts None to "", but None is not "". 

133 continue 

134 self.log.debug("Type Failure: %s %s %s %s %s", attr, type(attrSelf), type(attrOther), 

135 attrSelf, attrOther) 

136 return False 

137 else: 

138 if attrSelf != attrOther: 

139 self.log.debug("Value Failure: %s %s %s", attr, attrSelf, attrOther) 

140 return False 

141 

142 return True 

143 

144 @property 

145 def requiredAttributes(self): 

146 return self._requiredAttributes 

147 

148 @requiredAttributes.setter 

149 def requiredAttributes(self, value): 

150 self._requiredAttributes = value 

151 

152 def getMetadata(self): 

153 """Retrieve metadata associated with this calibration. 

154 

155 Returns 

156 ------- 

157 meta : `lsst.daf.base.PropertyList` 

158 Metadata. The returned `~lsst.daf.base.PropertyList` can be 

159 modified by the caller and the changes will be written to 

160 external files. 

161 """ 

162 return self._metadata 

163 

164 def setMetadata(self, metadata): 

165 """Store a copy of the supplied metadata with this calibration. 

166 

167 Parameters 

168 ---------- 

169 metadata : `lsst.daf.base.PropertyList` 

170 Metadata to associate with the calibration. Will be copied and 

171 overwrite existing metadata. 

172 """ 

173 if metadata is not None: 

174 self._metadata.update(metadata) 

175 

176 # Ensure that we have the obs type required by calibration ingest 

177 self._metadata["OBSTYPE"] = self._OBSTYPE 

178 self._metadata[self._OBSTYPE + "_SCHEMA"] = self._SCHEMA 

179 self._metadata[self._OBSTYPE + "_VERSION"] = self._VERSION 

180 

181 if isinstance(metadata, dict): 

182 self.calibInfoFromDict(metadata) 

183 elif isinstance(metadata, PropertyList): 

184 self.calibInfoFromDict(metadata.toDict()) 

185 

186 def updateMetadata(self, camera=None, detector=None, filterName=None, 

187 setCalibId=False, setCalibInfo=False, setDate=False, 

188 **kwargs): 

189 """Update metadata keywords with new values. 

190 

191 Parameters 

192 ---------- 

193 camera : `lsst.afw.cameraGeom.Camera`, optional 

194 Reference camera to use to set ``_instrument`` field. 

195 detector : `lsst.afw.cameraGeom.Detector`, optional 

196 Reference detector to use to set ``_detector*`` fields. 

197 filterName : `str`, optional 

198 Filter name to assign to this calibration. 

199 setCalibId : `bool`, optional 

200 Construct the ``_calibId`` field from other fields. 

201 setCalibInfo : `bool`, optional 

202 Set calibration parameters from metadata. 

203 setDate : `bool`, optional 

204 Ensure the metadata ``CALIBDATE`` fields are set to the current 

205 datetime. 

206 kwargs : `dict` or `collections.abc.Mapping`, optional 

207 Set of ``key=value`` pairs to assign to the metadata. 

208 """ 

209 mdOriginal = self.getMetadata() 

210 mdSupplemental = dict() 

211 

212 for k, v in kwargs.items(): 

213 if isinstance(v, fits.card.Undefined): 

214 kwargs[k] = None 

215 

216 if setCalibInfo: 

217 self.calibInfoFromDict(kwargs) 

218 

219 if camera: 

220 self._instrument = camera.getName() 

221 

222 if detector: 

223 self._detectorName = detector.getName() 

224 self._detectorSerial = detector.getSerial() 

225 self._detectorId = detector.getId() 

226 if "_" in self._detectorName: 

227 (self._raftName, self._slotName) = self._detectorName.split("_") 

228 

229 if filterName: 

230 # TOD0 DM-28093: I think this whole comment can go away, if we 

231 # always use physicalLabel everywhere in ip_isr. 

232 # If set via: 

233 # exposure.getInfo().getFilter().getName() 

234 # then this will hold the abstract filter. 

235 self._filter = filterName 

236 

237 if setDate: 

238 date = datetime.datetime.now() 

239 mdSupplemental["CALIBDATE"] = date.isoformat() 

240 mdSupplemental["CALIB_CREATION_DATE"] = date.date().isoformat() 

241 mdSupplemental["CALIB_CREATION_TIME"] = date.time().isoformat() 

242 

243 if setCalibId: 

244 values = [] 

245 values.append(f"instrument={self._instrument}") if self._instrument else None 

246 values.append(f"raftName={self._raftName}") if self._raftName else None 

247 values.append(f"detectorName={self._detectorName}") if self._detectorName else None 

248 values.append(f"detector={self._detectorId}") if self._detectorId else None 

249 values.append(f"filter={self._filter}") if self._filter else None 

250 

251 calibDate = mdOriginal.get("CALIBDATE", mdSupplemental.get("CALIBDATE", None)) 

252 values.append(f"calibDate={calibDate}") if calibDate else None 

253 

254 self._calibId = " ".join(values) 

255 

256 self._metadata["INSTRUME"] = self._instrument if self._instrument else None 

257 self._metadata["RAFTNAME"] = self._raftName if self._raftName else None 

258 self._metadata["SLOTNAME"] = self._slotName if self._slotName else None 

259 self._metadata["DETECTOR"] = self._detectorId 

260 self._metadata["DET_NAME"] = self._detectorName if self._detectorName else None 

261 self._metadata["DET_SER"] = self._detectorSerial if self._detectorSerial else None 

262 self._metadata["FILTER"] = self._filter if self._filter else None 

263 self._metadata["CALIB_ID"] = self._calibId if self._calibId else None 

264 self._metadata["CALIBCLS"] = get_full_type_name(self) 

265 

266 mdSupplemental.update(kwargs) 

267 mdOriginal.update(mdSupplemental) 

268 

269 def calibInfoFromDict(self, dictionary): 

270 """Handle common keywords. 

271 

272 This isn't an ideal solution, but until all calibrations 

273 expect to find everything in the metadata, they still need to 

274 search through dictionaries. 

275 

276 Parameters 

277 ---------- 

278 dictionary : `dict` or `lsst.daf.base.PropertyList` 

279 Source for the common keywords. 

280 

281 Raises 

282 ------ 

283 RuntimeError 

284 Raised if the dictionary does not match the expected OBSTYPE. 

285 """ 

286 

287 def search(haystack, needles): 

288 """Search dictionary 'haystack' for an entry in 'needles' 

289 """ 

290 test = [haystack.get(x) for x in needles] 

291 test = set([x for x in test if x is not None]) 

292 if len(test) == 0: 

293 if "metadata" in haystack: 

294 return search(haystack["metadata"], needles) 

295 else: 

296 return None 

297 elif len(test) == 1: 

298 value = list(test)[0] 

299 if value == "": 

300 return None 

301 else: 

302 return value 

303 else: 

304 raise ValueError(f"Too many values found: {len(test)} {test} {needles}") 

305 

306 if "metadata" in dictionary: 

307 metadata = dictionary["metadata"] 

308 

309 if self._OBSTYPE != metadata["OBSTYPE"]: 

310 raise RuntimeError(f"Incorrect calibration supplied. Expected {self._OBSTYPE}, " 

311 f"found {metadata['OBSTYPE']}") 

312 

313 self._instrument = search(dictionary, ["INSTRUME", "instrument"]) 

314 self._raftName = search(dictionary, ["RAFTNAME"]) 

315 self._slotName = search(dictionary, ["SLOTNAME"]) 

316 self._detectorId = search(dictionary, ["DETECTOR", "detectorId"]) 

317 self._detectorName = search(dictionary, ["DET_NAME", "DETECTOR_NAME", "detectorName"]) 

318 self._detectorSerial = search(dictionary, ["DET_SER", "DETECTOR_SERIAL", "detectorSerial"]) 

319 self._filter = search(dictionary, ["FILTER", "filterName"]) 

320 self._calibId = search(dictionary, ["CALIB_ID"]) 

321 

322 @classmethod 

323 def determineCalibClass(cls, metadata, message): 

324 """Attempt to find calibration class in metadata. 

325 

326 Parameters 

327 ---------- 

328 metadata : `dict` or `lsst.daf.base.PropertyList` 

329 Metadata possibly containing a calibration class entry. 

330 message : `str` 

331 Message to include in any errors. 

332 

333 Returns 

334 ------- 

335 calibClass : `object` 

336 The class to use to read the file contents. Should be an 

337 `lsst.ip.isr.IsrCalib` subclass. 

338 

339 Raises 

340 ------ 

341 ValueError 

342 Raised if the resulting calibClass is the base 

343 `lsst.ip.isr.IsrClass` (which does not implement the 

344 content methods). 

345 """ 

346 calibClassName = metadata.get("CALIBCLS") 

347 calibClass = doImport(calibClassName) if calibClassName is not None else cls 

348 if calibClass is IsrCalib: 

349 raise ValueError(f"Cannot use base class to read calibration data: {message}") 

350 return calibClass 

351 

352 @classmethod 

353 def readText(cls, filename, **kwargs): 

354 """Read calibration representation from a yaml/ecsv file. 

355 

356 Parameters 

357 ---------- 

358 filename : `str` 

359 Name of the file containing the calibration definition. 

360 kwargs : `dict` or collections.abc.Mapping`, optional 

361 Set of key=value pairs to pass to the ``fromDict`` or 

362 ``fromTable`` methods. 

363 

364 Returns 

365 ------- 

366 calib : `~lsst.ip.isr.IsrCalibType` 

367 Calibration class. 

368 

369 Raises 

370 ------ 

371 RuntimeError 

372 Raised if the filename does not end in ".ecsv" or ".yaml". 

373 """ 

374 if filename.endswith((".ecsv", ".ECSV")): 

375 data = Table.read(filename, format="ascii.ecsv") 

376 calibClass = cls.determineCalibClass(data.meta, "readText/ECSV") 

377 return calibClass.fromTable([data], **kwargs) 

378 elif filename.endswith((".yaml", ".YAML")): 

379 with open(filename, "r") as f: 

380 data = yaml.load(f, Loader=yaml.CLoader) 

381 calibClass = cls.determineCalibClass(data["metadata"], "readText/YAML") 

382 return calibClass.fromDict(data, **kwargs) 

383 else: 

384 raise RuntimeError(f"Unknown filename extension: {filename}") 

385 

386 def writeText(self, filename, format="auto"): 

387 """Write the calibration data to a text file. 

388 

389 Parameters 

390 ---------- 

391 filename : `str` 

392 Name of the file to write. 

393 format : `str` 

394 Format to write the file as. Supported values are: 

395 ``"auto"`` : Determine filetype from filename. 

396 ``"yaml"`` : Write as yaml. 

397 ``"ecsv"`` : Write as ecsv. 

398 

399 Returns 

400 ------- 

401 used : `str` 

402 The name of the file used to write the data. This may 

403 differ from the input if the format is explicitly chosen. 

404 

405 Raises 

406 ------ 

407 RuntimeError 

408 Raised if filename does not end in a known extension, or 

409 if all information cannot be written. 

410 

411 Notes 

412 ----- 

413 The file is written to YAML/ECSV format and will include any 

414 associated metadata. 

415 """ 

416 if format == "yaml" or (format == "auto" and filename.lower().endswith((".yaml", ".YAML"))): 

417 outDict = self.toDict() 

418 path, ext = os.path.splitext(filename) 

419 filename = path + ".yaml" 

420 with open(filename, "w") as f: 

421 yaml.dump(outDict, f) 

422 elif format == "ecsv" or (format == "auto" and filename.lower().endswith((".ecsv", ".ECSV"))): 

423 tableList = self.toTable() 

424 if len(tableList) > 1: 

425 # ECSV doesn't support multiple tables per file, so we 

426 # can only write the first table. 

427 raise RuntimeError(f"Unable to persist {len(tableList)}tables in ECSV format.") 

428 

429 table = tableList[0] 

430 path, ext = os.path.splitext(filename) 

431 filename = path + ".ecsv" 

432 table.write(filename, format="ascii.ecsv") 

433 else: 

434 raise RuntimeError(f"Attempt to write to a file {filename} " 

435 "that does not end in '.yaml' or '.ecsv'") 

436 

437 return filename 

438 

439 @classmethod 

440 def readFits(cls, filename, **kwargs): 

441 """Read calibration data from a FITS file. 

442 

443 Parameters 

444 ---------- 

445 filename : `str` 

446 Filename to read data from. 

447 kwargs : `dict` or collections.abc.Mapping`, optional 

448 Set of key=value pairs to pass to the ``fromTable`` 

449 method. 

450 

451 Returns 

452 ------- 

453 calib : `lsst.ip.isr.IsrCalib` 

454 Calibration contained within the file. 

455 """ 

456 tableList = [] 

457 tableList.append(Table.read(filename, hdu=1)) 

458 extNum = 2 # Fits indices start at 1, we've read one already. 

459 keepTrying = True 

460 

461 while keepTrying: 

462 with warnings.catch_warnings(): 

463 warnings.simplefilter("error") 

464 try: 

465 newTable = Table.read(filename, hdu=extNum) 

466 tableList.append(newTable) 

467 extNum += 1 

468 except Exception: 

469 keepTrying = False 

470 

471 for table in tableList: 

472 for k, v in table.meta.items(): 

473 if isinstance(v, fits.card.Undefined): 

474 table.meta[k] = None 

475 

476 calibClass = cls.determineCalibClass(tableList[0].meta, "readFits") 

477 return calibClass.fromTable(tableList, **kwargs) 

478 

479 def writeFits(self, filename): 

480 """Write calibration data to a FITS file. 

481 

482 Parameters 

483 ---------- 

484 filename : `str` 

485 Filename to write data to. 

486 

487 Returns 

488 ------- 

489 used : `str` 

490 The name of the file used to write the data. 

491 """ 

492 tableList = self.toTable() 

493 with warnings.catch_warnings(): 

494 warnings.filterwarnings("ignore", category=Warning, module="astropy.io") 

495 astropyList = [fits.table_to_hdu(table) for table in tableList] 

496 astropyList.insert(0, fits.PrimaryHDU()) 

497 

498 writer = fits.HDUList(astropyList) 

499 writer.writeto(filename, overwrite=True) 

500 return filename 

501 

502 def fromDetector(self, detector): 

503 """Modify the calibration parameters to match the supplied detector. 

504 

505 Parameters 

506 ---------- 

507 detector : `lsst.afw.cameraGeom.Detector` 

508 Detector to use to set parameters from. 

509 

510 Raises 

511 ------ 

512 NotImplementedError 

513 Raised if not implemented by a subclass. 

514 This needs to be implemented by subclasses for each 

515 calibration type. 

516 """ 

517 raise NotImplementedError("Must be implemented by subclass.") 

518 

519 @classmethod 

520 def fromDict(cls, dictionary, **kwargs): 

521 """Construct a calibration from a dictionary of properties. 

522 

523 Must be implemented by the specific calibration subclasses. 

524 

525 Parameters 

526 ---------- 

527 dictionary : `dict` 

528 Dictionary of properties. 

529 kwargs : `dict` or collections.abc.Mapping`, optional 

530 Set of key=value options. 

531 

532 Returns 

533 ------- 

534 calib : `lsst.ip.isr.CalibType` 

535 Constructed calibration. 

536 

537 Raises 

538 ------ 

539 NotImplementedError 

540 Raised if not implemented. 

541 """ 

542 raise NotImplementedError("Must be implemented by subclass.") 

543 

544 def toDict(self): 

545 """Return a dictionary containing the calibration properties. 

546 

547 The dictionary should be able to be round-tripped through 

548 `fromDict`. 

549 

550 Returns 

551 ------- 

552 dictionary : `dict` 

553 Dictionary of properties. 

554 

555 Raises 

556 ------ 

557 NotImplementedError 

558 Raised if not implemented. 

559 """ 

560 raise NotImplementedError("Must be implemented by subclass.") 

561 

562 @classmethod 

563 def fromTable(cls, tableList, **kwargs): 

564 """Construct a calibration from a dictionary of properties. 

565 

566 Must be implemented by the specific calibration subclasses. 

567 

568 Parameters 

569 ---------- 

570 tableList : `list` [`lsst.afw.table.Table`] 

571 List of tables of properties. 

572 kwargs : `dict` or collections.abc.Mapping`, optional 

573 Set of key=value options. 

574 

575 Returns 

576 ------- 

577 calib : `lsst.ip.isr.CalibType` 

578 Constructed calibration. 

579 

580 Raises 

581 ------ 

582 NotImplementedError 

583 Raised if not implemented. 

584 """ 

585 raise NotImplementedError("Must be implemented by subclass.") 

586 

587 def toTable(self): 

588 """Return a list of tables containing the calibration properties. 

589 

590 The table list should be able to be round-tripped through 

591 `fromDict`. 

592 

593 Returns 

594 ------- 

595 tableList : `list` [`lsst.afw.table.Table`] 

596 List of tables of properties. 

597 

598 Raises 

599 ------ 

600 NotImplementedError 

601 Raised if not implemented. 

602 """ 

603 raise NotImplementedError("Must be implemented by subclass.") 

604 

605 def validate(self, other=None): 

606 """Validate that this calibration is defined and can be used. 

607 

608 Parameters 

609 ---------- 

610 other : `object`, optional 

611 Thing to validate against. 

612 

613 Returns 

614 ------- 

615 valid : `bool` 

616 Returns true if the calibration is valid and appropriate. 

617 """ 

618 return False 

619 

620 def apply(self, target): 

621 """Method to apply the calibration to the target object. 

622 

623 Parameters 

624 ---------- 

625 target : `object` 

626 Thing to validate against. 

627 

628 Returns 

629 ------- 

630 valid : `bool` 

631 Returns true if the calibration was applied correctly. 

632 

633 Raises 

634 ------ 

635 NotImplementedError 

636 Raised if not implemented. 

637 """ 

638 raise NotImplementedError("Must be implemented by subclass.") 

639 

640 

641class IsrProvenance(IsrCalib): 

642 """Class for the provenance of data used to construct calibration. 

643 

644 Provenance is not really a calibration, but we would like to 

645 record this when constructing the calibration, and it provides an 

646 example of the base calibration class. 

647 

648 Parameters 

649 ---------- 

650 instrument : `str`, optional 

651 Name of the instrument the data was taken with. 

652 calibType : `str`, optional 

653 Type of calibration this provenance was generated for. 

654 detectorName : `str`, optional 

655 Name of the detector this calibration is for. 

656 detectorSerial : `str`, optional 

657 Identifier for the detector. 

658 

659 """ 

660 _OBSTYPE = "IsrProvenance" 

661 

662 def __init__(self, calibType="unknown", 

663 **kwargs): 

664 self.calibType = calibType 

665 self.dimensions = set() 

666 self.dataIdList = list() 

667 

668 super().__init__(**kwargs) 

669 

670 self.requiredAttributes.update(["calibType", "dimensions", "dataIdList"]) 

671 

672 def __str__(self): 

673 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )" 

674 

675 def __eq__(self, other): 

676 return super().__eq__(other) 

677 

678 def updateMetadata(self, setDate=False, **kwargs): 

679 """Update calibration metadata. 

680 

681 Parameters 

682 ---------- 

683 setDate : `bool`, optional 

684 Update the ``CALIBDATE`` fields in the metadata to the current 

685 time. Defaults to False. 

686 kwargs : `dict` or `collections.abc.Mapping`, optional 

687 Other keyword parameters to set in the metadata. 

688 """ 

689 kwargs["calibType"] = self.calibType 

690 super().updateMetadata(setDate=setDate, **kwargs) 

691 

692 def fromDataIds(self, dataIdList): 

693 """Update provenance from dataId List. 

694 

695 Parameters 

696 ---------- 

697 dataIdList : `list` [`lsst.daf.butler.DataId`] 

698 List of dataIds used in generating this calibration. 

699 """ 

700 for dataId in dataIdList: 

701 for key in dataId: 

702 if key not in self.dimensions: 

703 self.dimensions.add(key) 

704 self.dataIdList.append(dataId) 

705 

706 @classmethod 

707 def fromTable(cls, tableList): 

708 """Construct provenance from table list. 

709 

710 Parameters 

711 ---------- 

712 tableList : `list` [`lsst.afw.table.Table`] 

713 List of tables to construct the provenance from. 

714 

715 Returns 

716 ------- 

717 provenance : `lsst.ip.isr.IsrProvenance` 

718 The provenance defined in the tables. 

719 """ 

720 table = tableList[0] 

721 metadata = table.meta 

722 inDict = dict() 

723 inDict["metadata"] = metadata 

724 inDict["calibType"] = metadata["calibType"] 

725 inDict["dimensions"] = set() 

726 inDict["dataIdList"] = list() 

727 

728 schema = dict() 

729 for colName in table.columns: 

730 schema[colName.lower()] = colName 

731 inDict["dimensions"].add(colName.lower()) 

732 inDict["dimensions"] = sorted(inDict["dimensions"]) 

733 

734 for row in table: 

735 entry = dict() 

736 for dim in sorted(inDict["dimensions"]): 

737 entry[dim] = row[schema[dim]] 

738 inDict["dataIdList"].append(entry) 

739 

740 return cls.fromDict(inDict) 

741 

742 @classmethod 

743 def fromDict(cls, dictionary): 

744 """Construct provenance from a dictionary. 

745 

746 Parameters 

747 ---------- 

748 dictionary : `dict` 

749 Dictionary of provenance parameters. 

750 

751 Returns 

752 ------- 

753 provenance : `lsst.ip.isr.IsrProvenance` 

754 The provenance defined in the tables. 

755 """ 

756 calib = cls() 

757 if calib._OBSTYPE != dictionary["metadata"]["OBSTYPE"]: 

758 raise RuntimeError(f"Incorrect calibration supplied. Expected {calib._OBSTYPE}, " 

759 f"found {dictionary['metadata']['OBSTYPE']}") 

760 calib.updateMetadata(setDate=False, setCalibInfo=True, **dictionary["metadata"]) 

761 

762 # These properties should be in the metadata, but occasionally 

763 # are found in the dictionary itself. Check both places, 

764 # ending with `None` if neither contains the information. 

765 calib.calibType = dictionary["calibType"] 

766 calib.dimensions = set(dictionary["dimensions"]) 

767 calib.dataIdList = dictionary["dataIdList"] 

768 

769 calib.updateMetadata() 

770 return calib 

771 

772 def toDict(self): 

773 """Return a dictionary containing the provenance information. 

774 

775 Returns 

776 ------- 

777 dictionary : `dict` 

778 Dictionary of provenance. 

779 """ 

780 self.updateMetadata() 

781 

782 outDict = {} 

783 

784 metadata = self.getMetadata() 

785 outDict["metadata"] = metadata 

786 outDict["detectorName"] = self._detectorName 

787 outDict["detectorSerial"] = self._detectorSerial 

788 outDict["detectorId"] = self._detectorId 

789 outDict["instrument"] = self._instrument 

790 outDict["calibType"] = self.calibType 

791 outDict["dimensions"] = list(self.dimensions) 

792 outDict["dataIdList"] = self.dataIdList 

793 

794 return outDict 

795 

796 def toTable(self): 

797 """Return a list of tables containing the provenance. 

798 

799 This seems inefficient and slow, so this may not be the best 

800 way to store the data. 

801 

802 Returns 

803 ------- 

804 tableList : `list` [`lsst.afw.table.Table`] 

805 List of tables containing the provenance information 

806 """ 

807 tableList = [] 

808 self.updateMetadata(setDate=True, setCalibInfo=True) 

809 

810 catalog = Table(rows=self.dataIdList, 

811 names=self.dimensions) 

812 filteredMetadata = {k: v for k, v in self.getMetadata().toDict().items() if v is not None} 

813 catalog.meta = filteredMetadata 

814 tableList.append(catalog) 

815 return tableList