Coverage for python/lsst/ip/isr/calibType.py: 15%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

308 statements  

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21import abc 

22import datetime 

23import logging 

24import os.path 

25import warnings 

26import yaml 

27import numpy as np 

28 

29from astropy.table import Table 

30from astropy.io import fits 

31 

32from lsst.daf.base import PropertyList 

33from lsst.daf.butler.core.utils import getFullTypeName 

34from lsst.utils import doImport 

35 

36 

37__all__ = ["IsrCalib", "IsrProvenance"] 

38 

39 

40class IsrCalib(abc.ABC): 

41 """Generic calibration type. 

42 

43 Subclasses must implement the toDict, fromDict, toTable, fromTable 

44 methods that allow the calibration information to be converted 

45 from dictionaries and afw tables. This will allow the calibration 

46 to be persisted using the base class read/write methods. 

47 

48 The validate method is intended to provide a common way to check 

49 that the calibration is valid (internally consistent) and 

50 appropriate (usable with the intended data). The apply method is 

51 intended to allow the calibration to be applied in a consistent 

52 manner. 

53 

54 Parameters 

55 ---------- 

56 camera : `lsst.afw.cameraGeom.Camera`, optional 

57 Camera to extract metadata from. 

58 detector : `lsst.afw.cameraGeom.Detector`, optional 

59 Detector to extract metadata from. 

60 log : `logging.Logger`, optional 

61 Log for messages. 

62 """ 

63 _OBSTYPE = "generic" 

64 _SCHEMA = "NO SCHEMA" 

65 _VERSION = 0 

66 

67 def __init__(self, camera=None, detector=None, log=None, **kwargs): 

68 self._instrument = None 

69 self._raftName = None 

70 self._slotName = None 

71 self._detectorName = None 

72 self._detectorSerial = None 

73 self._detectorId = None 

74 self._filter = None 

75 self._calibId = None 

76 self._metadata = PropertyList() 

77 self.setMetadata(PropertyList()) 

78 self.calibInfoFromDict(kwargs) 

79 

80 # Define the required attributes for this calibration. 

81 self.requiredAttributes = set(["_OBSTYPE", "_SCHEMA", "_VERSION"]) 

82 self.requiredAttributes.update(["_instrument", "_raftName", "_slotName", 

83 "_detectorName", "_detectorSerial", "_detectorId", 

84 "_filter", "_calibId", "_metadata"]) 

85 

86 self.log = log if log else logging.getLogger(__name__.partition(".")[2]) 

87 

88 if detector: 

89 self.fromDetector(detector) 

90 self.updateMetadata(camera=camera, detector=detector) 

91 

92 def __str__(self): 

93 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )" 

94 

95 def __eq__(self, other): 

96 """Calibration equivalence. 

97 

98 Running ``calib.log.setLevel(0)`` enables debug statements to 

99 identify problematic fields. 

100 """ 

101 if not isinstance(other, self.__class__): 

102 self.log.debug("Incorrect class type: %s %s", self.__class__, other.__class__) 

103 return False 

104 

105 for attr in self._requiredAttributes: 

106 attrSelf = getattr(self, attr) 

107 attrOther = getattr(other, attr) 

108 

109 if isinstance(attrSelf, dict): 

110 # Dictionary of arrays. 

111 if attrSelf.keys() != attrOther.keys(): 

112 self.log.debug("Dict Key Failure: %s %s %s", attr, attrSelf.keys(), attrOther.keys()) 

113 return False 

114 for key in attrSelf: 

115 if not np.allclose(attrSelf[key], attrOther[key], equal_nan=True): 

116 self.log.debug("Array Failure: %s %s %s", key, attrSelf[key], attrOther[key]) 

117 return False 

118 elif isinstance(attrSelf, np.ndarray): 

119 # Bare array. 

120 if not np.allclose(attrSelf, attrOther, equal_nan=True): 

121 self.log.debug("Array Failure: %s %s %s", attr, attrSelf, attrOther) 

122 return False 

123 elif type(attrSelf) != type(attrOther): 

124 if set([attrSelf, attrOther]) == set([None, ""]): 

125 # Fits converts None to "", but None is not "". 

126 continue 

127 self.log.debug("Type Failure: %s %s %s %s %s", attr, type(attrSelf), type(attrOther), 

128 attrSelf, attrOther) 

129 return False 

130 else: 

131 if attrSelf != attrOther: 

132 self.log.debug("Value Failure: %s %s %s", attr, attrSelf, attrOther) 

133 return False 

134 

135 return True 

136 

137 @property 

138 def requiredAttributes(self): 

139 return self._requiredAttributes 

140 

141 @requiredAttributes.setter 

142 def requiredAttributes(self, value): 

143 self._requiredAttributes = value 

144 

145 def getMetadata(self): 

146 """Retrieve metadata associated with this calibration. 

147 

148 Returns 

149 ------- 

150 meta : `lsst.daf.base.PropertyList` 

151 Metadata. The returned `~lsst.daf.base.PropertyList` can be 

152 modified by the caller and the changes will be written to 

153 external files. 

154 """ 

155 return self._metadata 

156 

157 def setMetadata(self, metadata): 

158 """Store a copy of the supplied metadata with this calibration. 

159 

160 Parameters 

161 ---------- 

162 metadata : `lsst.daf.base.PropertyList` 

163 Metadata to associate with the calibration. Will be copied and 

164 overwrite existing metadata. 

165 """ 

166 if metadata is not None: 

167 self._metadata.update(metadata) 

168 

169 # Ensure that we have the obs type required by calibration ingest 

170 self._metadata["OBSTYPE"] = self._OBSTYPE 

171 self._metadata[self._OBSTYPE + "_SCHEMA"] = self._SCHEMA 

172 self._metadata[self._OBSTYPE + "_VERSION"] = self._VERSION 

173 

174 if isinstance(metadata, dict): 

175 self.calibInfoFromDict(metadata) 

176 elif isinstance(metadata, PropertyList): 

177 self.calibInfoFromDict(metadata.toDict()) 

178 

179 def updateMetadata(self, camera=None, detector=None, filterName=None, 

180 setCalibId=False, setCalibInfo=False, setDate=False, 

181 **kwargs): 

182 """Update metadata keywords with new values. 

183 

184 Parameters 

185 ---------- 

186 camera : `lsst.afw.cameraGeom.Camera`, optional 

187 Reference camera to use to set _instrument field. 

188 detector : `lsst.afw.cameraGeom.Detector`, optional 

189 Reference detector to use to set _detector* fields. 

190 filterName : `str`, optional 

191 Filter name to assign to this calibration. 

192 setCalibId : `bool`, optional 

193 Construct the _calibId field from other fields. 

194 setCalibInfo : `bool`, optional 

195 Set calibration parameters from metadata. 

196 setDate : `bool`, optional 

197 Ensure the metadata CALIBDATE fields are set to the current datetime. 

198 kwargs : `dict` or `collections.abc.Mapping`, optional 

199 Set of key=value pairs to assign to the metadata. 

200 """ 

201 mdOriginal = self.getMetadata() 

202 mdSupplemental = dict() 

203 

204 for k, v in kwargs.items(): 

205 if isinstance(v, fits.card.Undefined): 

206 kwargs[k] = None 

207 

208 if setCalibInfo: 

209 self.calibInfoFromDict(kwargs) 

210 

211 if camera: 

212 self._instrument = camera.getName() 

213 

214 if detector: 

215 self._detectorName = detector.getName() 

216 self._detectorSerial = detector.getSerial() 

217 self._detectorId = detector.getId() 

218 if "_" in self._detectorName: 

219 (self._raftName, self._slotName) = self._detectorName.split("_") 

220 

221 if filterName: 

222 # TOD0 DM-28093: I think this whole comment can go away, if we 

223 # always use physicalLabel everywhere in ip_isr. 

224 # If set via: 

225 # exposure.getInfo().getFilter().getName() 

226 # then this will hold the abstract filter. 

227 self._filter = filterName 

228 

229 if setDate: 

230 date = datetime.datetime.now() 

231 mdSupplemental["CALIBDATE"] = date.isoformat() 

232 mdSupplemental["CALIB_CREATION_DATE"] = date.date().isoformat() 

233 mdSupplemental["CALIB_CREATION_TIME"] = date.time().isoformat() 

234 

235 if setCalibId: 

236 values = [] 

237 values.append(f"instrument={self._instrument}") if self._instrument else None 

238 values.append(f"raftName={self._raftName}") if self._raftName else None 

239 values.append(f"detectorName={self._detectorName}") if self._detectorName else None 

240 values.append(f"detector={self._detectorId}") if self._detectorId else None 

241 values.append(f"filter={self._filter}") if self._filter else None 

242 

243 calibDate = mdOriginal.get("CALIBDATE", mdSupplemental.get("CALIBDATE", None)) 

244 values.append(f"calibDate={calibDate}") if calibDate else None 

245 

246 self._calibId = " ".join(values) 

247 

248 self._metadata["INSTRUME"] = self._instrument if self._instrument else None 

249 self._metadata["RAFTNAME"] = self._raftName if self._raftName else None 

250 self._metadata["SLOTNAME"] = self._slotName if self._slotName else None 

251 self._metadata["DETECTOR"] = self._detectorId 

252 self._metadata["DET_NAME"] = self._detectorName if self._detectorName else None 

253 self._metadata["DET_SER"] = self._detectorSerial if self._detectorSerial else None 

254 self._metadata["FILTER"] = self._filter if self._filter else None 

255 self._metadata["CALIB_ID"] = self._calibId if self._calibId else None 

256 self._metadata["CALIBCLS"] = getFullTypeName(self) 

257 

258 mdSupplemental.update(kwargs) 

259 mdOriginal.update(mdSupplemental) 

260 

261 def calibInfoFromDict(self, dictionary): 

262 """Handle common keywords. 

263 

264 This isn't an ideal solution, but until all calibrations 

265 expect to find everything in the metadata, they still need to 

266 search through dictionaries. 

267 

268 Parameters 

269 ---------- 

270 dictionary : `dict` or `lsst.daf.base.PropertyList` 

271 Source for the common keywords. 

272 

273 Raises 

274 ------ 

275 RuntimeError : 

276 Raised if the dictionary does not match the expected OBSTYPE. 

277 

278 """ 

279 

280 def search(haystack, needles): 

281 """Search dictionary 'haystack' for an entry in 'needles' 

282 """ 

283 test = [haystack.get(x) for x in needles] 

284 test = set([x for x in test if x is not None]) 

285 if len(test) == 0: 

286 if "metadata" in haystack: 

287 return search(haystack["metadata"], needles) 

288 else: 

289 return None 

290 elif len(test) == 1: 

291 value = list(test)[0] 

292 if value == "": 

293 return None 

294 else: 

295 return value 

296 else: 

297 raise ValueError(f"Too many values found: {len(test)} {test} {needles}") 

298 

299 if "metadata" in dictionary: 

300 metadata = dictionary["metadata"] 

301 

302 if self._OBSTYPE != metadata["OBSTYPE"]: 

303 raise RuntimeError(f"Incorrect calibration supplied. Expected {self._OBSTYPE}, " 

304 f"found {metadata['OBSTYPE']}") 

305 

306 self._instrument = search(dictionary, ["INSTRUME", "instrument"]) 

307 self._raftName = search(dictionary, ["RAFTNAME"]) 

308 self._slotName = search(dictionary, ["SLOTNAME"]) 

309 self._detectorId = search(dictionary, ["DETECTOR", "detectorId"]) 

310 self._detectorName = search(dictionary, ["DET_NAME", "DETECTOR_NAME", "detectorName"]) 

311 self._detectorSerial = search(dictionary, ["DET_SER", "DETECTOR_SERIAL", "detectorSerial"]) 

312 self._filter = search(dictionary, ["FILTER", "filterName"]) 

313 self._calibId = search(dictionary, ["CALIB_ID"]) 

314 

315 @classmethod 

316 def determineCalibClass(cls, metadata, message): 

317 """Attempt to find calibration class in metadata. 

318 

319 Parameters 

320 ---------- 

321 metadata : `dict` or `lsst.daf.base.PropertyList` 

322 Metadata possibly containing a calibration class entry. 

323 message : `str` 

324 Message to include in any errors. 

325 

326 Returns 

327 ------- 

328 calibClass : `object` 

329 The class to use to read the file contents. Should be an 

330 `lsst.ip.isr.IsrCalib` subclass. 

331 

332 Raises 

333 ------ 

334 ValueError : 

335 Raised if the resulting calibClass is the base 

336 `lsst.ip.isr.IsrClass` (which does not implement the 

337 content methods). 

338 """ 

339 calibClassName = metadata.get("CALIBCLS") 

340 calibClass = doImport(calibClassName) if calibClassName is not None else cls 

341 if calibClass is IsrCalib: 

342 raise ValueError(f"Cannot use base class to read calibration data: {msg}") 

343 return calibClass 

344 

345 @classmethod 

346 def readText(cls, filename, **kwargs): 

347 """Read calibration representation from a yaml/ecsv file. 

348 

349 Parameters 

350 ---------- 

351 filename : `str` 

352 Name of the file containing the calibration definition. 

353 kwargs : `dict` or collections.abc.Mapping`, optional 

354 Set of key=value pairs to pass to the ``fromDict`` or 

355 ``fromTable`` methods. 

356 

357 Returns 

358 ------- 

359 calib : `~lsst.ip.isr.IsrCalibType` 

360 Calibration class. 

361 

362 Raises 

363 ------ 

364 RuntimeError : 

365 Raised if the filename does not end in ".ecsv" or ".yaml". 

366 """ 

367 if filename.endswith((".ecsv", ".ECSV")): 

368 data = Table.read(filename, format="ascii.ecsv") 

369 calibClass = cls.determineCalibClass(data.meta, "readText/ECSV") 

370 return calibClass.fromTable([data], **kwargs) 

371 elif filename.endswith((".yaml", ".YAML")): 

372 with open(filename, "r") as f: 

373 data = yaml.load(f, Loader=yaml.CLoader) 

374 calibClass = cls.determineCalibClass(data["metadata"], "readText/YAML") 

375 return calibClass.fromDict(data, **kwargs) 

376 else: 

377 raise RuntimeError(f"Unknown filename extension: {filename}") 

378 

379 def writeText(self, filename, format="auto"): 

380 """Write the calibration data to a text file. 

381 

382 Parameters 

383 ---------- 

384 filename : `str` 

385 Name of the file to write. 

386 format : `str` 

387 Format to write the file as. Supported values are: 

388 ``"auto"`` : Determine filetype from filename. 

389 ``"yaml"`` : Write as yaml. 

390 ``"ecsv"`` : Write as ecsv. 

391 Returns 

392 ------- 

393 used : `str` 

394 The name of the file used to write the data. This may 

395 differ from the input if the format is explicitly chosen. 

396 

397 Raises 

398 ------ 

399 RuntimeError : 

400 Raised if filename does not end in a known extension, or 

401 if all information cannot be written. 

402 

403 Notes 

404 ----- 

405 The file is written to YAML/ECSV format and will include any 

406 associated metadata. 

407 """ 

408 if format == "yaml" or (format == "auto" and filename.lower().endswith((".yaml", ".YAML"))): 

409 outDict = self.toDict() 

410 path, ext = os.path.splitext(filename) 

411 filename = path + ".yaml" 

412 with open(filename, "w") as f: 

413 yaml.dump(outDict, f) 

414 elif format == "ecsv" or (format == "auto" and filename.lower().endswith((".ecsv", ".ECSV"))): 

415 tableList = self.toTable() 

416 if len(tableList) > 1: 

417 # ECSV doesn't support multiple tables per file, so we 

418 # can only write the first table. 

419 raise RuntimeError(f"Unable to persist {len(tableList)}tables in ECSV format.") 

420 

421 table = tableList[0] 

422 path, ext = os.path.splitext(filename) 

423 filename = path + ".ecsv" 

424 table.write(filename, format="ascii.ecsv") 

425 else: 

426 raise RuntimeError(f"Attempt to write to a file {filename} " 

427 "that does not end in '.yaml' or '.ecsv'") 

428 

429 return filename 

430 

431 @classmethod 

432 def readFits(cls, filename, **kwargs): 

433 """Read calibration data from a FITS file. 

434 

435 Parameters 

436 ---------- 

437 filename : `str` 

438 Filename to read data from. 

439 kwargs : `dict` or collections.abc.Mapping`, optional 

440 Set of key=value pairs to pass to the ``fromTable`` 

441 method. 

442 

443 Returns 

444 ------- 

445 calib : `lsst.ip.isr.IsrCalib` 

446 Calibration contained within the file. 

447 """ 

448 tableList = [] 

449 tableList.append(Table.read(filename, hdu=1)) 

450 extNum = 2 # Fits indices start at 1, we've read one already. 

451 keepTrying = True 

452 

453 while keepTrying: 

454 with warnings.catch_warnings(): 

455 warnings.simplefilter("error") 

456 try: 

457 newTable = Table.read(filename, hdu=extNum) 

458 tableList.append(newTable) 

459 extNum += 1 

460 except Exception: 

461 keepTrying = False 

462 

463 for table in tableList: 

464 for k, v in table.meta.items(): 

465 if isinstance(v, fits.card.Undefined): 

466 table.meta[k] = None 

467 

468 calibClass = cls.determineCalibClass(tableList[0].meta, "readFits") 

469 return calibClass.fromTable(tableList, **kwargs) 

470 

471 def writeFits(self, filename): 

472 """Write calibration data to a FITS file. 

473 

474 Parameters 

475 ---------- 

476 filename : `str` 

477 Filename to write data to. 

478 

479 Returns 

480 ------- 

481 used : `str` 

482 The name of the file used to write the data. 

483 

484 """ 

485 tableList = self.toTable() 

486 with warnings.catch_warnings(): 

487 warnings.filterwarnings("ignore", category=Warning, module="astropy.io") 

488 astropyList = [fits.table_to_hdu(table) for table in tableList] 

489 astropyList.insert(0, fits.PrimaryHDU()) 

490 

491 writer = fits.HDUList(astropyList) 

492 writer.writeto(filename, overwrite=True) 

493 return filename 

494 

495 def fromDetector(self, detector): 

496 """Modify the calibration parameters to match the supplied detector. 

497 

498 Parameters 

499 ---------- 

500 detector : `lsst.afw.cameraGeom.Detector` 

501 Detector to use to set parameters from. 

502 

503 Raises 

504 ------ 

505 NotImplementedError 

506 This needs to be implemented by subclasses for each 

507 calibration type. 

508 """ 

509 raise NotImplementedError("Must be implemented by subclass.") 

510 

511 @classmethod 

512 def fromDict(cls, dictionary, **kwargs): 

513 """Construct a calibration from a dictionary of properties. 

514 

515 Must be implemented by the specific calibration subclasses. 

516 

517 Parameters 

518 ---------- 

519 dictionary : `dict` 

520 Dictionary of properties. 

521 kwargs : `dict` or collections.abc.Mapping`, optional 

522 Set of key=value options. 

523 

524 Returns 

525 ------ 

526 calib : `lsst.ip.isr.CalibType` 

527 Constructed calibration. 

528 

529 Raises 

530 ------ 

531 NotImplementedError : 

532 Raised if not implemented. 

533 """ 

534 raise NotImplementedError("Must be implemented by subclass.") 

535 

536 def toDict(self): 

537 """Return a dictionary containing the calibration properties. 

538 

539 The dictionary should be able to be round-tripped through 

540 `fromDict`. 

541 

542 Returns 

543 ------- 

544 dictionary : `dict` 

545 Dictionary of properties. 

546 

547 Raises 

548 ------ 

549 NotImplementedError : 

550 Raised if not implemented. 

551 """ 

552 raise NotImplementedError("Must be implemented by subclass.") 

553 

554 @classmethod 

555 def fromTable(cls, tableList, **kwargs): 

556 """Construct a calibration from a dictionary of properties. 

557 

558 Must be implemented by the specific calibration subclasses. 

559 

560 Parameters 

561 ---------- 

562 tableList : `list` [`lsst.afw.table.Table`] 

563 List of tables of properties. 

564 kwargs : `dict` or collections.abc.Mapping`, optional 

565 Set of key=value options. 

566 

567 Returns 

568 ------ 

569 calib : `lsst.ip.isr.CalibType` 

570 Constructed calibration. 

571 

572 Raises 

573 ------ 

574 NotImplementedError : 

575 Raised if not implemented. 

576 """ 

577 raise NotImplementedError("Must be implemented by subclass.") 

578 

579 def toTable(self): 

580 """Return a list of tables containing the calibration properties. 

581 

582 The table list should be able to be round-tripped through 

583 `fromDict`. 

584 

585 Returns 

586 ------- 

587 tableList : `list` [`lsst.afw.table.Table`] 

588 List of tables of properties. 

589 

590 Raises 

591 ------ 

592 NotImplementedError : 

593 Raised if not implemented. 

594 """ 

595 raise NotImplementedError("Must be implemented by subclass.") 

596 

597 def validate(self, other=None): 

598 """Validate that this calibration is defined and can be used. 

599 

600 Parameters 

601 ---------- 

602 other : `object`, optional 

603 Thing to validate against. 

604 

605 Returns 

606 ------- 

607 valid : `bool` 

608 Returns true if the calibration is valid and appropriate. 

609 """ 

610 return False 

611 

612 def apply(self, target): 

613 """Method to apply the calibration to the target object. 

614 

615 Parameters 

616 ---------- 

617 target : `object` 

618 Thing to validate against. 

619 

620 Returns 

621 ------- 

622 valid : `bool` 

623 Returns true if the calibration was applied correctly. 

624 

625 Raises 

626 ------ 

627 NotImplementedError : 

628 Raised if not implemented. 

629 """ 

630 raise NotImplementedError("Must be implemented by subclass.") 

631 

632 

633class IsrProvenance(IsrCalib): 

634 """Class for the provenance of data used to construct calibration. 

635 

636 Provenance is not really a calibration, but we would like to 

637 record this when constructing the calibration, and it provides an 

638 example of the base calibration class. 

639 

640 Parameters 

641 ---------- 

642 instrument : `str`, optional 

643 Name of the instrument the data was taken with. 

644 calibType : `str`, optional 

645 Type of calibration this provenance was generated for. 

646 detectorName : `str`, optional 

647 Name of the detector this calibration is for. 

648 detectorSerial : `str`, optional 

649 Identifier for the detector. 

650 

651 """ 

652 _OBSTYPE = "IsrProvenance" 

653 

654 def __init__(self, calibType="unknown", 

655 **kwargs): 

656 self.calibType = calibType 

657 self.dimensions = set() 

658 self.dataIdList = list() 

659 

660 super().__init__(**kwargs) 

661 

662 self.requiredAttributes.update(["calibType", "dimensions", "dataIdList"]) 

663 

664 def __str__(self): 

665 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )" 

666 

667 def __eq__(self, other): 

668 return super().__eq__(other) 

669 

670 def updateMetadata(self, setDate=False, **kwargs): 

671 """Update calibration metadata. 

672 

673 Parameters 

674 ---------- 

675 setDate : `bool, optional 

676 Update the CALIBDATE fields in the metadata to the current 

677 time. Defaults to False. 

678 kwargs : `dict` or `collections.abc.Mapping`, optional 

679 Other keyword parameters to set in the metadata. 

680 """ 

681 kwargs["calibType"] = self.calibType 

682 super().updateMetadata(setDate=setDate, **kwargs) 

683 

684 def fromDataIds(self, dataIdList): 

685 """Update provenance from dataId List. 

686 

687 Parameters 

688 ---------- 

689 dataIdList : `list` [`lsst.daf.butler.DataId`] 

690 List of dataIds used in generating this calibration. 

691 """ 

692 for dataId in dataIdList: 

693 for key in dataId: 

694 if key not in self.dimensions: 

695 self.dimensions.add(key) 

696 self.dataIdList.append(dataId) 

697 

698 @classmethod 

699 def fromTable(cls, tableList): 

700 """Construct provenance from table list. 

701 

702 Parameters 

703 ---------- 

704 tableList : `list` [`lsst.afw.table.Table`] 

705 List of tables to construct the provenance from. 

706 

707 Returns 

708 ------- 

709 provenance : `lsst.ip.isr.IsrProvenance` 

710 The provenance defined in the tables. 

711 """ 

712 table = tableList[0] 

713 metadata = table.meta 

714 inDict = dict() 

715 inDict["metadata"] = metadata 

716 inDict["calibType"] = metadata["calibType"] 

717 inDict["dimensions"] = set() 

718 inDict["dataIdList"] = list() 

719 

720 schema = dict() 

721 for colName in table.columns: 

722 schema[colName.lower()] = colName 

723 inDict["dimensions"].add(colName.lower()) 

724 inDict["dimensions"] = sorted(inDict["dimensions"]) 

725 

726 for row in table: 

727 entry = dict() 

728 for dim in sorted(inDict["dimensions"]): 

729 entry[dim] = row[schema[dim]] 

730 inDict["dataIdList"].append(entry) 

731 

732 return cls.fromDict(inDict) 

733 

734 @classmethod 

735 def fromDict(cls, dictionary): 

736 """Construct provenance from a dictionary. 

737 

738 Parameters 

739 ---------- 

740 dictionary : `dict` 

741 Dictionary of provenance parameters. 

742 

743 Returns 

744 ------- 

745 provenance : `lsst.ip.isr.IsrProvenance` 

746 The provenance defined in the tables. 

747 """ 

748 calib = cls() 

749 if calib._OBSTYPE != dictionary["metadata"]["OBSTYPE"]: 

750 raise RuntimeError(f"Incorrect calibration supplied. Expected {calib._OBSTYPE}, " 

751 f"found {dictionary['metadata']['OBSTYPE']}") 

752 calib.updateMetadata(setDate=False, setCalibInfo=True, **dictionary["metadata"]) 

753 

754 # These properties should be in the metadata, but occasionally 

755 # are found in the dictionary itself. Check both places, 

756 # ending with `None` if neither contains the information. 

757 calib.calibType = dictionary["calibType"] 

758 calib.dimensions = set(dictionary["dimensions"]) 

759 calib.dataIdList = dictionary["dataIdList"] 

760 

761 calib.updateMetadata() 

762 return calib 

763 

764 def toDict(self): 

765 """Return a dictionary containing the provenance information. 

766 

767 Returns 

768 ------- 

769 dictionary : `dict` 

770 Dictionary of provenance. 

771 """ 

772 self.updateMetadata() 

773 

774 outDict = {} 

775 

776 metadata = self.getMetadata() 

777 outDict["metadata"] = metadata 

778 outDict["detectorName"] = self._detectorName 

779 outDict["detectorSerial"] = self._detectorSerial 

780 outDict["detectorId"] = self._detectorId 

781 outDict["instrument"] = self._instrument 

782 outDict["calibType"] = self.calibType 

783 outDict["dimensions"] = list(self.dimensions) 

784 outDict["dataIdList"] = self.dataIdList 

785 

786 return outDict 

787 

788 def toTable(self): 

789 """Return a list of tables containing the provenance. 

790 

791 This seems inefficient and slow, so this may not be the best 

792 way to store the data. 

793 

794 Returns 

795 ------- 

796 tableList : `list` [`lsst.afw.table.Table`] 

797 List of tables containing the provenance information 

798 

799 """ 

800 tableList = [] 

801 self.updateMetadata(setDate=True, setCalibInfo=True) 

802 

803 catalog = Table(rows=self.dataIdList, 

804 names=self.dimensions) 

805 filteredMetadata = {k: v for k, v in self.getMetadata().toDict().items() if v is not None} 

806 catalog.meta = filteredMetadata 

807 tableList.append(catalog) 

808 return tableList