Coverage for python/lsst/ip/isr/calibType.py: 15%

308 statements  

« prev     ^ index     » next       coverage.py v6.4.2, created at 2022-07-16 02:30 -0700

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21import abc 

22import datetime 

23import logging 

24import os.path 

25import warnings 

26import yaml 

27import numpy as np 

28 

29from astropy.table import Table 

30from astropy.io import fits 

31 

32from lsst.daf.base import PropertyList 

33from lsst.utils.introspection import get_full_type_name 

34from lsst.utils import doImport 

35 

36 

37__all__ = ["IsrCalib", "IsrProvenance"] 

38 

39 

40class IsrCalib(abc.ABC): 

41 """Generic calibration type. 

42 

43 Subclasses must implement the toDict, fromDict, toTable, fromTable 

44 methods that allow the calibration information to be converted 

45 from dictionaries and afw tables. This will allow the calibration 

46 to be persisted using the base class read/write methods. 

47 

48 The validate method is intended to provide a common way to check 

49 that the calibration is valid (internally consistent) and 

50 appropriate (usable with the intended data). The apply method is 

51 intended to allow the calibration to be applied in a consistent 

52 manner. 

53 

54 Parameters 

55 ---------- 

56 camera : `lsst.afw.cameraGeom.Camera`, optional 

57 Camera to extract metadata from. 

58 detector : `lsst.afw.cameraGeom.Detector`, optional 

59 Detector to extract metadata from. 

60 log : `logging.Logger`, optional 

61 Log for messages. 

62 """ 

63 _OBSTYPE = "generic" 

64 _SCHEMA = "NO SCHEMA" 

65 _VERSION = 0 

66 

67 def __init__(self, camera=None, detector=None, log=None, **kwargs): 

68 self._instrument = None 

69 self._raftName = None 

70 self._slotName = None 

71 self._detectorName = None 

72 self._detectorSerial = None 

73 self._detectorId = None 

74 self._filter = None 

75 self._calibId = None 

76 self._metadata = PropertyList() 

77 self.setMetadata(PropertyList()) 

78 self.calibInfoFromDict(kwargs) 

79 

80 # Define the required attributes for this calibration. 

81 self.requiredAttributes = set(["_OBSTYPE", "_SCHEMA", "_VERSION"]) 

82 self.requiredAttributes.update(["_instrument", "_raftName", "_slotName", 

83 "_detectorName", "_detectorSerial", "_detectorId", 

84 "_filter", "_calibId", "_metadata"]) 

85 

86 self.log = log if log else logging.getLogger(__name__) 

87 

88 if detector: 

89 self.fromDetector(detector) 

90 self.updateMetadata(camera=camera, detector=detector) 

91 

92 def __str__(self): 

93 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )" 

94 

95 def __eq__(self, other): 

96 """Calibration equivalence. 

97 

98 Running ``calib.log.setLevel(0)`` enables debug statements to 

99 identify problematic fields. 

100 """ 

101 if not isinstance(other, self.__class__): 

102 self.log.debug("Incorrect class type: %s %s", self.__class__, other.__class__) 

103 return False 

104 

105 for attr in self._requiredAttributes: 

106 attrSelf = getattr(self, attr) 

107 attrOther = getattr(other, attr) 

108 

109 if isinstance(attrSelf, dict): 

110 # Dictionary of arrays. 

111 if attrSelf.keys() != attrOther.keys(): 

112 self.log.debug("Dict Key Failure: %s %s %s", attr, attrSelf.keys(), attrOther.keys()) 

113 return False 

114 for key in attrSelf: 

115 if not np.allclose(attrSelf[key], attrOther[key], equal_nan=True): 

116 self.log.debug("Array Failure: %s %s %s", key, attrSelf[key], attrOther[key]) 

117 return False 

118 elif isinstance(attrSelf, np.ndarray): 

119 # Bare array. 

120 if not np.allclose(attrSelf, attrOther, equal_nan=True): 

121 self.log.debug("Array Failure: %s %s %s", attr, attrSelf, attrOther) 

122 return False 

123 elif type(attrSelf) != type(attrOther): 

124 if set([attrSelf, attrOther]) == set([None, ""]): 

125 # Fits converts None to "", but None is not "". 

126 continue 

127 self.log.debug("Type Failure: %s %s %s %s %s", attr, type(attrSelf), type(attrOther), 

128 attrSelf, attrOther) 

129 return False 

130 else: 

131 if attrSelf != attrOther: 

132 self.log.debug("Value Failure: %s %s %s", attr, attrSelf, attrOther) 

133 return False 

134 

135 return True 

136 

137 @property 

138 def requiredAttributes(self): 

139 return self._requiredAttributes 

140 

141 @requiredAttributes.setter 

142 def requiredAttributes(self, value): 

143 self._requiredAttributes = value 

144 

145 def getMetadata(self): 

146 """Retrieve metadata associated with this calibration. 

147 

148 Returns 

149 ------- 

150 meta : `lsst.daf.base.PropertyList` 

151 Metadata. The returned `~lsst.daf.base.PropertyList` can be 

152 modified by the caller and the changes will be written to 

153 external files. 

154 """ 

155 return self._metadata 

156 

157 def setMetadata(self, metadata): 

158 """Store a copy of the supplied metadata with this calibration. 

159 

160 Parameters 

161 ---------- 

162 metadata : `lsst.daf.base.PropertyList` 

163 Metadata to associate with the calibration. Will be copied and 

164 overwrite existing metadata. 

165 """ 

166 if metadata is not None: 

167 self._metadata.update(metadata) 

168 

169 # Ensure that we have the obs type required by calibration ingest 

170 self._metadata["OBSTYPE"] = self._OBSTYPE 

171 self._metadata[self._OBSTYPE + "_SCHEMA"] = self._SCHEMA 

172 self._metadata[self._OBSTYPE + "_VERSION"] = self._VERSION 

173 

174 if isinstance(metadata, dict): 

175 self.calibInfoFromDict(metadata) 

176 elif isinstance(metadata, PropertyList): 

177 self.calibInfoFromDict(metadata.toDict()) 

178 

179 def updateMetadata(self, camera=None, detector=None, filterName=None, 

180 setCalibId=False, setCalibInfo=False, setDate=False, 

181 **kwargs): 

182 """Update metadata keywords with new values. 

183 

184 Parameters 

185 ---------- 

186 camera : `lsst.afw.cameraGeom.Camera`, optional 

187 Reference camera to use to set _instrument field. 

188 detector : `lsst.afw.cameraGeom.Detector`, optional 

189 Reference detector to use to set _detector* fields. 

190 filterName : `str`, optional 

191 Filter name to assign to this calibration. 

192 setCalibId : `bool`, optional 

193 Construct the _calibId field from other fields. 

194 setCalibInfo : `bool`, optional 

195 Set calibration parameters from metadata. 

196 setDate : `bool`, optional 

197 Ensure the metadata CALIBDATE fields are set to the current 

198 datetime. 

199 kwargs : `dict` or `collections.abc.Mapping`, optional 

200 Set of key=value pairs to assign to the metadata. 

201 """ 

202 mdOriginal = self.getMetadata() 

203 mdSupplemental = dict() 

204 

205 for k, v in kwargs.items(): 

206 if isinstance(v, fits.card.Undefined): 

207 kwargs[k] = None 

208 

209 if setCalibInfo: 

210 self.calibInfoFromDict(kwargs) 

211 

212 if camera: 

213 self._instrument = camera.getName() 

214 

215 if detector: 

216 self._detectorName = detector.getName() 

217 self._detectorSerial = detector.getSerial() 

218 self._detectorId = detector.getId() 

219 if "_" in self._detectorName: 

220 (self._raftName, self._slotName) = self._detectorName.split("_") 

221 

222 if filterName: 

223 # TOD0 DM-28093: I think this whole comment can go away, if we 

224 # always use physicalLabel everywhere in ip_isr. 

225 # If set via: 

226 # exposure.getInfo().getFilter().getName() 

227 # then this will hold the abstract filter. 

228 self._filter = filterName 

229 

230 if setDate: 

231 date = datetime.datetime.now() 

232 mdSupplemental["CALIBDATE"] = date.isoformat() 

233 mdSupplemental["CALIB_CREATION_DATE"] = date.date().isoformat() 

234 mdSupplemental["CALIB_CREATION_TIME"] = date.time().isoformat() 

235 

236 if setCalibId: 

237 values = [] 

238 values.append(f"instrument={self._instrument}") if self._instrument else None 

239 values.append(f"raftName={self._raftName}") if self._raftName else None 

240 values.append(f"detectorName={self._detectorName}") if self._detectorName else None 

241 values.append(f"detector={self._detectorId}") if self._detectorId else None 

242 values.append(f"filter={self._filter}") if self._filter else None 

243 

244 calibDate = mdOriginal.get("CALIBDATE", mdSupplemental.get("CALIBDATE", None)) 

245 values.append(f"calibDate={calibDate}") if calibDate else None 

246 

247 self._calibId = " ".join(values) 

248 

249 self._metadata["INSTRUME"] = self._instrument if self._instrument else None 

250 self._metadata["RAFTNAME"] = self._raftName if self._raftName else None 

251 self._metadata["SLOTNAME"] = self._slotName if self._slotName else None 

252 self._metadata["DETECTOR"] = self._detectorId 

253 self._metadata["DET_NAME"] = self._detectorName if self._detectorName else None 

254 self._metadata["DET_SER"] = self._detectorSerial if self._detectorSerial else None 

255 self._metadata["FILTER"] = self._filter if self._filter else None 

256 self._metadata["CALIB_ID"] = self._calibId if self._calibId else None 

257 self._metadata["CALIBCLS"] = get_full_type_name(self) 

258 

259 mdSupplemental.update(kwargs) 

260 mdOriginal.update(mdSupplemental) 

261 

262 def calibInfoFromDict(self, dictionary): 

263 """Handle common keywords. 

264 

265 This isn't an ideal solution, but until all calibrations 

266 expect to find everything in the metadata, they still need to 

267 search through dictionaries. 

268 

269 Parameters 

270 ---------- 

271 dictionary : `dict` or `lsst.daf.base.PropertyList` 

272 Source for the common keywords. 

273 

274 Raises 

275 ------ 

276 RuntimeError : 

277 Raised if the dictionary does not match the expected OBSTYPE. 

278 

279 """ 

280 

281 def search(haystack, needles): 

282 """Search dictionary 'haystack' for an entry in 'needles' 

283 """ 

284 test = [haystack.get(x) for x in needles] 

285 test = set([x for x in test if x is not None]) 

286 if len(test) == 0: 

287 if "metadata" in haystack: 

288 return search(haystack["metadata"], needles) 

289 else: 

290 return None 

291 elif len(test) == 1: 

292 value = list(test)[0] 

293 if value == "": 

294 return None 

295 else: 

296 return value 

297 else: 

298 raise ValueError(f"Too many values found: {len(test)} {test} {needles}") 

299 

300 if "metadata" in dictionary: 

301 metadata = dictionary["metadata"] 

302 

303 if self._OBSTYPE != metadata["OBSTYPE"]: 

304 raise RuntimeError(f"Incorrect calibration supplied. Expected {self._OBSTYPE}, " 

305 f"found {metadata['OBSTYPE']}") 

306 

307 self._instrument = search(dictionary, ["INSTRUME", "instrument"]) 

308 self._raftName = search(dictionary, ["RAFTNAME"]) 

309 self._slotName = search(dictionary, ["SLOTNAME"]) 

310 self._detectorId = search(dictionary, ["DETECTOR", "detectorId"]) 

311 self._detectorName = search(dictionary, ["DET_NAME", "DETECTOR_NAME", "detectorName"]) 

312 self._detectorSerial = search(dictionary, ["DET_SER", "DETECTOR_SERIAL", "detectorSerial"]) 

313 self._filter = search(dictionary, ["FILTER", "filterName"]) 

314 self._calibId = search(dictionary, ["CALIB_ID"]) 

315 

316 @classmethod 

317 def determineCalibClass(cls, metadata, message): 

318 """Attempt to find calibration class in metadata. 

319 

320 Parameters 

321 ---------- 

322 metadata : `dict` or `lsst.daf.base.PropertyList` 

323 Metadata possibly containing a calibration class entry. 

324 message : `str` 

325 Message to include in any errors. 

326 

327 Returns 

328 ------- 

329 calibClass : `object` 

330 The class to use to read the file contents. Should be an 

331 `lsst.ip.isr.IsrCalib` subclass. 

332 

333 Raises 

334 ------ 

335 ValueError : 

336 Raised if the resulting calibClass is the base 

337 `lsst.ip.isr.IsrClass` (which does not implement the 

338 content methods). 

339 """ 

340 calibClassName = metadata.get("CALIBCLS") 

341 calibClass = doImport(calibClassName) if calibClassName is not None else cls 

342 if calibClass is IsrCalib: 

343 raise ValueError(f"Cannot use base class to read calibration data: {message}") 

344 return calibClass 

345 

346 @classmethod 

347 def readText(cls, filename, **kwargs): 

348 """Read calibration representation from a yaml/ecsv file. 

349 

350 Parameters 

351 ---------- 

352 filename : `str` 

353 Name of the file containing the calibration definition. 

354 kwargs : `dict` or collections.abc.Mapping`, optional 

355 Set of key=value pairs to pass to the ``fromDict`` or 

356 ``fromTable`` methods. 

357 

358 Returns 

359 ------- 

360 calib : `~lsst.ip.isr.IsrCalibType` 

361 Calibration class. 

362 

363 Raises 

364 ------ 

365 RuntimeError : 

366 Raised if the filename does not end in ".ecsv" or ".yaml". 

367 """ 

368 if filename.endswith((".ecsv", ".ECSV")): 

369 data = Table.read(filename, format="ascii.ecsv") 

370 calibClass = cls.determineCalibClass(data.meta, "readText/ECSV") 

371 return calibClass.fromTable([data], **kwargs) 

372 elif filename.endswith((".yaml", ".YAML")): 

373 with open(filename, "r") as f: 

374 data = yaml.load(f, Loader=yaml.CLoader) 

375 calibClass = cls.determineCalibClass(data["metadata"], "readText/YAML") 

376 return calibClass.fromDict(data, **kwargs) 

377 else: 

378 raise RuntimeError(f"Unknown filename extension: {filename}") 

379 

380 def writeText(self, filename, format="auto"): 

381 """Write the calibration data to a text file. 

382 

383 Parameters 

384 ---------- 

385 filename : `str` 

386 Name of the file to write. 

387 format : `str` 

388 Format to write the file as. Supported values are: 

389 ``"auto"`` : Determine filetype from filename. 

390 ``"yaml"`` : Write as yaml. 

391 ``"ecsv"`` : Write as ecsv. 

392 Returns 

393 ------- 

394 used : `str` 

395 The name of the file used to write the data. This may 

396 differ from the input if the format is explicitly chosen. 

397 

398 Raises 

399 ------ 

400 RuntimeError : 

401 Raised if filename does not end in a known extension, or 

402 if all information cannot be written. 

403 

404 Notes 

405 ----- 

406 The file is written to YAML/ECSV format and will include any 

407 associated metadata. 

408 """ 

409 if format == "yaml" or (format == "auto" and filename.lower().endswith((".yaml", ".YAML"))): 

410 outDict = self.toDict() 

411 path, ext = os.path.splitext(filename) 

412 filename = path + ".yaml" 

413 with open(filename, "w") as f: 

414 yaml.dump(outDict, f) 

415 elif format == "ecsv" or (format == "auto" and filename.lower().endswith((".ecsv", ".ECSV"))): 

416 tableList = self.toTable() 

417 if len(tableList) > 1: 

418 # ECSV doesn't support multiple tables per file, so we 

419 # can only write the first table. 

420 raise RuntimeError(f"Unable to persist {len(tableList)}tables in ECSV format.") 

421 

422 table = tableList[0] 

423 path, ext = os.path.splitext(filename) 

424 filename = path + ".ecsv" 

425 table.write(filename, format="ascii.ecsv") 

426 else: 

427 raise RuntimeError(f"Attempt to write to a file {filename} " 

428 "that does not end in '.yaml' or '.ecsv'") 

429 

430 return filename 

431 

432 @classmethod 

433 def readFits(cls, filename, **kwargs): 

434 """Read calibration data from a FITS file. 

435 

436 Parameters 

437 ---------- 

438 filename : `str` 

439 Filename to read data from. 

440 kwargs : `dict` or collections.abc.Mapping`, optional 

441 Set of key=value pairs to pass to the ``fromTable`` 

442 method. 

443 

444 Returns 

445 ------- 

446 calib : `lsst.ip.isr.IsrCalib` 

447 Calibration contained within the file. 

448 """ 

449 tableList = [] 

450 tableList.append(Table.read(filename, hdu=1)) 

451 extNum = 2 # Fits indices start at 1, we've read one already. 

452 keepTrying = True 

453 

454 while keepTrying: 

455 with warnings.catch_warnings(): 

456 warnings.simplefilter("error") 

457 try: 

458 newTable = Table.read(filename, hdu=extNum) 

459 tableList.append(newTable) 

460 extNum += 1 

461 except Exception: 

462 keepTrying = False 

463 

464 for table in tableList: 

465 for k, v in table.meta.items(): 

466 if isinstance(v, fits.card.Undefined): 

467 table.meta[k] = None 

468 

469 calibClass = cls.determineCalibClass(tableList[0].meta, "readFits") 

470 return calibClass.fromTable(tableList, **kwargs) 

471 

472 def writeFits(self, filename): 

473 """Write calibration data to a FITS file. 

474 

475 Parameters 

476 ---------- 

477 filename : `str` 

478 Filename to write data to. 

479 

480 Returns 

481 ------- 

482 used : `str` 

483 The name of the file used to write the data. 

484 

485 """ 

486 tableList = self.toTable() 

487 with warnings.catch_warnings(): 

488 warnings.filterwarnings("ignore", category=Warning, module="astropy.io") 

489 astropyList = [fits.table_to_hdu(table) for table in tableList] 

490 astropyList.insert(0, fits.PrimaryHDU()) 

491 

492 writer = fits.HDUList(astropyList) 

493 writer.writeto(filename, overwrite=True) 

494 return filename 

495 

496 def fromDetector(self, detector): 

497 """Modify the calibration parameters to match the supplied detector. 

498 

499 Parameters 

500 ---------- 

501 detector : `lsst.afw.cameraGeom.Detector` 

502 Detector to use to set parameters from. 

503 

504 Raises 

505 ------ 

506 NotImplementedError 

507 This needs to be implemented by subclasses for each 

508 calibration type. 

509 """ 

510 raise NotImplementedError("Must be implemented by subclass.") 

511 

512 @classmethod 

513 def fromDict(cls, dictionary, **kwargs): 

514 """Construct a calibration from a dictionary of properties. 

515 

516 Must be implemented by the specific calibration subclasses. 

517 

518 Parameters 

519 ---------- 

520 dictionary : `dict` 

521 Dictionary of properties. 

522 kwargs : `dict` or collections.abc.Mapping`, optional 

523 Set of key=value options. 

524 

525 Returns 

526 ------ 

527 calib : `lsst.ip.isr.CalibType` 

528 Constructed calibration. 

529 

530 Raises 

531 ------ 

532 NotImplementedError : 

533 Raised if not implemented. 

534 """ 

535 raise NotImplementedError("Must be implemented by subclass.") 

536 

537 def toDict(self): 

538 """Return a dictionary containing the calibration properties. 

539 

540 The dictionary should be able to be round-tripped through 

541 `fromDict`. 

542 

543 Returns 

544 ------- 

545 dictionary : `dict` 

546 Dictionary of properties. 

547 

548 Raises 

549 ------ 

550 NotImplementedError : 

551 Raised if not implemented. 

552 """ 

553 raise NotImplementedError("Must be implemented by subclass.") 

554 

555 @classmethod 

556 def fromTable(cls, tableList, **kwargs): 

557 """Construct a calibration from a dictionary of properties. 

558 

559 Must be implemented by the specific calibration subclasses. 

560 

561 Parameters 

562 ---------- 

563 tableList : `list` [`lsst.afw.table.Table`] 

564 List of tables of properties. 

565 kwargs : `dict` or collections.abc.Mapping`, optional 

566 Set of key=value options. 

567 

568 Returns 

569 ------ 

570 calib : `lsst.ip.isr.CalibType` 

571 Constructed calibration. 

572 

573 Raises 

574 ------ 

575 NotImplementedError : 

576 Raised if not implemented. 

577 """ 

578 raise NotImplementedError("Must be implemented by subclass.") 

579 

580 def toTable(self): 

581 """Return a list of tables containing the calibration properties. 

582 

583 The table list should be able to be round-tripped through 

584 `fromDict`. 

585 

586 Returns 

587 ------- 

588 tableList : `list` [`lsst.afw.table.Table`] 

589 List of tables of properties. 

590 

591 Raises 

592 ------ 

593 NotImplementedError : 

594 Raised if not implemented. 

595 """ 

596 raise NotImplementedError("Must be implemented by subclass.") 

597 

598 def validate(self, other=None): 

599 """Validate that this calibration is defined and can be used. 

600 

601 Parameters 

602 ---------- 

603 other : `object`, optional 

604 Thing to validate against. 

605 

606 Returns 

607 ------- 

608 valid : `bool` 

609 Returns true if the calibration is valid and appropriate. 

610 """ 

611 return False 

612 

613 def apply(self, target): 

614 """Method to apply the calibration to the target object. 

615 

616 Parameters 

617 ---------- 

618 target : `object` 

619 Thing to validate against. 

620 

621 Returns 

622 ------- 

623 valid : `bool` 

624 Returns true if the calibration was applied correctly. 

625 

626 Raises 

627 ------ 

628 NotImplementedError : 

629 Raised if not implemented. 

630 """ 

631 raise NotImplementedError("Must be implemented by subclass.") 

632 

633 

634class IsrProvenance(IsrCalib): 

635 """Class for the provenance of data used to construct calibration. 

636 

637 Provenance is not really a calibration, but we would like to 

638 record this when constructing the calibration, and it provides an 

639 example of the base calibration class. 

640 

641 Parameters 

642 ---------- 

643 instrument : `str`, optional 

644 Name of the instrument the data was taken with. 

645 calibType : `str`, optional 

646 Type of calibration this provenance was generated for. 

647 detectorName : `str`, optional 

648 Name of the detector this calibration is for. 

649 detectorSerial : `str`, optional 

650 Identifier for the detector. 

651 

652 """ 

653 _OBSTYPE = "IsrProvenance" 

654 

655 def __init__(self, calibType="unknown", 

656 **kwargs): 

657 self.calibType = calibType 

658 self.dimensions = set() 

659 self.dataIdList = list() 

660 

661 super().__init__(**kwargs) 

662 

663 self.requiredAttributes.update(["calibType", "dimensions", "dataIdList"]) 

664 

665 def __str__(self): 

666 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )" 

667 

668 def __eq__(self, other): 

669 return super().__eq__(other) 

670 

671 def updateMetadata(self, setDate=False, **kwargs): 

672 """Update calibration metadata. 

673 

674 Parameters 

675 ---------- 

676 setDate : `bool, optional 

677 Update the CALIBDATE fields in the metadata to the current 

678 time. Defaults to False. 

679 kwargs : `dict` or `collections.abc.Mapping`, optional 

680 Other keyword parameters to set in the metadata. 

681 """ 

682 kwargs["calibType"] = self.calibType 

683 super().updateMetadata(setDate=setDate, **kwargs) 

684 

685 def fromDataIds(self, dataIdList): 

686 """Update provenance from dataId List. 

687 

688 Parameters 

689 ---------- 

690 dataIdList : `list` [`lsst.daf.butler.DataId`] 

691 List of dataIds used in generating this calibration. 

692 """ 

693 for dataId in dataIdList: 

694 for key in dataId: 

695 if key not in self.dimensions: 

696 self.dimensions.add(key) 

697 self.dataIdList.append(dataId) 

698 

699 @classmethod 

700 def fromTable(cls, tableList): 

701 """Construct provenance from table list. 

702 

703 Parameters 

704 ---------- 

705 tableList : `list` [`lsst.afw.table.Table`] 

706 List of tables to construct the provenance from. 

707 

708 Returns 

709 ------- 

710 provenance : `lsst.ip.isr.IsrProvenance` 

711 The provenance defined in the tables. 

712 """ 

713 table = tableList[0] 

714 metadata = table.meta 

715 inDict = dict() 

716 inDict["metadata"] = metadata 

717 inDict["calibType"] = metadata["calibType"] 

718 inDict["dimensions"] = set() 

719 inDict["dataIdList"] = list() 

720 

721 schema = dict() 

722 for colName in table.columns: 

723 schema[colName.lower()] = colName 

724 inDict["dimensions"].add(colName.lower()) 

725 inDict["dimensions"] = sorted(inDict["dimensions"]) 

726 

727 for row in table: 

728 entry = dict() 

729 for dim in sorted(inDict["dimensions"]): 

730 entry[dim] = row[schema[dim]] 

731 inDict["dataIdList"].append(entry) 

732 

733 return cls.fromDict(inDict) 

734 

735 @classmethod 

736 def fromDict(cls, dictionary): 

737 """Construct provenance from a dictionary. 

738 

739 Parameters 

740 ---------- 

741 dictionary : `dict` 

742 Dictionary of provenance parameters. 

743 

744 Returns 

745 ------- 

746 provenance : `lsst.ip.isr.IsrProvenance` 

747 The provenance defined in the tables. 

748 """ 

749 calib = cls() 

750 if calib._OBSTYPE != dictionary["metadata"]["OBSTYPE"]: 

751 raise RuntimeError(f"Incorrect calibration supplied. Expected {calib._OBSTYPE}, " 

752 f"found {dictionary['metadata']['OBSTYPE']}") 

753 calib.updateMetadata(setDate=False, setCalibInfo=True, **dictionary["metadata"]) 

754 

755 # These properties should be in the metadata, but occasionally 

756 # are found in the dictionary itself. Check both places, 

757 # ending with `None` if neither contains the information. 

758 calib.calibType = dictionary["calibType"] 

759 calib.dimensions = set(dictionary["dimensions"]) 

760 calib.dataIdList = dictionary["dataIdList"] 

761 

762 calib.updateMetadata() 

763 return calib 

764 

765 def toDict(self): 

766 """Return a dictionary containing the provenance information. 

767 

768 Returns 

769 ------- 

770 dictionary : `dict` 

771 Dictionary of provenance. 

772 """ 

773 self.updateMetadata() 

774 

775 outDict = {} 

776 

777 metadata = self.getMetadata() 

778 outDict["metadata"] = metadata 

779 outDict["detectorName"] = self._detectorName 

780 outDict["detectorSerial"] = self._detectorSerial 

781 outDict["detectorId"] = self._detectorId 

782 outDict["instrument"] = self._instrument 

783 outDict["calibType"] = self.calibType 

784 outDict["dimensions"] = list(self.dimensions) 

785 outDict["dataIdList"] = self.dataIdList 

786 

787 return outDict 

788 

789 def toTable(self): 

790 """Return a list of tables containing the provenance. 

791 

792 This seems inefficient and slow, so this may not be the best 

793 way to store the data. 

794 

795 Returns 

796 ------- 

797 tableList : `list` [`lsst.afw.table.Table`] 

798 List of tables containing the provenance information 

799 

800 """ 

801 tableList = [] 

802 self.updateMetadata(setDate=True, setCalibInfo=True) 

803 

804 catalog = Table(rows=self.dataIdList, 

805 names=self.dimensions) 

806 filteredMetadata = {k: v for k, v in self.getMetadata().toDict().items() if v is not None} 

807 catalog.meta = filteredMetadata 

808 tableList.append(catalog) 

809 return tableList