Coverage for python/lsst/ip/isr/calibType.py: 14%

312 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-08-31 04:40 -0700

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21import abc 

22import datetime 

23import logging 

24import os.path 

25import warnings 

26import yaml 

27import numpy as np 

28 

29from astropy.table import Table 

30from astropy.io import fits 

31 

32from lsst.daf.base import PropertyList 

33from lsst.utils.introspection import get_full_type_name 

34from lsst.utils import doImport 

35 

36 

37__all__ = ["IsrCalib", "IsrProvenance"] 

38 

39 

40class IsrCalib(abc.ABC): 

41 """Generic calibration type. 

42 

43 Subclasses must implement the toDict, fromDict, toTable, fromTable 

44 methods that allow the calibration information to be converted 

45 from dictionaries and afw tables. This will allow the calibration 

46 to be persisted using the base class read/write methods. 

47 

48 The validate method is intended to provide a common way to check 

49 that the calibration is valid (internally consistent) and 

50 appropriate (usable with the intended data). The apply method is 

51 intended to allow the calibration to be applied in a consistent 

52 manner. 

53 

54 Parameters 

55 ---------- 

56 camera : `lsst.afw.cameraGeom.Camera`, optional 

57 Camera to extract metadata from. 

58 detector : `lsst.afw.cameraGeom.Detector`, optional 

59 Detector to extract metadata from. 

60 log : `logging.Logger`, optional 

61 Log for messages. 

62 """ 

63 _OBSTYPE = "generic" 

64 _SCHEMA = "NO SCHEMA" 

65 _VERSION = 0 

66 

67 def __init__(self, camera=None, detector=None, log=None, **kwargs): 

68 self._instrument = None 

69 self._raftName = None 

70 self._slotName = None 

71 self._detectorName = None 

72 self._detectorSerial = None 

73 self._detectorId = None 

74 self._filter = None 

75 self._calibId = None 

76 self._metadata = PropertyList() 

77 self.setMetadata(PropertyList()) 

78 self.calibInfoFromDict(kwargs) 

79 

80 # Define the required attributes for this calibration. 

81 self.requiredAttributes = set(["_OBSTYPE", "_SCHEMA", "_VERSION"]) 

82 self.requiredAttributes.update(["_instrument", "_raftName", "_slotName", 

83 "_detectorName", "_detectorSerial", "_detectorId", 

84 "_filter", "_calibId", "_metadata"]) 

85 

86 self.log = log if log else logging.getLogger(__name__) 

87 

88 if detector: 

89 self.fromDetector(detector) 

90 self.updateMetadata(camera=camera, detector=detector) 

91 

92 def __str__(self): 

93 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )" 

94 

95 def __eq__(self, other): 

96 """Calibration equivalence. 

97 

98 Running ``calib.log.setLevel(0)`` enables debug statements to 

99 identify problematic fields. 

100 """ 

101 if not isinstance(other, self.__class__): 

102 self.log.debug("Incorrect class type: %s %s", self.__class__, other.__class__) 

103 return False 

104 

105 for attr in self._requiredAttributes: 

106 attrSelf = getattr(self, attr) 

107 attrOther = getattr(other, attr) 

108 

109 if isinstance(attrSelf, dict): 

110 # Dictionary of arrays. 

111 if attrSelf.keys() != attrOther.keys(): 

112 self.log.debug("Dict Key Failure: %s %s %s", attr, attrSelf.keys(), attrOther.keys()) 

113 return False 

114 for key in attrSelf: 

115 try: 

116 if not np.allclose(attrSelf[key], attrOther[key], equal_nan=True): 

117 self.log.debug("Array Failure: %s %s %s", key, attrSelf[key], attrOther[key]) 

118 return False 

119 except TypeError: 

120 # If it is not something numpy can handle 

121 # (it's not a number or array of numbers), 

122 # then it needs to have its own equivalence 

123 # operator. 

124 if attrSelf[key] != attrOther[key]: 

125 return False 

126 elif isinstance(attrSelf, np.ndarray): 

127 # Bare array. 

128 if not np.allclose(attrSelf, attrOther, equal_nan=True): 

129 self.log.debug("Array Failure: %s %s %s", attr, attrSelf, attrOther) 

130 return False 

131 elif type(attrSelf) != type(attrOther): 

132 if set([attrSelf, attrOther]) == set([None, ""]): 

133 # Fits converts None to "", but None is not "". 

134 continue 

135 self.log.debug("Type Failure: %s %s %s %s %s", attr, type(attrSelf), type(attrOther), 

136 attrSelf, attrOther) 

137 return False 

138 else: 

139 if attrSelf != attrOther: 

140 self.log.debug("Value Failure: %s %s %s", attr, attrSelf, attrOther) 

141 return False 

142 

143 return True 

144 

145 @property 

146 def requiredAttributes(self): 

147 return self._requiredAttributes 

148 

149 @requiredAttributes.setter 

150 def requiredAttributes(self, value): 

151 self._requiredAttributes = value 

152 

153 def getMetadata(self): 

154 """Retrieve metadata associated with this calibration. 

155 

156 Returns 

157 ------- 

158 meta : `lsst.daf.base.PropertyList` 

159 Metadata. The returned `~lsst.daf.base.PropertyList` can be 

160 modified by the caller and the changes will be written to 

161 external files. 

162 """ 

163 return self._metadata 

164 

165 def setMetadata(self, metadata): 

166 """Store a copy of the supplied metadata with this calibration. 

167 

168 Parameters 

169 ---------- 

170 metadata : `lsst.daf.base.PropertyList` 

171 Metadata to associate with the calibration. Will be copied and 

172 overwrite existing metadata. 

173 """ 

174 if metadata is not None: 

175 self._metadata.update(metadata) 

176 

177 # Ensure that we have the obs type required by calibration ingest 

178 self._metadata["OBSTYPE"] = self._OBSTYPE 

179 self._metadata[self._OBSTYPE + "_SCHEMA"] = self._SCHEMA 

180 self._metadata[self._OBSTYPE + "_VERSION"] = self._VERSION 

181 

182 if isinstance(metadata, dict): 

183 self.calibInfoFromDict(metadata) 

184 elif isinstance(metadata, PropertyList): 

185 self.calibInfoFromDict(metadata.toDict()) 

186 

187 def updateMetadata(self, camera=None, detector=None, filterName=None, 

188 setCalibId=False, setCalibInfo=False, setDate=False, 

189 **kwargs): 

190 """Update metadata keywords with new values. 

191 

192 Parameters 

193 ---------- 

194 camera : `lsst.afw.cameraGeom.Camera`, optional 

195 Reference camera to use to set _instrument field. 

196 detector : `lsst.afw.cameraGeom.Detector`, optional 

197 Reference detector to use to set _detector* fields. 

198 filterName : `str`, optional 

199 Filter name to assign to this calibration. 

200 setCalibId : `bool`, optional 

201 Construct the _calibId field from other fields. 

202 setCalibInfo : `bool`, optional 

203 Set calibration parameters from metadata. 

204 setDate : `bool`, optional 

205 Ensure the metadata CALIBDATE fields are set to the current 

206 datetime. 

207 kwargs : `dict` or `collections.abc.Mapping`, optional 

208 Set of key=value pairs to assign to the metadata. 

209 """ 

210 mdOriginal = self.getMetadata() 

211 mdSupplemental = dict() 

212 

213 for k, v in kwargs.items(): 

214 if isinstance(v, fits.card.Undefined): 

215 kwargs[k] = None 

216 

217 if setCalibInfo: 

218 self.calibInfoFromDict(kwargs) 

219 

220 if camera: 

221 self._instrument = camera.getName() 

222 

223 if detector: 

224 self._detectorName = detector.getName() 

225 self._detectorSerial = detector.getSerial() 

226 self._detectorId = detector.getId() 

227 if "_" in self._detectorName: 

228 (self._raftName, self._slotName) = self._detectorName.split("_") 

229 

230 if filterName: 

231 # TOD0 DM-28093: I think this whole comment can go away, if we 

232 # always use physicalLabel everywhere in ip_isr. 

233 # If set via: 

234 # exposure.getInfo().getFilter().getName() 

235 # then this will hold the abstract filter. 

236 self._filter = filterName 

237 

238 if setDate: 

239 date = datetime.datetime.now() 

240 mdSupplemental["CALIBDATE"] = date.isoformat() 

241 mdSupplemental["CALIB_CREATION_DATE"] = date.date().isoformat() 

242 mdSupplemental["CALIB_CREATION_TIME"] = date.time().isoformat() 

243 

244 if setCalibId: 

245 values = [] 

246 values.append(f"instrument={self._instrument}") if self._instrument else None 

247 values.append(f"raftName={self._raftName}") if self._raftName else None 

248 values.append(f"detectorName={self._detectorName}") if self._detectorName else None 

249 values.append(f"detector={self._detectorId}") if self._detectorId else None 

250 values.append(f"filter={self._filter}") if self._filter else None 

251 

252 calibDate = mdOriginal.get("CALIBDATE", mdSupplemental.get("CALIBDATE", None)) 

253 values.append(f"calibDate={calibDate}") if calibDate else None 

254 

255 self._calibId = " ".join(values) 

256 

257 self._metadata["INSTRUME"] = self._instrument if self._instrument else None 

258 self._metadata["RAFTNAME"] = self._raftName if self._raftName else None 

259 self._metadata["SLOTNAME"] = self._slotName if self._slotName else None 

260 self._metadata["DETECTOR"] = self._detectorId 

261 self._metadata["DET_NAME"] = self._detectorName if self._detectorName else None 

262 self._metadata["DET_SER"] = self._detectorSerial if self._detectorSerial else None 

263 self._metadata["FILTER"] = self._filter if self._filter else None 

264 self._metadata["CALIB_ID"] = self._calibId if self._calibId else None 

265 self._metadata["CALIBCLS"] = get_full_type_name(self) 

266 

267 mdSupplemental.update(kwargs) 

268 mdOriginal.update(mdSupplemental) 

269 

270 def calibInfoFromDict(self, dictionary): 

271 """Handle common keywords. 

272 

273 This isn't an ideal solution, but until all calibrations 

274 expect to find everything in the metadata, they still need to 

275 search through dictionaries. 

276 

277 Parameters 

278 ---------- 

279 dictionary : `dict` or `lsst.daf.base.PropertyList` 

280 Source for the common keywords. 

281 

282 Raises 

283 ------ 

284 RuntimeError : 

285 Raised if the dictionary does not match the expected OBSTYPE. 

286 

287 """ 

288 

289 def search(haystack, needles): 

290 """Search dictionary 'haystack' for an entry in 'needles' 

291 """ 

292 test = [haystack.get(x) for x in needles] 

293 test = set([x for x in test if x is not None]) 

294 if len(test) == 0: 

295 if "metadata" in haystack: 

296 return search(haystack["metadata"], needles) 

297 else: 

298 return None 

299 elif len(test) == 1: 

300 value = list(test)[0] 

301 if value == "": 

302 return None 

303 else: 

304 return value 

305 else: 

306 raise ValueError(f"Too many values found: {len(test)} {test} {needles}") 

307 

308 if "metadata" in dictionary: 

309 metadata = dictionary["metadata"] 

310 

311 if self._OBSTYPE != metadata["OBSTYPE"]: 

312 raise RuntimeError(f"Incorrect calibration supplied. Expected {self._OBSTYPE}, " 

313 f"found {metadata['OBSTYPE']}") 

314 

315 self._instrument = search(dictionary, ["INSTRUME", "instrument"]) 

316 self._raftName = search(dictionary, ["RAFTNAME"]) 

317 self._slotName = search(dictionary, ["SLOTNAME"]) 

318 self._detectorId = search(dictionary, ["DETECTOR", "detectorId"]) 

319 self._detectorName = search(dictionary, ["DET_NAME", "DETECTOR_NAME", "detectorName"]) 

320 self._detectorSerial = search(dictionary, ["DET_SER", "DETECTOR_SERIAL", "detectorSerial"]) 

321 self._filter = search(dictionary, ["FILTER", "filterName"]) 

322 self._calibId = search(dictionary, ["CALIB_ID"]) 

323 

324 @classmethod 

325 def determineCalibClass(cls, metadata, message): 

326 """Attempt to find calibration class in metadata. 

327 

328 Parameters 

329 ---------- 

330 metadata : `dict` or `lsst.daf.base.PropertyList` 

331 Metadata possibly containing a calibration class entry. 

332 message : `str` 

333 Message to include in any errors. 

334 

335 Returns 

336 ------- 

337 calibClass : `object` 

338 The class to use to read the file contents. Should be an 

339 `lsst.ip.isr.IsrCalib` subclass. 

340 

341 Raises 

342 ------ 

343 ValueError : 

344 Raised if the resulting calibClass is the base 

345 `lsst.ip.isr.IsrClass` (which does not implement the 

346 content methods). 

347 """ 

348 calibClassName = metadata.get("CALIBCLS") 

349 calibClass = doImport(calibClassName) if calibClassName is not None else cls 

350 if calibClass is IsrCalib: 

351 raise ValueError(f"Cannot use base class to read calibration data: {message}") 

352 return calibClass 

353 

354 @classmethod 

355 def readText(cls, filename, **kwargs): 

356 """Read calibration representation from a yaml/ecsv file. 

357 

358 Parameters 

359 ---------- 

360 filename : `str` 

361 Name of the file containing the calibration definition. 

362 kwargs : `dict` or collections.abc.Mapping`, optional 

363 Set of key=value pairs to pass to the ``fromDict`` or 

364 ``fromTable`` methods. 

365 

366 Returns 

367 ------- 

368 calib : `~lsst.ip.isr.IsrCalibType` 

369 Calibration class. 

370 

371 Raises 

372 ------ 

373 RuntimeError : 

374 Raised if the filename does not end in ".ecsv" or ".yaml". 

375 """ 

376 if filename.endswith((".ecsv", ".ECSV")): 

377 data = Table.read(filename, format="ascii.ecsv") 

378 calibClass = cls.determineCalibClass(data.meta, "readText/ECSV") 

379 return calibClass.fromTable([data], **kwargs) 

380 elif filename.endswith((".yaml", ".YAML")): 

381 with open(filename, "r") as f: 

382 data = yaml.load(f, Loader=yaml.CLoader) 

383 calibClass = cls.determineCalibClass(data["metadata"], "readText/YAML") 

384 return calibClass.fromDict(data, **kwargs) 

385 else: 

386 raise RuntimeError(f"Unknown filename extension: {filename}") 

387 

388 def writeText(self, filename, format="auto"): 

389 """Write the calibration data to a text file. 

390 

391 Parameters 

392 ---------- 

393 filename : `str` 

394 Name of the file to write. 

395 format : `str` 

396 Format to write the file as. Supported values are: 

397 ``"auto"`` : Determine filetype from filename. 

398 ``"yaml"`` : Write as yaml. 

399 ``"ecsv"`` : Write as ecsv. 

400 Returns 

401 ------- 

402 used : `str` 

403 The name of the file used to write the data. This may 

404 differ from the input if the format is explicitly chosen. 

405 

406 Raises 

407 ------ 

408 RuntimeError : 

409 Raised if filename does not end in a known extension, or 

410 if all information cannot be written. 

411 

412 Notes 

413 ----- 

414 The file is written to YAML/ECSV format and will include any 

415 associated metadata. 

416 """ 

417 if format == "yaml" or (format == "auto" and filename.lower().endswith((".yaml", ".YAML"))): 

418 outDict = self.toDict() 

419 path, ext = os.path.splitext(filename) 

420 filename = path + ".yaml" 

421 with open(filename, "w") as f: 

422 yaml.dump(outDict, f) 

423 elif format == "ecsv" or (format == "auto" and filename.lower().endswith((".ecsv", ".ECSV"))): 

424 tableList = self.toTable() 

425 if len(tableList) > 1: 

426 # ECSV doesn't support multiple tables per file, so we 

427 # can only write the first table. 

428 raise RuntimeError(f"Unable to persist {len(tableList)}tables in ECSV format.") 

429 

430 table = tableList[0] 

431 path, ext = os.path.splitext(filename) 

432 filename = path + ".ecsv" 

433 table.write(filename, format="ascii.ecsv") 

434 else: 

435 raise RuntimeError(f"Attempt to write to a file {filename} " 

436 "that does not end in '.yaml' or '.ecsv'") 

437 

438 return filename 

439 

440 @classmethod 

441 def readFits(cls, filename, **kwargs): 

442 """Read calibration data from a FITS file. 

443 

444 Parameters 

445 ---------- 

446 filename : `str` 

447 Filename to read data from. 

448 kwargs : `dict` or collections.abc.Mapping`, optional 

449 Set of key=value pairs to pass to the ``fromTable`` 

450 method. 

451 

452 Returns 

453 ------- 

454 calib : `lsst.ip.isr.IsrCalib` 

455 Calibration contained within the file. 

456 """ 

457 tableList = [] 

458 tableList.append(Table.read(filename, hdu=1)) 

459 extNum = 2 # Fits indices start at 1, we've read one already. 

460 keepTrying = True 

461 

462 while keepTrying: 

463 with warnings.catch_warnings(): 

464 warnings.simplefilter("error") 

465 try: 

466 newTable = Table.read(filename, hdu=extNum) 

467 tableList.append(newTable) 

468 extNum += 1 

469 except Exception: 

470 keepTrying = False 

471 

472 for table in tableList: 

473 for k, v in table.meta.items(): 

474 if isinstance(v, fits.card.Undefined): 

475 table.meta[k] = None 

476 

477 calibClass = cls.determineCalibClass(tableList[0].meta, "readFits") 

478 return calibClass.fromTable(tableList, **kwargs) 

479 

480 def writeFits(self, filename): 

481 """Write calibration data to a FITS file. 

482 

483 Parameters 

484 ---------- 

485 filename : `str` 

486 Filename to write data to. 

487 

488 Returns 

489 ------- 

490 used : `str` 

491 The name of the file used to write the data. 

492 

493 """ 

494 tableList = self.toTable() 

495 with warnings.catch_warnings(): 

496 warnings.filterwarnings("ignore", category=Warning, module="astropy.io") 

497 astropyList = [fits.table_to_hdu(table) for table in tableList] 

498 astropyList.insert(0, fits.PrimaryHDU()) 

499 

500 writer = fits.HDUList(astropyList) 

501 writer.writeto(filename, overwrite=True) 

502 return filename 

503 

504 def fromDetector(self, detector): 

505 """Modify the calibration parameters to match the supplied detector. 

506 

507 Parameters 

508 ---------- 

509 detector : `lsst.afw.cameraGeom.Detector` 

510 Detector to use to set parameters from. 

511 

512 Raises 

513 ------ 

514 NotImplementedError 

515 This needs to be implemented by subclasses for each 

516 calibration type. 

517 """ 

518 raise NotImplementedError("Must be implemented by subclass.") 

519 

520 @classmethod 

521 def fromDict(cls, dictionary, **kwargs): 

522 """Construct a calibration from a dictionary of properties. 

523 

524 Must be implemented by the specific calibration subclasses. 

525 

526 Parameters 

527 ---------- 

528 dictionary : `dict` 

529 Dictionary of properties. 

530 kwargs : `dict` or collections.abc.Mapping`, optional 

531 Set of key=value options. 

532 

533 Returns 

534 ------ 

535 calib : `lsst.ip.isr.CalibType` 

536 Constructed calibration. 

537 

538 Raises 

539 ------ 

540 NotImplementedError : 

541 Raised if not implemented. 

542 """ 

543 raise NotImplementedError("Must be implemented by subclass.") 

544 

545 def toDict(self): 

546 """Return a dictionary containing the calibration properties. 

547 

548 The dictionary should be able to be round-tripped through 

549 `fromDict`. 

550 

551 Returns 

552 ------- 

553 dictionary : `dict` 

554 Dictionary of properties. 

555 

556 Raises 

557 ------ 

558 NotImplementedError : 

559 Raised if not implemented. 

560 """ 

561 raise NotImplementedError("Must be implemented by subclass.") 

562 

563 @classmethod 

564 def fromTable(cls, tableList, **kwargs): 

565 """Construct a calibration from a dictionary of properties. 

566 

567 Must be implemented by the specific calibration subclasses. 

568 

569 Parameters 

570 ---------- 

571 tableList : `list` [`lsst.afw.table.Table`] 

572 List of tables of properties. 

573 kwargs : `dict` or collections.abc.Mapping`, optional 

574 Set of key=value options. 

575 

576 Returns 

577 ------ 

578 calib : `lsst.ip.isr.CalibType` 

579 Constructed calibration. 

580 

581 Raises 

582 ------ 

583 NotImplementedError : 

584 Raised if not implemented. 

585 """ 

586 raise NotImplementedError("Must be implemented by subclass.") 

587 

588 def toTable(self): 

589 """Return a list of tables containing the calibration properties. 

590 

591 The table list should be able to be round-tripped through 

592 `fromDict`. 

593 

594 Returns 

595 ------- 

596 tableList : `list` [`lsst.afw.table.Table`] 

597 List of tables of properties. 

598 

599 Raises 

600 ------ 

601 NotImplementedError : 

602 Raised if not implemented. 

603 """ 

604 raise NotImplementedError("Must be implemented by subclass.") 

605 

606 def validate(self, other=None): 

607 """Validate that this calibration is defined and can be used. 

608 

609 Parameters 

610 ---------- 

611 other : `object`, optional 

612 Thing to validate against. 

613 

614 Returns 

615 ------- 

616 valid : `bool` 

617 Returns true if the calibration is valid and appropriate. 

618 """ 

619 return False 

620 

621 def apply(self, target): 

622 """Method to apply the calibration to the target object. 

623 

624 Parameters 

625 ---------- 

626 target : `object` 

627 Thing to validate against. 

628 

629 Returns 

630 ------- 

631 valid : `bool` 

632 Returns true if the calibration was applied correctly. 

633 

634 Raises 

635 ------ 

636 NotImplementedError : 

637 Raised if not implemented. 

638 """ 

639 raise NotImplementedError("Must be implemented by subclass.") 

640 

641 

642class IsrProvenance(IsrCalib): 

643 """Class for the provenance of data used to construct calibration. 

644 

645 Provenance is not really a calibration, but we would like to 

646 record this when constructing the calibration, and it provides an 

647 example of the base calibration class. 

648 

649 Parameters 

650 ---------- 

651 instrument : `str`, optional 

652 Name of the instrument the data was taken with. 

653 calibType : `str`, optional 

654 Type of calibration this provenance was generated for. 

655 detectorName : `str`, optional 

656 Name of the detector this calibration is for. 

657 detectorSerial : `str`, optional 

658 Identifier for the detector. 

659 

660 """ 

661 _OBSTYPE = "IsrProvenance" 

662 

663 def __init__(self, calibType="unknown", 

664 **kwargs): 

665 self.calibType = calibType 

666 self.dimensions = set() 

667 self.dataIdList = list() 

668 

669 super().__init__(**kwargs) 

670 

671 self.requiredAttributes.update(["calibType", "dimensions", "dataIdList"]) 

672 

673 def __str__(self): 

674 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )" 

675 

676 def __eq__(self, other): 

677 return super().__eq__(other) 

678 

679 def updateMetadata(self, setDate=False, **kwargs): 

680 """Update calibration metadata. 

681 

682 Parameters 

683 ---------- 

684 setDate : `bool, optional 

685 Update the CALIBDATE fields in the metadata to the current 

686 time. Defaults to False. 

687 kwargs : `dict` or `collections.abc.Mapping`, optional 

688 Other keyword parameters to set in the metadata. 

689 """ 

690 kwargs["calibType"] = self.calibType 

691 super().updateMetadata(setDate=setDate, **kwargs) 

692 

693 def fromDataIds(self, dataIdList): 

694 """Update provenance from dataId List. 

695 

696 Parameters 

697 ---------- 

698 dataIdList : `list` [`lsst.daf.butler.DataId`] 

699 List of dataIds used in generating this calibration. 

700 """ 

701 for dataId in dataIdList: 

702 for key in dataId: 

703 if key not in self.dimensions: 

704 self.dimensions.add(key) 

705 self.dataIdList.append(dataId) 

706 

707 @classmethod 

708 def fromTable(cls, tableList): 

709 """Construct provenance from table list. 

710 

711 Parameters 

712 ---------- 

713 tableList : `list` [`lsst.afw.table.Table`] 

714 List of tables to construct the provenance from. 

715 

716 Returns 

717 ------- 

718 provenance : `lsst.ip.isr.IsrProvenance` 

719 The provenance defined in the tables. 

720 """ 

721 table = tableList[0] 

722 metadata = table.meta 

723 inDict = dict() 

724 inDict["metadata"] = metadata 

725 inDict["calibType"] = metadata["calibType"] 

726 inDict["dimensions"] = set() 

727 inDict["dataIdList"] = list() 

728 

729 schema = dict() 

730 for colName in table.columns: 

731 schema[colName.lower()] = colName 

732 inDict["dimensions"].add(colName.lower()) 

733 inDict["dimensions"] = sorted(inDict["dimensions"]) 

734 

735 for row in table: 

736 entry = dict() 

737 for dim in sorted(inDict["dimensions"]): 

738 entry[dim] = row[schema[dim]] 

739 inDict["dataIdList"].append(entry) 

740 

741 return cls.fromDict(inDict) 

742 

743 @classmethod 

744 def fromDict(cls, dictionary): 

745 """Construct provenance from a dictionary. 

746 

747 Parameters 

748 ---------- 

749 dictionary : `dict` 

750 Dictionary of provenance parameters. 

751 

752 Returns 

753 ------- 

754 provenance : `lsst.ip.isr.IsrProvenance` 

755 The provenance defined in the tables. 

756 """ 

757 calib = cls() 

758 if calib._OBSTYPE != dictionary["metadata"]["OBSTYPE"]: 

759 raise RuntimeError(f"Incorrect calibration supplied. Expected {calib._OBSTYPE}, " 

760 f"found {dictionary['metadata']['OBSTYPE']}") 

761 calib.updateMetadata(setDate=False, setCalibInfo=True, **dictionary["metadata"]) 

762 

763 # These properties should be in the metadata, but occasionally 

764 # are found in the dictionary itself. Check both places, 

765 # ending with `None` if neither contains the information. 

766 calib.calibType = dictionary["calibType"] 

767 calib.dimensions = set(dictionary["dimensions"]) 

768 calib.dataIdList = dictionary["dataIdList"] 

769 

770 calib.updateMetadata() 

771 return calib 

772 

773 def toDict(self): 

774 """Return a dictionary containing the provenance information. 

775 

776 Returns 

777 ------- 

778 dictionary : `dict` 

779 Dictionary of provenance. 

780 """ 

781 self.updateMetadata() 

782 

783 outDict = {} 

784 

785 metadata = self.getMetadata() 

786 outDict["metadata"] = metadata 

787 outDict["detectorName"] = self._detectorName 

788 outDict["detectorSerial"] = self._detectorSerial 

789 outDict["detectorId"] = self._detectorId 

790 outDict["instrument"] = self._instrument 

791 outDict["calibType"] = self.calibType 

792 outDict["dimensions"] = list(self.dimensions) 

793 outDict["dataIdList"] = self.dataIdList 

794 

795 return outDict 

796 

797 def toTable(self): 

798 """Return a list of tables containing the provenance. 

799 

800 This seems inefficient and slow, so this may not be the best 

801 way to store the data. 

802 

803 Returns 

804 ------- 

805 tableList : `list` [`lsst.afw.table.Table`] 

806 List of tables containing the provenance information 

807 

808 """ 

809 tableList = [] 

810 self.updateMetadata(setDate=True, setCalibInfo=True) 

811 

812 catalog = Table(rows=self.dataIdList, 

813 names=self.dimensions) 

814 filteredMetadata = {k: v for k, v in self.getMetadata().toDict().items() if v is not None} 

815 catalog.meta = filteredMetadata 

816 tableList.append(catalog) 

817 return tableList