Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21import abc 

22import datetime 

23import os.path 

24import warnings 

25import yaml 

26from astropy.table import Table 

27from astropy.io import fits 

28 

29from lsst.log import Log 

30from lsst.daf.base import PropertyList 

31 

32 

33__all__ = ["IsrCalib", "IsrProvenance"] 

34 

35 

36class IsrCalib(abc.ABC): 

37 """Generic calibration type. 

38 

39 Subclasses must implement the toDict, fromDict, toTable, fromTable 

40 methods that allow the calibration information to be converted 

41 from dictionaries and afw tables. This will allow the calibration 

42 to be persisted using the base class read/write methods. 

43 

44 The validate method is intended to provide a common way to check 

45 that the calibration is valid (internally consistent) and 

46 appropriate (usable with the intended data). The apply method is 

47 intended to allow the calibration to be applied in a consistent 

48 manner. 

49 

50 Parameters 

51 ---------- 

52 camera : `lsst.afw.cameraGeom.Camera`, optional 

53 Camera to extract metadata from. 

54 detector : `lsst.afw.cameraGeom.Detector`, optional 

55 Detector to extract metadata from. 

56 log : `lsst.log.Log`, optional 

57 Log for messages. 

58 """ 

59 _OBSTYPE = 'generic' 

60 _SCHEMA = 'NO SCHEMA' 

61 _VERSION = 0 

62 

63 def __init__(self, camera=None, detector=None, log=None, **kwargs): 

64 self._instrument = None 

65 self._raftName = None 

66 self._slotName = None 

67 self._detectorName = None 

68 self._detectorSerial = None 

69 self._detectorId = None 

70 self._filter = None 

71 self._calibId = None 

72 self._metadata = PropertyList() 

73 self.setMetadata(PropertyList()) 

74 self.calibInfoFromDict(kwargs) 

75 

76 # Define the required attributes for this calibration. 

77 self.requiredAttributes = set(['_OBSTYPE', '_SCHEMA', '_VERSION']) 

78 self.requiredAttributes.update(['_instrument', '_raftName', '_slotName', 

79 '_detectorName', '_detectorSerial', '_detectorId', 

80 '_filter', '_calibId', '_metadata']) 

81 

82 self.log = log if log else Log.getLogger(__name__.partition(".")[2]) 

83 

84 if detector: 

85 self.fromDetector(detector) 

86 self.updateMetadata(camera=camera, detector=detector) 

87 

88 def __str__(self): 

89 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )" 

90 

91 def __eq__(self, other): 

92 """Calibration equivalence. 

93 

94 Subclasses will need to check specific sub-properties. The 

95 default is only to check common entries. 

96 """ 

97 if not isinstance(other, self.__class__): 

98 return False 

99 

100 for attr in self._requiredAttributes: 

101 if getattr(self, attr) != getattr(other, attr): 

102 return False 

103 

104 return True 

105 

106 @property 

107 def requiredAttributes(self): 

108 return self._requiredAttributes 

109 

110 @requiredAttributes.setter 

111 def requiredAttributes(self, value): 

112 self._requiredAttributes = value 

113 

114 def getMetadata(self): 

115 """Retrieve metadata associated with this calibration. 

116 

117 Returns 

118 ------- 

119 meta : `lsst.daf.base.PropertyList` 

120 Metadata. The returned `~lsst.daf.base.PropertyList` can be 

121 modified by the caller and the changes will be written to 

122 external files. 

123 """ 

124 return self._metadata 

125 

126 def setMetadata(self, metadata): 

127 """Store a copy of the supplied metadata with this calibration. 

128 

129 Parameters 

130 ---------- 

131 metadata : `lsst.daf.base.PropertyList` 

132 Metadata to associate with the calibration. Will be copied and 

133 overwrite existing metadata. 

134 """ 

135 if metadata is not None: 

136 self._metadata.update(metadata) 

137 

138 # Ensure that we have the obs type required by calibration ingest 

139 self._metadata["OBSTYPE"] = self._OBSTYPE 

140 self._metadata[self._OBSTYPE + "_SCHEMA"] = self._SCHEMA 

141 self._metadata[self._OBSTYPE + "_VERSION"] = self._VERSION 

142 

143 if isinstance(metadata, dict): 

144 self.calibInfoFromDict(metadata) 

145 elif isinstance(metadata, PropertyList): 

146 self.calibInfoFromDict(metadata.toDict()) 

147 

148 def updateMetadata(self, camera=None, detector=None, filterName=None, 

149 setCalibId=False, setCalibInfo=False, setDate=False, 

150 **kwargs): 

151 """Update metadata keywords with new values. 

152 

153 Parameters 

154 ---------- 

155 camera : `lsst.afw.cameraGeom.Camera`, optional 

156 Reference camera to use to set _instrument field. 

157 detector : `lsst.afw.cameraGeom.Detector`, optional 

158 Reference detector to use to set _detector* fields. 

159 filterName : `str`, optional 

160 Filter name to assign to this calibration. 

161 setCalibId : `bool` optional 

162 Construct the _calibId field from other fields. 

163 setDate : `bool`, optional 

164 Ensure the metadata CALIBDATE fields are set to the current datetime. 

165 kwargs : `dict` or `collections.abc.Mapping`, optional 

166 Set of key=value pairs to assign to the metadata. 

167 """ 

168 mdOriginal = self.getMetadata() 

169 mdSupplemental = dict() 

170 

171 if setCalibInfo: 

172 self.calibInfoFromDict(kwargs) 

173 

174 if camera: 

175 self._instrument = camera.getName() 

176 

177 if detector: 

178 self._detectorName = detector.getName() 

179 self._detectorSerial = detector.getSerial() 

180 self._detectorId = detector.getId() 

181 if "_" in self._detectorName: 

182 (self._raftName, self._slotName) = self._detectorName.split("_") 

183 

184 if filterName: 

185 # If set via: 

186 # exposure.getInfo().getFilter().getName() 

187 # then this will hold the abstract filter. 

188 self._filter = filterName 

189 

190 if setDate: 

191 date = datetime.datetime.now() 

192 mdSupplemental['CALIBDATE'] = date.isoformat() 

193 mdSupplemental['CALIB_CREATION_DATE'] = date.date().isoformat() 

194 mdSupplemental['CALIB_CREATION_TIME'] = date.time().isoformat() 

195 

196 if setCalibId: 

197 values = [] 

198 values.append(f"instrument={self._instrument}") if self._instrument else None 

199 values.append(f"raftName={self._raftName}") if self._raftName else None 

200 values.append(f"detectorName={self._detectorName}") if self._detectorName else None 

201 values.append(f"detector={self._detectorId}") if self._detectorId else None 

202 values.append(f"filter={self._filter}") if self._filter else None 

203 

204 calibDate = mdOriginal.get('CALIBDATE', mdSupplemental.get('CALIBDATE', None)) 

205 values.append(f"calibDate={calibDate}") if calibDate else None 

206 

207 self._calibId = " ".join(values) 

208 

209 self._metadata["INSTRUME"] = self._instrument if self._instrument else None 

210 self._metadata["RAFTNAME"] = self._raftName if self._raftName else None 

211 self._metadata["SLOTNAME"] = self._slotName if self._slotName else None 

212 self._metadata["DETECTOR"] = self._detectorId if self._detectorId else None 

213 self._metadata["DET_NAME"] = self._detectorName if self._detectorName else None 

214 self._metadata["DET_SER"] = self._detectorSerial if self._detectorSerial else None 

215 self._metadata["FILTER"] = self._filter if self._filter else None 

216 self._metadata["CALIB_ID"] = self._calibId if self._calibId else None 

217 

218 mdSupplemental.update(kwargs) 

219 mdOriginal.update(mdSupplemental) 

220 

221 def calibInfoFromDict(self, dictionary): 

222 """Handle common keywords. 

223 

224 This isn't an ideal solution, but until all calibrations 

225 expect to find everything in the metadata, they still need to 

226 search through dictionaries. 

227 

228 Parameters 

229 ---------- 

230 dictionary : `dict` or `lsst.daf.base.PropertyList` 

231 Source for the common keywords. 

232 

233 Raises 

234 ------ 

235 RuntimeError : 

236 Raised if the dictionary does not match the expected OBSTYPE. 

237 

238 """ 

239 

240 def search(haystack, needles): 

241 """Search dictionary 'haystack' for an entry in 'needles' 

242 """ 

243 test = [haystack.get(x) for x in needles] 

244 test = set([x for x in test if x is not None]) 

245 if len(test) == 0: 

246 if 'metadata' in haystack: 

247 return search(haystack['metadata'], needles) 

248 else: 

249 return None 

250 elif len(test) == 1: 

251 value = list(test)[0] 

252 if value == '': 

253 return None 

254 else: 

255 return value 

256 else: 

257 raise ValueError(f"Too many values found: {len(test)} {test} {needles}") 

258 

259 if 'metadata' in dictionary: 

260 metadata = dictionary['metadata'] 

261 

262 if self._OBSTYPE != metadata['OBSTYPE']: 

263 raise RuntimeError(f"Incorrect calibration supplied. Expected {self._OBSTYPE}, " 

264 f"found {metadata['OBSTYPE']}") 

265 

266 self._instrument = search(dictionary, ['INSTRUME', 'instrument']) 

267 self._raftName = search(dictionary, ['RAFTNAME']) 

268 self._slotName = search(dictionary, ['SLOTNAME']) 

269 self._detectorId = search(dictionary, ['DETECTOR', 'detectorId']) 

270 self._detectorName = search(dictionary, ['DET_NAME', 'DETECTOR_NAME', 'detectorName']) 

271 self._detectorSerial = search(dictionary, ['DET_SER', 'DETECTOR_SERIAL', 'detectorSerial']) 

272 self._filter = search(dictionary, ['FILTER', 'filterName']) 

273 self._calibId = search(dictionary, ['CALIB_ID']) 

274 

275 @classmethod 

276 def readText(cls, filename): 

277 """Read calibration representation from a yaml/ecsv file. 

278 

279 Parameters 

280 ---------- 

281 filename : `str` 

282 Name of the file containing the calibration definition. 

283 

284 Returns 

285 ------- 

286 calib : `~lsst.ip.isr.IsrCalibType` 

287 Calibration class. 

288 

289 Raises 

290 ------ 

291 RuntimeError : 

292 Raised if the filename does not end in ".ecsv" or ".yaml". 

293 """ 

294 if filename.endswith((".ecsv", ".ECSV")): 

295 data = Table.read(filename, format='ascii.ecsv') 

296 return cls.fromTable([data]) 

297 

298 elif filename.endswith((".yaml", ".YAML")): 

299 with open(filename, 'r') as f: 

300 data = yaml.load(f, Loader=yaml.CLoader) 

301 return cls.fromDict(data) 

302 else: 

303 raise RuntimeError(f"Unknown filename extension: {filename}") 

304 

305 def writeText(self, filename, format='auto'): 

306 """Write the calibration data to a text file. 

307 

308 Parameters 

309 ---------- 

310 filename : `str` 

311 Name of the file to write. 

312 format : `str` 

313 Format to write the file as. Supported values are: 

314 ``"auto"`` : Determine filetype from filename. 

315 ``"yaml"`` : Write as yaml. 

316 ``"ecsv"`` : Write as ecsv. 

317 Returns 

318 ------- 

319 used : `str` 

320 The name of the file used to write the data. This may 

321 differ from the input if the format is explicitly chosen. 

322 

323 Raises 

324 ------ 

325 RuntimeError : 

326 Raised if filename does not end in a known extension, or 

327 if all information cannot be written. 

328 

329 Notes 

330 ----- 

331 The file is written to YAML/ECSV format and will include any 

332 associated metadata. 

333 

334 """ 

335 if format == 'yaml' or (format == 'auto' and filename.lower().endswith((".yaml", ".YAML"))): 

336 outDict = self.toDict() 

337 path, ext = os.path.splitext(filename) 

338 filename = path + ".yaml" 

339 with open(filename, 'w') as f: 

340 yaml.dump(outDict, f) 

341 elif format == 'ecsv' or (format == 'auto' and filename.lower().endswith((".ecsv", ".ECSV"))): 

342 tableList = self.toTable() 

343 if len(tableList) > 1: 

344 # ECSV doesn't support multiple tables per file, so we 

345 # can only write the first table. 

346 raise RuntimeError(f"Unable to persist {len(tableList)}tables in ECSV format.") 

347 

348 table = tableList[0] 

349 path, ext = os.path.splitext(filename) 

350 filename = path + ".ecsv" 

351 table.write(filename, format="ascii.ecsv") 

352 else: 

353 raise RuntimeError(f"Attempt to write to a file {filename} " 

354 "that does not end in '.yaml' or '.ecsv'") 

355 

356 return filename 

357 

358 @classmethod 

359 def readFits(cls, filename): 

360 """Read calibration data from a FITS file. 

361 

362 Parameters 

363 ---------- 

364 filename : `str` 

365 Filename to read data from. 

366 

367 Returns 

368 ------- 

369 calib : `lsst.ip.isr.IsrCalib` 

370 Calibration contained within the file. 

371 """ 

372 tableList = [] 

373 tableList.append(Table.read(filename, hdu=1)) 

374 extNum = 2 # Fits indices start at 1, we've read one already. 

375 keepTrying = True 

376 

377 while keepTrying: 

378 with warnings.catch_warnings(): 

379 warnings.simplefilter("error") 

380 try: 

381 newTable = Table.read(filename, hdu=extNum) 

382 tableList.append(newTable) 

383 extNum += 1 

384 except Exception: 

385 keepTrying = False 

386 

387 for table in tableList: 

388 for k, v in table.meta.items(): 

389 if isinstance(v, fits.card.Undefined): 

390 table.meta[k] = None 

391 

392 return cls.fromTable(tableList) 

393 

394 def writeFits(self, filename): 

395 """Write calibration data to a FITS file. 

396 

397 Parameters 

398 ---------- 

399 filename : `str` 

400 Filename to write data to. 

401 

402 Returns 

403 ------- 

404 used : `str` 

405 The name of the file used to write the data. 

406 

407 """ 

408 tableList = self.toTable() 

409 with warnings.catch_warnings(): 

410 warnings.filterwarnings("ignore", category=Warning, module="astropy.io") 

411 astropyList = [fits.table_to_hdu(table) for table in tableList] 

412 astropyList.insert(0, fits.PrimaryHDU()) 

413 

414 writer = fits.HDUList(astropyList) 

415 writer.writeto(filename, overwrite=True) 

416 return filename 

417 

418 def fromDetector(self, detector): 

419 """Modify the calibration parameters to match the supplied detector. 

420 

421 Parameters 

422 ---------- 

423 detector : `lsst.afw.cameraGeom.Detector` 

424 Detector to use to set parameters from. 

425 

426 Raises 

427 ------ 

428 NotImplementedError 

429 This needs to be implemented by subclasses for each 

430 calibration type. 

431 """ 

432 raise NotImplementedError("Must be implemented by subclass.") 

433 

434 @classmethod 

435 def fromDict(cls, dictionary): 

436 """Construct a calibration from a dictionary of properties. 

437 

438 Must be implemented by the specific calibration subclasses. 

439 

440 Parameters 

441 ---------- 

442 dictionary : `dict` 

443 Dictionary of properties. 

444 

445 Returns 

446 ------ 

447 calib : `lsst.ip.isr.CalibType` 

448 Constructed calibration. 

449 

450 Raises 

451 ------ 

452 NotImplementedError : 

453 Raised if not implemented. 

454 """ 

455 raise NotImplementedError("Must be implemented by subclass.") 

456 

457 def toDict(self): 

458 """Return a dictionary containing the calibration properties. 

459 

460 The dictionary should be able to be round-tripped through 

461 `fromDict`. 

462 

463 Returns 

464 ------- 

465 dictionary : `dict` 

466 Dictionary of properties. 

467 

468 Raises 

469 ------ 

470 NotImplementedError : 

471 Raised if not implemented. 

472 """ 

473 raise NotImplementedError("Must be implemented by subclass.") 

474 

475 @classmethod 

476 def fromTable(cls, tableList): 

477 """Construct a calibration from a dictionary of properties. 

478 

479 Must be implemented by the specific calibration subclasses. 

480 

481 Parameters 

482 ---------- 

483 tableList : `list` [`lsst.afw.table.Table`] 

484 List of tables of properties. 

485 

486 Returns 

487 ------ 

488 calib : `lsst.ip.isr.CalibType` 

489 Constructed calibration. 

490 

491 Raises 

492 ------ 

493 NotImplementedError : 

494 Raised if not implemented. 

495 """ 

496 raise NotImplementedError("Must be implemented by subclass.") 

497 

498 def toTable(self): 

499 """Return a list of tables containing the calibration properties. 

500 

501 The table list should be able to be round-tripped through 

502 `fromDict`. 

503 

504 Returns 

505 ------- 

506 tableList : `list` [`lsst.afw.table.Table`] 

507 List of tables of properties. 

508 

509 Raises 

510 ------ 

511 NotImplementedError : 

512 Raised if not implemented. 

513 """ 

514 raise NotImplementedError("Must be implemented by subclass.") 

515 

516 def validate(self, other=None): 

517 """Validate that this calibration is defined and can be used. 

518 

519 Parameters 

520 ---------- 

521 other : `object`, optional 

522 Thing to validate against. 

523 

524 Returns 

525 ------- 

526 valid : `bool` 

527 Returns true if the calibration is valid and appropriate. 

528 """ 

529 return False 

530 

531 def apply(self, target): 

532 """Method to apply the calibration to the target object. 

533 

534 Parameters 

535 ---------- 

536 target : `object` 

537 Thing to validate against. 

538 

539 Returns 

540 ------- 

541 valid : `bool` 

542 Returns true if the calibration was applied correctly. 

543 

544 Raises 

545 ------ 

546 NotImplementedError : 

547 Raised if not implemented. 

548 """ 

549 raise NotImplementedError("Must be implemented by subclass.") 

550 

551 

552class IsrProvenance(IsrCalib): 

553 """Class for the provenance of data used to construct calibration. 

554 

555 Provenance is not really a calibration, but we would like to 

556 record this when constructing the calibration, and it provides an 

557 example of the base calibration class. 

558 

559 Parameters 

560 ---------- 

561 instrument : `str`, optional 

562 Name of the instrument the data was taken with. 

563 calibType : `str`, optional 

564 Type of calibration this provenance was generated for. 

565 detectorName : `str`, optional 

566 Name of the detector this calibration is for. 

567 detectorSerial : `str`, optional 

568 Identifier for the detector. 

569 

570 """ 

571 _OBSTYPE = 'IsrProvenance' 

572 

573 def __init__(self, calibType="unknown", 

574 **kwargs): 

575 self.calibType = calibType 

576 self.dimensions = set() 

577 self.dataIdList = list() 

578 

579 super().__init__(**kwargs) 

580 

581 self.requiredAttributes.update(['calibType', 'dimensions', 'dataIdList']) 

582 

583 def __str__(self): 

584 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )" 

585 

586 def __eq__(self, other): 

587 return super().__eq__(other) 

588 

589 def updateMetadata(self, setDate=False, **kwargs): 

590 """Update calibration metadata. 

591 

592 Parameters 

593 ---------- 

594 setDate : `bool, optional 

595 Update the CALIBDATE fields in the metadata to the current 

596 time. Defaults to False. 

597 kwargs : `dict` or `collections.abc.Mapping`, optional 

598 Other keyword parameters to set in the metadata. 

599 """ 

600 kwargs['calibType'] = self.calibType 

601 super().updateMetadata(setDate=setDate, **kwargs) 

602 

603 def fromDataIds(self, dataIdList): 

604 """Update provenance from dataId List. 

605 

606 Parameters 

607 ---------- 

608 dataIdList : `list` [`lsst.daf.butler.DataId`] 

609 List of dataIds used in generating this calibration. 

610 """ 

611 for dataId in dataIdList: 

612 for key in dataId: 

613 if key not in self.dimensions: 

614 self.dimensions.add(key) 

615 self.dataIdList.append(dataId) 

616 

617 @classmethod 

618 def fromTable(cls, tableList): 

619 """Construct provenance from table list. 

620 

621 Parameters 

622 ---------- 

623 tableList : `list` [`lsst.afw.table.Table`] 

624 List of tables to construct the provenance from. 

625 

626 Returns 

627 ------- 

628 provenance : `lsst.ip.isr.IsrProvenance` 

629 The provenance defined in the tables. 

630 """ 

631 table = tableList[0] 

632 metadata = table.meta 

633 inDict = dict() 

634 inDict['metadata'] = metadata 

635 inDict['calibType'] = metadata['calibType'] 

636 inDict['dimensions'] = set() 

637 inDict['dataIdList'] = list() 

638 

639 schema = dict() 

640 for colName in table.columns: 

641 schema[colName.lower()] = colName 

642 inDict['dimensions'].add(colName.lower()) 

643 inDict['dimensions'] = sorted(inDict['dimensions']) 

644 

645 for row in table: 

646 entry = dict() 

647 for dim in sorted(inDict['dimensions']): 

648 entry[dim] = row[schema[dim]] 

649 inDict['dataIdList'].append(entry) 

650 

651 return cls.fromDict(inDict) 

652 

653 @classmethod 

654 def fromDict(cls, dictionary): 

655 """Construct provenance from a dictionary. 

656 

657 Parameters 

658 ---------- 

659 dictionary : `dict` 

660 Dictionary of provenance parameters. 

661 

662 Returns 

663 ------- 

664 provenance : `lsst.ip.isr.IsrProvenance` 

665 The provenance defined in the tables. 

666 """ 

667 calib = cls() 

668 if calib._OBSTYPE != dictionary['metadata']['OBSTYPE']: 

669 raise RuntimeError(f"Incorrect calibration supplied. Expected {calib._OBSTYPE}, " 

670 f"found {dictionary['metadata']['OBSTYPE']}") 

671 

672 calib.setMetadata(dictionary['metadata']) 

673 

674 # These properties should be in the metadata, but occasionally 

675 # are found in the dictionary itself. Check both places, 

676 # ending with `None` if neither contains the information. 

677 calib.calibType = dictionary['calibType'] 

678 calib.dimensions = set(dictionary['dimensions']) 

679 calib.dataIdList = dictionary['dataIdList'] 

680 

681 calib.updateMetadata() 

682 return calib 

683 

684 def toDict(self): 

685 """Return a dictionary containing the provenance information. 

686 

687 Returns 

688 ------- 

689 dictionary : `dict` 

690 Dictionary of provenance. 

691 """ 

692 self.updateMetadata() 

693 

694 outDict = {} 

695 

696 metadata = self.getMetadata() 

697 outDict['metadata'] = metadata 

698 outDict['detectorName'] = self._detectorName 

699 outDict['detectorSerial'] = self._detectorSerial 

700 outDict['detectorId'] = self._detectorId 

701 outDict['instrument'] = self._instrument 

702 outDict['calibType'] = self.calibType 

703 outDict['dimensions'] = list(self.dimensions) 

704 outDict['dataIdList'] = self.dataIdList 

705 

706 return outDict 

707 

708 def toTable(self): 

709 """Return a list of tables containing the provenance. 

710 

711 This seems inefficient and slow, so this may not be the best 

712 way to store the data. 

713 

714 Returns 

715 ------- 

716 tableList : `list` [`lsst.afw.table.Table`] 

717 List of tables containing the provenance information 

718 

719 """ 

720 tableList = [] 

721 self.updateMetadata() 

722 catalog = Table(rows=self.dataIdList, 

723 names=self.dimensions) 

724 filteredMetadata = {k: v for k, v in self.getMetadata().toDict().items() if v is not None} 

725 catalog.meta = filteredMetadata 

726 tableList.append(catalog) 

727 return tableList