Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21import abc 

22import copy 

23import datetime 

24import os.path 

25import warnings 

26import yaml 

27from astropy.table import Table 

28from astropy.io import fits 

29 

30from lsst.log import Log 

31from lsst.daf.base import PropertyList 

32 

33 

34__all__ = ["IsrCalib", "IsrProvenance"] 

35 

36 

37class IsrCalib(abc.ABC): 

38 """Generic calibration type. 

39 

40 Subclasses must implement the toDict, fromDict, toTable, fromTable 

41 methods that allow the calibration information to be converted 

42 from dictionaries and afw tables. This will allow the calibration 

43 to be persisted using the base class read/write methods. 

44 

45 The validate method is intended to provide a common way to check 

46 that the calibration is valid (internally consistent) and 

47 appropriate (usable with the intended data). The apply method is 

48 intended to allow the calibration to be applied in a consistent 

49 manner. 

50 

51 Parameters 

52 ---------- 

53 camera : `lsst.afw.cameraGeom.Camera`, optional 

54 Camera to extract metadata from. 

55 detector : `lsst.afw.cameraGeom.Detector`, optional 

56 Detector to extract metadata from. 

57 log : `lsst.log.Log`, optional 

58 Log for messages. 

59 """ 

60 _OBSTYPE = 'generic' 

61 _SCHEMA = 'NO SCHEMA' 

62 _VERSION = 0 

63 

64 def __init__(self, camera=None, detector=None, detectorName=None, detectorId=None, log=None, **kwargs): 

65 self._instrument = None 

66 self._raftName = None 

67 self._slotName = None 

68 self._detectorName = detectorName 

69 self._detectorSerial = None 

70 self._detectorId = detectorId 

71 self._filter = None 

72 self._calibId = None 

73 

74 self.setMetadata(PropertyList()) 

75 

76 # Define the required attributes for this calibration. 

77 self.requiredAttributes = set(['_OBSTYPE', '_SCHEMA', '_VERSION']) 

78 self.requiredAttributes.update(['_instrument', '_raftName', '_slotName', 

79 '_detectorName', '_detectorSerial', '_detectorId', 

80 '_filter', '_calibId', '_metadata']) 

81 

82 self.log = log if log else Log.getLogger(__name__.partition(".")[2]) 

83 

84 if detector: 

85 self.fromDetector(detector) 

86 self.updateMetadata(camera=camera, detector=detector, setDate=False) 

87 

88 def __str__(self): 

89 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )" 

90 

91 def __eq__(self, other): 

92 """Calibration equivalence. 

93 

94 Subclasses will need to check specific sub-properties. The 

95 default is only to check common entries. 

96 """ 

97 if not isinstance(other, self.__class__): 

98 return False 

99 

100 for attr in self._requiredAttributes: 

101 if getattr(self, attr) != getattr(other, attr): 

102 return False 

103 

104 return True 

105 

106 @property 

107 def requiredAttributes(self): 

108 return self._requiredAttributes 

109 

110 @requiredAttributes.setter 

111 def requiredAttributes(self, value): 

112 self._requiredAttributes = value 

113 

114 def getMetadata(self): 

115 """Retrieve metadata associated with this calibration. 

116 

117 Returns 

118 ------- 

119 meta : `lsst.daf.base.PropertyList` 

120 Metadata. The returned `~lsst.daf.base.PropertyList` can be 

121 modified by the caller and the changes will be written to 

122 external files. 

123 """ 

124 return self._metadata 

125 

126 def setMetadata(self, metadata): 

127 """Store a copy of the supplied metadata with this calibration. 

128 

129 Parameters 

130 ---------- 

131 metadata : `lsst.daf.base.PropertyList` 

132 Metadata to associate with the calibration. Will be copied and 

133 overwrite existing metadata. 

134 """ 

135 if metadata is not None: 

136 self._metadata = copy.copy(metadata) 

137 

138 # Ensure that we have the obs type required by calibration ingest 

139 self._metadata["OBSTYPE"] = self._OBSTYPE 

140 self._metadata[self._OBSTYPE + "_SCHEMA"] = self._SCHEMA 

141 self._metadata[self._OBSTYPE + "_VERSION"] = self._VERSION 

142 

143 def updateMetadata(self, camera=None, detector=None, filterName=None, 

144 setCalibId=False, setDate=False, 

145 **kwargs): 

146 """Update metadata keywords with new values. 

147 

148 Parameters 

149 ---------- 

150 camera : `lsst.afw.cameraGeom.Camera`, optional 

151 Reference camera to use to set _instrument field. 

152 detector : `lsst.afw.cameraGeom.Detector`, optional 

153 Reference detector to use to set _detector* fields. 

154 filterName : `str`, optional 

155 Filter name to assign to this calibration. 

156 setCalibId : `bool` optional 

157 Construct the _calibId field from other fields. 

158 setDate : `bool`, optional 

159 Ensure the metadata CALIBDATE fields are set to the current datetime. 

160 kwargs : `dict` or `collections.abc.Mapping`, optional 

161 Set of key=value pairs to assign to the metadata. 

162 """ 

163 mdOriginal = self.getMetadata() 

164 mdSupplemental = dict() 

165 

166 if camera: 

167 self._instrument = camera.getName() 

168 

169 if detector: 

170 self._detectorName = detector.getName() 

171 self._detectorSerial = detector.getSerial() 

172 self._detectorId = detector.getId() 

173 if "_" in self._detectorName: 

174 (self._raftName, self._slotName) = self._detectorName.split("_") 

175 

176 if filterName: 

177 # If set via: 

178 # exposure.getInfo().getFilter().getName() 

179 # then this will hold the abstract filter. 

180 self._filter = filterName 

181 

182 if setCalibId: 

183 values = [] 

184 values.append(f"instrument={self._instrument}") if self._instrument else None 

185 values.append(f"raftName={self._raftName}") if self._raftName else None 

186 values.append(f"detectorName={self._detectorName}") if self._detectorName else None 

187 values.append(f"detector={self.detector}") if self._detector else None 

188 values.append(f"filter={self._filter}") if self._filter else None 

189 self._calibId = " ".join(values) 

190 

191 if setDate: 

192 date = datetime.datetime.now() 

193 mdSupplemental['CALIBDATE'] = date.isoformat() 

194 mdSupplemental['CALIB_CREATION_DATE'] = date.date().isoformat() 

195 mdSupplemental['CALIB_CREATION_TIME'] = date.time().isoformat() 

196 

197 self._metadata["INSTRUME"] = self._instrument if self._instrument else None 

198 self._metadata["RAFTNAME"] = self._raftName if self._raftName else None 

199 self._metadata["SLOTNAME"] = self._slotName if self._slotName else None 

200 self._metadata["DETECTOR"] = self._detectorId if self._detectorId else None 

201 self._metadata["DET_NAME"] = self._detectorName if self._detectorName else None 

202 self._metadata["DET_SER"] = self._detectorSerial if self._detectorSerial else None 

203 self._metadata["FILTER"] = self._filter if self._filter else None 

204 self._metadata["CALIB_ID"] = self._calibId if self._calibId else None 

205 

206 mdSupplemental.update(kwargs) 

207 mdOriginal.update(mdSupplemental) 

208 

209 @classmethod 

210 def readText(cls, filename): 

211 """Read calibration representation from a yaml/ecsv file. 

212 

213 Parameters 

214 ---------- 

215 filename : `str` 

216 Name of the file containing the calibration definition. 

217 

218 Returns 

219 ------- 

220 calib : `~lsst.ip.isr.IsrCalibType` 

221 Calibration class. 

222 

223 Raises 

224 ------ 

225 RuntimeError : 

226 Raised if the filename does not end in ".ecsv" or ".yaml". 

227 """ 

228 if filename.endswith((".ecsv", ".ECSV")): 

229 data = Table.read(filename, format='ascii.ecsv') 

230 return cls.fromTable([data]) 

231 

232 elif filename.endswith((".yaml", ".YAML")): 

233 with open(filename, 'r') as f: 

234 data = yaml.load(f, Loader=yaml.CLoader) 

235 return cls.fromDict(data) 

236 else: 

237 raise RuntimeError(f"Unknown filename extension: {filename}") 

238 

239 def writeText(self, filename, format='auto'): 

240 """Write the calibration data to a text file. 

241 

242 Parameters 

243 ---------- 

244 filename : `str` 

245 Name of the file to write. 

246 format : `str` 

247 Format to write the file as. Supported values are: 

248 ``"auto"`` : Determine filetype from filename. 

249 ``"yaml"`` : Write as yaml. 

250 ``"ecsv"`` : Write as ecsv. 

251 Returns 

252 ------- 

253 used : `str` 

254 The name of the file used to write the data. This may 

255 differ from the input if the format is explicitly chosen. 

256 

257 Raises 

258 ------ 

259 RuntimeError : 

260 Raised if filename does not end in a known extension, or 

261 if all information cannot be written. 

262 

263 Notes 

264 ----- 

265 The file is written to YAML/ECSV format and will include any 

266 associated metadata. 

267 

268 """ 

269 if format == 'yaml' or (format == 'auto' and filename.lower().endswith((".yaml", ".YAML"))): 

270 outDict = self.toDict() 

271 path, ext = os.path.splitext(filename) 

272 filename = path + ".yaml" 

273 with open(filename, 'w') as f: 

274 yaml.dump(outDict, f) 

275 elif format == 'ecsv' or (format == 'auto' and filename.lower().endswith((".ecsv", ".ECSV"))): 

276 tableList = self.toTable() 

277 if len(tableList) > 1: 

278 # ECSV doesn't support multiple tables per file, so we 

279 # can only write the first table. 

280 raise RuntimeError(f"Unable to persist {len(tableList)}tables in ECSV format.") 

281 

282 table = tableList[0] 

283 path, ext = os.path.splitext(filename) 

284 filename = path + ".ecsv" 

285 table.write(filename, format="ascii.ecsv") 

286 else: 

287 raise RuntimeError(f"Attempt to write to a file {filename} " 

288 "that does not end in '.yaml' or '.ecsv'") 

289 

290 return filename 

291 

292 @classmethod 

293 def readFits(cls, filename): 

294 """Read calibration data from a FITS file. 

295 

296 Parameters 

297 ---------- 

298 filename : `str` 

299 Filename to read data from. 

300 

301 Returns 

302 ------- 

303 calib : `lsst.ip.isr.IsrCalib` 

304 Calibration contained within the file. 

305 """ 

306 tableList = [] 

307 tableList.append(Table.read(filename, hdu=1)) 

308 extNum = 2 # Fits indices start at 1, we've read one already. 

309 try: 

310 with warnings.catch_warnings("error"): 

311 newTable = Table.read(filename, hdu=extNum) 

312 tableList.append(newTable) 

313 extNum += 1 

314 except Exception: 

315 pass 

316 

317 return cls.fromTable(tableList) 

318 

319 def writeFits(self, filename): 

320 """Write calibration data to a FITS file. 

321 

322 Parameters 

323 ---------- 

324 filename : `str` 

325 Filename to write data to. 

326 

327 Returns 

328 ------- 

329 used : `str` 

330 The name of the file used to write the data. 

331 

332 """ 

333 tableList = self.toTable() 

334 with warnings.catch_warnings(): 

335 warnings.filterwarnings("ignore", category=Warning, module="astropy.io") 

336 astropyList = [fits.table_to_hdu(table) for table in tableList] 

337 astropyList.insert(0, fits.PrimaryHDU()) 

338 

339 writer = fits.HDUList(astropyList) 

340 writer.writeto(filename, overwrite=True) 

341 return filename 

342 

343 def fromDetector(self, detector): 

344 """Modify the calibration parameters to match the supplied detector. 

345 

346 Parameters 

347 ---------- 

348 detector : `lsst.afw.cameraGeom.Detector` 

349 Detector to use to set parameters from. 

350 

351 Raises 

352 ------ 

353 NotImplementedError 

354 This needs to be implemented by subclasses for each 

355 calibration type. 

356 """ 

357 raise NotImplementedError("Must be implemented by subclass.") 

358 

359 @classmethod 

360 def fromDict(cls, dictionary): 

361 """Construct a calibration from a dictionary of properties. 

362 

363 Must be implemented by the specific calibration subclasses. 

364 

365 Parameters 

366 ---------- 

367 dictionary : `dict` 

368 Dictionary of properties. 

369 

370 Returns 

371 ------ 

372 calib : `lsst.ip.isr.CalibType` 

373 Constructed calibration. 

374 

375 Raises 

376 ------ 

377 NotImplementedError : 

378 Raised if not implemented. 

379 """ 

380 raise NotImplementedError("Must be implemented by subclass.") 

381 

382 def toDict(self): 

383 """Return a dictionary containing the calibration properties. 

384 

385 The dictionary should be able to be round-tripped through 

386 `fromDict`. 

387 

388 Returns 

389 ------- 

390 dictionary : `dict` 

391 Dictionary of properties. 

392 

393 Raises 

394 ------ 

395 NotImplementedError : 

396 Raised if not implemented. 

397 """ 

398 raise NotImplementedError("Must be implemented by subclass.") 

399 

400 @classmethod 

401 def fromTable(cls, tableList): 

402 """Construct a calibration from a dictionary of properties. 

403 

404 Must be implemented by the specific calibration subclasses. 

405 

406 Parameters 

407 ---------- 

408 tableList : `list` [`lsst.afw.table.Table`] 

409 List of tables of properties. 

410 

411 Returns 

412 ------ 

413 calib : `lsst.ip.isr.CalibType` 

414 Constructed calibration. 

415 

416 Raises 

417 ------ 

418 NotImplementedError : 

419 Raised if not implemented. 

420 """ 

421 raise NotImplementedError("Must be implemented by subclass.") 

422 

423 def toTable(self): 

424 """Return a list of tables containing the calibration properties. 

425 

426 The table list should be able to be round-tripped through 

427 `fromDict`. 

428 

429 Returns 

430 ------- 

431 tableList : `list` [`lsst.afw.table.Table`] 

432 List of tables of properties. 

433 

434 Raises 

435 ------ 

436 NotImplementedError : 

437 Raised if not implemented. 

438 """ 

439 raise NotImplementedError("Must be implemented by subclass.") 

440 

441 def validate(self, other=None): 

442 """Validate that this calibration is defined and can be used. 

443 

444 Parameters 

445 ---------- 

446 other : `object`, optional 

447 Thing to validate against. 

448 

449 Returns 

450 ------- 

451 valid : `bool` 

452 Returns true if the calibration is valid and appropriate. 

453 """ 

454 return False 

455 

456 def apply(self, target): 

457 """Method to apply the calibration to the target object. 

458 

459 Parameters 

460 ---------- 

461 target : `object` 

462 Thing to validate against. 

463 

464 Returns 

465 ------- 

466 valid : `bool` 

467 Returns true if the calibration was applied correctly. 

468 

469 Raises 

470 ------ 

471 NotImplementedError : 

472 Raised if not implemented. 

473 """ 

474 raise NotImplementedError("Must be implemented by subclass.") 

475 

476 

477class IsrProvenance(IsrCalib): 

478 """Class for the provenance of data used to construct calibration. 

479 

480 Provenance is not really a calibration, but we would like to 

481 record this when constructing the calibration, and it provides an 

482 example of the base calibration class. 

483 

484 Parameters 

485 ---------- 

486 instrument : `str`, optional 

487 Name of the instrument the data was taken with. 

488 calibType : `str`, optional 

489 Type of calibration this provenance was generated for. 

490 detectorName : `str`, optional 

491 Name of the detector this calibration is for. 

492 detectorSerial : `str`, optional 

493 Identifier for the detector. 

494 

495 """ 

496 _OBSTYPE = 'IsrProvenance' 

497 

498 def __init__(self, calibType="unknown", 

499 **kwargs): 

500 self.calibType = calibType 

501 self.dimensions = set() 

502 self.dataIdList = list() 

503 

504 super().__init__(**kwargs) 

505 

506 self.requiredAttributes.update(['calibType', 'dimensions', 'dataIdList']) 

507 

508 def __str__(self): 

509 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )" 

510 

511 def __eq__(self, other): 

512 return super().__eq__(other) 

513 

514 def updateMetadata(self, setDate=False, **kwargs): 

515 """Update calibration metadata. 

516 

517 Parameters 

518 ---------- 

519 setDate : `bool, optional 

520 Update the CALIBDATE fields in the metadata to the current 

521 time. Defaults to False. 

522 kwargs : `dict` or `collections.abc.Mapping`, optional 

523 Other keyword parameters to set in the metadata. 

524 """ 

525 kwargs['calibType'] = self.calibType 

526 super().updateMetadata(setDate=setDate, **kwargs) 

527 

528 def fromDataIds(self, dataIdList): 

529 """Update provenance from dataId List. 

530 

531 Parameters 

532 ---------- 

533 dataIdList : `list` [`lsst.daf.butler.DataId`] 

534 List of dataIds used in generating this calibration. 

535 """ 

536 for dataId in dataIdList: 

537 for key in dataId: 

538 if key not in self.dimensions: 

539 self.dimensions.add(key) 

540 self.dataIdList.append(dataId) 

541 

542 @classmethod 

543 def fromTable(cls, tableList): 

544 """Construct provenance from table list. 

545 

546 Parameters 

547 ---------- 

548 tableList : `list` [`lsst.afw.table.Table`] 

549 List of tables to construct the provenance from. 

550 

551 Returns 

552 ------- 

553 provenance : `lsst.ip.isr.IsrProvenance` 

554 The provenance defined in the tables. 

555 """ 

556 table = tableList[0] 

557 metadata = table.meta 

558 inDict = dict() 

559 inDict['metadata'] = metadata 

560 inDict['detectorName'] = metadata.get('DET_NAME', None) 

561 inDict['detectorSerial'] = metadata.get('DET_SER', None) 

562 inDict['instrument'] = metadata.get('INSTRUME', None) 

563 inDict['calibType'] = metadata['calibType'] 

564 inDict['dimensions'] = set() 

565 inDict['dataIdList'] = list() 

566 

567 schema = dict() 

568 for colName in table.columns: 

569 schema[colName.lower()] = colName 

570 inDict['dimensions'].add(colName.lower()) 

571 inDict['dimensions'] = sorted(inDict['dimensions']) 

572 

573 for row in table: 

574 entry = dict() 

575 for dim in sorted(inDict['dimensions']): 

576 entry[dim] = row[schema[dim]] 

577 inDict['dataIdList'].append(entry) 

578 

579 return cls.fromDict(inDict) 

580 

581 @classmethod 

582 def fromDict(cls, dictionary): 

583 """Construct provenance from a dictionary. 

584 

585 Parameters 

586 ---------- 

587 dictionary : `dict` 

588 Dictionary of provenance parameters. 

589 

590 Returns 

591 ------- 

592 provenance : `lsst.ip.isr.IsrProvenance` 

593 The provenance defined in the tables. 

594 """ 

595 calib = cls() 

596 calib.updateMetadata(setDate=False, **dictionary['metadata']) 

597 

598 # These properties should be in the metadata, but occasionally 

599 # are found in the dictionary itself. Check both places, 

600 # ending with `None` if neither contains the information. 

601 calib._detectorName = dictionary.get('detectorName', 

602 dictionary['metadata'].get('DET_NAME', None)) 

603 calib._detectorSerial = dictionary.get('detectorSerial', 

604 dictionary['metadata'].get('DET_SER', None)) 

605 calib._instrument = dictionary.get('instrument', 

606 dictionary['metadata'].get('INSTRUME', None)) 

607 calib.calibType = dictionary['calibType'] 

608 calib.dimensions = set(dictionary['dimensions']) 

609 calib.dataIdList = dictionary['dataIdList'] 

610 

611 calib.updateMetadata() 

612 return calib 

613 

614 def toDict(self): 

615 """Return a dictionary containing the provenance information. 

616 

617 Returns 

618 ------- 

619 dictionary : `dict` 

620 Dictionary of provenance. 

621 """ 

622 self.updateMetadata(setDate=True) 

623 

624 outDict = {} 

625 

626 metadata = self.getMetadata() 

627 outDict['metadata'] = metadata 

628 outDict['detectorName'] = self._detectorName 

629 outDict['detectorSerial'] = self._detectorSerial 

630 outDict['instrument'] = self._instrument 

631 outDict['calibType'] = self.calibType 

632 outDict['dimensions'] = list(self.dimensions) 

633 outDict['dataIdList'] = self.dataIdList 

634 

635 return outDict 

636 

637 def toTable(self): 

638 """Return a list of tables containing the provenance. 

639 

640 This seems inefficient and slow, so this may not be the best 

641 way to store the data. 

642 

643 Returns 

644 ------- 

645 tableList : `list` [`lsst.afw.table.Table`] 

646 List of tables containing the provenance information 

647 

648 """ 

649 tableList = [] 

650 self.updateMetadata(setDate=True) 

651 catalog = Table(rows=self.dataIdList, 

652 names=self.dimensions) 

653 filteredMetadata = {k: v for k, v in self.getMetadata().toDict().items() if v is not None} 

654 catalog.meta = filteredMetadata 

655 tableList.append(catalog) 

656 return tableList