Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21import abc 

22import datetime 

23import os.path 

24import warnings 

25import yaml 

26from astropy.table import Table 

27from astropy.io import fits 

28 

29from lsst.log import Log 

30from lsst.daf.base import PropertyList 

31 

32 

33__all__ = ["IsrCalib", "IsrProvenance"] 

34 

35 

36class IsrCalib(abc.ABC): 

37 """Generic calibration type. 

38 

39 Subclasses must implement the toDict, fromDict, toTable, fromTable 

40 methods that allow the calibration information to be converted 

41 from dictionaries and afw tables. This will allow the calibration 

42 to be persisted using the base class read/write methods. 

43 

44 The validate method is intended to provide a common way to check 

45 that the calibration is valid (internally consistent) and 

46 appropriate (usable with the intended data). The apply method is 

47 intended to allow the calibration to be applied in a consistent 

48 manner. 

49 

50 Parameters 

51 ---------- 

52 camera : `lsst.afw.cameraGeom.Camera`, optional 

53 Camera to extract metadata from. 

54 detector : `lsst.afw.cameraGeom.Detector`, optional 

55 Detector to extract metadata from. 

56 log : `lsst.log.Log`, optional 

57 Log for messages. 

58 """ 

59 _OBSTYPE = 'generic' 

60 _SCHEMA = 'NO SCHEMA' 

61 _VERSION = 0 

62 

63 def __init__(self, camera=None, detector=None, detectorName=None, detectorId=None, log=None, **kwargs): 

64 self._instrument = None 

65 self._raftName = None 

66 self._slotName = None 

67 self._detectorName = detectorName 

68 self._detectorSerial = None 

69 self._detectorId = detectorId 

70 self._filter = None 

71 self._calibId = None 

72 self._metadata = PropertyList() 

73 self.setMetadata(PropertyList()) 

74 

75 # Define the required attributes for this calibration. 

76 self.requiredAttributes = set(['_OBSTYPE', '_SCHEMA', '_VERSION']) 

77 self.requiredAttributes.update(['_instrument', '_raftName', '_slotName', 

78 '_detectorName', '_detectorSerial', '_detectorId', 

79 '_filter', '_calibId', '_metadata']) 

80 

81 self.log = log if log else Log.getLogger(__name__.partition(".")[2]) 

82 

83 if detector: 

84 self.fromDetector(detector) 

85 self.updateMetadata(camera=camera, detector=detector, setDate=False) 

86 

87 def __str__(self): 

88 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )" 

89 

90 def __eq__(self, other): 

91 """Calibration equivalence. 

92 

93 Subclasses will need to check specific sub-properties. The 

94 default is only to check common entries. 

95 """ 

96 if not isinstance(other, self.__class__): 

97 return False 

98 

99 for attr in self._requiredAttributes: 

100 if getattr(self, attr) != getattr(other, attr): 

101 return False 

102 

103 return True 

104 

105 @property 

106 def requiredAttributes(self): 

107 return self._requiredAttributes 

108 

109 @requiredAttributes.setter 

110 def requiredAttributes(self, value): 

111 self._requiredAttributes = value 

112 

113 def getMetadata(self): 

114 """Retrieve metadata associated with this calibration. 

115 

116 Returns 

117 ------- 

118 meta : `lsst.daf.base.PropertyList` 

119 Metadata. The returned `~lsst.daf.base.PropertyList` can be 

120 modified by the caller and the changes will be written to 

121 external files. 

122 """ 

123 return self._metadata 

124 

125 def setMetadata(self, metadata): 

126 """Store a copy of the supplied metadata with this calibration. 

127 

128 Parameters 

129 ---------- 

130 metadata : `lsst.daf.base.PropertyList` 

131 Metadata to associate with the calibration. Will be copied and 

132 overwrite existing metadata. 

133 """ 

134 if metadata is not None: 

135 self._metadata.update(metadata) 

136 

137 # Ensure that we have the obs type required by calibration ingest 

138 self._metadata["OBSTYPE"] = self._OBSTYPE 

139 self._metadata[self._OBSTYPE + "_SCHEMA"] = self._SCHEMA 

140 self._metadata[self._OBSTYPE + "_VERSION"] = self._VERSION 

141 

142 def updateMetadata(self, camera=None, detector=None, filterName=None, 

143 setCalibId=False, setDate=False, 

144 **kwargs): 

145 """Update metadata keywords with new values. 

146 

147 Parameters 

148 ---------- 

149 camera : `lsst.afw.cameraGeom.Camera`, optional 

150 Reference camera to use to set _instrument field. 

151 detector : `lsst.afw.cameraGeom.Detector`, optional 

152 Reference detector to use to set _detector* fields. 

153 filterName : `str`, optional 

154 Filter name to assign to this calibration. 

155 setCalibId : `bool` optional 

156 Construct the _calibId field from other fields. 

157 setDate : `bool`, optional 

158 Ensure the metadata CALIBDATE fields are set to the current datetime. 

159 kwargs : `dict` or `collections.abc.Mapping`, optional 

160 Set of key=value pairs to assign to the metadata. 

161 """ 

162 mdOriginal = self.getMetadata() 

163 mdSupplemental = dict() 

164 

165 if camera: 

166 self._instrument = camera.getName() 

167 

168 if detector: 

169 self._detectorName = detector.getName() 

170 self._detectorSerial = detector.getSerial() 

171 self._detectorId = detector.getId() 

172 if "_" in self._detectorName: 

173 (self._raftName, self._slotName) = self._detectorName.split("_") 

174 

175 if filterName: 

176 # If set via: 

177 # exposure.getInfo().getFilter().getName() 

178 # then this will hold the abstract filter. 

179 self._filter = filterName 

180 

181 if setCalibId: 

182 values = [] 

183 values.append(f"instrument={self._instrument}") if self._instrument else None 

184 values.append(f"raftName={self._raftName}") if self._raftName else None 

185 values.append(f"detectorName={self._detectorName}") if self._detectorName else None 

186 values.append(f"detector={self.detector}") if self._detector else None 

187 values.append(f"filter={self._filter}") if self._filter else None 

188 self._calibId = " ".join(values) 

189 

190 if setDate: 

191 date = datetime.datetime.now() 

192 mdSupplemental['CALIBDATE'] = date.isoformat() 

193 mdSupplemental['CALIB_CREATION_DATE'] = date.date().isoformat() 

194 mdSupplemental['CALIB_CREATION_TIME'] = date.time().isoformat() 

195 

196 self._metadata["INSTRUME"] = self._instrument if self._instrument else None 

197 self._metadata["RAFTNAME"] = self._raftName if self._raftName else None 

198 self._metadata["SLOTNAME"] = self._slotName if self._slotName else None 

199 self._metadata["DETECTOR"] = self._detectorId if self._detectorId else None 

200 self._metadata["DET_NAME"] = self._detectorName if self._detectorName else None 

201 self._metadata["DET_SER"] = self._detectorSerial if self._detectorSerial else None 

202 self._metadata["FILTER"] = self._filter if self._filter else None 

203 self._metadata["CALIB_ID"] = self._calibId if self._calibId else None 

204 

205 mdSupplemental.update(kwargs) 

206 mdOriginal.update(mdSupplemental) 

207 

208 @classmethod 

209 def readText(cls, filename): 

210 """Read calibration representation from a yaml/ecsv file. 

211 

212 Parameters 

213 ---------- 

214 filename : `str` 

215 Name of the file containing the calibration definition. 

216 

217 Returns 

218 ------- 

219 calib : `~lsst.ip.isr.IsrCalibType` 

220 Calibration class. 

221 

222 Raises 

223 ------ 

224 RuntimeError : 

225 Raised if the filename does not end in ".ecsv" or ".yaml". 

226 """ 

227 if filename.endswith((".ecsv", ".ECSV")): 

228 data = Table.read(filename, format='ascii.ecsv') 

229 return cls.fromTable([data]) 

230 

231 elif filename.endswith((".yaml", ".YAML")): 

232 with open(filename, 'r') as f: 

233 data = yaml.load(f, Loader=yaml.CLoader) 

234 return cls.fromDict(data) 

235 else: 

236 raise RuntimeError(f"Unknown filename extension: {filename}") 

237 

238 def writeText(self, filename, format='auto'): 

239 """Write the calibration data to a text file. 

240 

241 Parameters 

242 ---------- 

243 filename : `str` 

244 Name of the file to write. 

245 format : `str` 

246 Format to write the file as. Supported values are: 

247 ``"auto"`` : Determine filetype from filename. 

248 ``"yaml"`` : Write as yaml. 

249 ``"ecsv"`` : Write as ecsv. 

250 Returns 

251 ------- 

252 used : `str` 

253 The name of the file used to write the data. This may 

254 differ from the input if the format is explicitly chosen. 

255 

256 Raises 

257 ------ 

258 RuntimeError : 

259 Raised if filename does not end in a known extension, or 

260 if all information cannot be written. 

261 

262 Notes 

263 ----- 

264 The file is written to YAML/ECSV format and will include any 

265 associated metadata. 

266 

267 """ 

268 if format == 'yaml' or (format == 'auto' and filename.lower().endswith((".yaml", ".YAML"))): 

269 outDict = self.toDict() 

270 path, ext = os.path.splitext(filename) 

271 filename = path + ".yaml" 

272 with open(filename, 'w') as f: 

273 yaml.dump(outDict, f) 

274 elif format == 'ecsv' or (format == 'auto' and filename.lower().endswith((".ecsv", ".ECSV"))): 

275 tableList = self.toTable() 

276 if len(tableList) > 1: 

277 # ECSV doesn't support multiple tables per file, so we 

278 # can only write the first table. 

279 raise RuntimeError(f"Unable to persist {len(tableList)}tables in ECSV format.") 

280 

281 table = tableList[0] 

282 path, ext = os.path.splitext(filename) 

283 filename = path + ".ecsv" 

284 table.write(filename, format="ascii.ecsv") 

285 else: 

286 raise RuntimeError(f"Attempt to write to a file {filename} " 

287 "that does not end in '.yaml' or '.ecsv'") 

288 

289 return filename 

290 

291 @classmethod 

292 def readFits(cls, filename): 

293 """Read calibration data from a FITS file. 

294 

295 Parameters 

296 ---------- 

297 filename : `str` 

298 Filename to read data from. 

299 

300 Returns 

301 ------- 

302 calib : `lsst.ip.isr.IsrCalib` 

303 Calibration contained within the file. 

304 """ 

305 tableList = [] 

306 tableList.append(Table.read(filename, hdu=1)) 

307 extNum = 2 # Fits indices start at 1, we've read one already. 

308 try: 

309 with warnings.catch_warnings("error"): 

310 newTable = Table.read(filename, hdu=extNum) 

311 tableList.append(newTable) 

312 extNum += 1 

313 except Exception: 

314 pass 

315 

316 return cls.fromTable(tableList) 

317 

318 def writeFits(self, filename): 

319 """Write calibration data to a FITS file. 

320 

321 Parameters 

322 ---------- 

323 filename : `str` 

324 Filename to write data to. 

325 

326 Returns 

327 ------- 

328 used : `str` 

329 The name of the file used to write the data. 

330 

331 """ 

332 tableList = self.toTable() 

333 with warnings.catch_warnings(): 

334 warnings.filterwarnings("ignore", category=Warning, module="astropy.io") 

335 astropyList = [fits.table_to_hdu(table) for table in tableList] 

336 astropyList.insert(0, fits.PrimaryHDU()) 

337 

338 writer = fits.HDUList(astropyList) 

339 writer.writeto(filename, overwrite=True) 

340 return filename 

341 

342 def fromDetector(self, detector): 

343 """Modify the calibration parameters to match the supplied detector. 

344 

345 Parameters 

346 ---------- 

347 detector : `lsst.afw.cameraGeom.Detector` 

348 Detector to use to set parameters from. 

349 

350 Raises 

351 ------ 

352 NotImplementedError 

353 This needs to be implemented by subclasses for each 

354 calibration type. 

355 """ 

356 raise NotImplementedError("Must be implemented by subclass.") 

357 

358 @classmethod 

359 def fromDict(cls, dictionary): 

360 """Construct a calibration from a dictionary of properties. 

361 

362 Must be implemented by the specific calibration subclasses. 

363 

364 Parameters 

365 ---------- 

366 dictionary : `dict` 

367 Dictionary of properties. 

368 

369 Returns 

370 ------ 

371 calib : `lsst.ip.isr.CalibType` 

372 Constructed calibration. 

373 

374 Raises 

375 ------ 

376 NotImplementedError : 

377 Raised if not implemented. 

378 """ 

379 raise NotImplementedError("Must be implemented by subclass.") 

380 

381 def toDict(self): 

382 """Return a dictionary containing the calibration properties. 

383 

384 The dictionary should be able to be round-tripped through 

385 `fromDict`. 

386 

387 Returns 

388 ------- 

389 dictionary : `dict` 

390 Dictionary of properties. 

391 

392 Raises 

393 ------ 

394 NotImplementedError : 

395 Raised if not implemented. 

396 """ 

397 raise NotImplementedError("Must be implemented by subclass.") 

398 

399 @classmethod 

400 def fromTable(cls, tableList): 

401 """Construct a calibration from a dictionary of properties. 

402 

403 Must be implemented by the specific calibration subclasses. 

404 

405 Parameters 

406 ---------- 

407 tableList : `list` [`lsst.afw.table.Table`] 

408 List of tables of properties. 

409 

410 Returns 

411 ------ 

412 calib : `lsst.ip.isr.CalibType` 

413 Constructed calibration. 

414 

415 Raises 

416 ------ 

417 NotImplementedError : 

418 Raised if not implemented. 

419 """ 

420 raise NotImplementedError("Must be implemented by subclass.") 

421 

422 def toTable(self): 

423 """Return a list of tables containing the calibration properties. 

424 

425 The table list should be able to be round-tripped through 

426 `fromDict`. 

427 

428 Returns 

429 ------- 

430 tableList : `list` [`lsst.afw.table.Table`] 

431 List of tables of properties. 

432 

433 Raises 

434 ------ 

435 NotImplementedError : 

436 Raised if not implemented. 

437 """ 

438 raise NotImplementedError("Must be implemented by subclass.") 

439 

440 def validate(self, other=None): 

441 """Validate that this calibration is defined and can be used. 

442 

443 Parameters 

444 ---------- 

445 other : `object`, optional 

446 Thing to validate against. 

447 

448 Returns 

449 ------- 

450 valid : `bool` 

451 Returns true if the calibration is valid and appropriate. 

452 """ 

453 return False 

454 

455 def apply(self, target): 

456 """Method to apply the calibration to the target object. 

457 

458 Parameters 

459 ---------- 

460 target : `object` 

461 Thing to validate against. 

462 

463 Returns 

464 ------- 

465 valid : `bool` 

466 Returns true if the calibration was applied correctly. 

467 

468 Raises 

469 ------ 

470 NotImplementedError : 

471 Raised if not implemented. 

472 """ 

473 raise NotImplementedError("Must be implemented by subclass.") 

474 

475 

476class IsrProvenance(IsrCalib): 

477 """Class for the provenance of data used to construct calibration. 

478 

479 Provenance is not really a calibration, but we would like to 

480 record this when constructing the calibration, and it provides an 

481 example of the base calibration class. 

482 

483 Parameters 

484 ---------- 

485 instrument : `str`, optional 

486 Name of the instrument the data was taken with. 

487 calibType : `str`, optional 

488 Type of calibration this provenance was generated for. 

489 detectorName : `str`, optional 

490 Name of the detector this calibration is for. 

491 detectorSerial : `str`, optional 

492 Identifier for the detector. 

493 

494 """ 

495 _OBSTYPE = 'IsrProvenance' 

496 

497 def __init__(self, calibType="unknown", 

498 **kwargs): 

499 self.calibType = calibType 

500 self.dimensions = set() 

501 self.dataIdList = list() 

502 

503 super().__init__(**kwargs) 

504 

505 self.requiredAttributes.update(['calibType', 'dimensions', 'dataIdList']) 

506 

507 def __str__(self): 

508 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )" 

509 

510 def __eq__(self, other): 

511 return super().__eq__(other) 

512 

513 def updateMetadata(self, setDate=False, **kwargs): 

514 """Update calibration metadata. 

515 

516 Parameters 

517 ---------- 

518 setDate : `bool, optional 

519 Update the CALIBDATE fields in the metadata to the current 

520 time. Defaults to False. 

521 kwargs : `dict` or `collections.abc.Mapping`, optional 

522 Other keyword parameters to set in the metadata. 

523 """ 

524 kwargs['calibType'] = self.calibType 

525 super().updateMetadata(setDate=setDate, **kwargs) 

526 

527 def fromDataIds(self, dataIdList): 

528 """Update provenance from dataId List. 

529 

530 Parameters 

531 ---------- 

532 dataIdList : `list` [`lsst.daf.butler.DataId`] 

533 List of dataIds used in generating this calibration. 

534 """ 

535 for dataId in dataIdList: 

536 for key in dataId: 

537 if key not in self.dimensions: 

538 self.dimensions.add(key) 

539 self.dataIdList.append(dataId) 

540 

541 @classmethod 

542 def fromTable(cls, tableList): 

543 """Construct provenance from table list. 

544 

545 Parameters 

546 ---------- 

547 tableList : `list` [`lsst.afw.table.Table`] 

548 List of tables to construct the provenance from. 

549 

550 Returns 

551 ------- 

552 provenance : `lsst.ip.isr.IsrProvenance` 

553 The provenance defined in the tables. 

554 """ 

555 table = tableList[0] 

556 metadata = table.meta 

557 inDict = dict() 

558 inDict['metadata'] = metadata 

559 inDict['detectorName'] = metadata.get('DET_NAME', None) 

560 inDict['detectorSerial'] = metadata.get('DET_SER', None) 

561 inDict['instrument'] = metadata.get('INSTRUME', None) 

562 inDict['calibType'] = metadata['calibType'] 

563 inDict['dimensions'] = set() 

564 inDict['dataIdList'] = list() 

565 

566 schema = dict() 

567 for colName in table.columns: 

568 schema[colName.lower()] = colName 

569 inDict['dimensions'].add(colName.lower()) 

570 inDict['dimensions'] = sorted(inDict['dimensions']) 

571 

572 for row in table: 

573 entry = dict() 

574 for dim in sorted(inDict['dimensions']): 

575 entry[dim] = row[schema[dim]] 

576 inDict['dataIdList'].append(entry) 

577 

578 return cls.fromDict(inDict) 

579 

580 @classmethod 

581 def fromDict(cls, dictionary): 

582 """Construct provenance from a dictionary. 

583 

584 Parameters 

585 ---------- 

586 dictionary : `dict` 

587 Dictionary of provenance parameters. 

588 

589 Returns 

590 ------- 

591 provenance : `lsst.ip.isr.IsrProvenance` 

592 The provenance defined in the tables. 

593 """ 

594 calib = cls() 

595 calib.updateMetadata(setDate=False, **dictionary['metadata']) 

596 

597 # These properties should be in the metadata, but occasionally 

598 # are found in the dictionary itself. Check both places, 

599 # ending with `None` if neither contains the information. 

600 calib._detectorName = dictionary.get('detectorName', 

601 dictionary['metadata'].get('DET_NAME', None)) 

602 calib._detectorSerial = dictionary.get('detectorSerial', 

603 dictionary['metadata'].get('DET_SER', None)) 

604 calib._instrument = dictionary.get('instrument', 

605 dictionary['metadata'].get('INSTRUME', None)) 

606 calib.calibType = dictionary['calibType'] 

607 calib.dimensions = set(dictionary['dimensions']) 

608 calib.dataIdList = dictionary['dataIdList'] 

609 

610 calib.updateMetadata() 

611 return calib 

612 

613 def toDict(self): 

614 """Return a dictionary containing the provenance information. 

615 

616 Returns 

617 ------- 

618 dictionary : `dict` 

619 Dictionary of provenance. 

620 """ 

621 self.updateMetadata(setDate=True) 

622 

623 outDict = {} 

624 

625 metadata = self.getMetadata() 

626 outDict['metadata'] = metadata 

627 outDict['detectorName'] = self._detectorName 

628 outDict['detectorSerial'] = self._detectorSerial 

629 outDict['instrument'] = self._instrument 

630 outDict['calibType'] = self.calibType 

631 outDict['dimensions'] = list(self.dimensions) 

632 outDict['dataIdList'] = self.dataIdList 

633 

634 return outDict 

635 

636 def toTable(self): 

637 """Return a list of tables containing the provenance. 

638 

639 This seems inefficient and slow, so this may not be the best 

640 way to store the data. 

641 

642 Returns 

643 ------- 

644 tableList : `list` [`lsst.afw.table.Table`] 

645 List of tables containing the provenance information 

646 

647 """ 

648 tableList = [] 

649 self.updateMetadata(setDate=True) 

650 catalog = Table(rows=self.dataIdList, 

651 names=self.dimensions) 

652 filteredMetadata = {k: v for k, v in self.getMetadata().toDict().items() if v is not None} 

653 catalog.meta = filteredMetadata 

654 tableList.append(catalog) 

655 return tableList