Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21import abc 

22import copy 

23import datetime 

24import os.path 

25import warnings 

26import yaml 

27from astropy.table import Table 

28from astropy.io import fits 

29 

30from lsst.log import Log 

31from lsst.daf.base import PropertyList 

32 

33 

34__all__ = ["IsrCalib", "IsrProvenance"] 

35 

36 

37class IsrCalib(abc.ABC): 

38 """Generic calibration type. 

39 

40 Subclasses must implement the toDict, fromDict, toTable, fromTable 

41 methods that allow the calibration information to be converted 

42 from dictionaries and afw tables. This will allow the calibration 

43 to be persisted using the base class read/write methods. 

44 

45 The validate method is intended to provide a common way to check 

46 that the calibration is valid (internally consistent) and 

47 appropriate (usable with the intended data). The apply method is 

48 intended to allow the calibration to be applied in a consistent 

49 manner. 

50 

51 Parameters 

52 ---------- 

53 detectorName : `str`, optional 

54 Name of the detector this calibration is for. 

55 detectorSerial : `str`, optional 

56 Identifier for the detector. 

57 detector : `lsst.afw.cameraGeom.Detector`, optional 

58 Detector to extract metadata from. 

59 log : `lsst.log.Log`, optional 

60 Log for messages. 

61 

62 """ 

63 _OBSTYPE = 'generic' 

64 _SCHEMA = 'NO SCHEMA' 

65 _VERSION = 0 

66 

67 def __init__(self, detectorName=None, detectorSerial=None, detector=None, log=None, **kwargs): 

68 self._detectorName = detectorName 

69 self._detectorSerial = detectorSerial 

70 self.setMetadata(PropertyList()) 

71 

72 # Define the required attributes for this calibration. 

73 self.requiredAttributes = set(['_OBSTYPE', '_SCHEMA', '_VERSION']) 

74 self.requiredAttributes.update(['_detectorName', '_detectorSerial', '_metadata']) 

75 

76 self.log = log if log else Log.getLogger(__name__.partition(".")[2]) 

77 

78 if detector: 

79 self.fromDetector(detector) 

80 self.updateMetadata(setDate=False) 

81 

82 def __str__(self): 

83 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )" 

84 

85 def __eq__(self, other): 

86 """Calibration equivalence. 

87 

88 Subclasses will need to check specific sub-properties. The 

89 default is only to check common entries. 

90 """ 

91 if not isinstance(other, self.__class__): 

92 return False 

93 

94 for attr in self._requiredAttributes: 

95 if getattr(self, attr) != getattr(other, attr): 

96 return False 

97 

98 return True 

99 

100 @property 

101 def requiredAttributes(self): 

102 return self._requiredAttributes 

103 

104 @requiredAttributes.setter 

105 def requiredAttributes(self, value): 

106 self._requiredAttributes = value 

107 

108 def getMetadata(self): 

109 

110 """Retrieve metadata associated with this calibration. 

111 

112 Returns 

113 ------- 

114 meta : `lsst.daf.base.PropertyList` 

115 Metadata. The returned `~lsst.daf.base.PropertyList` can be 

116 modified by the caller and the changes will be written to 

117 external files. 

118 """ 

119 return self._metadata 

120 

121 def setMetadata(self, metadata): 

122 """Store a copy of the supplied metadata with this calibration. 

123 

124 Parameters 

125 ---------- 

126 metadata : `lsst.daf.base.PropertyList` 

127 Metadata to associate with the calibration. Will be copied and 

128 overwrite existing metadata. 

129 """ 

130 if metadata is not None: 

131 self._metadata = copy.copy(metadata) 

132 

133 # Ensure that we have the obs type required by calibration ingest 

134 self._metadata["OBSTYPE"] = self._OBSTYPE 

135 self._metadata[self._OBSTYPE + "_SCHEMA"] = self._SCHEMA 

136 self._metadata[self._OBSTYPE + "_VERSION"] = self._VERSION 

137 

138 def updateMetadata(self, setDate=False, **kwargs): 

139 """Update metadata keywords with new values. 

140 

141 Parameters 

142 ---------- 

143 setDate : `bool`, optional 

144 Ensure the metadata CALIBDATE fields are set to the current datetime. 

145 kwargs : `dict` or `collections.abc.Mapping`, optional 

146 Set of key=value pairs to assign to the metadata. 

147 """ 

148 mdOriginal = self.getMetadata() 

149 mdSupplemental = dict() 

150 

151 self._metadata["DETECTOR"] = self._detectorName 

152 self._metadata["DETECTOR_SERIAL"] = self._detectorSerial 

153 

154 if setDate: 

155 date = datetime.datetime.now() 

156 mdSupplemental['CALIBDATE'] = date.isoformat() 

157 mdSupplemental['CALIB_CREATION_DATE'] = date.date().isoformat() 

158 mdSupplemental['CALIB_CREATION_TIME'] = date.time().isoformat() 

159 

160 mdSupplemental.update(kwargs) 

161 mdOriginal.update(mdSupplemental) 

162 

163 @classmethod 

164 def readText(cls, filename): 

165 """Read calibration representation from a yaml/ecsv file. 

166 

167 Parameters 

168 ---------- 

169 filename : `str` 

170 Name of the file containing the calibration definition. 

171 

172 Returns 

173 ------- 

174 calib : `~lsst.ip.isr.IsrCalibType` 

175 Calibration class. 

176 

177 Raises 

178 ------ 

179 RuntimeError : 

180 Raised if the filename does not end in ".ecsv" or ".yaml". 

181 """ 

182 if filename.endswith((".ecsv", ".ECSV")): 

183 data = Table.read(filename, format='ascii.ecsv') 

184 return cls.fromTable([data]) 

185 

186 elif filename.endswith((".yaml", ".YAML")): 

187 with open(filename, 'r') as f: 

188 data = yaml.load(f, Loader=yaml.CLoader) 

189 return cls.fromDict(data) 

190 else: 

191 raise RuntimeError(f"Unknown filename extension: {filename}") 

192 

193 def writeText(self, filename, format='auto'): 

194 """Write the calibration data to a text file. 

195 

196 Parameters 

197 ---------- 

198 filename : `str` 

199 Name of the file to write. 

200 format : `str` 

201 Format to write the file as. Supported values are: 

202 ``"auto"`` : Determine filetype from filename. 

203 ``"yaml"`` : Write as yaml. 

204 ``"ecsv"`` : Write as ecsv. 

205 Returns 

206 ------- 

207 used : `str` 

208 The name of the file used to write the data. This may 

209 differ from the input if the format is explicitly chosen. 

210 

211 Raises 

212 ------ 

213 RuntimeError : 

214 Raised if filename does not end in a known extension, or 

215 if all information cannot be written. 

216 

217 Notes 

218 ----- 

219 The file is written to YAML/ECSV format and will include any 

220 associated metadata. 

221 

222 """ 

223 if format == 'yaml' or (format == 'auto' and filename.lower().endswith((".yaml", ".YAML"))): 

224 outDict = self.toDict() 

225 path, ext = os.path.splitext(filename) 

226 filename = path + ".yaml" 

227 with open(filename, 'w') as f: 

228 yaml.dump(outDict, f) 

229 elif format == 'ecsv' or (format == 'auto' and filename.lower().endswith((".ecsv", ".ECSV"))): 

230 tableList = self.toTable() 

231 if len(tableList) > 1: 

232 # ECSV doesn't support multiple tables per file, so we 

233 # can only write the first table. 

234 raise RuntimeError(f"Unable to persist {len(tableList)}tables in ECSV format.") 

235 

236 table = tableList[0] 

237 path, ext = os.path.splitext(filename) 

238 filename = path + ".ecsv" 

239 table.write(filename, format="ascii.ecsv") 

240 else: 

241 raise RuntimeError(f"Attempt to write to a file {filename} " 

242 "that does not end in '.yaml' or '.ecsv'") 

243 

244 return filename 

245 

246 @classmethod 

247 def readFits(cls, filename): 

248 """Read calibration data from a FITS file. 

249 

250 Parameters 

251 ---------- 

252 filename : `str` 

253 Filename to read data from. 

254 

255 Returns 

256 ------- 

257 calib : `lsst.ip.isr.IsrCalib` 

258 Calibration contained within the file. 

259 """ 

260 tableList = [] 

261 tableList.append(Table.read(filename, hdu=1)) 

262 extNum = 2 # Fits indices start at 1, we've read one already. 

263 try: 

264 with warnings.catch_warnings("error"): 

265 newTable = Table.read(filename, hdu=extNum) 

266 tableList.append(newTable) 

267 extNum += 1 

268 except Exception: 

269 pass 

270 

271 return cls.fromTable(tableList) 

272 

273 def writeFits(self, filename): 

274 """Write calibration data to a FITS file. 

275 

276 Parameters 

277 ---------- 

278 filename : `str` 

279 Filename to write data to. 

280 

281 Returns 

282 ------- 

283 used : `str` 

284 The name of the file used to write the data. 

285 

286 """ 

287 tableList = self.toTable() 

288 with warnings.catch_warnings(): 

289 warnings.filterwarnings("ignore", category=Warning, module="astropy.io") 

290 astropyList = [fits.table_to_hdu(table) for table in tableList] 

291 astropyList.insert(0, fits.PrimaryHDU()) 

292 

293 writer = fits.HDUList(astropyList) 

294 writer.writeto(filename, overwrite=True) 

295 return filename 

296 

297 def fromDetector(self, detector): 

298 """Modify the calibration parameters to match the supplied detector. 

299 

300 Parameters 

301 ---------- 

302 detector : `lsst.afw.cameraGeom.Detector` 

303 Detector to use to set parameters from. 

304 

305 Raises 

306 ------ 

307 NotImplementedError 

308 This needs to be implemented by subclasses for each 

309 calibration type. 

310 """ 

311 raise NotImplementedError("Must be implemented by subclass.") 

312 

313 @classmethod 

314 def fromDict(cls, dictionary): 

315 """Construct a calibration from a dictionary of properties. 

316 

317 Must be implemented by the specific calibration subclasses. 

318 

319 Parameters 

320 ---------- 

321 dictionary : `dict` 

322 Dictionary of properties. 

323 

324 Returns 

325 ------ 

326 calib : `lsst.ip.isr.CalibType` 

327 Constructed calibration. 

328 

329 Raises 

330 ------ 

331 NotImplementedError : 

332 Raised if not implemented. 

333 """ 

334 raise NotImplementedError("Must be implemented by subclass.") 

335 

336 def toDict(self): 

337 """Return a dictionary containing the calibration properties. 

338 

339 The dictionary should be able to be round-tripped through 

340 `fromDict`. 

341 

342 Returns 

343 ------- 

344 dictionary : `dict` 

345 Dictionary of properties. 

346 

347 Raises 

348 ------ 

349 NotImplementedError : 

350 Raised if not implemented. 

351 """ 

352 raise NotImplementedError("Must be implemented by subclass.") 

353 

354 @classmethod 

355 def fromTable(cls, tableList): 

356 """Construct a calibration from a dictionary of properties. 

357 

358 Must be implemented by the specific calibration subclasses. 

359 

360 Parameters 

361 ---------- 

362 tableList : `list` [`lsst.afw.table.Table`] 

363 List of tables of properties. 

364 

365 Returns 

366 ------ 

367 calib : `lsst.ip.isr.CalibType` 

368 Constructed calibration. 

369 

370 Raises 

371 ------ 

372 NotImplementedError : 

373 Raised if not implemented. 

374 """ 

375 raise NotImplementedError("Must be implemented by subclass.") 

376 

377 def toTable(self): 

378 """Return a list of tables containing the calibration properties. 

379 

380 The table list should be able to be round-tripped through 

381 `fromDict`. 

382 

383 Returns 

384 ------- 

385 tableList : `list` [`lsst.afw.table.Table`] 

386 List of tables of properties. 

387 

388 Raises 

389 ------ 

390 NotImplementedError : 

391 Raised if not implemented. 

392 """ 

393 raise NotImplementedError("Must be implemented by subclass.") 

394 

395 def validate(self, other=None): 

396 """Validate that this calibration is defined and can be used. 

397 

398 Parameters 

399 ---------- 

400 other : `object`, optional 

401 Thing to validate against. 

402 

403 Returns 

404 ------- 

405 valid : `bool` 

406 Returns true if the calibration is valid and appropriate. 

407 """ 

408 return False 

409 

410 def apply(self, target): 

411 """Method to apply the calibration to the target object. 

412 

413 Parameters 

414 ---------- 

415 target : `object` 

416 Thing to validate against. 

417 

418 Returns 

419 ------- 

420 valid : `bool` 

421 Returns true if the calibration was applied correctly. 

422 

423 Raises 

424 ------ 

425 NotImplementedError : 

426 Raised if not implemented. 

427 """ 

428 raise NotImplementedError("Must be implemented by subclass.") 

429 

430 

431class IsrProvenance(IsrCalib): 

432 """Class for the provenance of data used to construct calibration. 

433 

434 Provenance is not really a calibration, but we would like to 

435 record this when constructing the calibration, and it provides an 

436 example of the base calibration class. 

437 

438 Parameters 

439 ---------- 

440 instrument : `str`, optional 

441 Name of the instrument the data was taken with. 

442 calibType : `str`, optional 

443 Type of calibration this provenance was generated for. 

444 detectorName : `str`, optional 

445 Name of the detector this calibration is for. 

446 detectorSerial : `str`, optional 

447 Identifier for the detector. 

448 

449 """ 

450 _OBSTYPE = 'IsrProvenance' 

451 

452 def __init__(self, instrument="unknown", calibType="unknown", 

453 **kwargs): 

454 self.instrument = instrument 

455 self.calibType = calibType 

456 self.dimensions = set() 

457 self.dataIdList = list() 

458 

459 super().__init__(**kwargs) 

460 

461 self.requiredAttributes.update(['instrument', 'calibType', 'dimensions', 'dataIdList']) 

462 

463 def __str__(self): 

464 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )" 

465 

466 def __eq__(self, other): 

467 return super().__eq__(other) 

468 

469 def updateMetadata(self, setDate=False, **kwargs): 

470 """Update calibration metadata. 

471 

472 Parameters 

473 ---------- 

474 setDate : `bool, optional 

475 Update the CALIBDATE fields in the metadata to the current 

476 time. Defaults to False. 

477 kwargs : `dict` or `collections.abc.Mapping`, optional 

478 Other keyword parameters to set in the metadata. 

479 """ 

480 kwargs["DETECTOR"] = self._detectorName 

481 kwargs["DETECTOR_SERIAL"] = self._detectorSerial 

482 

483 kwargs['INSTRUME'] = self.instrument 

484 kwargs['calibType'] = self.calibType 

485 super().updateMetadata(setDate=setDate, **kwargs) 

486 

487 def fromDataIds(self, dataIdList): 

488 """Update provenance from dataId List. 

489 

490 Parameters 

491 ---------- 

492 dataIdList : `list` [`lsst.daf.butler.DataId`] 

493 List of dataIds used in generating this calibration. 

494 """ 

495 for dataId in dataIdList: 

496 for key in dataId: 

497 if key not in self.dimensions: 

498 self.dimensions.add(key) 

499 self.dataIdList.append(dataId) 

500 

501 @classmethod 

502 def fromTable(cls, tableList): 

503 """Construct provenance from table list. 

504 

505 Parameters 

506 ---------- 

507 tableList : `list` [`lsst.afw.table.Table`] 

508 List of tables to construct the provenance from. 

509 

510 Returns 

511 ------- 

512 provenance : `lsst.ip.isr.IsrProvenance` 

513 The provenance defined in the tables. 

514 """ 

515 table = tableList[0] 

516 metadata = table.meta 

517 inDict = dict() 

518 inDict['metadata'] = metadata 

519 inDict['detectorName'] = metadata['DETECTOR'] 

520 inDict['detectorSerial'] = metadata['DETECTOR_SERIAL'] 

521 inDict['instrument'] = metadata['INSTRUME'] 

522 inDict['calibType'] = metadata['calibType'] 

523 inDict['dimensions'] = set() 

524 inDict['dataIdList'] = list() 

525 

526 schema = dict() 

527 for colName in table.columns: 

528 schema[colName.lower()] = colName 

529 inDict['dimensions'].add(colName.lower()) 

530 inDict['dimensions'] = sorted(inDict['dimensions']) 

531 

532 for row in table: 

533 entry = dict() 

534 for dim in sorted(inDict['dimensions']): 

535 entry[dim] = row[schema[dim]] 

536 inDict['dataIdList'].append(entry) 

537 

538 return cls.fromDict(inDict) 

539 

540 @classmethod 

541 def fromDict(cls, dictionary): 

542 """Construct provenance from a dictionary. 

543 

544 Parameters 

545 ---------- 

546 dictionary : `dict` 

547 Dictionary of provenance parameters. 

548 

549 Returns 

550 ------- 

551 provenance : `lsst.ip.isr.IsrProvenance` 

552 The provenance defined in the tables. 

553 """ 

554 calib = cls() 

555 calib.updateMetadata(setDate=False, **dictionary['metadata']) 

556 calib._detectorName = dictionary['detectorName'] 

557 calib._detectorSerial = dictionary['detectorSerial'] 

558 calib.instrument = dictionary['instrument'] 

559 calib.calibType = dictionary['calibType'] 

560 calib.dimensions = set(dictionary['dimensions']) 

561 calib.dataIdList = dictionary['dataIdList'] 

562 

563 calib.updateMetadata() 

564 return calib 

565 

566 def toDict(self): 

567 """Return a dictionary containing the provenance information. 

568 

569 Returns 

570 ------- 

571 dictionary : `dict` 

572 Dictionary of provenance. 

573 """ 

574 self.updateMetadata(setDate=True) 

575 

576 outDict = {} 

577 

578 metadata = self.getMetadata() 

579 outDict['metadata'] = metadata 

580 outDict['detectorName'] = self._detectorName 

581 outDict['detectorSerial'] = self._detectorSerial 

582 outDict['instrument'] = self.instrument 

583 outDict['calibType'] = self.calibType 

584 outDict['dimensions'] = list(self.dimensions) 

585 outDict['dataIdList'] = self.dataIdList 

586 

587 return outDict 

588 

589 def toTable(self): 

590 """Return a list of tables containing the provenance. 

591 

592 This seems inefficient and slow, so this may not be the best 

593 way to store the data. 

594 

595 Returns 

596 ------- 

597 tableList : `list` [`lsst.afw.table.Table`] 

598 List of tables containing the provenance information 

599 

600 """ 

601 tableList = [] 

602 self.updateMetadata(setDate=True) 

603 catalog = Table(rows=self.dataIdList, 

604 names=self.dimensions) 

605 catalog.meta = self.getMetadata().toDict() 

606 tableList.append(catalog) 

607 return tableList