Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21import abc 

22import copy 

23import datetime 

24import os.path 

25import warnings 

26import yaml 

27from astropy.table import Table 

28 

29from lsst.log import Log 

30from lsst.daf.base import PropertyList 

31 

32 

33__all__ = ["IsrCalib", "IsrProvenance"] 

34 

35 

36class IsrCalib(abc.ABC): 

37 """Generic calibration type. 

38 

39 Subclasses must implement the toDict, fromDict, toTable, fromTable 

40 methods that allow the calibration information to be converted 

41 from dictionaries and afw tables. This will allow the calibration 

42 to be persisted using the base class read/write methods. 

43 

44 The validate method is intended to provide a common way to check 

45 that the calibration is valid (internally consistent) and 

46 appropriate (usable with the intended data). The apply method is 

47 intended to allow the calibration to be applied in a consistent 

48 manner. 

49 

50 Parameters 

51 ---------- 

52 detectorName : `str`, optional 

53 Name of the detector this calibration is for. 

54 detectorSerial : `str`, optional 

55 Identifier for the detector. 

56 detector : `lsst.afw.cameraGeom.Detector`, optional 

57 Detector to extract metadata from. 

58 log : `lsst.log.Log`, optional 

59 Log for messages. 

60 

61 """ 

62 _OBSTYPE = 'generic' 

63 _SCHEMA = 'NO SCHEMA' 

64 _VERSION = 0 

65 

66 def __init__(self, detectorName=None, detectorSerial=None, detector=None, log=None, **kwargs): 

67 self._detectorName = detectorName 

68 self._detectorSerial = detectorSerial 

69 self.setMetadata(PropertyList()) 

70 

71 # Define the required attributes for this calibration. 

72 self.requiredAttributes = set(['_OBSTYPE', '_SCHEMA', '_VERSION']) 

73 self.requiredAttributes.update(['_detectorName', '_detectorSerial', '_metadata']) 

74 

75 self.log = log if log else Log.getLogger(__name__.partition(".")[2]) 

76 

77 if detector: 

78 self.fromDetector(detector) 

79 self.updateMetadata(setDate=False) 

80 

81 def __str__(self): 

82 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )" 

83 

84 def __eq__(self, other): 

85 """Calibration equivalence. 

86 

87 Subclasses will need to check specific sub-properties. The 

88 default is only to check common entries. 

89 """ 

90 if not isinstance(other, self.__class__): 

91 return False 

92 

93 for attr in self._requiredAttributes: 

94 if getattr(self, attr) != getattr(other, attr): 

95 return False 

96 

97 return True 

98 

99 @property 

100 def requiredAttributes(self): 

101 return self._requiredAttributes 

102 

103 @requiredAttributes.setter 

104 def requiredAttributes(self, value): 

105 self._requiredAttributes = value 

106 

107 def getMetadata(self): 

108 

109 """Retrieve metadata associated with this calibration. 

110 

111 Returns 

112 ------- 

113 meta : `lsst.daf.base.PropertyList` 

114 Metadata. The returned `~lsst.daf.base.PropertyList` can be 

115 modified by the caller and the changes will be written to 

116 external files. 

117 """ 

118 return self._metadata 

119 

120 def setMetadata(self, metadata): 

121 """Store a copy of the supplied metadata with this calibration. 

122 

123 Parameters 

124 ---------- 

125 metadata : `lsst.daf.base.PropertyList` 

126 Metadata to associate with the calibration. Will be copied and 

127 overwrite existing metadata. 

128 """ 

129 if metadata is not None: 

130 self._metadata = copy.copy(metadata) 

131 

132 # Ensure that we have the obs type required by calibration ingest 

133 self._metadata["OBSTYPE"] = self._OBSTYPE 

134 self._metadata[self._OBSTYPE + "_SCHEMA"] = self._SCHEMA 

135 self._metadata[self._OBSTYPE + "_VERSION"] = self._VERSION 

136 

137 def updateMetadata(self, setDate=False, **kwargs): 

138 """Update metadata keywords with new values. 

139 

140 Parameters 

141 ---------- 

142 setDate : `bool`, optional 

143 Ensure the metadata CALIBDATE fields are set to the current datetime. 

144 kwargs : 

145 Set of key=value pairs to assign to the metadata. 

146 """ 

147 mdOriginal = self.getMetadata() 

148 mdSupplemental = dict() 

149 

150 self._metadata["DETECTOR"] = self._detectorName 

151 self._metadata["DETECTOR_SERIAL"] = self._detectorSerial 

152 

153 if setDate: 

154 date = datetime.datetime.now() 

155 mdSupplemental['CALIBDATE'] = date.isoformat() 

156 mdSupplemental['CALIB_CREATION_DATE'] = date.date().isoformat() 

157 mdSupplemental['CALIB_CREATION_TIME'] = date.time().isoformat() 

158 

159 mdSupplemental.update(kwargs) 

160 mdOriginal.update(mdSupplemental) 

161 

162 @classmethod 

163 def readText(cls, filename): 

164 """Read calibration representation from a yaml/ecsv file. 

165 

166 Parameters 

167 ---------- 

168 filename : `str` 

169 Name of the file containing the calibration definition. 

170 

171 Returns 

172 ------- 

173 calib : `~lsst.ip.isr.IsrCalibType` 

174 Calibration class. 

175 

176 Raises 

177 ------ 

178 RuntimeError : 

179 Raised if the filename does not end in ".ecsv" or ".yaml". 

180 """ 

181 if filename.endswith((".ecsv", ".ECSV")): 

182 data = Table.read(filename, format='ascii.ecsv') 

183 return cls.fromTable([data]) 

184 

185 elif filename.endswith((".yaml", ".YAML")): 

186 with open(filename, 'r') as f: 

187 data = yaml.load(f, Loader=yaml.CLoader) 

188 return cls.fromDict(data) 

189 else: 

190 raise RuntimeError(f"Unknown filename extension: {filename}") 

191 

192 def writeText(self, filename, format='auto'): 

193 """Write the calibration data to a text file. 

194 

195 Parameters 

196 ---------- 

197 filename : `str` 

198 Name of the file to write. 

199 format : `str` 

200 Format to write the file as. Supported values are: 

201 ``"auto"`` : Determine filetype from filename. 

202 ``"yaml"`` : Write as yaml. 

203 ``"ecsv"`` : Write as ecsv. 

204 Returns 

205 ------- 

206 used : `str` 

207 The name of the file used to write the data. This may 

208 differ from the input if the format is explicitly chosen. 

209 

210 Raises 

211 ------ 

212 RuntimeError : 

213 Raised if filename does not end in a known extension, or 

214 if all information cannot be written. 

215 

216 Notes 

217 ----- 

218 The file is written to YAML/ECSV format and will include any 

219 associated metadata. 

220 

221 """ 

222 if format == 'yaml' or (format == 'auto' and filename.lower().endswith((".yaml", ".YAML"))): 

223 outDict = self.toDict() 

224 path, ext = os.path.splitext(filename) 

225 filename = path + ".yaml" 

226 with open(filename, 'w') as f: 

227 yaml.dump(outDict, f) 

228 elif format == 'ecsv' or (format == 'auto' and filename.lower().endswith((".ecsv", ".ECSV"))): 

229 tableList = self.toTable() 

230 if len(tableList) > 1: 

231 # ECSV doesn't support multiple tables per file, so we 

232 # can only write the first table. 

233 raise RuntimeError(f"Unable to persist {len(tableList)}tables in ECSV format.") 

234 

235 table = tableList[0] 

236 path, ext = os.path.splitext(filename) 

237 filename = path + ".ecsv" 

238 table.write(filename, format="ascii.ecsv") 

239 else: 

240 raise RuntimeError(f"Attempt to write to a file {filename} " 

241 "that does not end in '.yaml' or '.ecsv'") 

242 

243 return filename 

244 

245 @classmethod 

246 def readFits(cls, filename): 

247 """Read calibration data from a FITS file. 

248 

249 Parameters 

250 ---------- 

251 filename : `str` 

252 Filename to read data from. 

253 

254 Returns 

255 ------- 

256 calib : `lsst.ip.isr.IsrCalib` 

257 Calibration contained within the file. 

258 """ 

259 tableList = [] 

260 tableList.append(Table.read(filename, hdu=1)) 

261 extNum = 2 # Fits indices start at 1, we've read one already. 

262 try: 

263 with warnings.catch_warnings("error"): 

264 newTable = Table.read(filename, hdu=extNum) 

265 tableList.append(newTable) 

266 extNum += 1 

267 except Exception: 

268 pass 

269 

270 return cls.fromTable(tableList) 

271 

272 def writeFits(self, filename): 

273 """Write calibration data to a FITS file. 

274 

275 Parameters 

276 ---------- 

277 filename : `str` 

278 Filename to write data to. 

279 

280 Returns 

281 ------- 

282 used : `str` 

283 The name of the file used to write the data. 

284 

285 """ 

286 tableList = self.toTable() 

287 

288 with warnings.catch_warnings(): 

289 warnings.filterwarnings("ignore", category=Warning, module="astropy.io") 

290 tableList[0].write(filename) 

291 for table in tableList[1:]: 

292 table.write(filename, append=True) 

293 

294 return filename 

295 

296 def fromDetector(self, detector): 

297 """Modify the calibration parameters to match the supplied detector. 

298 

299 Parameters 

300 ---------- 

301 detector : `lsst.afw.cameraGeom.Detector` 

302 Detector to use to set parameters from. 

303 

304 Raises 

305 ------ 

306 NotImplementedError 

307 This needs to be implemented by subclasses for each 

308 calibration type. 

309 """ 

310 raise NotImplementedError("Must be implemented by subclass.") 

311 

312 @classmethod 

313 def fromDict(cls, dictionary): 

314 """Construct a calibration from a dictionary of properties. 

315 

316 Must be implemented by the specific calibration subclasses. 

317 

318 Parameters 

319 ---------- 

320 dictionary : `dict` 

321 Dictionary of properties. 

322 

323 Returns 

324 ------ 

325 calib : `lsst.ip.isr.CalibType` 

326 Constructed calibration. 

327 

328 Raises 

329 ------ 

330 NotImplementedError : 

331 Raised if not implemented. 

332 """ 

333 raise NotImplementedError("Must be implemented by subclass.") 

334 

335 def toDict(self): 

336 """Return a dictionary containing the calibration properties. 

337 

338 The dictionary should be able to be round-tripped through 

339 `fromDict`. 

340 

341 Returns 

342 ------- 

343 dictionary : `dict` 

344 Dictionary of properties. 

345 

346 Raises 

347 ------ 

348 NotImplementedError : 

349 Raised if not implemented. 

350 """ 

351 raise NotImplementedError("Must be implemented by subclass.") 

352 

353 @classmethod 

354 def fromTable(cls, tableList): 

355 """Construct a calibration from a dictionary of properties. 

356 

357 Must be implemented by the specific calibration subclasses. 

358 

359 Parameters 

360 ---------- 

361 tableList : `list` [`lsst.afw.table.Table`] 

362 List of tables of properties. 

363 

364 Returns 

365 ------ 

366 calib : `lsst.ip.isr.CalibType` 

367 Constructed calibration. 

368 

369 Raises 

370 ------ 

371 NotImplementedError : 

372 Raised if not implemented. 

373 """ 

374 raise NotImplementedError("Must be implemented by subclass.") 

375 

376 def toTable(self): 

377 """Return a list of tables containing the calibration properties. 

378 

379 The table list should be able to be round-tripped through 

380 `fromDict`. 

381 

382 Returns 

383 ------- 

384 tableList : `list` [`lsst.afw.table.Table`] 

385 List of tables of properties. 

386 

387 Raises 

388 ------ 

389 NotImplementedError : 

390 Raised if not implemented. 

391 """ 

392 raise NotImplementedError("Must be implemented by subclass.") 

393 

394 def validate(self, other=None): 

395 """Validate that this calibration is defined and can be used. 

396 

397 Parameters 

398 ---------- 

399 other : `object`, optional 

400 Thing to validate against. 

401 

402 Returns 

403 ------- 

404 valid : `bool` 

405 Returns true if the calibration is valid and appropriate. 

406 """ 

407 return False 

408 

409 def apply(self, target): 

410 """Method to apply the calibration to the target object. 

411 

412 Parameters 

413 ---------- 

414 target : `object` 

415 Thing to validate against. 

416 

417 Returns 

418 ------- 

419 valid : `bool` 

420 Returns true if the calibration was applied correctly. 

421 

422 Raises 

423 ------ 

424 NotImplementedError : 

425 Raised if not implemented. 

426 """ 

427 raise NotImplementedError("Must be implemented by subclass.") 

428 

429 

430class IsrProvenance(IsrCalib): 

431 """Class for the provenance of data used to construct calibration. 

432 

433 Provenance is not really a calibration, but we would like to 

434 record this when constructing the calibration, and it provides an 

435 example of the base calibration class. 

436 

437 Parameters 

438 ---------- 

439 instrument : `str`, optional 

440 Name of the instrument the data was taken with. 

441 calibType : `str`, optional 

442 Type of calibration this provenance was generated for. 

443 detectorName : `str`, optional 

444 Name of the detector this calibration is for. 

445 detectorSerial : `str`, optional 

446 Identifier for the detector. 

447 

448 """ 

449 _OBSTYPE = 'IsrProvenance' 

450 

451 def __init__(self, instrument="unknown", calibType="unknown", 

452 **kwargs): 

453 self.instrument = instrument 

454 self.calibType = calibType 

455 self.dimensions = set() 

456 self.dataIdList = list() 

457 

458 super().__init__(**kwargs) 

459 

460 self.requiredAttributes.update(['instrument', 'calibType', 'dimensions', 'dataIdList']) 

461 

462 def __str__(self): 

463 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )" 

464 

465 def __eq__(self, other): 

466 return super().__eq__(other) 

467 

468 def updateMetadata(self, setDate=False, **kwargs): 

469 """Update calibration metadata. 

470 

471 Parameters 

472 ---------- 

473 setDate : `bool, optional 

474 Update the CALIBDATE fields in the metadata to the current 

475 time. Defaults to False. 

476 kwargs : 

477 Other keyword parameters to set in the metadata. 

478 """ 

479 kwargs["DETECTOR"] = self._detectorName 

480 kwargs["DETECTOR_SERIAL"] = self._detectorSerial 

481 

482 kwargs['INSTRUME'] = self.instrument 

483 kwargs['calibType'] = self.calibType 

484 super().updateMetadata(setDate=setDate, **kwargs) 

485 

486 def fromDataIds(self, dataIdList): 

487 """Update provenance from dataId List. 

488 

489 Parameters 

490 ---------- 

491 dataIdList : `list` [`lsst.daf.butler.DataId`] 

492 List of dataIds used in generating this calibration. 

493 """ 

494 for dataId in dataIdList: 

495 for key in dataId: 

496 if key not in self.dimensions: 

497 self.dimensions.add(key) 

498 self.dataIdList.append(dataId) 

499 

500 @classmethod 

501 def fromTable(cls, tableList): 

502 """Construct provenance from table list. 

503 

504 Parameters 

505 ---------- 

506 tableList : `list` [`lsst.afw.table.Table`] 

507 List of tables to construct the provenance from. 

508 

509 Returns 

510 ------- 

511 provenance : `lsst.ip.isr.IsrProvenance` 

512 The provenance defined in the tables. 

513 """ 

514 table = tableList[0] 

515 metadata = table.meta 

516 inDict = dict() 

517 inDict['metadata'] = metadata 

518 inDict['detectorName'] = metadata['DETECTOR'] 

519 inDict['detectorSerial'] = metadata['DETECTOR_SERIAL'] 

520 inDict['instrument'] = metadata['INSTRUME'] 

521 inDict['calibType'] = metadata['calibType'] 

522 inDict['dimensions'] = set() 

523 inDict['dataIdList'] = list() 

524 

525 schema = dict() 

526 for colName in table.columns: 

527 schema[colName.lower()] = colName 

528 inDict['dimensions'].add(colName.lower()) 

529 inDict['dimensions'] = sorted(inDict['dimensions']) 

530 

531 for row in table: 

532 entry = dict() 

533 for dim in sorted(inDict['dimensions']): 

534 entry[dim] = row[schema[dim]] 

535 inDict['dataIdList'].append(entry) 

536 

537 return cls.fromDict(inDict) 

538 

539 @classmethod 

540 def fromDict(cls, dictionary): 

541 """Construct provenance from a dictionary. 

542 

543 Parameters 

544 ---------- 

545 dictionary : `dict` 

546 Dictionary of provenance parameters. 

547 

548 Returns 

549 ------- 

550 provenance : `lsst.ip.isr.IsrProvenance` 

551 The provenance defined in the tables. 

552 """ 

553 calib = cls() 

554 calib.updateMetadata(setDate=False, **dictionary['metadata']) 

555 calib._detectorName = dictionary['detectorName'] 

556 calib._detectorSerial = dictionary['detectorSerial'] 

557 calib.instrument = dictionary['instrument'] 

558 calib.calibType = dictionary['calibType'] 

559 calib.dimensions = set(dictionary['dimensions']) 

560 calib.dataIdList = dictionary['dataIdList'] 

561 

562 calib.updateMetadata() 

563 return calib 

564 

565 def toDict(self): 

566 """Return a dictionary containing the provenance information. 

567 

568 Returns 

569 ------- 

570 dictionary : `dict` 

571 Dictionary of provenance. 

572 """ 

573 self.updateMetadata(setDate=True) 

574 

575 outDict = {} 

576 

577 metadata = self.getMetadata() 

578 outDict['metadata'] = metadata 

579 outDict['detectorName'] = self._detectorName 

580 outDict['detectorSerial'] = self._detectorSerial 

581 outDict['instrument'] = self.instrument 

582 outDict['calibType'] = self.calibType 

583 outDict['dimensions'] = list(self.dimensions) 

584 outDict['dataIdList'] = self.dataIdList 

585 

586 return outDict 

587 

588 def toTable(self): 

589 """Return a list of tables containing the provenance. 

590 

591 This seems inefficient and slow, so this may not be the best 

592 way to store the data. 

593 

594 Returns 

595 ------- 

596 tableList : `list` [`lsst.afw.table.Table`] 

597 List of tables containing the provenance information 

598 

599 """ 

600 tableList = [] 

601 self.updateMetadata(setDate=True) 

602 catalog = Table(rows=self.dataIdList, 

603 names=self.dimensions) 

604 catalog.meta = self.getMetadata().toDict() 

605 tableList.append(catalog) 

606 return tableList