Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21import abc 

22import datetime 

23import os.path 

24import warnings 

25import yaml 

26from astropy.table import Table 

27from astropy.io import fits 

28 

29from lsst.log import Log 

30from lsst.daf.base import PropertyList 

31 

32 

33__all__ = ["IsrCalib", "IsrProvenance"] 

34 

35 

36class IsrCalib(abc.ABC): 

37 """Generic calibration type. 

38 

39 Subclasses must implement the toDict, fromDict, toTable, fromTable 

40 methods that allow the calibration information to be converted 

41 from dictionaries and afw tables. This will allow the calibration 

42 to be persisted using the base class read/write methods. 

43 

44 The validate method is intended to provide a common way to check 

45 that the calibration is valid (internally consistent) and 

46 appropriate (usable with the intended data). The apply method is 

47 intended to allow the calibration to be applied in a consistent 

48 manner. 

49 

50 Parameters 

51 ---------- 

52 camera : `lsst.afw.cameraGeom.Camera`, optional 

53 Camera to extract metadata from. 

54 detector : `lsst.afw.cameraGeom.Detector`, optional 

55 Detector to extract metadata from. 

56 log : `lsst.log.Log`, optional 

57 Log for messages. 

58 """ 

59 _OBSTYPE = 'generic' 

60 _SCHEMA = 'NO SCHEMA' 

61 _VERSION = 0 

62 

63 def __init__(self, camera=None, detector=None, detectorName=None, detectorId=None, log=None, **kwargs): 

64 self._instrument = None 

65 self._raftName = None 

66 self._slotName = None 

67 self._detectorName = detectorName 

68 self._detectorSerial = None 

69 self._detectorId = detectorId 

70 self._filter = None 

71 self._calibId = None 

72 self._metadata = PropertyList() 

73 self.setMetadata(PropertyList()) 

74 

75 # Define the required attributes for this calibration. 

76 self.requiredAttributes = set(['_OBSTYPE', '_SCHEMA', '_VERSION']) 

77 self.requiredAttributes.update(['_instrument', '_raftName', '_slotName', 

78 '_detectorName', '_detectorSerial', '_detectorId', 

79 '_filter', '_calibId', '_metadata']) 

80 

81 self.log = log if log else Log.getLogger(__name__.partition(".")[2]) 

82 

83 if detector: 

84 self.fromDetector(detector) 

85 self.updateMetadata(camera=camera, detector=detector, setDate=False) 

86 

87 def __str__(self): 

88 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )" 

89 

90 def __eq__(self, other): 

91 """Calibration equivalence. 

92 

93 Subclasses will need to check specific sub-properties. The 

94 default is only to check common entries. 

95 """ 

96 if not isinstance(other, self.__class__): 

97 return False 

98 

99 for attr in self._requiredAttributes: 

100 if getattr(self, attr) != getattr(other, attr): 

101 return False 

102 

103 return True 

104 

105 @property 

106 def requiredAttributes(self): 

107 return self._requiredAttributes 

108 

109 @requiredAttributes.setter 

110 def requiredAttributes(self, value): 

111 self._requiredAttributes = value 

112 

113 def getMetadata(self): 

114 """Retrieve metadata associated with this calibration. 

115 

116 Returns 

117 ------- 

118 meta : `lsst.daf.base.PropertyList` 

119 Metadata. The returned `~lsst.daf.base.PropertyList` can be 

120 modified by the caller and the changes will be written to 

121 external files. 

122 """ 

123 return self._metadata 

124 

125 def setMetadata(self, metadata): 

126 """Store a copy of the supplied metadata with this calibration. 

127 

128 Parameters 

129 ---------- 

130 metadata : `lsst.daf.base.PropertyList` 

131 Metadata to associate with the calibration. Will be copied and 

132 overwrite existing metadata. 

133 """ 

134 if metadata is not None: 

135 self._metadata.update(metadata) 

136 

137 # Ensure that we have the obs type required by calibration ingest 

138 self._metadata["OBSTYPE"] = self._OBSTYPE 

139 self._metadata[self._OBSTYPE + "_SCHEMA"] = self._SCHEMA 

140 self._metadata[self._OBSTYPE + "_VERSION"] = self._VERSION 

141 

142 def updateMetadata(self, camera=None, detector=None, filterName=None, 

143 setCalibId=False, setDate=False, 

144 **kwargs): 

145 """Update metadata keywords with new values. 

146 

147 Parameters 

148 ---------- 

149 camera : `lsst.afw.cameraGeom.Camera`, optional 

150 Reference camera to use to set _instrument field. 

151 detector : `lsst.afw.cameraGeom.Detector`, optional 

152 Reference detector to use to set _detector* fields. 

153 filterName : `str`, optional 

154 Filter name to assign to this calibration. 

155 setCalibId : `bool` optional 

156 Construct the _calibId field from other fields. 

157 setDate : `bool`, optional 

158 Ensure the metadata CALIBDATE fields are set to the current datetime. 

159 kwargs : `dict` or `collections.abc.Mapping`, optional 

160 Set of key=value pairs to assign to the metadata. 

161 """ 

162 mdOriginal = self.getMetadata() 

163 mdSupplemental = dict() 

164 

165 if camera: 

166 self._instrument = camera.getName() 

167 

168 if detector: 

169 self._detectorName = detector.getName() 

170 self._detectorSerial = detector.getSerial() 

171 self._detectorId = detector.getId() 

172 if "_" in self._detectorName: 

173 (self._raftName, self._slotName) = self._detectorName.split("_") 

174 

175 if filterName: 

176 # If set via: 

177 # exposure.getInfo().getFilter().getName() 

178 # then this will hold the abstract filter. 

179 self._filter = filterName 

180 

181 if setCalibId: 

182 values = [] 

183 values.append(f"instrument={self._instrument}") if self._instrument else None 

184 values.append(f"raftName={self._raftName}") if self._raftName else None 

185 values.append(f"detectorName={self._detectorName}") if self._detectorName else None 

186 values.append(f"detector={self.detector}") if self._detector else None 

187 values.append(f"filter={self._filter}") if self._filter else None 

188 self._calibId = " ".join(values) 

189 

190 if setDate: 

191 date = datetime.datetime.now() 

192 mdSupplemental['CALIBDATE'] = date.isoformat() 

193 mdSupplemental['CALIB_CREATION_DATE'] = date.date().isoformat() 

194 mdSupplemental['CALIB_CREATION_TIME'] = date.time().isoformat() 

195 

196 self._metadata["INSTRUME"] = self._instrument if self._instrument else None 

197 self._metadata["RAFTNAME"] = self._raftName if self._raftName else None 

198 self._metadata["SLOTNAME"] = self._slotName if self._slotName else None 

199 self._metadata["DETECTOR"] = self._detectorId if self._detectorId else None 

200 self._metadata["DET_NAME"] = self._detectorName if self._detectorName else None 

201 self._metadata["DET_SER"] = self._detectorSerial if self._detectorSerial else None 

202 self._metadata["FILTER"] = self._filter if self._filter else None 

203 self._metadata["CALIB_ID"] = self._calibId if self._calibId else None 

204 

205 mdSupplemental.update(kwargs) 

206 mdOriginal.update(mdSupplemental) 

207 

208 @classmethod 

209 def readText(cls, filename): 

210 """Read calibration representation from a yaml/ecsv file. 

211 

212 Parameters 

213 ---------- 

214 filename : `str` 

215 Name of the file containing the calibration definition. 

216 

217 Returns 

218 ------- 

219 calib : `~lsst.ip.isr.IsrCalibType` 

220 Calibration class. 

221 

222 Raises 

223 ------ 

224 RuntimeError : 

225 Raised if the filename does not end in ".ecsv" or ".yaml". 

226 """ 

227 if filename.endswith((".ecsv", ".ECSV")): 

228 data = Table.read(filename, format='ascii.ecsv') 

229 return cls.fromTable([data]) 

230 

231 elif filename.endswith((".yaml", ".YAML")): 

232 with open(filename, 'r') as f: 

233 data = yaml.load(f, Loader=yaml.CLoader) 

234 return cls.fromDict(data) 

235 else: 

236 raise RuntimeError(f"Unknown filename extension: {filename}") 

237 

238 def writeText(self, filename, format='auto'): 

239 """Write the calibration data to a text file. 

240 

241 Parameters 

242 ---------- 

243 filename : `str` 

244 Name of the file to write. 

245 format : `str` 

246 Format to write the file as. Supported values are: 

247 ``"auto"`` : Determine filetype from filename. 

248 ``"yaml"`` : Write as yaml. 

249 ``"ecsv"`` : Write as ecsv. 

250 Returns 

251 ------- 

252 used : `str` 

253 The name of the file used to write the data. This may 

254 differ from the input if the format is explicitly chosen. 

255 

256 Raises 

257 ------ 

258 RuntimeError : 

259 Raised if filename does not end in a known extension, or 

260 if all information cannot be written. 

261 

262 Notes 

263 ----- 

264 The file is written to YAML/ECSV format and will include any 

265 associated metadata. 

266 

267 """ 

268 if format == 'yaml' or (format == 'auto' and filename.lower().endswith((".yaml", ".YAML"))): 

269 outDict = self.toDict() 

270 path, ext = os.path.splitext(filename) 

271 filename = path + ".yaml" 

272 with open(filename, 'w') as f: 

273 yaml.dump(outDict, f) 

274 elif format == 'ecsv' or (format == 'auto' and filename.lower().endswith((".ecsv", ".ECSV"))): 

275 tableList = self.toTable() 

276 if len(tableList) > 1: 

277 # ECSV doesn't support multiple tables per file, so we 

278 # can only write the first table. 

279 raise RuntimeError(f"Unable to persist {len(tableList)}tables in ECSV format.") 

280 

281 table = tableList[0] 

282 path, ext = os.path.splitext(filename) 

283 filename = path + ".ecsv" 

284 table.write(filename, format="ascii.ecsv") 

285 else: 

286 raise RuntimeError(f"Attempt to write to a file {filename} " 

287 "that does not end in '.yaml' or '.ecsv'") 

288 

289 return filename 

290 

291 @classmethod 

292 def readFits(cls, filename): 

293 """Read calibration data from a FITS file. 

294 

295 Parameters 

296 ---------- 

297 filename : `str` 

298 Filename to read data from. 

299 

300 Returns 

301 ------- 

302 calib : `lsst.ip.isr.IsrCalib` 

303 Calibration contained within the file. 

304 """ 

305 tableList = [] 

306 tableList.append(Table.read(filename, hdu=1)) 

307 extNum = 2 # Fits indices start at 1, we've read one already. 

308 keepTrying = True 

309 

310 while keepTrying: 

311 with warnings.catch_warnings(): 

312 warnings.simplefilter("error") 

313 try: 

314 newTable = Table.read(filename, hdu=extNum) 

315 tableList.append(newTable) 

316 extNum += 1 

317 except Exception: 

318 keepTrying = False 

319 

320 return cls.fromTable(tableList) 

321 

322 def writeFits(self, filename): 

323 """Write calibration data to a FITS file. 

324 

325 Parameters 

326 ---------- 

327 filename : `str` 

328 Filename to write data to. 

329 

330 Returns 

331 ------- 

332 used : `str` 

333 The name of the file used to write the data. 

334 

335 """ 

336 tableList = self.toTable() 

337 with warnings.catch_warnings(): 

338 warnings.filterwarnings("ignore", category=Warning, module="astropy.io") 

339 astropyList = [fits.table_to_hdu(table) for table in tableList] 

340 astropyList.insert(0, fits.PrimaryHDU()) 

341 

342 writer = fits.HDUList(astropyList) 

343 writer.writeto(filename, overwrite=True) 

344 return filename 

345 

346 def fromDetector(self, detector): 

347 """Modify the calibration parameters to match the supplied detector. 

348 

349 Parameters 

350 ---------- 

351 detector : `lsst.afw.cameraGeom.Detector` 

352 Detector to use to set parameters from. 

353 

354 Raises 

355 ------ 

356 NotImplementedError 

357 This needs to be implemented by subclasses for each 

358 calibration type. 

359 """ 

360 raise NotImplementedError("Must be implemented by subclass.") 

361 

362 @classmethod 

363 def fromDict(cls, dictionary): 

364 """Construct a calibration from a dictionary of properties. 

365 

366 Must be implemented by the specific calibration subclasses. 

367 

368 Parameters 

369 ---------- 

370 dictionary : `dict` 

371 Dictionary of properties. 

372 

373 Returns 

374 ------ 

375 calib : `lsst.ip.isr.CalibType` 

376 Constructed calibration. 

377 

378 Raises 

379 ------ 

380 NotImplementedError : 

381 Raised if not implemented. 

382 """ 

383 raise NotImplementedError("Must be implemented by subclass.") 

384 

385 def toDict(self): 

386 """Return a dictionary containing the calibration properties. 

387 

388 The dictionary should be able to be round-tripped through 

389 `fromDict`. 

390 

391 Returns 

392 ------- 

393 dictionary : `dict` 

394 Dictionary of properties. 

395 

396 Raises 

397 ------ 

398 NotImplementedError : 

399 Raised if not implemented. 

400 """ 

401 raise NotImplementedError("Must be implemented by subclass.") 

402 

403 @classmethod 

404 def fromTable(cls, tableList): 

405 """Construct a calibration from a dictionary of properties. 

406 

407 Must be implemented by the specific calibration subclasses. 

408 

409 Parameters 

410 ---------- 

411 tableList : `list` [`lsst.afw.table.Table`] 

412 List of tables of properties. 

413 

414 Returns 

415 ------ 

416 calib : `lsst.ip.isr.CalibType` 

417 Constructed calibration. 

418 

419 Raises 

420 ------ 

421 NotImplementedError : 

422 Raised if not implemented. 

423 """ 

424 raise NotImplementedError("Must be implemented by subclass.") 

425 

426 def toTable(self): 

427 """Return a list of tables containing the calibration properties. 

428 

429 The table list should be able to be round-tripped through 

430 `fromDict`. 

431 

432 Returns 

433 ------- 

434 tableList : `list` [`lsst.afw.table.Table`] 

435 List of tables of properties. 

436 

437 Raises 

438 ------ 

439 NotImplementedError : 

440 Raised if not implemented. 

441 """ 

442 raise NotImplementedError("Must be implemented by subclass.") 

443 

444 def validate(self, other=None): 

445 """Validate that this calibration is defined and can be used. 

446 

447 Parameters 

448 ---------- 

449 other : `object`, optional 

450 Thing to validate against. 

451 

452 Returns 

453 ------- 

454 valid : `bool` 

455 Returns true if the calibration is valid and appropriate. 

456 """ 

457 return False 

458 

459 def apply(self, target): 

460 """Method to apply the calibration to the target object. 

461 

462 Parameters 

463 ---------- 

464 target : `object` 

465 Thing to validate against. 

466 

467 Returns 

468 ------- 

469 valid : `bool` 

470 Returns true if the calibration was applied correctly. 

471 

472 Raises 

473 ------ 

474 NotImplementedError : 

475 Raised if not implemented. 

476 """ 

477 raise NotImplementedError("Must be implemented by subclass.") 

478 

479 

480class IsrProvenance(IsrCalib): 

481 """Class for the provenance of data used to construct calibration. 

482 

483 Provenance is not really a calibration, but we would like to 

484 record this when constructing the calibration, and it provides an 

485 example of the base calibration class. 

486 

487 Parameters 

488 ---------- 

489 instrument : `str`, optional 

490 Name of the instrument the data was taken with. 

491 calibType : `str`, optional 

492 Type of calibration this provenance was generated for. 

493 detectorName : `str`, optional 

494 Name of the detector this calibration is for. 

495 detectorSerial : `str`, optional 

496 Identifier for the detector. 

497 

498 """ 

499 _OBSTYPE = 'IsrProvenance' 

500 

501 def __init__(self, calibType="unknown", 

502 **kwargs): 

503 self.calibType = calibType 

504 self.dimensions = set() 

505 self.dataIdList = list() 

506 

507 super().__init__(**kwargs) 

508 

509 self.requiredAttributes.update(['calibType', 'dimensions', 'dataIdList']) 

510 

511 def __str__(self): 

512 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )" 

513 

514 def __eq__(self, other): 

515 return super().__eq__(other) 

516 

517 def updateMetadata(self, setDate=False, **kwargs): 

518 """Update calibration metadata. 

519 

520 Parameters 

521 ---------- 

522 setDate : `bool, optional 

523 Update the CALIBDATE fields in the metadata to the current 

524 time. Defaults to False. 

525 kwargs : `dict` or `collections.abc.Mapping`, optional 

526 Other keyword parameters to set in the metadata. 

527 """ 

528 kwargs['calibType'] = self.calibType 

529 super().updateMetadata(setDate=setDate, **kwargs) 

530 

531 def fromDataIds(self, dataIdList): 

532 """Update provenance from dataId List. 

533 

534 Parameters 

535 ---------- 

536 dataIdList : `list` [`lsst.daf.butler.DataId`] 

537 List of dataIds used in generating this calibration. 

538 """ 

539 for dataId in dataIdList: 

540 for key in dataId: 

541 if key not in self.dimensions: 

542 self.dimensions.add(key) 

543 self.dataIdList.append(dataId) 

544 

545 @classmethod 

546 def fromTable(cls, tableList): 

547 """Construct provenance from table list. 

548 

549 Parameters 

550 ---------- 

551 tableList : `list` [`lsst.afw.table.Table`] 

552 List of tables to construct the provenance from. 

553 

554 Returns 

555 ------- 

556 provenance : `lsst.ip.isr.IsrProvenance` 

557 The provenance defined in the tables. 

558 """ 

559 table = tableList[0] 

560 metadata = table.meta 

561 inDict = dict() 

562 inDict['metadata'] = metadata 

563 inDict['detectorName'] = metadata.get('DET_NAME', None) 

564 inDict['detectorSerial'] = metadata.get('DET_SER', None) 

565 inDict['instrument'] = metadata.get('INSTRUME', None) 

566 inDict['calibType'] = metadata['calibType'] 

567 inDict['dimensions'] = set() 

568 inDict['dataIdList'] = list() 

569 

570 schema = dict() 

571 for colName in table.columns: 

572 schema[colName.lower()] = colName 

573 inDict['dimensions'].add(colName.lower()) 

574 inDict['dimensions'] = sorted(inDict['dimensions']) 

575 

576 for row in table: 

577 entry = dict() 

578 for dim in sorted(inDict['dimensions']): 

579 entry[dim] = row[schema[dim]] 

580 inDict['dataIdList'].append(entry) 

581 

582 return cls.fromDict(inDict) 

583 

584 @classmethod 

585 def fromDict(cls, dictionary): 

586 """Construct provenance from a dictionary. 

587 

588 Parameters 

589 ---------- 

590 dictionary : `dict` 

591 Dictionary of provenance parameters. 

592 

593 Returns 

594 ------- 

595 provenance : `lsst.ip.isr.IsrProvenance` 

596 The provenance defined in the tables. 

597 """ 

598 calib = cls() 

599 calib.updateMetadata(setDate=False, **dictionary['metadata']) 

600 

601 # These properties should be in the metadata, but occasionally 

602 # are found in the dictionary itself. Check both places, 

603 # ending with `None` if neither contains the information. 

604 calib._detectorName = dictionary.get('detectorName', 

605 dictionary['metadata'].get('DET_NAME', None)) 

606 calib._detectorSerial = dictionary.get('detectorSerial', 

607 dictionary['metadata'].get('DET_SER', None)) 

608 calib._instrument = dictionary.get('instrument', 

609 dictionary['metadata'].get('INSTRUME', None)) 

610 calib.calibType = dictionary['calibType'] 

611 calib.dimensions = set(dictionary['dimensions']) 

612 calib.dataIdList = dictionary['dataIdList'] 

613 

614 calib.updateMetadata() 

615 return calib 

616 

617 def toDict(self): 

618 """Return a dictionary containing the provenance information. 

619 

620 Returns 

621 ------- 

622 dictionary : `dict` 

623 Dictionary of provenance. 

624 """ 

625 self.updateMetadata(setDate=True) 

626 

627 outDict = {} 

628 

629 metadata = self.getMetadata() 

630 outDict['metadata'] = metadata 

631 outDict['detectorName'] = self._detectorName 

632 outDict['detectorSerial'] = self._detectorSerial 

633 outDict['instrument'] = self._instrument 

634 outDict['calibType'] = self.calibType 

635 outDict['dimensions'] = list(self.dimensions) 

636 outDict['dataIdList'] = self.dataIdList 

637 

638 return outDict 

639 

640 def toTable(self): 

641 """Return a list of tables containing the provenance. 

642 

643 This seems inefficient and slow, so this may not be the best 

644 way to store the data. 

645 

646 Returns 

647 ------- 

648 tableList : `list` [`lsst.afw.table.Table`] 

649 List of tables containing the provenance information 

650 

651 """ 

652 tableList = [] 

653 self.updateMetadata(setDate=True) 

654 catalog = Table(rows=self.dataIdList, 

655 names=self.dimensions) 

656 filteredMetadata = {k: v for k, v in self.getMetadata().toDict().items() if v is not None} 

657 catalog.meta = filteredMetadata 

658 tableList.append(catalog) 

659 return tableList