Coverage for python/lsst/ip/isr/brighterFatterKernel.py: 7%

231 statements  

« prev     ^ index     » next       coverage.py v7.4.3, created at 2024-03-02 11:55 +0000

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22"""Brighter Fatter Kernel calibration definition.""" 

23 

24 

25__all__ = ['BrighterFatterKernel'] 

26 

27 

28import numpy as np 

29from astropy.table import Table 

30import lsst.afw.math as afwMath 

31from . import IsrCalib 

32 

33 

34class BrighterFatterKernel(IsrCalib): 

35 """Calibration of brighter-fatter kernels for an instrument. 

36 

37 ampKernels are the kernels for each amplifier in a detector, as 

38 generated by having ``level == 'AMP'``. 

39 

40 detectorKernel is the kernel generated for a detector as a 

41 whole, as generated by having ``level == 'DETECTOR'``. 

42 

43 makeDetectorKernelFromAmpwiseKernels is a method to generate the 

44 kernel for a detector, constructed by averaging together the 

45 ampwise kernels in the detector. The existing application code is 

46 only defined for kernels with ``level == 'DETECTOR'``, so this method 

47 is used if the supplied kernel was built with ``level == 'AMP'``. 

48 

49 Parameters 

50 ---------- 

51 camera : `lsst.afw.cameraGeom.Camera` 

52 Camera describing detector geometry. 

53 level : `str` 

54 Level the kernels will be generated for. 

55 log : `logging.Logger`, optional 

56 Log to write messages to. 

57 **kwargs : 

58 Parameters to pass to parent constructor. 

59 

60 Notes 

61 ----- 

62 Version 1.1 adds the `expIdMask` property, and substitutes 

63 `means` and `variances` for `rawMeans` and `rawVariances` 

64 from the PTC dataset. 

65 

66 expIdMask : `dict`, [`str`,`numpy.ndarray`] 

67 Dictionary keyed by amp names containing the mask produced after 

68 outlier rejection. 

69 rawMeans : `dict`, [`str`, `numpy.ndarray`] 

70 Dictionary keyed by amp names containing the unmasked average of the 

71 means of the exposures in each flat pair. 

72 rawVariances : `dict`, [`str`, `numpy.ndarray`] 

73 Dictionary keyed by amp names containing the variance of the 

74 difference image of the exposures in each flat pair. 

75 Corresponds to rawVars of PTC. 

76 rawXcorrs : `dict`, [`str`, `numpy.ndarray`] 

77 Dictionary keyed by amp names containing an array of measured 

78 covariances per mean flux. 

79 Corresponds to covariances of PTC. 

80 badAmps : `list` 

81 List of bad amplifiers names. 

82 shape : `tuple` 

83 Tuple of the shape of the BFK kernels. 

84 gain : `dict`, [`str`,`float`] 

85 Dictionary keyed by amp names containing the fitted gains. 

86 noise : `dict`, [`str`,`float`] 

87 Dictionary keyed by amp names containing the fitted noise. 

88 meanXcorrs : `dict`, [`str`,`numpy.ndarray`] 

89 Dictionary keyed by amp names containing the averaged 

90 cross-correlations. 

91 valid : `dict`, [`str`,`bool`] 

92 Dictionary keyed by amp names containing validity of data. 

93 ampKernels : `dict`, [`str`, `numpy.ndarray`] 

94 Dictionary keyed by amp names containing the BF kernels. 

95 detKernels : `dict` 

96 Dictionary keyed by detector names containing the BF kernels. 

97 """ 

98 _OBSTYPE = 'bfk' 

99 _SCHEMA = 'Brighter-fatter kernel' 

100 _VERSION = 1.1 

101 

102 def __init__(self, camera=None, level=None, **kwargs): 

103 self.level = level 

104 

105 # Things inherited from the PTC 

106 self.expIdMask = dict() 

107 self.rawMeans = dict() 

108 self.rawVariances = dict() 

109 self.rawXcorrs = dict() 

110 self.badAmps = list() 

111 self.shape = (17, 17) 

112 self.gain = dict() 

113 self.noise = dict() 

114 

115 # Things calculated from the PTC 

116 self.meanXcorrs = dict() 

117 self.valid = dict() 

118 

119 # Things that are used downstream 

120 self.ampKernels = dict() 

121 self.detKernels = dict() 

122 

123 super().__init__(**kwargs) 

124 

125 if camera: 

126 self.initFromCamera(camera, detectorId=kwargs.get('detectorId', None)) 

127 

128 self.requiredAttributes.update(['level', 'expIdMask', 'rawMeans', 'rawVariances', 'rawXcorrs', 

129 'badAmps', 'gain', 'noise', 'meanXcorrs', 'valid', 

130 'ampKernels', 'detKernels']) 

131 

132 def updateMetadata(self, setDate=False, **kwargs): 

133 """Update calibration metadata. 

134 

135 This calls the base class's method after ensuring the required 

136 calibration keywords will be saved. 

137 

138 Parameters 

139 ---------- 

140 setDate : `bool`, optional 

141 Update the CALIBDATE fields in the metadata to the current 

142 time. Defaults to False. 

143 kwargs : 

144 Other keyword parameters to set in the metadata. 

145 """ 

146 kwargs['LEVEL'] = self.level 

147 kwargs['KERNEL_DX'] = self.shape[0] 

148 kwargs['KERNEL_DY'] = self.shape[1] 

149 

150 super().updateMetadata(setDate=setDate, **kwargs) 

151 

152 def initFromCamera(self, camera, detectorId=None): 

153 """Initialize kernel structure from camera. 

154 

155 Parameters 

156 ---------- 

157 camera : `lsst.afw.cameraGeom.Camera` 

158 Camera to use to define geometry. 

159 detectorId : `int`, optional 

160 Index of the detector to generate. 

161 

162 Returns 

163 ------- 

164 calib : `lsst.ip.isr.BrighterFatterKernel` 

165 The initialized calibration. 

166 

167 Raises 

168 ------ 

169 RuntimeError 

170 Raised if no detectorId is supplied for a calibration with 

171 ``level='AMP'``. 

172 """ 

173 self._instrument = camera.getName() 

174 

175 if detectorId is not None: 

176 detector = camera[detectorId] 

177 self._detectorId = detectorId 

178 self._detectorName = detector.getName() 

179 self._detectorSerial = detector.getSerial() 

180 

181 if self.level == 'AMP': 

182 if detectorId is None: 

183 raise RuntimeError("A detectorId must be supplied if level='AMP'.") 

184 

185 self.badAmps = [] 

186 

187 for amp in detector: 

188 ampName = amp.getName() 

189 self.expIdMask[ampName] = [] 

190 self.rawMeans[ampName] = [] 

191 self.rawVariances[ampName] = [] 

192 self.rawXcorrs[ampName] = [] 

193 self.gain[ampName] = amp.getGain() 

194 self.noise[ampName] = amp.getReadNoise() 

195 self.meanXcorrs[ampName] = [] 

196 self.ampKernels[ampName] = [] 

197 self.valid[ampName] = [] 

198 elif self.level == 'DETECTOR': 

199 if detectorId is None: 

200 for det in camera: 

201 detName = det.getName() 

202 self.detKernels[detName] = [] 

203 else: 

204 self.detKernels[self._detectorName] = [] 

205 

206 return self 

207 

208 def getLengths(self): 

209 """Return the set of lengths needed for reshaping components. 

210 

211 Returns 

212 ------- 

213 kernelLength : `int` 

214 Product of the elements of self.shape. 

215 smallLength : `int` 

216 Size of an untiled covariance. 

217 nObs : `int` 

218 Number of observation pairs used in the kernel. 

219 """ 

220 kernelLength = self.shape[0] * self.shape[1] 

221 smallLength = int((self.shape[0] - 1)*(self.shape[1] - 1)/4) 

222 if self.level == 'AMP': 

223 nObservations = set([len(self.rawMeans[amp]) for amp in self.rawMeans]) 

224 if len(nObservations) != 1: 

225 raise RuntimeError("Inconsistent number of observations found.") 

226 nObs = nObservations.pop() 

227 else: 

228 nObs = 0 

229 

230 return (kernelLength, smallLength, nObs) 

231 

232 @classmethod 

233 def fromDict(cls, dictionary): 

234 """Construct a calibration from a dictionary of properties. 

235 

236 Parameters 

237 ---------- 

238 dictionary : `dict` 

239 Dictionary of properties. 

240 

241 Returns 

242 ------- 

243 calib : `lsst.ip.isr.BrighterFatterKernel` 

244 Constructed calibration. 

245 

246 Raises 

247 ------ 

248 RuntimeError 

249 Raised if the supplied dictionary is for a different 

250 calibration. 

251 Raised if the version of the supplied dictionary is 1.0. 

252 """ 

253 calib = cls() 

254 

255 if calib._OBSTYPE != (found := dictionary['metadata']['OBSTYPE']): 

256 raise RuntimeError(f"Incorrect brighter-fatter kernel supplied. Expected {calib._OBSTYPE}, " 

257 f"found {found}") 

258 calib.setMetadata(dictionary['metadata']) 

259 calib.calibInfoFromDict(dictionary) 

260 

261 calib.level = dictionary['metadata'].get('LEVEL', 'AMP') 

262 calib.shape = (dictionary['metadata'].get('KERNEL_DX', 0), 

263 dictionary['metadata'].get('KERNEL_DY', 0)) 

264 

265 calibVersion = dictionary['metadata']['bfk_VERSION'] 

266 if calibVersion == 1.0: 

267 calib.log.debug("Old Version of brighter-fatter kernel found. Current version: " 

268 f"{calib._VERSION}. The new attribute 'expIdMask' will be " 

269 "populated with 'True' values, and the new attributes 'rawMeans' " 

270 "and 'rawVariances' will be populated with the masked 'means' " 

271 "and 'variances' values." 

272 ) 

273 # use 'means', because 'expIdMask' does not exist. 

274 calib.expIdMask = {amp: np.repeat(True, len(dictionary['means'][amp])) for amp in 

275 dictionary['means']} 

276 calib.rawMeans = {amp: np.array(dictionary['means'][amp]) for amp in dictionary['means']} 

277 calib.rawVariances = {amp: np.array(dictionary['variances'][amp]) for amp in 

278 dictionary['variances']} 

279 elif calibVersion == 1.1: 

280 calib.expIdMask = {amp: np.array(dictionary['expIdMask'][amp]) for amp in dictionary['expIdMask']} 

281 calib.rawMeans = {amp: np.array(dictionary['rawMeans'][amp]) for amp in dictionary['rawMeans']} 

282 calib.rawVariances = {amp: np.array(dictionary['rawVariances'][amp]) for amp in 

283 dictionary['rawVariances']} 

284 else: 

285 raise RuntimeError(f"Unknown version for brighter-fatter kernel: {calibVersion}") 

286 

287 # Lengths for reshape: 

288 _, smallLength, nObs = calib.getLengths() 

289 smallShapeSide = int(np.sqrt(smallLength)) 

290 

291 calib.rawXcorrs = {amp: np.array(dictionary['rawXcorrs'][amp]).reshape((nObs, 

292 smallShapeSide, 

293 smallShapeSide)) 

294 for amp in dictionary['rawXcorrs']} 

295 

296 calib.gain = dictionary['gain'] 

297 calib.noise = dictionary['noise'] 

298 

299 calib.meanXcorrs = {amp: np.array(dictionary['meanXcorrs'][amp]).reshape(calib.shape) 

300 for amp in dictionary['rawXcorrs']} 

301 calib.ampKernels = {amp: np.array(dictionary['ampKernels'][amp]).reshape(calib.shape) 

302 for amp in dictionary['ampKernels']} 

303 calib.valid = {amp: bool(value) for amp, value in dictionary['valid'].items()} 

304 calib.badAmps = [amp for amp, valid in dictionary['valid'].items() if valid is False] 

305 

306 calib.detKernels = {det: np.array(dictionary['detKernels'][det]).reshape(calib.shape) 

307 for det in dictionary['detKernels']} 

308 

309 calib.updateMetadata() 

310 return calib 

311 

312 def toDict(self): 

313 """Return a dictionary containing the calibration properties. 

314 

315 The dictionary should be able to be round-tripped through 

316 `fromDict`. 

317 

318 Returns 

319 ------- 

320 dictionary : `dict` 

321 Dictionary of properties. 

322 """ 

323 self.updateMetadata() 

324 

325 outDict = {} 

326 metadata = self.getMetadata() 

327 outDict['metadata'] = metadata 

328 

329 # Lengths for ravel: 

330 kernelLength, smallLength, nObs = self.getLengths() 

331 

332 outDict['expIdMask'] = {amp: np.array(self.expIdMask[amp]).tolist() for amp in self.expIdMask} 

333 outDict['rawMeans'] = {amp: np.array(self.rawMeans[amp]).tolist() for amp in self.rawMeans} 

334 outDict['rawVariances'] = {amp: np.array(self.rawVariances[amp]).tolist() for amp in 

335 self.rawVariances} 

336 

337 for amp in self.rawXcorrs.keys(): 

338 # Check to see if we need to repack the data. 

339 correlationShape = np.array(self.rawXcorrs[amp]).shape 

340 if nObs != correlationShape[0]: 

341 if correlationShape[0] == np.sum(self.expIdMask[amp]): 

342 # Repack data. 

343 self.repackCorrelations(amp, correlationShape) 

344 else: 

345 raise ValueError("Could not coerce rawXcorrs into appropriate shape " 

346 "(have %d correlations, but expect to see %d.", 

347 correlationShape[0], np.sum(self.expIdMask[amp])) 

348 

349 outDict['rawXcorrs'] = {amp: np.array(self.rawXcorrs[amp]).reshape(nObs*smallLength).tolist() 

350 for amp in self.rawXcorrs} 

351 outDict['badAmps'] = self.badAmps 

352 outDict['gain'] = self.gain 

353 outDict['noise'] = self.noise 

354 

355 outDict['meanXcorrs'] = {amp: self.meanXcorrs[amp].reshape(kernelLength).tolist() 

356 for amp in self.meanXcorrs} 

357 outDict['ampKernels'] = {amp: self.ampKernels[amp].reshape(kernelLength).tolist() 

358 for amp in self.ampKernels} 

359 outDict['valid'] = self.valid 

360 

361 outDict['detKernels'] = {det: self.detKernels[det].reshape(kernelLength).tolist() 

362 for det in self.detKernels} 

363 return outDict 

364 

365 @classmethod 

366 def fromTable(cls, tableList): 

367 """Construct calibration from a list of tables. 

368 

369 This method uses the `fromDict` method to create the 

370 calibration, after constructing an appropriate dictionary from 

371 the input tables. 

372 

373 Parameters 

374 ---------- 

375 tableList : `list` [`astropy.table.Table`] 

376 List of tables to use to construct the brighter-fatter 

377 calibration. 

378 

379 Returns 

380 ------- 

381 calib : `lsst.ip.isr.BrighterFatterKernel` 

382 The calibration defined in the tables. 

383 """ 

384 ampTable = tableList[0] 

385 

386 metadata = ampTable.meta 

387 inDict = dict() 

388 inDict['metadata'] = metadata 

389 

390 amps = ampTable['AMPLIFIER'] 

391 

392 # Determine version for expected values. The ``fromDict`` 

393 # method can unpack either, but the appropriate fields need to 

394 # be supplied. 

395 calibVersion = metadata['bfk_VERSION'] 

396 

397 if calibVersion == 1.0: 

398 # We expect to find ``means`` and ``variances`` for this 

399 # case, and will construct an ``expIdMask`` from these 

400 # parameters in the ``fromDict`` method. 

401 rawMeanList = ampTable['MEANS'] 

402 rawVarianceList = ampTable['VARIANCES'] 

403 

404 inDict['means'] = {amp: mean for amp, mean in zip(amps, rawMeanList)} 

405 inDict['variances'] = {amp: var for amp, var in zip(amps, rawVarianceList)} 

406 elif calibVersion == 1.1: 

407 # This will have ``rawMeans`` and ``rawVariances``, which 

408 # are filtered via the ``expIdMask`` fields. 

409 expIdMaskList = ampTable['EXP_ID_MASK'] 

410 rawMeanList = ampTable['RAW_MEANS'] 

411 rawVarianceList = ampTable['RAW_VARIANCES'] 

412 

413 inDict['expIdMask'] = {amp: mask for amp, mask in zip(amps, expIdMaskList)} 

414 inDict['rawMeans'] = {amp: mean for amp, mean in zip(amps, rawMeanList)} 

415 inDict['rawVariances'] = {amp: var for amp, var in zip(amps, rawVarianceList)} 

416 else: 

417 raise RuntimeError(f"Unknown version for brighter-fatter kernel: {calibVersion}") 

418 

419 rawXcorrs = ampTable['RAW_XCORRS'] 

420 gainList = ampTable['GAIN'] 

421 noiseList = ampTable['NOISE'] 

422 

423 meanXcorrs = ampTable['MEAN_XCORRS'] 

424 ampKernels = ampTable['KERNEL'] 

425 validList = ampTable['VALID'] 

426 

427 inDict['rawXcorrs'] = {amp: kernel for amp, kernel in zip(amps, rawXcorrs)} 

428 inDict['gain'] = {amp: gain for amp, gain in zip(amps, gainList)} 

429 inDict['noise'] = {amp: noise for amp, noise in zip(amps, noiseList)} 

430 inDict['meanXcorrs'] = {amp: kernel for amp, kernel in zip(amps, meanXcorrs)} 

431 inDict['ampKernels'] = {amp: kernel for amp, kernel in zip(amps, ampKernels)} 

432 inDict['valid'] = {amp: bool(valid) for amp, valid in zip(amps, validList)} 

433 

434 inDict['badAmps'] = [amp for amp, valid in inDict['valid'].items() if valid is False] 

435 

436 if len(tableList) > 1: 

437 detTable = tableList[1] 

438 inDict['detKernels'] = {det: kernel for det, kernel 

439 in zip(detTable['DETECTOR'], detTable['KERNEL'])} 

440 else: 

441 inDict['detKernels'] = {} 

442 

443 return cls.fromDict(inDict) 

444 

445 def toTable(self): 

446 """Construct a list of tables containing the information in this 

447 calibration. 

448 

449 The list of tables should create an identical calibration 

450 after being passed to this class's fromTable method. 

451 

452 Returns 

453 ------- 

454 tableList : `list` [`lsst.afw.table.Table`] 

455 List of tables containing the crosstalk calibration 

456 information. 

457 

458 """ 

459 tableList = [] 

460 self.updateMetadata() 

461 

462 # Lengths 

463 kernelLength, smallLength, nObs = self.getLengths() 

464 

465 ampList = [] 

466 expIdMaskList = [] 

467 rawMeanList = [] 

468 rawVarianceList = [] 

469 rawXcorrs = [] 

470 gainList = [] 

471 noiseList = [] 

472 

473 meanXcorrsList = [] 

474 kernelList = [] 

475 validList = [] 

476 

477 if self.level == 'AMP': 

478 for amp in self.rawMeans.keys(): 

479 ampList.append(amp) 

480 expIdMaskList.append(self.expIdMask[amp]) 

481 rawMeanList.append(self.rawMeans[amp]) 

482 rawVarianceList.append(self.rawVariances[amp]) 

483 

484 correlationShape = np.array(self.rawXcorrs[amp]).shape 

485 if nObs != correlationShape[0]: 

486 if correlationShape[0] == np.sum(self.expIdMask[amp]): 

487 # Repack data. 

488 self.repackCorrelations(amp, correlationShape) 

489 else: 

490 raise ValueError("Could not coerce rawXcorrs into appropriate shape " 

491 "(have %d correlations, but expect to see %d.", 

492 correlationShape[0], np.sum(self.expIdMask[amp])) 

493 

494 rawXcorrs.append(np.array(self.rawXcorrs[amp]).reshape(nObs*smallLength).tolist()) 

495 gainList.append(self.gain[amp]) 

496 noiseList.append(self.noise[amp]) 

497 

498 meanXcorrsList.append(self.meanXcorrs[amp].reshape(kernelLength).tolist()) 

499 kernelList.append(self.ampKernels[amp].reshape(kernelLength).tolist()) 

500 validList.append(int(self.valid[amp] and not (amp in self.badAmps))) 

501 

502 ampTable = Table({'AMPLIFIER': ampList, 

503 'EXP_ID_MASK': expIdMaskList, 

504 'RAW_MEANS': rawMeanList, 

505 'RAW_VARIANCES': rawVarianceList, 

506 'RAW_XCORRS': rawXcorrs, 

507 'GAIN': gainList, 

508 'NOISE': noiseList, 

509 'MEAN_XCORRS': meanXcorrsList, 

510 'KERNEL': kernelList, 

511 'VALID': validList, 

512 }) 

513 

514 ampTable.meta = self.getMetadata().toDict() 

515 tableList.append(ampTable) 

516 

517 if len(self.detKernels): 

518 detList = [] 

519 kernelList = [] 

520 for det in self.detKernels.keys(): 

521 detList.append(det) 

522 kernelList.append(self.detKernels[det].reshape(kernelLength).tolist()) 

523 

524 detTable = Table({'DETECTOR': detList, 

525 'KERNEL': kernelList}) 

526 detTable.meta = self.getMetadata().toDict() 

527 tableList.append(detTable) 

528 

529 return tableList 

530 

531 def repackCorrelations(self, amp, correlationShape): 

532 """If the correlations were masked, they need to be repacked into the 

533 correct shape. 

534 

535 Parameters 

536 ---------- 

537 amp : `str` 

538 Amplifier needing repacked. 

539 correlationShape : `tuple` [`int`], (3, ) 

540 Shape the correlations are expected to take. 

541 """ 

542 repackedCorrelations = [] 

543 idx = 0 

544 for maskValue in self.expIdMask[amp]: 

545 if maskValue: 

546 repackedCorrelations.append(self.rawXcorrs[amp][idx]) 

547 idx += 1 

548 else: 

549 repackedCorrelations.append(np.full((correlationShape[1], correlationShape[2]), np.nan)) 

550 self.rawXcorrs[amp] = repackedCorrelations 

551 

552 # Implementation methods 

553 def makeDetectorKernelFromAmpwiseKernels(self, detectorName, ampsToExclude=[]): 

554 """Average the amplifier level kernels to create a detector level 

555 kernel. There is no change in index ordering/orientation from 

556 this averaging. 

557 

558 Parameters 

559 ---------- 

560 detectorName : `str` 

561 Detector for which the averaged kernel will be used. 

562 ampsToExclude : `list` [`str`], optional 

563 Amps that should not be included in the average. 

564 """ 

565 inKernels = np.array([self.ampKernels[amp] for amp in 

566 self.ampKernels if amp not in ampsToExclude]) 

567 avgKernel = np.zeros_like(inKernels[0]) 

568 sctrl = afwMath.StatisticsControl() 

569 sctrl.setNumSigmaClip(5.0) 

570 for i in range(np.shape(avgKernel)[0]): 

571 for j in range(np.shape(avgKernel)[1]): 

572 avgKernel[i, j] = afwMath.makeStatistics(inKernels[:, i, j], 

573 afwMath.MEANCLIP, sctrl).getValue() 

574 

575 self.detKernels[detectorName] = avgKernel 

576 

577 def replaceDetectorKernelWithAmpKernel(self, ampName, detectorName): 

578 self.detKernel[detectorName] = self.ampKernel[ampName]