Coverage for python/lsst/ip/isr/brighterFatterKernel.py: 7%

212 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-09-20 02:39 -0700

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22"""Brighter Fatter Kernel calibration definition.""" 

23 

24 

25__all__ = ['BrighterFatterKernel'] 

26 

27 

28import numpy as np 

29from astropy.table import Table 

30import lsst.afw.math as afwMath 

31from . import IsrCalib 

32 

33 

34class BrighterFatterKernel(IsrCalib): 

35 """Calibration of brighter-fatter kernels for an instrument. 

36 

37 ampKernels are the kernels for each amplifier in a detector, as 

38 generated by having ``level == 'AMP'``. 

39 

40 detectorKernel is the kernel generated for a detector as a 

41 whole, as generated by having ``level == 'DETECTOR'``. 

42 

43 makeDetectorKernelFromAmpwiseKernels is a method to generate the 

44 kernel for a detector, constructed by averaging together the 

45 ampwise kernels in the detector. The existing application code is 

46 only defined for kernels with ``level == 'DETECTOR'``, so this method 

47 is used if the supplied kernel was built with ``level == 'AMP'``. 

48 

49 Parameters 

50 ---------- 

51 camera : `lsst.afw.cameraGeom.Camera` 

52 Camera describing detector geometry. 

53 level : `str` 

54 Level the kernels will be generated for. 

55 log : `logging.Logger`, optional 

56 Log to write messages to. 

57 **kwargs : 

58 Parameters to pass to parent constructor. 

59 

60 Notes 

61 ----- 

62 Version 1.1 adds the `expIdMask` property, and substitutes 

63 `means` and `variances` for `rawMeans` and `rawVariances` 

64 from the PTC dataset. 

65 

66 expIdMask : `dict`, [`str`,`numpy.ndarray`] 

67 Dictionary keyed by amp names containing the mask produced after 

68 outlier rejection. 

69 rawMeans : `dict`, [`str`, `numpy.ndarray`] 

70 Dictionary keyed by amp names containing the unmasked average of the 

71 means of the exposures in each flat pair. 

72 rawVariances : `dict`, [`str`, `numpy.ndarray`] 

73 Dictionary keyed by amp names containing the variance of the 

74 difference image of the exposures in each flat pair. 

75 Corresponds to rawVars of PTC. 

76 rawXcorrs : `dict`, [`str`, `numpy.ndarray`] 

77 Dictionary keyed by amp names containing an array of measured 

78 covariances per mean flux. 

79 Corresponds to covariances of PTC. 

80 badAmps : `list` 

81 List of bad amplifiers names. 

82 shape : `tuple` 

83 Tuple of the shape of the BFK kernels. 

84 gain : `dict`, [`str`,`float`] 

85 Dictionary keyed by amp names containing the fitted gains. 

86 noise : `dict`, [`str`,`float`] 

87 Dictionary keyed by amp names containing the fitted noise. 

88 meanXcorrs : `dict`, [`str`,`numpy.ndarray`] 

89 Dictionary keyed by amp names containing the averaged 

90 cross-correlations. 

91 valid : `dict`, [`str`,`bool`] 

92 Dictionary keyed by amp names containing validity of data. 

93 ampKernels : `dict`, [`str`, `numpy.ndarray`] 

94 Dictionary keyed by amp names containing the BF kernels. 

95 detKernels : `dict` 

96 Dictionary keyed by detector names containing the BF kernels. 

97 """ 

98 _OBSTYPE = 'bfk' 

99 _SCHEMA = 'Brighter-fatter kernel' 

100 _VERSION = 1.1 

101 

102 def __init__(self, camera=None, level=None, **kwargs): 

103 self.level = level 

104 

105 # Things inherited from the PTC 

106 self.expIdMask = dict() 

107 self.rawMeans = dict() 

108 self.rawVariances = dict() 

109 self.rawXcorrs = dict() 

110 self.badAmps = list() 

111 self.shape = (17, 17) 

112 self.gain = dict() 

113 self.noise = dict() 

114 

115 # Things calculated from the PTC 

116 self.meanXcorrs = dict() 

117 self.valid = dict() 

118 

119 # Things that are used downstream 

120 self.ampKernels = dict() 

121 self.detKernels = dict() 

122 

123 super().__init__(**kwargs) 

124 

125 if camera: 

126 self.initFromCamera(camera, detectorId=kwargs.get('detectorId', None)) 

127 

128 self.requiredAttributes.update(['level', 'expIdMask', 'rawMeans', 'rawVariances', 'rawXcorrs', 

129 'badAmps', 'gain', 'noise', 'meanXcorrs', 'valid', 

130 'ampKernels', 'detKernels']) 

131 

132 def updateMetadata(self, setDate=False, **kwargs): 

133 """Update calibration metadata. 

134 

135 This calls the base class's method after ensuring the required 

136 calibration keywords will be saved. 

137 

138 Parameters 

139 ---------- 

140 setDate : `bool`, optional 

141 Update the CALIBDATE fields in the metadata to the current 

142 time. Defaults to False. 

143 kwargs : 

144 Other keyword parameters to set in the metadata. 

145 """ 

146 kwargs['LEVEL'] = self.level 

147 kwargs['KERNEL_DX'] = self.shape[0] 

148 kwargs['KERNEL_DY'] = self.shape[1] 

149 

150 super().updateMetadata(setDate=setDate, **kwargs) 

151 

152 def initFromCamera(self, camera, detectorId=None): 

153 """Initialize kernel structure from camera. 

154 

155 Parameters 

156 ---------- 

157 camera : `lsst.afw.cameraGeom.Camera` 

158 Camera to use to define geometry. 

159 detectorId : `int`, optional 

160 Index of the detector to generate. 

161 

162 Returns 

163 ------- 

164 calib : `lsst.ip.isr.BrighterFatterKernel` 

165 The initialized calibration. 

166 

167 Raises 

168 ------ 

169 RuntimeError 

170 Raised if no detectorId is supplied for a calibration with 

171 ``level='AMP'``. 

172 """ 

173 self._instrument = camera.getName() 

174 

175 if detectorId is not None: 

176 detector = camera[detectorId] 

177 self._detectorId = detectorId 

178 self._detectorName = detector.getName() 

179 self._detectorSerial = detector.getSerial() 

180 

181 if self.level == 'AMP': 

182 if detectorId is None: 

183 raise RuntimeError("A detectorId must be supplied if level='AMP'.") 

184 

185 self.badAmps = [] 

186 

187 for amp in detector: 

188 ampName = amp.getName() 

189 self.expIdMask[ampName] = [] 

190 self.rawMeans[ampName] = [] 

191 self.rawVariances[ampName] = [] 

192 self.rawXcorrs[ampName] = [] 

193 self.gain[ampName] = amp.getGain() 

194 self.noise[ampName] = amp.getReadNoise() 

195 self.meanXcorrs[ampName] = [] 

196 self.ampKernels[ampName] = [] 

197 self.valid[ampName] = [] 

198 elif self.level == 'DETECTOR': 

199 if detectorId is None: 

200 for det in camera: 

201 detName = det.getName() 

202 self.detKernels[detName] = [] 

203 else: 

204 self.detKernels[self._detectorName] = [] 

205 

206 return self 

207 

208 def getLengths(self): 

209 """Return the set of lengths needed for reshaping components. 

210 

211 Returns 

212 ------- 

213 kernelLength : `int` 

214 Product of the elements of self.shape. 

215 smallLength : `int` 

216 Size of an untiled covariance. 

217 nObs : `int` 

218 Number of observation pairs used in the kernel. 

219 """ 

220 kernelLength = self.shape[0] * self.shape[1] 

221 smallLength = int((self.shape[0] - 1)*(self.shape[1] - 1)/4) 

222 if self.level == 'AMP': 

223 nObservations = set([len(self.rawMeans[amp]) for amp in self.rawMeans]) 

224 if len(nObservations) != 1: 

225 raise RuntimeError("Inconsistent number of observations found.") 

226 nObs = nObservations.pop() 

227 else: 

228 nObs = 0 

229 

230 return (kernelLength, smallLength, nObs) 

231 

232 @classmethod 

233 def fromDict(cls, dictionary): 

234 """Construct a calibration from a dictionary of properties. 

235 

236 Parameters 

237 ---------- 

238 dictionary : `dict` 

239 Dictionary of properties. 

240 

241 Returns 

242 ------- 

243 calib : `lsst.ip.isr.BrighterFatterKernel` 

244 Constructed calibration. 

245 

246 Raises 

247 ------ 

248 RuntimeError 

249 Raised if the supplied dictionary is for a different 

250 calibration. 

251 Raised if the version of the supplied dictionary is 1.0. 

252 """ 

253 calib = cls() 

254 

255 if calib._OBSTYPE != (found := dictionary['metadata']['OBSTYPE']): 

256 raise RuntimeError(f"Incorrect brighter-fatter kernel supplied. Expected {calib._OBSTYPE}, " 

257 f"found {found}") 

258 calib.setMetadata(dictionary['metadata']) 

259 calib.calibInfoFromDict(dictionary) 

260 

261 calib.level = dictionary['metadata'].get('LEVEL', 'AMP') 

262 calib.shape = (dictionary['metadata'].get('KERNEL_DX', 0), 

263 dictionary['metadata'].get('KERNEL_DY', 0)) 

264 

265 calibVersion = dictionary['metadata']['bfk_VERSION'] 

266 if calibVersion == 1.0: 

267 calib.log.warning("Old Version of brighter-fatter kernel found. Current version: " 

268 f"{calib._VERSION}. The new attribute 'expIdMask' will be " 

269 "populated with 'True' values, and the new attributes 'rawMeans'" 

270 "and 'rawVariances' will be populated with the masked 'means'." 

271 "and 'variances' values." 

272 ) 

273 # use 'means', because 'expIdMask' does not exist. 

274 calib.expIdMask = {amp: np.repeat(True, len(dictionary['means'][amp])) for amp in 

275 dictionary['means']} 

276 calib.rawMeans = {amp: np.array(dictionary['means'][amp]) for amp in dictionary['means']} 

277 calib.rawVariances = {amp: np.array(dictionary['variances'][amp]) for amp in 

278 dictionary['variances']} 

279 elif calibVersion == 1.1: 

280 calib.expIdMask = {amp: np.array(dictionary['expIdMask'][amp]) for amp in dictionary['expIdMask']} 

281 calib.rawMeans = {amp: np.array(dictionary['rawMeans'][amp]) for amp in dictionary['rawMeans']} 

282 calib.rawVariances = {amp: np.array(dictionary['rawVariances'][amp]) for amp in 

283 dictionary['rawVariances']} 

284 else: 

285 raise RuntimeError(f"Unknown version for brighter-fatter kernel: {calibVersion}") 

286 

287 # Lengths for reshape: 

288 _, smallLength, nObs = calib.getLengths() 

289 smallShapeSide = int(np.sqrt(smallLength)) 

290 

291 calib.rawXcorrs = {amp: np.array(dictionary['rawXcorrs'][amp]).reshape((nObs, 

292 smallShapeSide, 

293 smallShapeSide)) 

294 for amp in dictionary['rawXcorrs']} 

295 

296 calib.gain = dictionary['gain'] 

297 calib.noise = dictionary['noise'] 

298 

299 calib.meanXcorrs = {amp: np.array(dictionary['meanXcorrs'][amp]).reshape(calib.shape) 

300 for amp in dictionary['rawXcorrs']} 

301 calib.ampKernels = {amp: np.array(dictionary['ampKernels'][amp]).reshape(calib.shape) 

302 for amp in dictionary['ampKernels']} 

303 calib.valid = {amp: bool(value) for amp, value in dictionary['valid'].items()} 

304 calib.badAmps = [amp for amp, valid in dictionary['valid'].items() if valid is False] 

305 

306 calib.detKernels = {det: np.array(dictionary['detKernels'][det]).reshape(calib.shape) 

307 for det in dictionary['detKernels']} 

308 

309 calib.updateMetadata() 

310 return calib 

311 

312 def toDict(self): 

313 """Return a dictionary containing the calibration properties. 

314 

315 The dictionary should be able to be round-tripped through 

316 `fromDict`. 

317 

318 Returns 

319 ------- 

320 dictionary : `dict` 

321 Dictionary of properties. 

322 """ 

323 self.updateMetadata() 

324 

325 outDict = {} 

326 metadata = self.getMetadata() 

327 outDict['metadata'] = metadata 

328 

329 # Lengths for ravel: 

330 kernelLength, smallLength, nObs = self.getLengths() 

331 

332 outDict['expIdMask'] = {amp: np.array(self.expIdMask[amp]).tolist() for amp in self.expIdMask} 

333 outDict['rawMeans'] = {amp: np.array(self.rawMeans[amp]).tolist() for amp in self.rawMeans} 

334 outDict['rawVariances'] = {amp: np.array(self.rawVariances[amp]).tolist() for amp in 

335 self.rawVariances} 

336 outDict['rawXcorrs'] = {amp: np.array(self.rawXcorrs[amp]).reshape(nObs*smallLength).tolist() 

337 for amp in self.rawXcorrs} 

338 outDict['badAmps'] = self.badAmps 

339 outDict['gain'] = self.gain 

340 outDict['noise'] = self.noise 

341 

342 outDict['meanXcorrs'] = {amp: self.meanXcorrs[amp].reshape(kernelLength).tolist() 

343 for amp in self.meanXcorrs} 

344 outDict['ampKernels'] = {amp: self.ampKernels[amp].reshape(kernelLength).tolist() 

345 for amp in self.ampKernels} 

346 outDict['valid'] = self.valid 

347 

348 outDict['detKernels'] = {det: self.detKernels[det].reshape(kernelLength).tolist() 

349 for det in self.detKernels} 

350 return outDict 

351 

352 @classmethod 

353 def fromTable(cls, tableList): 

354 """Construct calibration from a list of tables. 

355 

356 This method uses the `fromDict` method to create the 

357 calibration, after constructing an appropriate dictionary from 

358 the input tables. 

359 

360 Parameters 

361 ---------- 

362 tableList : `list` [`astropy.table.Table`] 

363 List of tables to use to construct the brighter-fatter 

364 calibration. 

365 

366 Returns 

367 ------- 

368 calib : `lsst.ip.isr.BrighterFatterKernel` 

369 The calibration defined in the tables. 

370 """ 

371 ampTable = tableList[0] 

372 

373 metadata = ampTable.meta 

374 inDict = dict() 

375 inDict['metadata'] = metadata 

376 

377 amps = ampTable['AMPLIFIER'] 

378 

379 # Determine version for expected values. The ``fromDict`` 

380 # method can unpack either, but the appropriate fields need to 

381 # be supplied. 

382 calibVersion = metadata['bfk_VERSION'] 

383 

384 if calibVersion == 1.0: 

385 # We expect to find ``means`` and ``variances`` for this 

386 # case, and will construct an ``expIdMask`` from these 

387 # parameters in the ``fromDict`` method. 

388 rawMeanList = ampTable['MEANS'] 

389 rawVarianceList = ampTable['VARIANCES'] 

390 

391 inDict['means'] = {amp: mean for amp, mean in zip(amps, rawMeanList)} 

392 inDict['variances'] = {amp: var for amp, var in zip(amps, rawVarianceList)} 

393 elif calibVersion == 1.1: 

394 # This will have ``rawMeans`` and ``rawVariances``, which 

395 # are filtered via the ``expIdMask`` fields. 

396 expIdMaskList = ampTable['EXP_ID_MASK'] 

397 rawMeanList = ampTable['RAW_MEANS'] 

398 rawVarianceList = ampTable['RAW_VARIANCES'] 

399 

400 inDict['expIdMask'] = {amp: mask for amp, mask in zip(amps, expIdMaskList)} 

401 inDict['rawMeans'] = {amp: mean for amp, mean in zip(amps, rawMeanList)} 

402 inDict['rawVariances'] = {amp: var for amp, var in zip(amps, rawVarianceList)} 

403 else: 

404 raise RuntimeError(f"Unknown version for brighter-fatter kernel: {calibVersion}") 

405 

406 rawXcorrs = ampTable['RAW_XCORRS'] 

407 gainList = ampTable['GAIN'] 

408 noiseList = ampTable['NOISE'] 

409 

410 meanXcorrs = ampTable['MEAN_XCORRS'] 

411 ampKernels = ampTable['KERNEL'] 

412 validList = ampTable['VALID'] 

413 

414 inDict['rawXcorrs'] = {amp: kernel for amp, kernel in zip(amps, rawXcorrs)} 

415 inDict['gain'] = {amp: gain for amp, gain in zip(amps, gainList)} 

416 inDict['noise'] = {amp: noise for amp, noise in zip(amps, noiseList)} 

417 inDict['meanXcorrs'] = {amp: kernel for amp, kernel in zip(amps, meanXcorrs)} 

418 inDict['ampKernels'] = {amp: kernel for amp, kernel in zip(amps, ampKernels)} 

419 inDict['valid'] = {amp: bool(valid) for amp, valid in zip(amps, validList)} 

420 

421 inDict['badAmps'] = [amp for amp, valid in inDict['valid'].items() if valid is False] 

422 

423 if len(tableList) > 1: 

424 detTable = tableList[1] 

425 inDict['detKernels'] = {det: kernel for det, kernel 

426 in zip(detTable['DETECTOR'], detTable['KERNEL'])} 

427 else: 

428 inDict['detKernels'] = {} 

429 

430 return cls.fromDict(inDict) 

431 

432 def toTable(self): 

433 """Construct a list of tables containing the information in this 

434 calibration. 

435 

436 The list of tables should create an identical calibration 

437 after being passed to this class's fromTable method. 

438 

439 Returns 

440 ------- 

441 tableList : `list` [`lsst.afw.table.Table`] 

442 List of tables containing the crosstalk calibration 

443 information. 

444 

445 """ 

446 tableList = [] 

447 self.updateMetadata() 

448 

449 # Lengths 

450 kernelLength, smallLength, nObs = self.getLengths() 

451 

452 ampList = [] 

453 expIdMaskList = [] 

454 rawMeanList = [] 

455 rawVarianceList = [] 

456 rawXcorrs = [] 

457 gainList = [] 

458 noiseList = [] 

459 

460 meanXcorrsList = [] 

461 kernelList = [] 

462 validList = [] 

463 

464 if self.level == 'AMP': 

465 for amp in self.rawMeans.keys(): 

466 ampList.append(amp) 

467 expIdMaskList.append(self.expIdMask[amp]) 

468 rawMeanList.append(self.rawMeans[amp]) 

469 rawVarianceList.append(self.rawVariances[amp]) 

470 rawXcorrs.append(np.array(self.rawXcorrs[amp]).reshape(nObs*smallLength).tolist()) 

471 gainList.append(self.gain[amp]) 

472 noiseList.append(self.noise[amp]) 

473 

474 meanXcorrsList.append(self.meanXcorrs[amp].reshape(kernelLength).tolist()) 

475 kernelList.append(self.ampKernels[amp].reshape(kernelLength).tolist()) 

476 validList.append(int(self.valid[amp] and not (amp in self.badAmps))) 

477 

478 ampTable = Table({'AMPLIFIER': ampList, 

479 'EXP_ID_MASK': expIdMaskList, 

480 'RAW_MEANS': rawMeanList, 

481 'RAW_VARIANCES': rawVarianceList, 

482 'RAW_XCORRS': rawXcorrs, 

483 'GAIN': gainList, 

484 'NOISE': noiseList, 

485 'MEAN_XCORRS': meanXcorrsList, 

486 'KERNEL': kernelList, 

487 'VALID': validList, 

488 }) 

489 

490 ampTable.meta = self.getMetadata().toDict() 

491 tableList.append(ampTable) 

492 

493 if len(self.detKernels): 

494 detList = [] 

495 kernelList = [] 

496 for det in self.detKernels.keys(): 

497 detList.append(det) 

498 kernelList.append(self.detKernels[det].reshape(kernelLength).tolist()) 

499 

500 detTable = Table({'DETECTOR': detList, 

501 'KERNEL': kernelList}) 

502 detTable.meta = self.getMetadata().toDict() 

503 tableList.append(detTable) 

504 

505 return tableList 

506 

507 # Implementation methods 

508 def makeDetectorKernelFromAmpwiseKernels(self, detectorName, ampsToExclude=[]): 

509 """Average the amplifier level kernels to create a detector level 

510 kernel. 

511 """ 

512 inKernels = np.array([self.ampKernels[amp] for amp in 

513 self.ampKernels if amp not in ampsToExclude]) 

514 averagingList = np.transpose(inKernels) 

515 avgKernel = np.zeros_like(inKernels[0]) 

516 sctrl = afwMath.StatisticsControl() 

517 sctrl.setNumSigmaClip(5.0) 

518 for i in range(np.shape(avgKernel)[0]): 

519 for j in range(np.shape(avgKernel)[1]): 

520 avgKernel[i, j] = afwMath.makeStatistics(averagingList[i, j], 

521 afwMath.MEANCLIP, sctrl).getValue() 

522 

523 self.detKernels[detectorName] = avgKernel 

524 

525 def replaceDetectorKernelWithAmpKernel(self, ampName, detectorName): 

526 self.detKernel[detectorName] = self.ampKernel[ampName]