Coverage for python/lsst/ip/isr/brighterFatterKernel.py: 7%

212 statements  

« prev     ^ index     » next       coverage.py v6.4.1, created at 2022-06-29 03:04 -0700

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22"""Brighter Fatter Kernel calibration definition.""" 

23 

24 

25__all__ = ['BrighterFatterKernel'] 

26 

27 

28import numpy as np 

29from astropy.table import Table 

30import lsst.afw.math as afwMath 

31from . import IsrCalib 

32 

33 

34class BrighterFatterKernel(IsrCalib): 

35 """Calibration of brighter-fatter kernels for an instrument. 

36 

37 ampKernels are the kernels for each amplifier in a detector, as 

38 generated by having level == 'AMP' 

39 

40 detectorKernel is the kernel generated for a detector as a 

41 whole, as generated by having level == 'DETECTOR' 

42 

43 makeDetectorKernelFromAmpwiseKernels is a method to generate the 

44 kernel for a detector, constructed by averaging together the 

45 ampwise kernels in the detector. The existing application code is 

46 only defined for kernels with level == 'DETECTOR', so this method 

47 is used if the supplied kernel was built with level == 'AMP'. 

48 

49 Parameters 

50 ---------- 

51 level : `str` 

52 Level the kernels will be generated for. 

53 log : `logging.Logger`, optional 

54 Log to write messages to. 

55 **kwargs : 

56 Parameters to pass to parent constructor. 

57 

58 Notes 

59 ----- 

60 TODO: DM-35260 

61 Document what is stored in the BFK calibration. 

62 

63 Version 1.1 adds the `expIdMask` property, and substitutes 

64 `means` and `variances` for `rawMeans` and `rawVariances` 

65 from the PTC dataset. 

66 """ 

67 _OBSTYPE = 'bfk' 

68 _SCHEMA = 'Brighter-fatter kernel' 

69 _VERSION = 1.1 

70 

71 def __init__(self, camera=None, level=None, **kwargs): 

72 self.level = level 

73 

74 # Things inherited from the PTC 

75 self.expIdMask = dict() 

76 self.rawMeans = dict() 

77 self.rawVariances = dict() 

78 self.rawXcorrs = dict() 

79 self.badAmps = list() 

80 self.shape = (17, 17) 

81 self.gain = dict() 

82 self.noise = dict() 

83 

84 # Things calculated from the PTC 

85 self.meanXcorrs = dict() 

86 self.valid = dict() 

87 

88 # Things that are used downstream 

89 self.ampKernels = dict() 

90 self.detKernels = dict() 

91 

92 super().__init__(**kwargs) 

93 

94 if camera: 

95 self.initFromCamera(camera, detectorId=kwargs.get('detectorId', None)) 

96 

97 self.requiredAttributes.update(['level', 'expIdMask', 'rawMeans', 'rawVariances', 'rawXcorrs', 

98 'badAmps', 'gain', 'noise', 'meanXcorrs', 'valid', 

99 'ampKernels', 'detKernels']) 

100 

101 def updateMetadata(self, setDate=False, **kwargs): 

102 """Update calibration metadata. 

103 

104 This calls the base class's method after ensuring the required 

105 calibration keywords will be saved. 

106 

107 Parameters 

108 ---------- 

109 setDate : `bool`, optional 

110 Update the CALIBDATE fields in the metadata to the current 

111 time. Defaults to False. 

112 kwargs : 

113 Other keyword parameters to set in the metadata. 

114 """ 

115 kwargs['LEVEL'] = self.level 

116 kwargs['KERNEL_DX'] = self.shape[0] 

117 kwargs['KERNEL_DY'] = self.shape[1] 

118 

119 super().updateMetadata(setDate=setDate, **kwargs) 

120 

121 def initFromCamera(self, camera, detectorId=None): 

122 """Initialize kernel structure from camera. 

123 

124 Parameters 

125 ---------- 

126 camera : `lsst.afw.cameraGeom.Camera` 

127 Camera to use to define geometry. 

128 detectorId : `int`, optional 

129 Index of the detector to generate. 

130 

131 Returns 

132 ------- 

133 calib : `lsst.ip.isr.BrighterFatterKernel` 

134 The initialized calibration. 

135 

136 Raises 

137 ------ 

138 RuntimeError : 

139 Raised if no detectorId is supplied for a calibration with 

140 level='AMP'. 

141 """ 

142 self._instrument = camera.getName() 

143 

144 if detectorId is not None: 

145 detector = camera[detectorId] 

146 self._detectorId = detectorId 

147 self._detectorName = detector.getName() 

148 self._detectorSerial = detector.getSerial() 

149 

150 if self.level == 'AMP': 

151 if detectorId is None: 

152 raise RuntimeError("A detectorId must be supplied if level='AMP'.") 

153 

154 self.badAmps = [] 

155 

156 for amp in detector: 

157 ampName = amp.getName() 

158 self.expIdMask[ampName] = [] 

159 self.rawMeans[ampName] = [] 

160 self.rawVariances[ampName] = [] 

161 self.rawXcorrs[ampName] = [] 

162 self.gain[ampName] = amp.getGain() 

163 self.noise[ampName] = amp.getReadNoise() 

164 self.meanXcorrs[ampName] = [] 

165 self.ampKernels[ampName] = [] 

166 self.valid[ampName] = [] 

167 elif self.level == 'DETECTOR': 

168 if detectorId is None: 

169 for det in camera: 

170 detName = det.getName() 

171 self.detKernels[detName] = [] 

172 else: 

173 self.detKernels[self._detectorName] = [] 

174 

175 return self 

176 

177 def getLengths(self): 

178 """Return the set of lengths needed for reshaping components. 

179 

180 Returns 

181 ------- 

182 kernelLength : `int` 

183 Product of the elements of self.shape. 

184 smallLength : `int` 

185 Size of an untiled covariance. 

186 nObs : `int` 

187 Number of observation pairs used in the kernel. 

188 """ 

189 kernelLength = self.shape[0] * self.shape[1] 

190 smallLength = int((self.shape[0] - 1)*(self.shape[1] - 1)/4) 

191 if self.level == 'AMP': 

192 nObservations = set([len(self.rawMeans[amp]) for amp in self.rawMeans]) 

193 if len(nObservations) != 1: 

194 raise RuntimeError("Inconsistent number of observations found.") 

195 nObs = nObservations.pop() 

196 else: 

197 nObs = 0 

198 

199 return (kernelLength, smallLength, nObs) 

200 

201 @classmethod 

202 def fromDict(cls, dictionary): 

203 """Construct a calibration from a dictionary of properties. 

204 

205 Parameters 

206 ---------- 

207 dictionary : `dict` 

208 Dictionary of properties. 

209 

210 Returns 

211 ------- 

212 calib : `lsst.ip.isr.BrighterFatterKernel 

213 Constructed calibration. 

214 

215 Raises 

216 ------ 

217 RuntimeError : 

218 Raised if the supplied dictionary is for a different 

219 calibration. 

220 Raised if the version of the supplied dictionary is 1.0. 

221 """ 

222 calib = cls() 

223 

224 if calib._OBSTYPE != (found := dictionary['metadata']['OBSTYPE']): 

225 raise RuntimeError(f"Incorrect brighter-fatter kernel supplied. Expected {calib._OBSTYPE}, " 

226 f"found {found}") 

227 calib.setMetadata(dictionary['metadata']) 

228 calib.calibInfoFromDict(dictionary) 

229 

230 calib.level = dictionary['metadata'].get('LEVEL', 'AMP') 

231 calib.shape = (dictionary['metadata'].get('KERNEL_DX', 0), 

232 dictionary['metadata'].get('KERNEL_DY', 0)) 

233 

234 calibVersion = dictionary['metadata']['bfk_VERSION'] 

235 if calibVersion == 1.0: 

236 calib.log.warning("Old Version of brighter-fatter kernel found. Current version: " 

237 f"{calib._VERSION}. The new attribute 'expIdMask' will be " 

238 "populated with 'True' values, and the new attributes 'rawMeans'" 

239 "and 'rawVariances' will be populated with the masked 'means'." 

240 "and 'variances' values." 

241 ) 

242 # use 'means', because 'expIdMask' does not exist. 

243 calib.expIdMask = {amp: np.repeat(True, len(dictionary['means'][amp])) for amp in 

244 dictionary['means']} 

245 calib.rawMeans = {amp: np.array(dictionary['means'][amp]) for amp in dictionary['means']} 

246 calib.rawVariances = {amp: np.array(dictionary['variances'][amp]) for amp in 

247 dictionary['variances']} 

248 elif calibVersion == 1.1: 

249 calib.expIdMask = {amp: np.array(dictionary['expIdMask'][amp]) for amp in dictionary['expIdMask']} 

250 calib.rawMeans = {amp: np.array(dictionary['rawMeans'][amp]) for amp in dictionary['rawMeans']} 

251 calib.rawVariances = {amp: np.array(dictionary['rawVariances'][amp]) for amp in 

252 dictionary['rawVariances']} 

253 else: 

254 raise RuntimeError(f"Unknown version for brighter-fatter kernel: {calibVersion}") 

255 

256 # Lengths for reshape: 

257 _, smallLength, nObs = calib.getLengths() 

258 smallShapeSide = int(np.sqrt(smallLength)) 

259 

260 calib.rawXcorrs = {amp: np.array(dictionary['rawXcorrs'][amp]).reshape((nObs, 

261 smallShapeSide, 

262 smallShapeSide)) 

263 for amp in dictionary['rawXcorrs']} 

264 

265 calib.gain = dictionary['gain'] 

266 calib.noise = dictionary['noise'] 

267 

268 calib.meanXcorrs = {amp: np.array(dictionary['meanXcorrs'][amp]).reshape(calib.shape) 

269 for amp in dictionary['rawXcorrs']} 

270 calib.ampKernels = {amp: np.array(dictionary['ampKernels'][amp]).reshape(calib.shape) 

271 for amp in dictionary['ampKernels']} 

272 calib.valid = {amp: bool(value) for amp, value in dictionary['valid'].items()} 

273 calib.badAmps = [amp for amp, valid in dictionary['valid'].items() if valid is False] 

274 

275 calib.detKernels = {det: np.array(dictionary['detKernels'][det]).reshape(calib.shape) 

276 for det in dictionary['detKernels']} 

277 

278 calib.updateMetadata() 

279 return calib 

280 

281 def toDict(self): 

282 """Return a dictionary containing the calibration properties. 

283 

284 The dictionary should be able to be round-tripped through 

285 `fromDict`. 

286 

287 Returns 

288 ------- 

289 dictionary : `dict` 

290 Dictionary of properties. 

291 """ 

292 self.updateMetadata() 

293 

294 outDict = {} 

295 metadata = self.getMetadata() 

296 outDict['metadata'] = metadata 

297 

298 # Lengths for ravel: 

299 kernelLength, smallLength, nObs = self.getLengths() 

300 

301 outDict['expIdMask'] = {amp: np.array(self.expIdMask[amp]).tolist() for amp in self.expIdMask} 

302 outDict['rawMeans'] = {amp: np.array(self.rawMeans[amp]).tolist() for amp in self.rawMeans} 

303 outDict['rawVariances'] = {amp: np.array(self.rawVariances[amp]).tolist() for amp in 

304 self.rawVariances} 

305 outDict['rawXcorrs'] = {amp: np.array(self.rawXcorrs[amp]).reshape(nObs*smallLength).tolist() 

306 for amp in self.rawXcorrs} 

307 outDict['badAmps'] = self.badAmps 

308 outDict['gain'] = self.gain 

309 outDict['noise'] = self.noise 

310 

311 outDict['meanXcorrs'] = {amp: self.meanXcorrs[amp].reshape(kernelLength).tolist() 

312 for amp in self.meanXcorrs} 

313 outDict['ampKernels'] = {amp: self.ampKernels[amp].reshape(kernelLength).tolist() 

314 for amp in self.ampKernels} 

315 outDict['valid'] = self.valid 

316 

317 outDict['detKernels'] = {det: self.detKernels[det].reshape(kernelLength).tolist() 

318 for det in self.detKernels} 

319 return outDict 

320 

321 @classmethod 

322 def fromTable(cls, tableList): 

323 """Construct calibration from a list of tables. 

324 

325 This method uses the `fromDict` method to create the 

326 calibration, after constructing an appropriate dictionary from 

327 the input tables. 

328 

329 Parameters 

330 ---------- 

331 tableList : `list` [`astropy.table.Table`] 

332 List of tables to use to construct the brighter-fatter 

333 calibration. 

334 

335 Returns 

336 ------- 

337 calib : `lsst.ip.isr.BrighterFatterKernel` 

338 The calibration defined in the tables. 

339 """ 

340 ampTable = tableList[0] 

341 

342 metadata = ampTable.meta 

343 inDict = dict() 

344 inDict['metadata'] = metadata 

345 

346 amps = ampTable['AMPLIFIER'] 

347 

348 # Determine version for expected values. The ``fromDict`` 

349 # method can unpack either, but the appropriate fields need to 

350 # be supplied. 

351 calibVersion = metadata['bfk_VERSION'] 

352 

353 if calibVersion == 1.0: 

354 # We expect to find ``means`` and ``variances`` for this 

355 # case, and will construct an ``expIdMask`` from these 

356 # parameters in the ``fromDict`` method. 

357 rawMeanList = ampTable['MEANS'] 

358 rawVarianceList = ampTable['VARIANCES'] 

359 

360 inDict['means'] = {amp: mean for amp, mean in zip(amps, rawMeanList)} 

361 inDict['variances'] = {amp: var for amp, var in zip(amps, rawVarianceList)} 

362 elif calibVersion == 1.1: 

363 # This will have ``rawMeans`` and ``rawVariances``, which 

364 # are filtered via the ``expIdMask`` fields. 

365 expIdMaskList = ampTable['EXP_ID_MASK'] 

366 rawMeanList = ampTable['RAW_MEANS'] 

367 rawVarianceList = ampTable['RAW_VARIANCES'] 

368 

369 inDict['expIdMask'] = {amp: mask for amp, mask in zip(amps, expIdMaskList)} 

370 inDict['rawMeans'] = {amp: mean for amp, mean in zip(amps, rawMeanList)} 

371 inDict['rawVariances'] = {amp: var for amp, var in zip(amps, rawVarianceList)} 

372 else: 

373 raise RuntimeError(f"Unknown version for brighter-fatter kernel: {calibVersion}") 

374 

375 rawXcorrs = ampTable['RAW_XCORRS'] 

376 gainList = ampTable['GAIN'] 

377 noiseList = ampTable['NOISE'] 

378 

379 meanXcorrs = ampTable['MEAN_XCORRS'] 

380 ampKernels = ampTable['KERNEL'] 

381 validList = ampTable['VALID'] 

382 

383 inDict['rawXcorrs'] = {amp: kernel for amp, kernel in zip(amps, rawXcorrs)} 

384 inDict['gain'] = {amp: gain for amp, gain in zip(amps, gainList)} 

385 inDict['noise'] = {amp: noise for amp, noise in zip(amps, noiseList)} 

386 inDict['meanXcorrs'] = {amp: kernel for amp, kernel in zip(amps, meanXcorrs)} 

387 inDict['ampKernels'] = {amp: kernel for amp, kernel in zip(amps, ampKernels)} 

388 inDict['valid'] = {amp: bool(valid) for amp, valid in zip(amps, validList)} 

389 

390 inDict['badAmps'] = [amp for amp, valid in inDict['valid'].items() if valid is False] 

391 

392 if len(tableList) > 1: 

393 detTable = tableList[1] 

394 inDict['detKernels'] = {det: kernel for det, kernel 

395 in zip(detTable['DETECTOR'], detTable['KERNEL'])} 

396 else: 

397 inDict['detKernels'] = {} 

398 

399 return cls.fromDict(inDict) 

400 

401 def toTable(self): 

402 """Construct a list of tables containing the information in this 

403 calibration. 

404 

405 The list of tables should create an identical calibration 

406 after being passed to this class's fromTable method. 

407 

408 Returns 

409 ------- 

410 tableList : `list` [`lsst.afw.table.Table`] 

411 List of tables containing the crosstalk calibration 

412 information. 

413 

414 """ 

415 tableList = [] 

416 self.updateMetadata() 

417 

418 # Lengths 

419 kernelLength, smallLength, nObs = self.getLengths() 

420 

421 ampList = [] 

422 expIdMaskList = [] 

423 rawMeanList = [] 

424 rawVarianceList = [] 

425 rawXcorrs = [] 

426 gainList = [] 

427 noiseList = [] 

428 

429 meanXcorrsList = [] 

430 kernelList = [] 

431 validList = [] 

432 

433 if self.level == 'AMP': 

434 for amp in self.rawMeans.keys(): 

435 ampList.append(amp) 

436 expIdMaskList.append(self.expIdMask[amp]) 

437 rawMeanList.append(self.rawMeans[amp]) 

438 rawVarianceList.append(self.rawVariances[amp]) 

439 rawXcorrs.append(np.array(self.rawXcorrs[amp]).reshape(nObs*smallLength).tolist()) 

440 gainList.append(self.gain[amp]) 

441 noiseList.append(self.noise[amp]) 

442 

443 meanXcorrsList.append(self.meanXcorrs[amp].reshape(kernelLength).tolist()) 

444 kernelList.append(self.ampKernels[amp].reshape(kernelLength).tolist()) 

445 validList.append(int(self.valid[amp] and not (amp in self.badAmps))) 

446 

447 ampTable = Table({'AMPLIFIER': ampList, 

448 'EXP_ID_MASK': expIdMaskList, 

449 'RAW_MEANS': rawMeanList, 

450 'RAW_VARIANCES': rawVarianceList, 

451 'RAW_XCORRS': rawXcorrs, 

452 'GAIN': gainList, 

453 'NOISE': noiseList, 

454 'MEAN_XCORRS': meanXcorrsList, 

455 'KERNEL': kernelList, 

456 'VALID': validList, 

457 }) 

458 

459 ampTable.meta = self.getMetadata().toDict() 

460 tableList.append(ampTable) 

461 

462 if len(self.detKernels): 

463 detList = [] 

464 kernelList = [] 

465 for det in self.detKernels.keys(): 

466 detList.append(det) 

467 kernelList.append(self.detKernels[det].reshape(kernelLength).tolist()) 

468 

469 detTable = Table({'DETECTOR': detList, 

470 'KERNEL': kernelList}) 

471 detTable.meta = self.getMetadata().toDict() 

472 tableList.append(detTable) 

473 

474 return tableList 

475 

476 # Implementation methods 

477 def makeDetectorKernelFromAmpwiseKernels(self, detectorName, ampsToExclude=[]): 

478 """Average the amplifier level kernels to create a detector level 

479 kernel. 

480 """ 

481 inKernels = np.array([self.ampKernels[amp] for amp in 

482 self.ampKernels if amp not in ampsToExclude]) 

483 averagingList = np.transpose(inKernels) 

484 avgKernel = np.zeros_like(inKernels[0]) 

485 sctrl = afwMath.StatisticsControl() 

486 sctrl.setNumSigmaClip(5.0) 

487 for i in range(np.shape(avgKernel)[0]): 

488 for j in range(np.shape(avgKernel)[1]): 

489 avgKernel[i, j] = afwMath.makeStatistics(averagingList[i, j], 

490 afwMath.MEANCLIP, sctrl).getValue() 

491 

492 self.detKernels[detectorName] = avgKernel 

493 

494 def replaceDetectorKernelWithAmpKernel(self, ampName, detectorName): 

495 self.detKernel[detectorName] = self.ampKernel[ampName]