Coverage for python/lsst/cp/pipe/makeBrighterFatterKernel.py: 13%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

199 statements  

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22"""Calculation of brighter-fatter effect correlations and kernels.""" 

23 

24__all__ = ['BrighterFatterKernelSolveTask', 

25 'BrighterFatterKernelSolveConfig'] 

26 

27import numpy as np 

28 

29import lsst.afw.math as afwMath 

30import lsst.pex.config as pexConfig 

31import lsst.pipe.base as pipeBase 

32import lsst.pipe.base.connectionTypes as cT 

33 

34from lsst.ip.isr import (BrighterFatterKernel) 

35from .utils import (funcPolynomial, irlsFit) 

36from ._lookupStaticCalibration import lookupStaticCalibration 

37 

38 

39class BrighterFatterKernelSolveConnections(pipeBase.PipelineTaskConnections, 

40 dimensions=("instrument", "exposure", "detector")): 

41 dummy = cT.Input( 

42 name="raw", 

43 doc="Dummy exposure.", 

44 storageClass='Exposure', 

45 dimensions=("instrument", "exposure", "detector"), 

46 multiple=True, 

47 deferLoad=True, 

48 ) 

49 camera = cT.PrerequisiteInput( 

50 name="camera", 

51 doc="Camera associated with this data.", 

52 storageClass="Camera", 

53 dimensions=("instrument", ), 

54 isCalibration=True, 

55 lookupFunction=lookupStaticCalibration, 

56 ) 

57 inputPtc = cT.PrerequisiteInput( 

58 name="ptc", 

59 doc="Photon transfer curve dataset.", 

60 storageClass="PhotonTransferCurveDataset", 

61 dimensions=("instrument", "detector"), 

62 isCalibration=True, 

63 ) 

64 

65 outputBFK = cT.Output( 

66 name="brighterFatterKernel", 

67 doc="Output measured brighter-fatter kernel.", 

68 storageClass="BrighterFatterKernel", 

69 dimensions=("instrument", "detector"), 

70 isCalibration=True, 

71 ) 

72 

73 

74class BrighterFatterKernelSolveConfig(pipeBase.PipelineTaskConfig, 

75 pipelineConnections=BrighterFatterKernelSolveConnections): 

76 level = pexConfig.ChoiceField( 

77 doc="The level at which to calculate the brighter-fatter kernels", 

78 dtype=str, 

79 default="AMP", 

80 allowed={ 

81 "AMP": "Every amplifier treated separately", 

82 "DETECTOR": "One kernel per detector", 

83 } 

84 ) 

85 ignoreAmpsForAveraging = pexConfig.ListField( 

86 dtype=str, 

87 doc="List of amp names to ignore when averaging the amplifier kernels into the detector" 

88 " kernel. Only relevant for level = DETECTOR", 

89 default=[] 

90 ) 

91 xcorrCheckRejectLevel = pexConfig.Field( 

92 dtype=float, 

93 doc="Rejection level for the sum of the input cross-correlations. Arrays which " 

94 "sum to greater than this are discarded before the clipped mean is calculated.", 

95 default=2.0 

96 ) 

97 nSigmaClip = pexConfig.Field( 

98 dtype=float, 

99 doc="Number of sigma to clip when calculating means for the cross-correlation", 

100 default=5 

101 ) 

102 forceZeroSum = pexConfig.Field( 

103 dtype=bool, 

104 doc="Force the correlation matrix to have zero sum by adjusting the (0,0) value?", 

105 default=False, 

106 ) 

107 useAmatrix = pexConfig.Field( 

108 dtype=bool, 

109 doc="Use the PTC 'a' matrix (Astier et al. 2019 equation 20) " 

110 "instead of the average of measured covariances?", 

111 default=False, 

112 ) 

113 

114 maxIterSuccessiveOverRelaxation = pexConfig.Field( 

115 dtype=int, 

116 doc="The maximum number of iterations allowed for the successive over-relaxation method", 

117 default=10000 

118 ) 

119 eLevelSuccessiveOverRelaxation = pexConfig.Field( 

120 dtype=float, 

121 doc="The target residual error for the successive over-relaxation method", 

122 default=5.0e-14 

123 ) 

124 

125 correlationQuadraticFit = pexConfig.Field( 

126 dtype=bool, 

127 doc="Use a quadratic fit to find the correlations instead of simple averaging?", 

128 default=False, 

129 ) 

130 correlationModelRadius = pexConfig.Field( 

131 dtype=int, 

132 doc="Build a model of the correlation coefficients for radii larger than this value in pixels?", 

133 default=100, 

134 ) 

135 correlationModelSlope = pexConfig.Field( 

136 dtype=float, 

137 doc="Slope of the correlation model for radii larger than correlationModelRadius", 

138 default=-1.35, 

139 ) 

140 

141 

142class BrighterFatterKernelSolveTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

143 """Measure appropriate Brighter-Fatter Kernel from the PTC dataset. 

144 """ 

145 

146 ConfigClass = BrighterFatterKernelSolveConfig 

147 _DefaultName = 'cpBfkMeasure' 

148 

149 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

150 """Ensure that the input and output dimensions are passed along. 

151 

152 Parameters 

153 ---------- 

154 butlerQC : `lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext` 

155 Butler to operate on. 

156 inputRefs : `lsst.pipe.base.connections.InputQuantizedConnection` 

157 Input data refs to load. 

158 ouptutRefs : `lsst.pipe.base.connections.OutputQuantizedConnection` 

159 Output data refs to persist. 

160 """ 

161 inputs = butlerQC.get(inputRefs) 

162 

163 # Use the dimensions to set calib/provenance information. 

164 inputs['inputDims'] = inputRefs.inputPtc.dataId.byName() 

165 

166 outputs = self.run(**inputs) 

167 butlerQC.put(outputs, outputRefs) 

168 

169 def run(self, inputPtc, dummy, camera, inputDims): 

170 """Combine covariance information from PTC into brighter-fatter 

171 kernels. 

172 

173 Parameters 

174 ---------- 

175 inputPtc : `lsst.ip.isr.PhotonTransferCurveDataset` 

176 PTC data containing per-amplifier covariance measurements. 

177 dummy : `lsst.afw.image.Exposure` 

178 The exposure used to select the appropriate PTC dataset. 

179 In almost all circumstances, one of the input exposures 

180 used to generate the PTC dataset is the best option. 

181 camera : `lsst.afw.cameraGeom.Camera` 

182 Camera to use for camera geometry information. 

183 inputDims : `lsst.daf.butler.DataCoordinate` or `dict` 

184 DataIds to use to populate the output calibration. 

185 

186 Returns 

187 ------- 

188 results : `lsst.pipe.base.Struct` 

189 The resulst struct containing: 

190 

191 ``outputBfk`` 

192 Resulting Brighter-Fatter Kernel 

193 (`lsst.ip.isr.BrighterFatterKernel`). 

194 """ 

195 if len(dummy) == 0: 

196 self.log.warn("No dummy exposure found.") 

197 

198 detector = camera[inputDims['detector']] 

199 detName = detector.getName() 

200 

201 if self.config.level == 'DETECTOR': 

202 detectorCorrList = list() 

203 

204 bfk = BrighterFatterKernel(camera=camera, detectorId=detector.getId(), level=self.config.level) 

205 bfk.means = inputPtc.finalMeans # ADU 

206 bfk.variances = inputPtc.finalVars # ADU^2 

207 # Use the PTC covariances as the cross-correlations. These 

208 # are scaled before the kernel is generated, which performs 

209 # the conversion. 

210 bfk.rawXcorrs = inputPtc.covariances # ADU^2 

211 bfk.badAmps = inputPtc.badAmps 

212 bfk.shape = (inputPtc.covMatrixSide*2 + 1, inputPtc.covMatrixSide*2 + 1) 

213 bfk.gain = inputPtc.gain 

214 bfk.noise = inputPtc.noise 

215 bfk.meanXcorrs = dict() 

216 bfk.valid = dict() 

217 

218 for amp in detector: 

219 ampName = amp.getName() 

220 gain = bfk.gain[ampName] 

221 

222 # Using the inputPtc.expIdMask works if the covariance 

223 # array has the same length as the rawMeans/rawVars. This 

224 # isn't the case, as it's the same size as the 

225 # finalMeans/finalVars. However, these arrays (and the 

226 # covariance) are padded with NAN values to match the 

227 # longest amplifier vector. We do not want to include 

228 # these NAN values, so we construct a mask for all non-NAN 

229 # values in finalMeans, and use that to filter finalVars 

230 # and the covariances. 

231 mask = np.isfinite(bfk.means[ampName]) 

232 fluxes = np.array(bfk.means[ampName])[mask] 

233 variances = np.array(bfk.variances[ampName])[mask] 

234 xCorrList = np.array([np.array(xcorr) for xcorr in bfk.rawXcorrs[ampName]])[mask] 

235 

236 if gain <= 0: 

237 # We've received very bad data. 

238 self.log.warn("Impossible gain recieved from PTC for %s: %f. Skipping amplifier.", 

239 ampName, gain) 

240 bfk.meanXcorrs[ampName] = np.zeros(bfk.shape) 

241 bfk.ampKernels[ampName] = np.zeros(bfk.shape) 

242 bfk.valid[ampName] = False 

243 continue 

244 

245 fluxes = np.array([flux*gain for flux in fluxes]) # Now in e^- 

246 variances = np.array([variance*gain*gain for variance in variances]) # Now in e^2- 

247 

248 # This should duplicate Coulton et al. 2017 Equation 22-29 

249 # (arxiv:1711.06273) 

250 scaledCorrList = list() 

251 for xcorrNum, (xcorr, flux, var) in enumerate(zip(xCorrList, fluxes, variances), 1): 

252 q = np.array(xcorr) * gain * gain # xcorr now in e^- 

253 q *= 2.0 # Remove factor of 1/2 applied in PTC. 

254 self.log.info("Amp: %s %d/%d Flux: %f Var: %f Q(0,0): %g Q(1,0): %g Q(0,1): %g", 

255 ampName, xcorrNum, len(xCorrList), flux, var, q[0][0], q[1][0], q[0][1]) 

256 

257 # Normalize by the flux, which removes the (0,0) 

258 # component attributable to Poisson noise. This 

259 # contains the two "t I delta(x - x')" terms in 

260 # Coulton et al. 2017 equation 29 

261 q[0][0] -= 2.0*(flux) 

262 

263 if q[0][0] > 0.0: 

264 self.log.warn("Amp: %s %d skipped due to value of (variance-mean)=%f", 

265 ampName, xcorrNum, q[0][0]) 

266 continue 

267 

268 # This removes the "t (I_a^2 + I_b^2)" factor in 

269 # Coulton et al. 2017 equation 29. 

270 q /= -2.0*(flux**2) 

271 scaled = self._tileArray(q) 

272 

273 xcorrCheck = np.abs(np.sum(scaled))/np.sum(np.abs(scaled)) 

274 if (xcorrCheck > self.config.xcorrCheckRejectLevel) or not (np.isfinite(xcorrCheck)): 

275 self.log.warn("Amp: %s %d skipped due to value of triangle-inequality sum %f", 

276 ampName, xcorrNum, xcorrCheck) 

277 continue 

278 

279 scaledCorrList.append(scaled) 

280 self.log.info("Amp: %s %d/%d Final: %g XcorrCheck: %f", 

281 ampName, xcorrNum, len(xCorrList), q[0][0], xcorrCheck) 

282 

283 if len(scaledCorrList) == 0: 

284 self.log.warn("Amp: %s All inputs rejected for amp!", ampName) 

285 bfk.meanXcorrs[ampName] = np.zeros(bfk.shape) 

286 bfk.ampKernels[ampName] = np.zeros(bfk.shape) 

287 bfk.valid[ampName] = False 

288 continue 

289 

290 if self.config.useAmatrix: 

291 # Use the aMatrix, ignoring the meanXcorr generated above. 

292 preKernel = np.pad(self._tileArray(np.array(inputPtc.aMatrix[ampName])), ((1, 1))) 

293 elif self.config.correlationQuadraticFit: 

294 # Use a quadratic fit to the correlations as a 

295 # function of flux. 

296 preKernel = self.quadraticCorrelations(scaledCorrList, fluxes, f"Amp: {ampName}") 

297 else: 

298 # Use a simple average of the measured correlations. 

299 preKernel = self.averageCorrelations(scaledCorrList, f"Amp: {ampName}") 

300 

301 center = int((bfk.shape[0] - 1) / 2) 

302 

303 if self.config.forceZeroSum: 

304 totalSum = np.sum(preKernel) 

305 

306 if self.config.correlationModelRadius < (preKernel.shape[0] - 1) / 2: 

307 # Assume a correlation model of 

308 # Corr(r) = -preFactor * r^(2 * slope) 

309 preFactor = np.sqrt(preKernel[center, center + 1] * preKernel[center + 1, center]) 

310 slopeFactor = 2.0 * np.abs(self.config.correlationModelSlope) 

311 totalSum += 2.0*np.pi*(preFactor / (slopeFactor*(center + 0.5))**slopeFactor) 

312 

313 preKernel[center, center] -= totalSum 

314 self.log.info("%s Zero-Sum Scale: %g", ampName, totalSum) 

315 

316 finalSum = np.sum(preKernel) 

317 bfk.meanXcorrs[ampName] = preKernel 

318 

319 postKernel = self.successiveOverRelax(preKernel) 

320 bfk.ampKernels[ampName] = postKernel 

321 if self.config.level == 'DETECTOR': 

322 detectorCorrList.extend(scaledCorrList) 

323 bfk.valid[ampName] = True 

324 self.log.info("Amp: %s Sum: %g Center Info Pre: %g Post: %g", 

325 ampName, finalSum, preKernel[center, center], postKernel[center, center]) 

326 

327 # Assemble a detector kernel? 

328 if self.config.level == 'DETECTOR': 

329 preKernel = self.averageCorrelations(detectorCorrList, f"Det: {detName}") 

330 finalSum = np.sum(preKernel) 

331 center = int((bfk.shape[0] - 1) / 2) 

332 

333 postKernel = self.successiveOverRelax(preKernel) 

334 bfk.detKernels[detName] = postKernel 

335 self.log.info("Det: %s Sum: %g Center Info Pre: %g Post: %g", 

336 detName, finalSum, preKernel[center, center], postKernel[center, center]) 

337 

338 return pipeBase.Struct( 

339 outputBFK=bfk, 

340 ) 

341 

342 def averageCorrelations(self, xCorrList, name): 

343 """Average input correlations. 

344 

345 Parameters 

346 ---------- 

347 xCorrList : `list` [`numpy.array`] 

348 List of cross-correlations. These are expected to be 

349 square arrays. 

350 name : `str` 

351 Name for log messages. 

352 

353 Returns 

354 ------- 

355 meanXcorr : `numpy.array`, (N, N) 

356 The averaged cross-correlation. 

357 """ 

358 meanXcorr = np.zeros_like(xCorrList[0]) 

359 xCorrList = np.transpose(xCorrList) 

360 sctrl = afwMath.StatisticsControl() 

361 sctrl.setNumSigmaClip(self.config.nSigmaClip) 

362 for i in range(np.shape(meanXcorr)[0]): 

363 for j in range(np.shape(meanXcorr)[1]): 

364 meanXcorr[i, j] = afwMath.makeStatistics(xCorrList[i, j], 

365 afwMath.MEANCLIP, sctrl).getValue() 

366 

367 # To match previous definitions, pad by one element. 

368 meanXcorr = np.pad(meanXcorr, ((1, 1))) 

369 

370 return meanXcorr 

371 

372 def quadraticCorrelations(self, xCorrList, fluxList, name): 

373 """Measure a quadratic correlation model. 

374 

375 Parameters 

376 ---------- 

377 xCorrList : `list` [`numpy.array`] 

378 List of cross-correlations. These are expected to be 

379 square arrays. 

380 fluxList : `numpy.array`, (Nflux,) 

381 Associated list of fluxes. 

382 name : `str` 

383 Name for log messages. 

384 

385 Returns 

386 ------- 

387 meanXcorr : `numpy.array`, (N, N) 

388 The averaged cross-correlation. 

389 """ 

390 meanXcorr = np.zeros_like(xCorrList[0]) 

391 fluxList = np.square(fluxList) 

392 xCorrList = np.array(xCorrList) 

393 

394 for i in range(np.shape(meanXcorr)[0]): 

395 for j in range(np.shape(meanXcorr)[1]): 

396 # Fit corrlation_i(x, y) = a0 + a1 * (flux_i)^2 The 

397 # i,j indices are inverted to apply the transposition, 

398 # as is done in the averaging case. 

399 linearFit, linearFitErr, chiSq, weights = irlsFit([0.0, 1e-4], fluxList, 

400 xCorrList[:, j, i], funcPolynomial) 

401 meanXcorr[i, j] = linearFit[1] # Discard the intercept. 

402 self.log.debug("Quad fit meanXcorr[%d,%d] = %g", i, j, linearFit[1]) 

403 

404 # To match previous definitions, pad by one element. 

405 meanXcorr = np.pad(meanXcorr, ((1, 1))) 

406 

407 return meanXcorr 

408 

409 @staticmethod 

410 def _tileArray(in_array): 

411 """Given an input quarter-image, tile/mirror it and return full image. 

412 

413 Given a square input of side-length n, of the form 

414 

415 input = array([[1, 2, 3], 

416 [4, 5, 6], 

417 [7, 8, 9]]) 

418 

419 return an array of size 2n-1 as 

420 

421 output = array([[ 9, 8, 7, 8, 9], 

422 [ 6, 5, 4, 5, 6], 

423 [ 3, 2, 1, 2, 3], 

424 [ 6, 5, 4, 5, 6], 

425 [ 9, 8, 7, 8, 9]]) 

426 

427 Parameters 

428 ---------- 

429 input : `np.array`, (N, N) 

430 The square input quarter-array 

431 

432 Returns 

433 ------- 

434 output : `np.array`, (2*N + 1, 2*N + 1) 

435 The full, tiled array 

436 """ 

437 assert(in_array.shape[0] == in_array.shape[1]) 

438 length = in_array.shape[0] - 1 

439 output = np.zeros((2*length + 1, 2*length + 1)) 

440 

441 for i in range(length + 1): 

442 for j in range(length + 1): 

443 output[i + length, j + length] = in_array[i, j] 

444 output[-i + length, j + length] = in_array[i, j] 

445 output[i + length, -j + length] = in_array[i, j] 

446 output[-i + length, -j + length] = in_array[i, j] 

447 return output 

448 

449 def successiveOverRelax(self, source, maxIter=None, eLevel=None): 

450 """An implementation of the successive over relaxation (SOR) method. 

451 

452 A numerical method for solving a system of linear equations 

453 with faster convergence than the Gauss-Seidel method. 

454 

455 Parameters 

456 ---------- 

457 source : `numpy.ndarray`, (N, N) 

458 The input array. 

459 maxIter : `int`, optional 

460 Maximum number of iterations to attempt before aborting. 

461 eLevel : `float`, optional 

462 The target error level at which we deem convergence to have 

463 occurred. 

464 

465 Returns 

466 ------- 

467 output : `numpy.ndarray`, (N, N) 

468 The solution. 

469 """ 

470 if not maxIter: 

471 maxIter = self.config.maxIterSuccessiveOverRelaxation 

472 if not eLevel: 

473 eLevel = self.config.eLevelSuccessiveOverRelaxation 

474 

475 assert source.shape[0] == source.shape[1], "Input array must be square" 

476 # initialize, and set boundary conditions 

477 func = np.zeros([source.shape[0] + 2, source.shape[1] + 2]) 

478 resid = np.zeros([source.shape[0] + 2, source.shape[1] + 2]) 

479 rhoSpe = np.cos(np.pi/source.shape[0]) # Here a square grid is assumed 

480 

481 # Calculate the initial error 

482 for i in range(1, func.shape[0] - 1): 

483 for j in range(1, func.shape[1] - 1): 

484 resid[i, j] = (func[i, j - 1] + func[i, j + 1] + func[i - 1, j] 

485 + func[i + 1, j] - 4*func[i, j] - source[i - 1, j - 1]) 

486 inError = np.sum(np.abs(resid)) 

487 

488 # Iterate until convergence 

489 # We perform two sweeps per cycle, 

490 # updating 'odd' and 'even' points separately 

491 nIter = 0 

492 omega = 1.0 

493 dx = 1.0 

494 while nIter < maxIter*2: 

495 outError = 0 

496 if nIter%2 == 0: 

497 for i in range(1, func.shape[0] - 1, 2): 

498 for j in range(1, func.shape[1] - 1, 2): 

499 resid[i, j] = float(func[i, j-1] + func[i, j + 1] + func[i - 1, j] 

500 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1]) 

501 func[i, j] += omega*resid[i, j]*.25 

502 for i in range(2, func.shape[0] - 1, 2): 

503 for j in range(2, func.shape[1] - 1, 2): 

504 resid[i, j] = float(func[i, j - 1] + func[i, j + 1] + func[i - 1, j] 

505 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1]) 

506 func[i, j] += omega*resid[i, j]*.25 

507 else: 

508 for i in range(1, func.shape[0] - 1, 2): 

509 for j in range(2, func.shape[1] - 1, 2): 

510 resid[i, j] = float(func[i, j - 1] + func[i, j + 1] + func[i - 1, j] 

511 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1]) 

512 func[i, j] += omega*resid[i, j]*.25 

513 for i in range(2, func.shape[0] - 1, 2): 

514 for j in range(1, func.shape[1] - 1, 2): 

515 resid[i, j] = float(func[i, j - 1] + func[i, j + 1] + func[i - 1, j] 

516 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1]) 

517 func[i, j] += omega*resid[i, j]*.25 

518 outError = np.sum(np.abs(resid)) 

519 if outError < inError*eLevel: 

520 break 

521 if nIter == 0: 

522 omega = 1.0/(1 - rhoSpe*rhoSpe/2.0) 

523 else: 

524 omega = 1.0/(1 - rhoSpe*rhoSpe*omega/4.0) 

525 nIter += 1 

526 

527 if nIter >= maxIter*2: 

528 self.log.warn("Failure: SuccessiveOverRelaxation did not converge in %s iterations." 

529 "\noutError: %s, inError: %s," % (nIter//2, outError, inError*eLevel)) 

530 else: 

531 self.log.info("Success: SuccessiveOverRelaxation converged in %s iterations." 

532 "\noutError: %s, inError: %s", nIter//2, outError, inError*eLevel) 

533 return func[1: -1, 1: -1]