Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22"""Calculation of brighter-fatter effect correlations and kernels.""" 

23 

24__all__ = ['BrighterFatterKernelSolveTask', 

25 'BrighterFatterKernelSolveConfig'] 

26 

27import numpy as np 

28 

29import lsst.afw.math as afwMath 

30import lsst.pex.config as pexConfig 

31import lsst.pipe.base as pipeBase 

32import lsst.pipe.base.connectionTypes as cT 

33 

34from lsst.ip.isr import (BrighterFatterKernel) 

35from .utils import (funcPolynomial, irlsFit) 

36from ._lookupStaticCalibration import lookupStaticCalibration 

37 

38 

39class BrighterFatterKernelSolveConnections(pipeBase.PipelineTaskConnections, 

40 dimensions=("instrument", "exposure", "detector")): 

41 dummy = cT.Input( 

42 name="raw", 

43 doc="Dummy exposure.", 

44 storageClass='Exposure', 

45 dimensions=("instrument", "exposure", "detector"), 

46 multiple=True, 

47 deferLoad=True, 

48 ) 

49 camera = cT.PrerequisiteInput( 

50 name="camera", 

51 doc="Camera associated with this data.", 

52 storageClass="Camera", 

53 dimensions=("instrument", ), 

54 isCalibration=True, 

55 lookupFunction=lookupStaticCalibration, 

56 ) 

57 inputPtc = cT.PrerequisiteInput( 

58 name="ptc", 

59 doc="Photon transfer curve dataset.", 

60 storageClass="PhotonTransferCurveDataset", 

61 dimensions=("instrument", "detector"), 

62 isCalibration=True, 

63 ) 

64 

65 outputBFK = cT.Output( 

66 name="brighterFatterKernel", 

67 doc="Output measured brighter-fatter kernel.", 

68 storageClass="BrighterFatterKernel", 

69 dimensions=("instrument", "detector"), 

70 isCalibration=True, 

71 ) 

72 

73 

74class BrighterFatterKernelSolveConfig(pipeBase.PipelineTaskConfig, 

75 pipelineConnections=BrighterFatterKernelSolveConnections): 

76 level = pexConfig.ChoiceField( 

77 doc="The level at which to calculate the brighter-fatter kernels", 

78 dtype=str, 

79 default="AMP", 

80 allowed={ 

81 "AMP": "Every amplifier treated separately", 

82 "DETECTOR": "One kernel per detector", 

83 } 

84 ) 

85 ignoreAmpsForAveraging = pexConfig.ListField( 

86 dtype=str, 

87 doc="List of amp names to ignore when averaging the amplifier kernels into the detector" 

88 " kernel. Only relevant for level = DETECTOR", 

89 default=[] 

90 ) 

91 xcorrCheckRejectLevel = pexConfig.Field( 

92 dtype=float, 

93 doc="Rejection level for the sum of the input cross-correlations. Arrays which " 

94 "sum to greater than this are discarded before the clipped mean is calculated.", 

95 default=2.0 

96 ) 

97 nSigmaClip = pexConfig.Field( 

98 dtype=float, 

99 doc="Number of sigma to clip when calculating means for the cross-correlation", 

100 default=5 

101 ) 

102 forceZeroSum = pexConfig.Field( 

103 dtype=bool, 

104 doc="Force the correlation matrix to have zero sum by adjusting the (0,0) value?", 

105 default=False, 

106 ) 

107 useAmatrix = pexConfig.Field( 

108 dtype=bool, 

109 doc="Use the PTC 'a' matrix (Astier et al. 2019 equation 20) " 

110 "instead of the average of measured covariances?", 

111 default=False, 

112 ) 

113 

114 maxIterSuccessiveOverRelaxation = pexConfig.Field( 

115 dtype=int, 

116 doc="The maximum number of iterations allowed for the successive over-relaxation method", 

117 default=10000 

118 ) 

119 eLevelSuccessiveOverRelaxation = pexConfig.Field( 

120 dtype=float, 

121 doc="The target residual error for the successive over-relaxation method", 

122 default=5.0e-14 

123 ) 

124 

125 correlationQuadraticFit = pexConfig.Field( 

126 dtype=bool, 

127 doc="Use a quadratic fit to find the correlations instead of simple averaging?", 

128 default=False, 

129 ) 

130 correlationModelRadius = pexConfig.Field( 

131 dtype=int, 

132 doc="Build a model of the correlation coefficients for radii larger than this value in pixels?", 

133 default=100, 

134 ) 

135 correlationModelSlope = pexConfig.Field( 

136 dtype=float, 

137 doc="Slope of the correlation model for radii larger than correlationModelRadius", 

138 default=-1.35, 

139 ) 

140 

141 

142class BrighterFatterKernelSolveTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

143 """Measure appropriate Brighter-Fatter Kernel from the PTC dataset. 

144 

145 """ 

146 ConfigClass = BrighterFatterKernelSolveConfig 

147 _DefaultName = 'cpBfkMeasure' 

148 

149 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

150 """Ensure that the input and output dimensions are passed along. 

151 

152 Parameters 

153 ---------- 

154 butlerQC : `lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext` 

155 Butler to operate on. 

156 inputRefs : `lsst.pipe.base.connections.InputQuantizedConnection` 

157 Input data refs to load. 

158 ouptutRefs : `lsst.pipe.base.connections.OutputQuantizedConnection` 

159 Output data refs to persist. 

160 """ 

161 inputs = butlerQC.get(inputRefs) 

162 

163 # Use the dimensions to set calib/provenance information. 

164 inputs['inputDims'] = inputRefs.inputPtc.dataId.byName() 

165 

166 outputs = self.run(**inputs) 

167 butlerQC.put(outputs, outputRefs) 

168 

169 def run(self, inputPtc, dummy, camera, inputDims): 

170 """Combine covariance information from PTC into brighter-fatter kernels. 

171 

172 Parameters 

173 ---------- 

174 inputPtc : `lsst.ip.isr.PhotonTransferCurveDataset` 

175 PTC data containing per-amplifier covariance measurements. 

176 dummy : `lsst.afw.image.Exposure` 

177 The exposure used to select the appropriate PTC dataset. 

178 In almost all circumstances, one of the input exposures 

179 used to generate the PTC dataset is the best option. 

180 camera : `lsst.afw.cameraGeom.Camera` 

181 Camera to use for camera geometry information. 

182 inputDims : `lsst.daf.butler.DataCoordinate` or `dict` 

183 DataIds to use to populate the output calibration. 

184 

185 Returns 

186 ------- 

187 results : `lsst.pipe.base.Struct` 

188 The resulst struct containing: 

189 

190 ``outputBfk`` : `lsst.ip.isr.BrighterFatterKernel` 

191 Resulting Brighter-Fatter Kernel. 

192 """ 

193 if len(dummy) == 0: 

194 self.log.warn("No dummy exposure found.") 

195 

196 detector = camera[inputDims['detector']] 

197 detName = detector.getName() 

198 

199 if self.config.level == 'DETECTOR': 

200 detectorCorrList = list() 

201 

202 bfk = BrighterFatterKernel(camera=camera, detectorId=detector.getId(), level=self.config.level) 

203 bfk.means = inputPtc.finalMeans # ADU 

204 bfk.variances = inputPtc.finalVars # ADU^2 

205 # Use the PTC covariances as the cross-correlations. These 

206 # are scaled before the kernel is generated, which performs 

207 # the conversion. 

208 bfk.rawXcorrs = inputPtc.covariances # ADU^2 

209 bfk.badAmps = inputPtc.badAmps 

210 bfk.shape = (inputPtc.covMatrixSide*2 + 1, inputPtc.covMatrixSide*2 + 1) 

211 bfk.gain = inputPtc.gain 

212 bfk.noise = inputPtc.noise 

213 bfk.meanXcorrs = dict() 

214 bfk.valid = dict() 

215 

216 for amp in detector: 

217 ampName = amp.getName() 

218 mask = np.array(inputPtc.expIdMask[ampName], dtype=bool) 

219 

220 gain = bfk.gain[ampName] 

221 fluxes = np.array(bfk.means[ampName])[mask] 

222 variances = np.array(bfk.variances[ampName])[mask] 

223 xCorrList = [np.array(xcorr) for xcorr in bfk.rawXcorrs[ampName]] 

224 xCorrList = np.array(xCorrList)[mask] 

225 

226 if gain <= 0: 

227 # We've received very bad data. 

228 self.log.warn("Impossible gain recieved from PTC for %s: %f. Skipping amplifier.", 

229 ampName, gain) 

230 bfk.meanXcorrs[ampName] = np.zeros(bfk.shape) 

231 bfk.ampKernels[ampName] = np.zeros(bfk.shape) 

232 bfk.valid[ampName] = False 

233 continue 

234 

235 fluxes = np.array([flux*gain for flux in fluxes]) # Now in e^- 

236 variances = np.array([variance*gain*gain for variance in variances]) # Now in e^2- 

237 

238 # This should duplicate Coulton et al. 2017 Equation 22-29 (arxiv:1711.06273) 

239 scaledCorrList = list() 

240 for xcorrNum, (xcorr, flux, var) in enumerate(zip(xCorrList, fluxes, variances), 1): 

241 q = np.array(xcorr) * gain * gain # xcorr now in e^- 

242 q *= 2.0 # Remove factor of 1/2 applied in PTC. 

243 self.log.info("Amp: %s %d/%d Flux: %f Var: %f Q(0,0): %g Q(1,0): %g Q(0,1): %g", 

244 ampName, xcorrNum, len(xCorrList), flux, var, q[0][0], q[1][0], q[0][1]) 

245 

246 # Normalize by the flux, which removes the (0,0) 

247 # component attributable to Poisson noise. This 

248 # contains the two "t I delta(x - x')" terms in Coulton et al. 2017 equation 29 

249 q[0][0] -= 2.0*(flux) 

250 

251 if q[0][0] > 0.0: 

252 self.log.warn("Amp: %s %d skipped due to value of (variance-mean)=%f", 

253 ampName, xcorrNum, q[0][0]) 

254 continue 

255 

256 # This removes the "t (I_a^2 + I_b^2)" factor in Coulton et al. 2017 equation 29. 

257 q /= -2.0*(flux**2) 

258 scaled = self._tileArray(q) 

259 

260 xcorrCheck = np.abs(np.sum(scaled))/np.sum(np.abs(scaled)) 

261 if (xcorrCheck > self.config.xcorrCheckRejectLevel) or not (np.isfinite(xcorrCheck)): 

262 self.log.warn("Amp: %s %d skipped due to value of triangle-inequality sum %f", 

263 ampName, xcorrNum, xcorrCheck) 

264 continue 

265 

266 scaledCorrList.append(scaled) 

267 self.log.info("Amp: %s %d/%d Final: %g XcorrCheck: %f", 

268 ampName, xcorrNum, len(xCorrList), q[0][0], xcorrCheck) 

269 

270 if len(scaledCorrList) == 0: 

271 self.log.warn("Amp: %s All inputs rejected for amp!", ampName) 

272 bfk.meanXcorrs[ampName] = np.zeros(bfk.shape) 

273 bfk.ampKernels[ampName] = np.zeros(bfk.shape) 

274 bfk.valid[ampName] = False 

275 continue 

276 

277 if self.config.useAmatrix: 

278 # Use the aMatrix, ignoring the meanXcorr generated above. 

279 preKernel = np.pad(self._tileArray(np.array(inputPtc.aMatrix[ampName])), ((1, 1))) 

280 elif self.config.correlationQuadraticFit: 

281 # Use a quadratic fit to the correlations as a function of flux. 

282 preKernel = self.quadraticCorrelations(scaledCorrList, fluxes, f"Amp: {ampName}") 

283 else: 

284 # Use a simple average of the measured correlations. 

285 preKernel = self.averageCorrelations(scaledCorrList, f"Amp: {ampName}") 

286 

287 center = int((bfk.shape[0] - 1) / 2) 

288 

289 if self.config.forceZeroSum: 

290 totalSum = np.sum(preKernel) 

291 

292 if self.config.correlationModelRadius < (preKernel.shape[0] - 1) / 2: 

293 # Assume a correlation model of Corr(r) = -preFactor * r^(2 * slope) 

294 preFactor = np.sqrt(preKernel[center, center + 1] * preKernel[center + 1, center]) 

295 slopeFactor = 2.0 * np.abs(self.config.correlationModelSlope) 

296 totalSum += 2.0*np.pi*(preFactor / (slopeFactor*(center + 0.5))**slopeFactor) 

297 

298 preKernel[center, center] -= totalSum 

299 self.log.info("%s Zero-Sum Scale: %g", ampName, totalSum) 

300 

301 finalSum = np.sum(preKernel) 

302 bfk.meanXcorrs[ampName] = preKernel 

303 

304 postKernel = self.successiveOverRelax(preKernel) 

305 bfk.ampKernels[ampName] = postKernel 

306 if self.config.level == 'DETECTOR': 

307 detectorCorrList.extend(scaledCorrList) 

308 bfk.valid[ampName] = True 

309 self.log.info("Amp: %s Sum: %g Center Info Pre: %g Post: %g", 

310 ampName, finalSum, preKernel[center, center], postKernel[center, center]) 

311 

312 # Assemble a detector kernel? 

313 if self.config.level == 'DETECTOR': 

314 preKernel = self.averageCorrelations(detectorCorrList, f"Det: {detName}") 

315 finalSum = np.sum(preKernel) 

316 center = int((bfk.shape[0] - 1) / 2) 

317 

318 postKernel = self.successiveOverRelax(preKernel) 

319 bfk.detKernels[detName] = postKernel 

320 self.log.info("Det: %s Sum: %g Center Info Pre: %g Post: %g", 

321 detName, finalSum, preKernel[center, center], postKernel[center, center]) 

322 

323 return pipeBase.Struct( 

324 outputBFK=bfk, 

325 ) 

326 

327 def averageCorrelations(self, xCorrList, name): 

328 """Average input correlations. 

329 

330 Parameters 

331 ---------- 

332 xCorrList : `list` [`numpy.array`] 

333 List of cross-correlations. 

334 name : `str` 

335 Name for log messages. 

336 

337 Returns 

338 ------- 

339 meanXcorr : `numpy.array` 

340 The averaged cross-correlation. 

341 """ 

342 meanXcorr = np.zeros_like(xCorrList[0]) 

343 xCorrList = np.transpose(xCorrList) 

344 sctrl = afwMath.StatisticsControl() 

345 sctrl.setNumSigmaClip(self.config.nSigmaClip) 

346 for i in range(np.shape(meanXcorr)[0]): 

347 for j in range(np.shape(meanXcorr)[1]): 

348 meanXcorr[i, j] = afwMath.makeStatistics(xCorrList[i, j], 

349 afwMath.MEANCLIP, sctrl).getValue() 

350 

351 # To match previous definitions, pad by one element. 

352 meanXcorr = np.pad(meanXcorr, ((1, 1))) 

353 

354 return meanXcorr 

355 

356 def quadraticCorrelations(self, xCorrList, fluxList, name): 

357 """Measure a quadratic correlation model. 

358 

359 Parameters 

360 ---------- 

361 xCorrList : `list` [`numpy.array`] 

362 List of cross-correlations. 

363 fluxList : `numpy.array` 

364 Associated list of fluxes. 

365 name : `str` 

366 Name for log messages. 

367 

368 Returns 

369 ------- 

370 meanXcorr : `numpy.array` 

371 The averaged cross-correlation. 

372 """ 

373 meanXcorr = np.zeros_like(xCorrList[0]) 

374 fluxList = np.square(fluxList) 

375 xCorrList = np.array(xCorrList) 

376 

377 for i in range(np.shape(meanXcorr)[0]): 

378 for j in range(np.shape(meanXcorr)[1]): 

379 # Fit corrlation_i(x, y) = a0 + a1 * (flux_i)^2 The 

380 # i,j indices are inverted to apply the transposition, 

381 # as is done in the averaging case. 

382 linearFit, linearFitErr, chiSq, weights = irlsFit([0.0, 1e-4], fluxList, 

383 xCorrList[:, j, i], funcPolynomial) 

384 meanXcorr[i, j] = linearFit[1] # Discard the intercept. 

385 self.log.debug("Quad fit meanXcorr[%d,%d] = %g", i, j, linearFit[1]) 

386 

387 # To match previous definitions, pad by one element. 

388 meanXcorr = np.pad(meanXcorr, ((1, 1))) 

389 

390 return meanXcorr 

391 

392 @staticmethod 

393 def _tileArray(in_array): 

394 """Given an input quarter-image, tile/mirror it and return full image. 

395 

396 Given a square input of side-length n, of the form 

397 

398 input = array([[1, 2, 3], 

399 [4, 5, 6], 

400 [7, 8, 9]]) 

401 

402 return an array of size 2n-1 as 

403 

404 output = array([[ 9, 8, 7, 8, 9], 

405 [ 6, 5, 4, 5, 6], 

406 [ 3, 2, 1, 2, 3], 

407 [ 6, 5, 4, 5, 6], 

408 [ 9, 8, 7, 8, 9]]) 

409 

410 Parameters: 

411 ----------- 

412 input : `np.array` 

413 The square input quarter-array 

414 

415 Returns: 

416 -------- 

417 output : `np.array` 

418 The full, tiled array 

419 """ 

420 assert(in_array.shape[0] == in_array.shape[1]) 

421 length = in_array.shape[0] - 1 

422 output = np.zeros((2*length + 1, 2*length + 1)) 

423 

424 for i in range(length + 1): 

425 for j in range(length + 1): 

426 output[i + length, j + length] = in_array[i, j] 

427 output[-i + length, j + length] = in_array[i, j] 

428 output[i + length, -j + length] = in_array[i, j] 

429 output[-i + length, -j + length] = in_array[i, j] 

430 return output 

431 

432 def successiveOverRelax(self, source, maxIter=None, eLevel=None): 

433 """An implementation of the successive over relaxation (SOR) method. 

434 

435 A numerical method for solving a system of linear equations 

436 with faster convergence than the Gauss-Seidel method. 

437 

438 Parameters: 

439 ----------- 

440 source : `numpy.ndarray` 

441 The input array. 

442 maxIter : `int`, optional 

443 Maximum number of iterations to attempt before aborting. 

444 eLevel : `float`, optional 

445 The target error level at which we deem convergence to have 

446 occurred. 

447 

448 Returns: 

449 -------- 

450 output : `numpy.ndarray` 

451 The solution. 

452 """ 

453 if not maxIter: 

454 maxIter = self.config.maxIterSuccessiveOverRelaxation 

455 if not eLevel: 

456 eLevel = self.config.eLevelSuccessiveOverRelaxation 

457 

458 assert source.shape[0] == source.shape[1], "Input array must be square" 

459 # initialize, and set boundary conditions 

460 func = np.zeros([source.shape[0] + 2, source.shape[1] + 2]) 

461 resid = np.zeros([source.shape[0] + 2, source.shape[1] + 2]) 

462 rhoSpe = np.cos(np.pi/source.shape[0]) # Here a square grid is assumed 

463 

464 # Calculate the initial error 

465 for i in range(1, func.shape[0] - 1): 

466 for j in range(1, func.shape[1] - 1): 

467 resid[i, j] = (func[i, j - 1] + func[i, j + 1] + func[i - 1, j] 

468 + func[i + 1, j] - 4*func[i, j] - source[i - 1, j - 1]) 

469 inError = np.sum(np.abs(resid)) 

470 

471 # Iterate until convergence 

472 # We perform two sweeps per cycle, 

473 # updating 'odd' and 'even' points separately 

474 nIter = 0 

475 omega = 1.0 

476 dx = 1.0 

477 while nIter < maxIter*2: 

478 outError = 0 

479 if nIter%2 == 0: 

480 for i in range(1, func.shape[0] - 1, 2): 

481 for j in range(1, func.shape[1] - 1, 2): 

482 resid[i, j] = float(func[i, j-1] + func[i, j + 1] + func[i - 1, j] 

483 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1]) 

484 func[i, j] += omega*resid[i, j]*.25 

485 for i in range(2, func.shape[0] - 1, 2): 

486 for j in range(2, func.shape[1] - 1, 2): 

487 resid[i, j] = float(func[i, j - 1] + func[i, j + 1] + func[i - 1, j] 

488 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1]) 

489 func[i, j] += omega*resid[i, j]*.25 

490 else: 

491 for i in range(1, func.shape[0] - 1, 2): 

492 for j in range(2, func.shape[1] - 1, 2): 

493 resid[i, j] = float(func[i, j - 1] + func[i, j + 1] + func[i - 1, j] 

494 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1]) 

495 func[i, j] += omega*resid[i, j]*.25 

496 for i in range(2, func.shape[0] - 1, 2): 

497 for j in range(1, func.shape[1] - 1, 2): 

498 resid[i, j] = float(func[i, j - 1] + func[i, j + 1] + func[i - 1, j] 

499 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1]) 

500 func[i, j] += omega*resid[i, j]*.25 

501 outError = np.sum(np.abs(resid)) 

502 if outError < inError*eLevel: 

503 break 

504 if nIter == 0: 

505 omega = 1.0/(1 - rhoSpe*rhoSpe/2.0) 

506 else: 

507 omega = 1.0/(1 - rhoSpe*rhoSpe*omega/4.0) 

508 nIter += 1 

509 

510 if nIter >= maxIter*2: 

511 self.log.warn("Failure: SuccessiveOverRelaxation did not converge in %s iterations." 

512 "\noutError: %s, inError: %s," % (nIter//2, outError, inError*eLevel)) 

513 else: 

514 self.log.info("Success: SuccessiveOverRelaxation converged in %s iterations." 

515 "\noutError: %s, inError: %s", nIter//2, outError, inError*eLevel) 

516 return func[1: -1, 1: -1]