Coverage for python/lsst/cp/pipe/makeBrighterFatterKernel.py : 13%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22"""Calculation of brighter-fatter effect correlations and kernels."""
24__all__ = ['BrighterFatterKernelSolveTask',
25 'BrighterFatterKernelSolveConfig']
27import numpy as np
29import lsst.afw.math as afwMath
30import lsst.pex.config as pexConfig
31import lsst.pipe.base as pipeBase
32import lsst.pipe.base.connectionTypes as cT
34from lsst.ip.isr import (BrighterFatterKernel)
35from .utils import (funcPolynomial, irlsFit)
36from ._lookupStaticCalibration import lookupStaticCalibration
39class BrighterFatterKernelSolveConnections(pipeBase.PipelineTaskConnections,
40 dimensions=("instrument", "exposure", "detector")):
41 dummy = cT.Input(
42 name="raw",
43 doc="Dummy exposure.",
44 storageClass='Exposure',
45 dimensions=("instrument", "exposure", "detector"),
46 multiple=True,
47 deferLoad=True,
48 )
49 camera = cT.PrerequisiteInput(
50 name="camera",
51 doc="Camera associated with this data.",
52 storageClass="Camera",
53 dimensions=("instrument", ),
54 isCalibration=True,
55 lookupFunction=lookupStaticCalibration,
56 )
57 inputPtc = cT.PrerequisiteInput(
58 name="ptc",
59 doc="Photon transfer curve dataset.",
60 storageClass="PhotonTransferCurveDataset",
61 dimensions=("instrument", "detector"),
62 isCalibration=True,
63 )
65 outputBFK = cT.Output(
66 name="brighterFatterKernel",
67 doc="Output measured brighter-fatter kernel.",
68 storageClass="BrighterFatterKernel",
69 dimensions=("instrument", "detector"),
70 isCalibration=True,
71 )
74class BrighterFatterKernelSolveConfig(pipeBase.PipelineTaskConfig,
75 pipelineConnections=BrighterFatterKernelSolveConnections):
76 level = pexConfig.ChoiceField(
77 doc="The level at which to calculate the brighter-fatter kernels",
78 dtype=str,
79 default="AMP",
80 allowed={
81 "AMP": "Every amplifier treated separately",
82 "DETECTOR": "One kernel per detector",
83 }
84 )
85 ignoreAmpsForAveraging = pexConfig.ListField(
86 dtype=str,
87 doc="List of amp names to ignore when averaging the amplifier kernels into the detector"
88 " kernel. Only relevant for level = DETECTOR",
89 default=[]
90 )
91 xcorrCheckRejectLevel = pexConfig.Field(
92 dtype=float,
93 doc="Rejection level for the sum of the input cross-correlations. Arrays which "
94 "sum to greater than this are discarded before the clipped mean is calculated.",
95 default=2.0
96 )
97 nSigmaClip = pexConfig.Field(
98 dtype=float,
99 doc="Number of sigma to clip when calculating means for the cross-correlation",
100 default=5
101 )
102 forceZeroSum = pexConfig.Field(
103 dtype=bool,
104 doc="Force the correlation matrix to have zero sum by adjusting the (0,0) value?",
105 default=False,
106 )
107 useAmatrix = pexConfig.Field(
108 dtype=bool,
109 doc="Use the PTC 'a' matrix (Astier et al. 2019 equation 20) "
110 "instead of the average of measured covariances?",
111 default=False,
112 )
114 maxIterSuccessiveOverRelaxation = pexConfig.Field(
115 dtype=int,
116 doc="The maximum number of iterations allowed for the successive over-relaxation method",
117 default=10000
118 )
119 eLevelSuccessiveOverRelaxation = pexConfig.Field(
120 dtype=float,
121 doc="The target residual error for the successive over-relaxation method",
122 default=5.0e-14
123 )
125 correlationQuadraticFit = pexConfig.Field(
126 dtype=bool,
127 doc="Use a quadratic fit to find the correlations instead of simple averaging?",
128 default=False,
129 )
130 correlationModelRadius = pexConfig.Field(
131 dtype=int,
132 doc="Build a model of the correlation coefficients for radii larger than this value in pixels?",
133 default=100,
134 )
135 correlationModelSlope = pexConfig.Field(
136 dtype=float,
137 doc="Slope of the correlation model for radii larger than correlationModelRadius",
138 default=-1.35,
139 )
142class BrighterFatterKernelSolveTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
143 """Measure appropriate Brighter-Fatter Kernel from the PTC dataset.
145 """
146 ConfigClass = BrighterFatterKernelSolveConfig
147 _DefaultName = 'cpBfkMeasure'
149 def runQuantum(self, butlerQC, inputRefs, outputRefs):
150 """Ensure that the input and output dimensions are passed along.
152 Parameters
153 ----------
154 butlerQC : `lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
155 Butler to operate on.
156 inputRefs : `lsst.pipe.base.connections.InputQuantizedConnection`
157 Input data refs to load.
158 ouptutRefs : `lsst.pipe.base.connections.OutputQuantizedConnection`
159 Output data refs to persist.
160 """
161 inputs = butlerQC.get(inputRefs)
163 # Use the dimensions to set calib/provenance information.
164 inputs['inputDims'] = inputRefs.inputPtc.dataId.byName()
166 outputs = self.run(**inputs)
167 butlerQC.put(outputs, outputRefs)
169 def run(self, inputPtc, dummy, camera, inputDims):
170 """Combine covariance information from PTC into brighter-fatter kernels.
172 Parameters
173 ----------
174 inputPtc : `lsst.ip.isr.PhotonTransferCurveDataset`
175 PTC data containing per-amplifier covariance measurements.
176 dummy : `lsst.afw.image.Exposure`
177 The exposure used to select the appropriate PTC dataset.
178 In almost all circumstances, one of the input exposures
179 used to generate the PTC dataset is the best option.
180 camera : `lsst.afw.cameraGeom.Camera`
181 Camera to use for camera geometry information.
182 inputDims : `lsst.daf.butler.DataCoordinate` or `dict`
183 DataIds to use to populate the output calibration.
185 Returns
186 -------
187 results : `lsst.pipe.base.Struct`
188 The resulst struct containing:
190 ``outputBfk`` : `lsst.ip.isr.BrighterFatterKernel`
191 Resulting Brighter-Fatter Kernel.
192 """
193 if len(dummy) == 0:
194 self.log.warn("No dummy exposure found.")
196 detector = camera[inputDims['detector']]
197 detName = detector.getName()
199 if self.config.level == 'DETECTOR':
200 detectorCorrList = list()
202 bfk = BrighterFatterKernel(camera=camera, detectorId=detector.getId(), level=self.config.level)
203 bfk.means = inputPtc.finalMeans # ADU
204 bfk.variances = inputPtc.finalVars # ADU^2
205 # Use the PTC covariances as the cross-correlations. These
206 # are scaled before the kernel is generated, which performs
207 # the conversion.
208 bfk.rawXcorrs = inputPtc.covariances # ADU^2
209 bfk.badAmps = inputPtc.badAmps
210 bfk.shape = (inputPtc.covMatrixSide*2 + 1, inputPtc.covMatrixSide*2 + 1)
211 bfk.gain = inputPtc.gain
212 bfk.noise = inputPtc.noise
213 bfk.meanXcorrs = dict()
214 bfk.valid = dict()
216 for amp in detector:
217 ampName = amp.getName()
218 mask = np.array(inputPtc.expIdMask[ampName], dtype=bool)
220 gain = bfk.gain[ampName]
221 fluxes = np.array(bfk.means[ampName])[mask]
222 variances = np.array(bfk.variances[ampName])[mask]
223 xCorrList = [np.array(xcorr) for xcorr in bfk.rawXcorrs[ampName]]
224 xCorrList = np.array(xCorrList)[mask]
226 if gain <= 0:
227 # We've received very bad data.
228 self.log.warn("Impossible gain recieved from PTC for %s: %f. Skipping amplifier.",
229 ampName, gain)
230 bfk.meanXcorrs[ampName] = np.zeros(bfk.shape)
231 bfk.ampKernels[ampName] = np.zeros(bfk.shape)
232 bfk.valid[ampName] = False
233 continue
235 fluxes = np.array([flux*gain for flux in fluxes]) # Now in e^-
236 variances = np.array([variance*gain*gain for variance in variances]) # Now in e^2-
238 # This should duplicate Coulton et al. 2017 Equation 22-29 (arxiv:1711.06273)
239 scaledCorrList = list()
240 for xcorrNum, (xcorr, flux, var) in enumerate(zip(xCorrList, fluxes, variances), 1):
241 q = np.array(xcorr) * gain * gain # xcorr now in e^-
242 q *= 2.0 # Remove factor of 1/2 applied in PTC.
243 self.log.info("Amp: %s %d/%d Flux: %f Var: %f Q(0,0): %g Q(1,0): %g Q(0,1): %g",
244 ampName, xcorrNum, len(xCorrList), flux, var, q[0][0], q[1][0], q[0][1])
246 # Normalize by the flux, which removes the (0,0)
247 # component attributable to Poisson noise. This
248 # contains the two "t I delta(x - x')" terms in Coulton et al. 2017 equation 29
249 q[0][0] -= 2.0*(flux)
251 if q[0][0] > 0.0:
252 self.log.warn("Amp: %s %d skipped due to value of (variance-mean)=%f",
253 ampName, xcorrNum, q[0][0])
254 continue
256 # This removes the "t (I_a^2 + I_b^2)" factor in Coulton et al. 2017 equation 29.
257 q /= -2.0*(flux**2)
258 scaled = self._tileArray(q)
260 xcorrCheck = np.abs(np.sum(scaled))/np.sum(np.abs(scaled))
261 if (xcorrCheck > self.config.xcorrCheckRejectLevel) or not (np.isfinite(xcorrCheck)):
262 self.log.warn("Amp: %s %d skipped due to value of triangle-inequality sum %f",
263 ampName, xcorrNum, xcorrCheck)
264 continue
266 scaledCorrList.append(scaled)
267 self.log.info("Amp: %s %d/%d Final: %g XcorrCheck: %f",
268 ampName, xcorrNum, len(xCorrList), q[0][0], xcorrCheck)
270 if len(scaledCorrList) == 0:
271 self.log.warn("Amp: %s All inputs rejected for amp!", ampName)
272 bfk.meanXcorrs[ampName] = np.zeros(bfk.shape)
273 bfk.ampKernels[ampName] = np.zeros(bfk.shape)
274 bfk.valid[ampName] = False
275 continue
277 if self.config.useAmatrix:
278 # Use the aMatrix, ignoring the meanXcorr generated above.
279 preKernel = np.pad(self._tileArray(np.array(inputPtc.aMatrix[ampName])), ((1, 1)))
280 elif self.config.correlationQuadraticFit:
281 # Use a quadratic fit to the correlations as a function of flux.
282 preKernel = self.quadraticCorrelations(scaledCorrList, fluxes, f"Amp: {ampName}")
283 else:
284 # Use a simple average of the measured correlations.
285 preKernel = self.averageCorrelations(scaledCorrList, f"Amp: {ampName}")
287 center = int((bfk.shape[0] - 1) / 2)
289 if self.config.forceZeroSum:
290 totalSum = np.sum(preKernel)
292 if self.config.correlationModelRadius < (preKernel.shape[0] - 1) / 2:
293 # Assume a correlation model of Corr(r) = -preFactor * r^(2 * slope)
294 preFactor = np.sqrt(preKernel[center, center + 1] * preKernel[center + 1, center])
295 slopeFactor = 2.0 * np.abs(self.config.correlationModelSlope)
296 totalSum += 2.0*np.pi*(preFactor / (slopeFactor*(center + 0.5))**slopeFactor)
298 preKernel[center, center] -= totalSum
299 self.log.info("%s Zero-Sum Scale: %g", ampName, totalSum)
301 finalSum = np.sum(preKernel)
302 bfk.meanXcorrs[ampName] = preKernel
304 postKernel = self.successiveOverRelax(preKernel)
305 bfk.ampKernels[ampName] = postKernel
306 if self.config.level == 'DETECTOR':
307 detectorCorrList.extend(scaledCorrList)
308 bfk.valid[ampName] = True
309 self.log.info("Amp: %s Sum: %g Center Info Pre: %g Post: %g",
310 ampName, finalSum, preKernel[center, center], postKernel[center, center])
312 # Assemble a detector kernel?
313 if self.config.level == 'DETECTOR':
314 preKernel = self.averageCorrelations(detectorCorrList, f"Det: {detName}")
315 finalSum = np.sum(preKernel)
316 center = int((bfk.shape[0] - 1) / 2)
318 postKernel = self.successiveOverRelax(preKernel)
319 bfk.detKernels[detName] = postKernel
320 self.log.info("Det: %s Sum: %g Center Info Pre: %g Post: %g",
321 detName, finalSum, preKernel[center, center], postKernel[center, center])
323 return pipeBase.Struct(
324 outputBFK=bfk,
325 )
327 def averageCorrelations(self, xCorrList, name):
328 """Average input correlations.
330 Parameters
331 ----------
332 xCorrList : `list` [`numpy.array`]
333 List of cross-correlations.
334 name : `str`
335 Name for log messages.
337 Returns
338 -------
339 meanXcorr : `numpy.array`
340 The averaged cross-correlation.
341 """
342 meanXcorr = np.zeros_like(xCorrList[0])
343 xCorrList = np.transpose(xCorrList)
344 sctrl = afwMath.StatisticsControl()
345 sctrl.setNumSigmaClip(self.config.nSigmaClip)
346 for i in range(np.shape(meanXcorr)[0]):
347 for j in range(np.shape(meanXcorr)[1]):
348 meanXcorr[i, j] = afwMath.makeStatistics(xCorrList[i, j],
349 afwMath.MEANCLIP, sctrl).getValue()
351 # To match previous definitions, pad by one element.
352 meanXcorr = np.pad(meanXcorr, ((1, 1)))
354 return meanXcorr
356 def quadraticCorrelations(self, xCorrList, fluxList, name):
357 """Measure a quadratic correlation model.
359 Parameters
360 ----------
361 xCorrList : `list` [`numpy.array`]
362 List of cross-correlations.
363 fluxList : `numpy.array`
364 Associated list of fluxes.
365 name : `str`
366 Name for log messages.
368 Returns
369 -------
370 meanXcorr : `numpy.array`
371 The averaged cross-correlation.
372 """
373 meanXcorr = np.zeros_like(xCorrList[0])
374 fluxList = np.square(fluxList)
375 xCorrList = np.array(xCorrList)
377 for i in range(np.shape(meanXcorr)[0]):
378 for j in range(np.shape(meanXcorr)[1]):
379 # Fit corrlation_i(x, y) = a0 + a1 * (flux_i)^2 The
380 # i,j indices are inverted to apply the transposition,
381 # as is done in the averaging case.
382 linearFit, linearFitErr, chiSq, weights = irlsFit([0.0, 1e-4], fluxList,
383 xCorrList[:, j, i], funcPolynomial)
384 meanXcorr[i, j] = linearFit[1] # Discard the intercept.
385 self.log.debug("Quad fit meanXcorr[%d,%d] = %g", i, j, linearFit[1])
387 # To match previous definitions, pad by one element.
388 meanXcorr = np.pad(meanXcorr, ((1, 1)))
390 return meanXcorr
392 @staticmethod
393 def _tileArray(in_array):
394 """Given an input quarter-image, tile/mirror it and return full image.
396 Given a square input of side-length n, of the form
398 input = array([[1, 2, 3],
399 [4, 5, 6],
400 [7, 8, 9]])
402 return an array of size 2n-1 as
404 output = array([[ 9, 8, 7, 8, 9],
405 [ 6, 5, 4, 5, 6],
406 [ 3, 2, 1, 2, 3],
407 [ 6, 5, 4, 5, 6],
408 [ 9, 8, 7, 8, 9]])
410 Parameters:
411 -----------
412 input : `np.array`
413 The square input quarter-array
415 Returns:
416 --------
417 output : `np.array`
418 The full, tiled array
419 """
420 assert(in_array.shape[0] == in_array.shape[1])
421 length = in_array.shape[0] - 1
422 output = np.zeros((2*length + 1, 2*length + 1))
424 for i in range(length + 1):
425 for j in range(length + 1):
426 output[i + length, j + length] = in_array[i, j]
427 output[-i + length, j + length] = in_array[i, j]
428 output[i + length, -j + length] = in_array[i, j]
429 output[-i + length, -j + length] = in_array[i, j]
430 return output
432 def successiveOverRelax(self, source, maxIter=None, eLevel=None):
433 """An implementation of the successive over relaxation (SOR) method.
435 A numerical method for solving a system of linear equations
436 with faster convergence than the Gauss-Seidel method.
438 Parameters:
439 -----------
440 source : `numpy.ndarray`
441 The input array.
442 maxIter : `int`, optional
443 Maximum number of iterations to attempt before aborting.
444 eLevel : `float`, optional
445 The target error level at which we deem convergence to have
446 occurred.
448 Returns:
449 --------
450 output : `numpy.ndarray`
451 The solution.
452 """
453 if not maxIter:
454 maxIter = self.config.maxIterSuccessiveOverRelaxation
455 if not eLevel:
456 eLevel = self.config.eLevelSuccessiveOverRelaxation
458 assert source.shape[0] == source.shape[1], "Input array must be square"
459 # initialize, and set boundary conditions
460 func = np.zeros([source.shape[0] + 2, source.shape[1] + 2])
461 resid = np.zeros([source.shape[0] + 2, source.shape[1] + 2])
462 rhoSpe = np.cos(np.pi/source.shape[0]) # Here a square grid is assumed
464 # Calculate the initial error
465 for i in range(1, func.shape[0] - 1):
466 for j in range(1, func.shape[1] - 1):
467 resid[i, j] = (func[i, j - 1] + func[i, j + 1] + func[i - 1, j]
468 + func[i + 1, j] - 4*func[i, j] - source[i - 1, j - 1])
469 inError = np.sum(np.abs(resid))
471 # Iterate until convergence
472 # We perform two sweeps per cycle,
473 # updating 'odd' and 'even' points separately
474 nIter = 0
475 omega = 1.0
476 dx = 1.0
477 while nIter < maxIter*2:
478 outError = 0
479 if nIter%2 == 0:
480 for i in range(1, func.shape[0] - 1, 2):
481 for j in range(1, func.shape[1] - 1, 2):
482 resid[i, j] = float(func[i, j-1] + func[i, j + 1] + func[i - 1, j]
483 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
484 func[i, j] += omega*resid[i, j]*.25
485 for i in range(2, func.shape[0] - 1, 2):
486 for j in range(2, func.shape[1] - 1, 2):
487 resid[i, j] = float(func[i, j - 1] + func[i, j + 1] + func[i - 1, j]
488 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
489 func[i, j] += omega*resid[i, j]*.25
490 else:
491 for i in range(1, func.shape[0] - 1, 2):
492 for j in range(2, func.shape[1] - 1, 2):
493 resid[i, j] = float(func[i, j - 1] + func[i, j + 1] + func[i - 1, j]
494 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
495 func[i, j] += omega*resid[i, j]*.25
496 for i in range(2, func.shape[0] - 1, 2):
497 for j in range(1, func.shape[1] - 1, 2):
498 resid[i, j] = float(func[i, j - 1] + func[i, j + 1] + func[i - 1, j]
499 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
500 func[i, j] += omega*resid[i, j]*.25
501 outError = np.sum(np.abs(resid))
502 if outError < inError*eLevel:
503 break
504 if nIter == 0:
505 omega = 1.0/(1 - rhoSpe*rhoSpe/2.0)
506 else:
507 omega = 1.0/(1 - rhoSpe*rhoSpe*omega/4.0)
508 nIter += 1
510 if nIter >= maxIter*2:
511 self.log.warn("Failure: SuccessiveOverRelaxation did not converge in %s iterations."
512 "\noutError: %s, inError: %s," % (nIter//2, outError, inError*eLevel))
513 else:
514 self.log.info("Success: SuccessiveOverRelaxation converged in %s iterations."
515 "\noutError: %s, inError: %s", nIter//2, outError, inError*eLevel)
516 return func[1: -1, 1: -1]