Coverage for python/lsst/cp/pipe/makeBrighterFatterKernel.py: 15%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22"""Calculation of brighter-fatter effect correlations and kernels."""
24__all__ = ['BrighterFatterKernelSolveTask',
25 'BrighterFatterKernelSolveConfig']
27import numpy as np
29import lsst.afw.math as afwMath
30import lsst.pex.config as pexConfig
31import lsst.pipe.base as pipeBase
32import lsst.pipe.base.connectionTypes as cT
34from lsst.ip.isr import (BrighterFatterKernel)
35from .utils import (funcPolynomial, irlsFit)
36from ._lookupStaticCalibration import lookupStaticCalibration
39class BrighterFatterKernelSolveConnections(pipeBase.PipelineTaskConnections,
40 dimensions=("instrument", "exposure", "detector")):
41 dummy = cT.Input(
42 name="raw",
43 doc="Dummy exposure.",
44 storageClass='Exposure',
45 dimensions=("instrument", "exposure", "detector"),
46 multiple=True,
47 deferLoad=True,
48 )
49 camera = cT.PrerequisiteInput(
50 name="camera",
51 doc="Camera associated with this data.",
52 storageClass="Camera",
53 dimensions=("instrument", ),
54 isCalibration=True,
55 lookupFunction=lookupStaticCalibration,
56 )
57 inputPtc = cT.PrerequisiteInput(
58 name="ptc",
59 doc="Photon transfer curve dataset.",
60 storageClass="PhotonTransferCurveDataset",
61 dimensions=("instrument", "detector"),
62 isCalibration=True,
63 )
65 outputBFK = cT.Output(
66 name="brighterFatterKernel",
67 doc="Output measured brighter-fatter kernel.",
68 storageClass="BrighterFatterKernel",
69 dimensions=("instrument", "detector"),
70 isCalibration=True,
71 )
74class BrighterFatterKernelSolveConfig(pipeBase.PipelineTaskConfig,
75 pipelineConnections=BrighterFatterKernelSolveConnections):
76 level = pexConfig.ChoiceField(
77 doc="The level at which to calculate the brighter-fatter kernels",
78 dtype=str,
79 default="AMP",
80 allowed={
81 "AMP": "Every amplifier treated separately",
82 "DETECTOR": "One kernel per detector",
83 }
84 )
85 ignoreAmpsForAveraging = pexConfig.ListField(
86 dtype=str,
87 doc="List of amp names to ignore when averaging the amplifier kernels into the detector"
88 " kernel. Only relevant for level = DETECTOR",
89 default=[]
90 )
91 xcorrCheckRejectLevel = pexConfig.Field(
92 dtype=float,
93 doc="Rejection level for the sum of the input cross-correlations. Arrays which "
94 "sum to greater than this are discarded before the clipped mean is calculated.",
95 default=2.0
96 )
97 nSigmaClip = pexConfig.Field(
98 dtype=float,
99 doc="Number of sigma to clip when calculating means for the cross-correlation",
100 default=5
101 )
102 forceZeroSum = pexConfig.Field(
103 dtype=bool,
104 doc="Force the correlation matrix to have zero sum by adjusting the (0,0) value?",
105 default=False,
106 )
107 useAmatrix = pexConfig.Field(
108 dtype=bool,
109 doc="Use the PTC 'a' matrix (Astier et al. 2019 equation 20) "
110 "instead of the average of measured covariances?",
111 default=False,
112 )
114 maxIterSuccessiveOverRelaxation = pexConfig.Field(
115 dtype=int,
116 doc="The maximum number of iterations allowed for the successive over-relaxation method",
117 default=10000
118 )
119 eLevelSuccessiveOverRelaxation = pexConfig.Field(
120 dtype=float,
121 doc="The target residual error for the successive over-relaxation method",
122 default=5.0e-14
123 )
125 correlationQuadraticFit = pexConfig.Field(
126 dtype=bool,
127 doc="Use a quadratic fit to find the correlations instead of simple averaging?",
128 default=False,
129 )
130 correlationModelRadius = pexConfig.Field(
131 dtype=int,
132 doc="Build a model of the correlation coefficients for radii larger than this value in pixels?",
133 default=100,
134 )
135 correlationModelSlope = pexConfig.Field(
136 dtype=float,
137 doc="Slope of the correlation model for radii larger than correlationModelRadius",
138 default=-1.35,
139 )
142class BrighterFatterKernelSolveTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
143 """Measure appropriate Brighter-Fatter Kernel from the PTC dataset.
144 """
146 ConfigClass = BrighterFatterKernelSolveConfig
147 _DefaultName = 'cpBfkMeasure'
149 def runQuantum(self, butlerQC, inputRefs, outputRefs):
150 """Ensure that the input and output dimensions are passed along.
152 Parameters
153 ----------
154 butlerQC : `lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
155 Butler to operate on.
156 inputRefs : `lsst.pipe.base.connections.InputQuantizedConnection`
157 Input data refs to load.
158 ouptutRefs : `lsst.pipe.base.connections.OutputQuantizedConnection`
159 Output data refs to persist.
160 """
161 inputs = butlerQC.get(inputRefs)
163 # Use the dimensions to set calib/provenance information.
164 inputs['inputDims'] = inputRefs.inputPtc.dataId.byName()
166 outputs = self.run(**inputs)
167 butlerQC.put(outputs, outputRefs)
169 def run(self, inputPtc, dummy, camera, inputDims):
170 """Combine covariance information from PTC into brighter-fatter
171 kernels.
173 Parameters
174 ----------
175 inputPtc : `lsst.ip.isr.PhotonTransferCurveDataset`
176 PTC data containing per-amplifier covariance measurements.
177 dummy : `lsst.afw.image.Exposure`
178 The exposure used to select the appropriate PTC dataset.
179 In almost all circumstances, one of the input exposures
180 used to generate the PTC dataset is the best option.
181 camera : `lsst.afw.cameraGeom.Camera`
182 Camera to use for camera geometry information.
183 inputDims : `lsst.daf.butler.DataCoordinate` or `dict`
184 DataIds to use to populate the output calibration.
186 Returns
187 -------
188 results : `lsst.pipe.base.Struct`
189 The resulst struct containing:
191 ``outputBfk``
192 Resulting Brighter-Fatter Kernel
193 (`lsst.ip.isr.BrighterFatterKernel`).
194 """
195 if len(dummy) == 0:
196 self.log.warning("No dummy exposure found.")
198 detector = camera[inputDims['detector']]
199 detName = detector.getName()
201 if self.config.level == 'DETECTOR':
202 detectorCorrList = list()
203 detectorFluxes = list()
205 bfk = BrighterFatterKernel(camera=camera, detectorId=detector.getId(), level=self.config.level)
206 bfk.means = inputPtc.finalMeans # ADU
207 bfk.variances = inputPtc.finalVars # ADU^2
208 # Use the PTC covariances as the cross-correlations. These
209 # are scaled before the kernel is generated, which performs
210 # the conversion.
211 bfk.rawXcorrs = inputPtc.covariances # ADU^2
212 bfk.badAmps = inputPtc.badAmps
213 bfk.shape = (inputPtc.covMatrixSide*2 + 1, inputPtc.covMatrixSide*2 + 1)
214 bfk.gain = inputPtc.gain
215 bfk.noise = inputPtc.noise
216 bfk.meanXcorrs = dict()
217 bfk.valid = dict()
219 for amp in detector:
220 ampName = amp.getName()
221 gain = bfk.gain[ampName]
223 # Using the inputPtc.expIdMask works if the covariance
224 # array has the same length as the rawMeans/rawVars. This
225 # isn't the case, as it's the same size as the
226 # finalMeans/finalVars. However, these arrays (and the
227 # covariance) are padded with NAN values to match the
228 # longest amplifier vector. We do not want to include
229 # these NAN values, so we construct a mask for all non-NAN
230 # values in finalMeans, and use that to filter finalVars
231 # and the covariances.
232 mask = np.isfinite(bfk.means[ampName])
233 fluxes = np.array(bfk.means[ampName])[mask]
234 variances = np.array(bfk.variances[ampName])[mask]
235 xCorrList = np.array([np.array(xcorr) for xcorr in bfk.rawXcorrs[ampName]])[mask]
237 if gain <= 0:
238 # We've received very bad data.
239 self.log.warning("Impossible gain recieved from PTC for %s: %f. Skipping amplifier.",
240 ampName, gain)
241 bfk.meanXcorrs[ampName] = np.zeros(bfk.shape)
242 bfk.ampKernels[ampName] = np.zeros(bfk.shape)
243 bfk.valid[ampName] = False
244 continue
246 fluxes = np.array([flux*gain for flux in fluxes]) # Now in e^-
247 variances = np.array([variance*gain*gain for variance in variances]) # Now in e^2-
249 # This should duplicate Coulton et al. 2017 Equation 22-29
250 # (arxiv:1711.06273)
251 scaledCorrList = list()
252 for xcorrNum, (xcorr, flux, var) in enumerate(zip(xCorrList, fluxes, variances), 1):
253 q = np.array(xcorr) * gain * gain # xcorr now in e^-
254 q *= 2.0 # Remove factor of 1/2 applied in PTC.
255 self.log.info("Amp: %s %d/%d Flux: %f Var: %f Q(0,0): %g Q(1,0): %g Q(0,1): %g",
256 ampName, xcorrNum, len(xCorrList), flux, var, q[0][0], q[1][0], q[0][1])
258 # Normalize by the flux, which removes the (0,0)
259 # component attributable to Poisson noise. This
260 # contains the two "t I delta(x - x')" terms in
261 # Coulton et al. 2017 equation 29
262 q[0][0] -= 2.0*(flux)
264 if q[0][0] > 0.0:
265 self.log.warning("Amp: %s %d skipped due to value of (variance-mean)=%f",
266 ampName, xcorrNum, q[0][0])
267 # If we drop an element of ``scaledCorrList``
268 # (which is what this does), we need to ensure we
269 # drop the flux entry as well.
270 fluxes = np.delete(fluxes, xcorrNum)
271 continue
273 # This removes the "t (I_a^2 + I_b^2)" factor in
274 # Coulton et al. 2017 equation 29.
275 q /= -2.0*(flux**2)
276 scaled = self._tileArray(q)
278 xcorrCheck = np.abs(np.sum(scaled))/np.sum(np.abs(scaled))
279 if (xcorrCheck > self.config.xcorrCheckRejectLevel) or not (np.isfinite(xcorrCheck)):
280 self.log.warning("Amp: %s %d skipped due to value of triangle-inequality sum %f",
281 ampName, xcorrNum, xcorrCheck)
282 fluxes = np.delete(fluxes, xcorrNum)
283 continue
285 scaledCorrList.append(scaled)
286 self.log.info("Amp: %s %d/%d Final: %g XcorrCheck: %f",
287 ampName, xcorrNum, len(xCorrList), q[0][0], xcorrCheck)
289 if len(scaledCorrList) == 0:
290 self.log.warning("Amp: %s All inputs rejected for amp!", ampName)
291 bfk.meanXcorrs[ampName] = np.zeros(bfk.shape)
292 bfk.ampKernels[ampName] = np.zeros(bfk.shape)
293 bfk.valid[ampName] = False
294 continue
296 if self.config.useAmatrix:
297 # Use the aMatrix, ignoring the meanXcorr generated above.
298 preKernel = np.pad(self._tileArray(np.array(inputPtc.aMatrix[ampName])), ((1, 1)))
299 elif self.config.correlationQuadraticFit:
300 # Use a quadratic fit to the correlations as a
301 # function of flux.
302 preKernel = self.quadraticCorrelations(scaledCorrList, fluxes, f"Amp: {ampName}")
303 else:
304 # Use a simple average of the measured correlations.
305 preKernel = self.averageCorrelations(scaledCorrList, f"Amp: {ampName}")
307 center = int((bfk.shape[0] - 1) / 2)
309 if self.config.forceZeroSum:
310 totalSum = np.sum(preKernel)
312 if self.config.correlationModelRadius < (preKernel.shape[0] - 1) / 2:
313 # Assume a correlation model of
314 # Corr(r) = -preFactor * r^(2 * slope)
315 preFactor = np.sqrt(preKernel[center, center + 1] * preKernel[center + 1, center])
316 slopeFactor = 2.0 * np.abs(self.config.correlationModelSlope)
317 totalSum += 2.0*np.pi*(preFactor / (slopeFactor*(center + 0.5))**slopeFactor)
319 preKernel[center, center] -= totalSum
320 self.log.info("%s Zero-Sum Scale: %g", ampName, totalSum)
322 finalSum = np.sum(preKernel)
323 bfk.meanXcorrs[ampName] = preKernel
325 postKernel = self.successiveOverRelax(preKernel)
326 bfk.ampKernels[ampName] = postKernel
327 if self.config.level == 'DETECTOR':
328 detectorCorrList.extend(scaledCorrList)
329 detectorFluxes.extend(fluxes)
330 bfk.valid[ampName] = True
331 self.log.info("Amp: %s Sum: %g Center Info Pre: %g Post: %g",
332 ampName, finalSum, preKernel[center, center], postKernel[center, center])
334 # Assemble a detector kernel?
335 if self.config.level == 'DETECTOR':
336 if self.config.correlationQuadraticFit:
337 preKernel = self.quadraticCorrelations(detectorCorrList, detectorFluxes, f"Amp: {ampName}")
338 else:
339 preKernel = self.averageCorrelations(detectorCorrList, f"Det: {detName}")
340 finalSum = np.sum(preKernel)
341 center = int((bfk.shape[0] - 1) / 2)
343 postKernel = self.successiveOverRelax(preKernel)
344 bfk.detKernels[detName] = postKernel
345 self.log.info("Det: %s Sum: %g Center Info Pre: %g Post: %g",
346 detName, finalSum, preKernel[center, center], postKernel[center, center])
348 return pipeBase.Struct(
349 outputBFK=bfk,
350 )
352 def averageCorrelations(self, xCorrList, name):
353 """Average input correlations.
355 Parameters
356 ----------
357 xCorrList : `list` [`numpy.array`]
358 List of cross-correlations. These are expected to be
359 square arrays.
360 name : `str`
361 Name for log messages.
363 Returns
364 -------
365 meanXcorr : `numpy.array`, (N, N)
366 The averaged cross-correlation.
367 """
368 meanXcorr = np.zeros_like(xCorrList[0])
369 xCorrList = np.transpose(xCorrList)
370 sctrl = afwMath.StatisticsControl()
371 sctrl.setNumSigmaClip(self.config.nSigmaClip)
372 for i in range(np.shape(meanXcorr)[0]):
373 for j in range(np.shape(meanXcorr)[1]):
374 meanXcorr[i, j] = afwMath.makeStatistics(xCorrList[i, j],
375 afwMath.MEANCLIP, sctrl).getValue()
377 # To match previous definitions, pad by one element.
378 meanXcorr = np.pad(meanXcorr, ((1, 1)))
380 return meanXcorr
382 def quadraticCorrelations(self, xCorrList, fluxList, name):
383 """Measure a quadratic correlation model.
385 Parameters
386 ----------
387 xCorrList : `list` [`numpy.array`]
388 List of cross-correlations. These are expected to be
389 square arrays.
390 fluxList : `numpy.array`, (Nflux,)
391 Associated list of fluxes.
392 name : `str`
393 Name for log messages.
395 Returns
396 -------
397 meanXcorr : `numpy.array`, (N, N)
398 The averaged cross-correlation.
399 """
400 meanXcorr = np.zeros_like(xCorrList[0])
401 fluxList = np.square(fluxList)
402 xCorrList = np.array(xCorrList)
404 for i in range(np.shape(meanXcorr)[0]):
405 for j in range(np.shape(meanXcorr)[1]):
406 # Fit corrlation_i(x, y) = a0 + a1 * (flux_i)^2 The
407 # i,j indices are inverted to apply the transposition,
408 # as is done in the averaging case.
409 linearFit, linearFitErr, chiSq, weights = irlsFit([0.0, 1e-4], fluxList,
410 xCorrList[:, j, i], funcPolynomial,
411 scaleResidual=False)
412 meanXcorr[i, j] = linearFit[1] # Discard the intercept.
413 self.log.info("Quad fit meanXcorr[%d,%d] = %g", i, j, linearFit[1])
415 # To match previous definitions, pad by one element.
416 meanXcorr = np.pad(meanXcorr, ((1, 1)))
418 return meanXcorr
420 @staticmethod
421 def _tileArray(in_array):
422 """Given an input quarter-image, tile/mirror it and return full image.
424 Given a square input of side-length n, of the form
426 input = array([[1, 2, 3],
427 [4, 5, 6],
428 [7, 8, 9]])
430 return an array of size 2n-1 as
432 output = array([[ 9, 8, 7, 8, 9],
433 [ 6, 5, 4, 5, 6],
434 [ 3, 2, 1, 2, 3],
435 [ 6, 5, 4, 5, 6],
436 [ 9, 8, 7, 8, 9]])
438 Parameters
439 ----------
440 input : `np.array`, (N, N)
441 The square input quarter-array
443 Returns
444 -------
445 output : `np.array`, (2*N + 1, 2*N + 1)
446 The full, tiled array
447 """
448 assert(in_array.shape[0] == in_array.shape[1])
449 length = in_array.shape[0] - 1
450 output = np.zeros((2*length + 1, 2*length + 1))
452 for i in range(length + 1):
453 for j in range(length + 1):
454 output[i + length, j + length] = in_array[i, j]
455 output[-i + length, j + length] = in_array[i, j]
456 output[i + length, -j + length] = in_array[i, j]
457 output[-i + length, -j + length] = in_array[i, j]
458 return output
460 def successiveOverRelax(self, source, maxIter=None, eLevel=None):
461 """An implementation of the successive over relaxation (SOR) method.
463 A numerical method for solving a system of linear equations
464 with faster convergence than the Gauss-Seidel method.
466 Parameters
467 ----------
468 source : `numpy.ndarray`, (N, N)
469 The input array.
470 maxIter : `int`, optional
471 Maximum number of iterations to attempt before aborting.
472 eLevel : `float`, optional
473 The target error level at which we deem convergence to have
474 occurred.
476 Returns
477 -------
478 output : `numpy.ndarray`, (N, N)
479 The solution.
480 """
481 if not maxIter:
482 maxIter = self.config.maxIterSuccessiveOverRelaxation
483 if not eLevel:
484 eLevel = self.config.eLevelSuccessiveOverRelaxation
486 assert source.shape[0] == source.shape[1], "Input array must be square"
487 # initialize, and set boundary conditions
488 func = np.zeros([source.shape[0] + 2, source.shape[1] + 2])
489 resid = np.zeros([source.shape[0] + 2, source.shape[1] + 2])
490 rhoSpe = np.cos(np.pi/source.shape[0]) # Here a square grid is assumed
492 # Calculate the initial error
493 for i in range(1, func.shape[0] - 1):
494 for j in range(1, func.shape[1] - 1):
495 resid[i, j] = (func[i, j - 1] + func[i, j + 1] + func[i - 1, j]
496 + func[i + 1, j] - 4*func[i, j] - source[i - 1, j - 1])
497 inError = np.sum(np.abs(resid))
499 # Iterate until convergence
500 # We perform two sweeps per cycle,
501 # updating 'odd' and 'even' points separately
502 nIter = 0
503 omega = 1.0
504 dx = 1.0
505 while nIter < maxIter*2:
506 outError = 0
507 if nIter%2 == 0:
508 for i in range(1, func.shape[0] - 1, 2):
509 for j in range(1, func.shape[1] - 1, 2):
510 resid[i, j] = float(func[i, j-1] + func[i, j + 1] + func[i - 1, j]
511 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
512 func[i, j] += omega*resid[i, j]*.25
513 for i in range(2, func.shape[0] - 1, 2):
514 for j in range(2, func.shape[1] - 1, 2):
515 resid[i, j] = float(func[i, j - 1] + func[i, j + 1] + func[i - 1, j]
516 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
517 func[i, j] += omega*resid[i, j]*.25
518 else:
519 for i in range(1, func.shape[0] - 1, 2):
520 for j in range(2, func.shape[1] - 1, 2):
521 resid[i, j] = float(func[i, j - 1] + func[i, j + 1] + func[i - 1, j]
522 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
523 func[i, j] += omega*resid[i, j]*.25
524 for i in range(2, func.shape[0] - 1, 2):
525 for j in range(1, func.shape[1] - 1, 2):
526 resid[i, j] = float(func[i, j - 1] + func[i, j + 1] + func[i - 1, j]
527 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
528 func[i, j] += omega*resid[i, j]*.25
529 outError = np.sum(np.abs(resid))
530 if outError < inError*eLevel:
531 break
532 if nIter == 0:
533 omega = 1.0/(1 - rhoSpe*rhoSpe/2.0)
534 else:
535 omega = 1.0/(1 - rhoSpe*rhoSpe*omega/4.0)
536 nIter += 1
538 if nIter >= maxIter*2:
539 self.log.warning("Failure: SuccessiveOverRelaxation did not converge in %s iterations."
540 "\noutError: %s, inError: %s,", nIter//2, outError, inError*eLevel)
541 else:
542 self.log.info("Success: SuccessiveOverRelaxation converged in %s iterations."
543 "\noutError: %s, inError: %s", nIter//2, outError, inError*eLevel)
544 return func[1: -1, 1: -1]