Coverage for python/lsst/cp/pipe/makeBrighterFatterKernel.py: 12%
215 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-03-23 02:55 -0700
« prev ^ index » next coverage.py v6.5.0, created at 2023-03-23 02:55 -0700
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22"""Calculation of brighter-fatter effect correlations and kernels."""
24__all__ = ['BrighterFatterKernelSolveTask',
25 'BrighterFatterKernelSolveConfig']
27import numpy as np
29import lsst.afw.math as afwMath
30import lsst.pex.config as pexConfig
31import lsst.pipe.base as pipeBase
32import lsst.pipe.base.connectionTypes as cT
34from lsst.ip.isr import (BrighterFatterKernel)
35from .utils import (funcPolynomial, irlsFit)
36from ._lookupStaticCalibration import lookupStaticCalibration
39class BrighterFatterKernelSolveConnections(pipeBase.PipelineTaskConnections,
40 dimensions=("instrument", "exposure", "detector")):
41 dummy = cT.Input(
42 name="raw",
43 doc="Dummy exposure.",
44 storageClass='Exposure',
45 dimensions=("instrument", "exposure", "detector"),
46 multiple=True,
47 deferLoad=True,
48 )
49 camera = cT.PrerequisiteInput(
50 name="camera",
51 doc="Camera associated with this data.",
52 storageClass="Camera",
53 dimensions=("instrument", ),
54 isCalibration=True,
55 lookupFunction=lookupStaticCalibration,
56 )
57 inputPtc = cT.PrerequisiteInput(
58 name="ptc",
59 doc="Photon transfer curve dataset.",
60 storageClass="PhotonTransferCurveDataset",
61 dimensions=("instrument", "detector"),
62 isCalibration=True,
63 )
65 outputBFK = cT.Output(
66 name="brighterFatterKernel",
67 doc="Output measured brighter-fatter kernel.",
68 storageClass="BrighterFatterKernel",
69 dimensions=("instrument", "detector"),
70 isCalibration=True,
71 )
74class BrighterFatterKernelSolveConfig(pipeBase.PipelineTaskConfig,
75 pipelineConnections=BrighterFatterKernelSolveConnections):
76 level = pexConfig.ChoiceField(
77 doc="The level at which to calculate the brighter-fatter kernels",
78 dtype=str,
79 default="AMP",
80 allowed={
81 "AMP": "Every amplifier treated separately",
82 "DETECTOR": "One kernel per detector",
83 }
84 )
85 ignoreAmpsForAveraging = pexConfig.ListField(
86 dtype=str,
87 doc="List of amp names to ignore when averaging the amplifier kernels into the detector"
88 " kernel. Only relevant for level = DETECTOR",
89 default=[]
90 )
91 xcorrCheckRejectLevel = pexConfig.Field(
92 dtype=float,
93 doc="Rejection level for the sum of the input cross-correlations. Arrays which "
94 "sum to greater than this are discarded before the clipped mean is calculated.",
95 default=2.0
96 )
97 nSigmaClip = pexConfig.Field(
98 dtype=float,
99 doc="Number of sigma to clip when calculating means for the cross-correlation",
100 default=5
101 )
102 forceZeroSum = pexConfig.Field(
103 dtype=bool,
104 doc="Force the correlation matrix to have zero sum by adjusting the (0,0) value?",
105 default=False,
106 )
107 useAmatrix = pexConfig.Field(
108 dtype=bool,
109 doc="Use the PTC 'a' matrix (Astier et al. 2019 equation 20) "
110 "instead of the average of measured covariances?",
111 default=False,
112 )
114 maxIterSuccessiveOverRelaxation = pexConfig.Field(
115 dtype=int,
116 doc="The maximum number of iterations allowed for the successive over-relaxation method",
117 default=10000
118 )
119 eLevelSuccessiveOverRelaxation = pexConfig.Field(
120 dtype=float,
121 doc="The target residual error for the successive over-relaxation method",
122 default=5.0e-14
123 )
125 correlationQuadraticFit = pexConfig.Field(
126 dtype=bool,
127 doc="Use a quadratic fit to find the correlations instead of simple averaging?",
128 default=False,
129 )
130 correlationModelRadius = pexConfig.Field(
131 dtype=int,
132 doc="Build a model of the correlation coefficients for radii larger than this value in pixels?",
133 default=100,
134 )
135 correlationModelSlope = pexConfig.Field(
136 dtype=float,
137 doc="Slope of the correlation model for radii larger than correlationModelRadius",
138 default=-1.35,
139 )
142class BrighterFatterKernelSolveTask(pipeBase.PipelineTask):
143 """Measure appropriate Brighter-Fatter Kernel from the PTC dataset.
144 """
146 ConfigClass = BrighterFatterKernelSolveConfig
147 _DefaultName = 'cpBfkMeasure'
149 def runQuantum(self, butlerQC, inputRefs, outputRefs):
150 """Ensure that the input and output dimensions are passed along.
152 Parameters
153 ----------
154 butlerQC : `lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
155 Butler to operate on.
156 inputRefs : `lsst.pipe.base.connections.InputQuantizedConnection`
157 Input data refs to load.
158 ouptutRefs : `lsst.pipe.base.connections.OutputQuantizedConnection`
159 Output data refs to persist.
160 """
161 inputs = butlerQC.get(inputRefs)
163 # Use the dimensions to set calib/provenance information.
164 inputs['inputDims'] = inputRefs.inputPtc.dataId.byName()
166 outputs = self.run(**inputs)
167 butlerQC.put(outputs, outputRefs)
169 def run(self, inputPtc, dummy, camera, inputDims):
170 """Combine covariance information from PTC into brighter-fatter
171 kernels.
173 Parameters
174 ----------
175 inputPtc : `lsst.ip.isr.PhotonTransferCurveDataset`
176 PTC data containing per-amplifier covariance measurements.
177 dummy : `lsst.afw.image.Exposure`
178 The exposure used to select the appropriate PTC dataset.
179 In almost all circumstances, one of the input exposures
180 used to generate the PTC dataset is the best option.
181 camera : `lsst.afw.cameraGeom.Camera`
182 Camera to use for camera geometry information.
183 inputDims : `lsst.daf.butler.DataCoordinate` or `dict`
184 DataIds to use to populate the output calibration.
186 Returns
187 -------
188 results : `lsst.pipe.base.Struct`
189 The resulst struct containing:
191 ``outputBfk``
192 Resulting Brighter-Fatter Kernel
193 (`lsst.ip.isr.BrighterFatterKernel`).
194 """
195 if len(dummy) == 0:
196 self.log.warning("No dummy exposure found.")
198 detector = camera[inputDims['detector']]
199 detName = detector.getName()
201 if self.config.level == 'DETECTOR':
202 detectorCorrList = list()
203 detectorFluxes = list()
205 bfk = BrighterFatterKernel(camera=camera, detectorId=detector.getId(), level=self.config.level)
206 bfk.rawMeans = inputPtc.rawMeans # ADU
207 bfk.rawVariances = inputPtc.rawVars # ADU^2
208 bfk.expIdMask = inputPtc.expIdMask
210 # Use the PTC covariances as the cross-correlations. These
211 # are scaled before the kernel is generated, which performs
212 # the conversion.
213 bfk.rawXcorrs = inputPtc.covariances # ADU^2
214 bfk.badAmps = inputPtc.badAmps
215 bfk.shape = (inputPtc.covMatrixSide*2 + 1, inputPtc.covMatrixSide*2 + 1)
216 bfk.gain = inputPtc.gain
217 bfk.noise = inputPtc.noise
218 bfk.meanXcorrs = dict()
219 bfk.valid = dict()
220 bfk.updateMetadataFromExposures([inputPtc])
222 for amp in detector:
223 ampName = amp.getName()
224 gain = bfk.gain[ampName]
225 mask = inputPtc.expIdMask[ampName]
226 if gain <= 0:
227 # We've received very bad data.
228 self.log.warning("Impossible gain recieved from PTC for %s: %f. Skipping bad amplifier.",
229 ampName, gain)
230 bfk.meanXcorrs[ampName] = np.zeros(bfk.shape)
231 bfk.ampKernels[ampName] = np.zeros(bfk.shape)
232 bfk.rawXcorrs[ampName] = np.zeros((len(mask), inputPtc.covMatrixSide, inputPtc.covMatrixSide))
233 bfk.valid[ampName] = False
234 continue
236 # Use inputPtc.expIdMask to get the means, variances, and
237 # covariances that were not masked after PTC. The
238 # covariances may now have the mask already applied.
239 fluxes = np.array(bfk.rawMeans[ampName])[mask]
240 variances = np.array(bfk.rawVariances[ampName])[mask]
241 xCorrList = np.array([np.array(xcorr) for xcorr in bfk.rawXcorrs[ampName]])
242 if np.sum(mask) < len(xCorrList):
243 # Only apply the mask if needed.
244 xCorrList = xCorrList[mask]
246 fluxes = np.array([flux*gain for flux in fluxes]) # Now in e^-
247 variances = np.array([variance*gain*gain for variance in variances]) # Now in e^2-
249 # This should duplicate Coulton et al. 2017 Equation 22-29
250 # (arxiv:1711.06273)
251 scaledCorrList = list()
252 corrList = list()
253 truncatedFluxes = list()
254 for xcorrNum, (xcorr, flux, var) in enumerate(zip(xCorrList, fluxes, variances), 1):
255 q = np.array(xcorr) * gain * gain # xcorr now in e^-
256 q *= 2.0 # Remove factor of 1/2 applied in PTC.
257 self.log.info("Amp: %s %d/%d Flux: %f Var: %f Q(0,0): %g Q(1,0): %g Q(0,1): %g",
258 ampName, xcorrNum, len(xCorrList), flux, var, q[0][0], q[1][0], q[0][1])
260 # Normalize by the flux, which removes the (0,0)
261 # component attributable to Poisson noise. This
262 # contains the two "t I delta(x - x')" terms in
263 # Coulton et al. 2017 equation 29
264 q[0][0] -= 2.0*(flux)
266 if q[0][0] > 0.0:
267 self.log.warning("Amp: %s %d skipped due to value of (variance-mean)=%f",
268 ampName, xcorrNum, q[0][0])
269 # If we drop an element of ``scaledCorrList``
270 # (which is what this does), we need to ensure we
271 # drop the flux entry as well.
272 continue
274 # This removes the "t (I_a^2 + I_b^2)" factor in
275 # Coulton et al. 2017 equation 29.
276 # The quadratic fit option needs the correlations unscaled
277 q /= -2.0
278 unscaled = self._tileArray(q)
279 q /= flux**2
280 scaled = self._tileArray(q)
281 xcorrCheck = np.abs(np.sum(scaled))/np.sum(np.abs(scaled))
282 if (xcorrCheck > self.config.xcorrCheckRejectLevel) or not (np.isfinite(xcorrCheck)):
283 self.log.warning("Amp: %s %d skipped due to value of triangle-inequality sum %f",
284 ampName, xcorrNum, xcorrCheck)
285 continue
287 scaledCorrList.append(scaled)
288 corrList.append(unscaled)
289 truncatedFluxes.append(flux)
290 self.log.info("Amp: %s %d/%d Final: %g XcorrCheck: %f",
291 ampName, xcorrNum, len(xCorrList), q[0][0], xcorrCheck)
293 fluxes = np.array(truncatedFluxes)
295 if len(scaledCorrList) == 0:
296 self.log.warning("Amp: %s All inputs rejected for amp!", ampName)
297 bfk.meanXcorrs[ampName] = np.zeros(bfk.shape)
298 bfk.ampKernels[ampName] = np.zeros(bfk.shape)
299 bfk.valid[ampName] = False
300 continue
302 if self.config.useAmatrix:
303 # Use the aMatrix, ignoring the meanXcorr generated above.
304 preKernel = np.pad(self._tileArray(np.array(inputPtc.aMatrix[ampName])), ((1, 1)))
305 elif self.config.correlationQuadraticFit:
306 # Use a quadratic fit to the correlations as a
307 # function of flux.
308 preKernel = self.quadraticCorrelations(corrList, fluxes, f"Amp: {ampName}")
309 else:
310 # Use a simple average of the measured correlations.
311 preKernel = self.averageCorrelations(scaledCorrList, f"Amp: {ampName}")
313 center = int((bfk.shape[0] - 1) / 2)
315 if self.config.forceZeroSum:
316 totalSum = np.sum(preKernel)
318 if self.config.correlationModelRadius < (preKernel.shape[0] - 1) / 2:
319 # Assume a correlation model of
320 # Corr(r) = -preFactor * r^(2 * slope)
321 preFactor = np.sqrt(preKernel[center, center + 1] * preKernel[center + 1, center])
322 slopeFactor = 2.0 * np.abs(self.config.correlationModelSlope)
323 totalSum += 2.0*np.pi*(preFactor / (slopeFactor*(center + 0.5))**slopeFactor)
325 preKernel[center, center] -= totalSum
326 self.log.info("%s Zero-Sum Scale: %g", ampName, totalSum)
328 finalSum = np.sum(preKernel)
329 bfk.meanXcorrs[ampName] = preKernel
331 postKernel = self.successiveOverRelax(preKernel)
332 bfk.ampKernels[ampName] = postKernel
333 if self.config.level == 'DETECTOR' and ampName not in self.config.ignoreAmpsForAveraging:
334 detectorCorrList.extend(scaledCorrList)
335 detectorFluxes.extend(fluxes)
336 bfk.valid[ampName] = True
337 self.log.info("Amp: %s Sum: %g Center Info Pre: %g Post: %g",
338 ampName, finalSum, preKernel[center, center], postKernel[center, center])
340 # Assemble a detector kernel?
341 if self.config.level == 'DETECTOR':
342 if self.config.correlationQuadraticFit:
343 preKernel = self.quadraticCorrelations(detectorCorrList, detectorFluxes, f"Amp: {ampName}")
344 else:
345 preKernel = self.averageCorrelations(detectorCorrList, f"Det: {detName}")
346 finalSum = np.sum(preKernel)
347 center = int((bfk.shape[0] - 1) / 2)
349 postKernel = self.successiveOverRelax(preKernel)
350 bfk.detKernels[detName] = postKernel
351 self.log.info("Det: %s Sum: %g Center Info Pre: %g Post: %g",
352 detName, finalSum, preKernel[center, center], postKernel[center, center])
354 return pipeBase.Struct(
355 outputBFK=bfk,
356 )
358 def averageCorrelations(self, xCorrList, name):
359 """Average input correlations.
361 Parameters
362 ----------
363 xCorrList : `list` [`numpy.array`]
364 List of cross-correlations. These are expected to be
365 square arrays.
366 name : `str`
367 Name for log messages.
369 Returns
370 -------
371 meanXcorr : `numpy.array`, (N, N)
372 The averaged cross-correlation.
373 """
374 meanXcorr = np.zeros_like(xCorrList[0])
375 xCorrList = np.transpose(xCorrList)
376 sctrl = afwMath.StatisticsControl()
377 sctrl.setNumSigmaClip(self.config.nSigmaClip)
378 for i in range(np.shape(meanXcorr)[0]):
379 for j in range(np.shape(meanXcorr)[1]):
380 meanXcorr[i, j] = afwMath.makeStatistics(xCorrList[i, j],
381 afwMath.MEANCLIP, sctrl).getValue()
383 # To match previous definitions, pad by one element.
384 meanXcorr = np.pad(meanXcorr, ((1, 1)))
386 return meanXcorr
388 def quadraticCorrelations(self, xCorrList, fluxList, name):
389 """Measure a quadratic correlation model.
391 Parameters
392 ----------
393 xCorrList : `list` [`numpy.array`]
394 List of cross-correlations. These are expected to be
395 square arrays.
396 fluxList : `numpy.array`, (Nflux,)
397 Associated list of fluxes.
398 name : `str`
399 Name for log messages.
401 Returns
402 -------
403 meanXcorr : `numpy.array`, (N, N)
404 The averaged cross-correlation.
405 """
406 meanXcorr = np.zeros_like(xCorrList[0])
407 fluxList = np.square(fluxList)
408 xCorrList = np.array(xCorrList)
410 for i in range(np.shape(meanXcorr)[0]):
411 for j in range(np.shape(meanXcorr)[1]):
412 # Fit corrlation_i(x, y) = a0 + a1 * (flux_i)^2 The
413 # i,j indices are inverted to apply the transposition,
414 # as is done in the averaging case.
415 linearFit, linearFitErr, chiSq, weights = irlsFit([0.0, 1e-4], fluxList,
416 xCorrList[:, j, i], funcPolynomial,
417 scaleResidual=False)
418 meanXcorr[i, j] = linearFit[1] # Discard the intercept.
419 self.log.info("Quad fit meanXcorr[%d,%d] = %g", i, j, linearFit[1])
421 # To match previous definitions, pad by one element.
422 meanXcorr = np.pad(meanXcorr, ((1, 1)))
424 return meanXcorr
426 @staticmethod
427 def _tileArray(in_array):
428 """Given an input quarter-image, tile/mirror it and return full image.
430 Given a square input of side-length n, of the form
432 input = array([[1, 2, 3],
433 [4, 5, 6],
434 [7, 8, 9]])
436 return an array of size 2n-1 as
438 output = array([[ 9, 8, 7, 8, 9],
439 [ 6, 5, 4, 5, 6],
440 [ 3, 2, 1, 2, 3],
441 [ 6, 5, 4, 5, 6],
442 [ 9, 8, 7, 8, 9]])
444 Parameters
445 ----------
446 input : `np.array`, (N, N)
447 The square input quarter-array
449 Returns
450 -------
451 output : `np.array`, (2*N + 1, 2*N + 1)
452 The full, tiled array
453 """
454 assert(in_array.shape[0] == in_array.shape[1])
455 length = in_array.shape[0] - 1
456 output = np.zeros((2*length + 1, 2*length + 1))
458 for i in range(length + 1):
459 for j in range(length + 1):
460 output[i + length, j + length] = in_array[i, j]
461 output[-i + length, j + length] = in_array[i, j]
462 output[i + length, -j + length] = in_array[i, j]
463 output[-i + length, -j + length] = in_array[i, j]
464 return output
466 def successiveOverRelax(self, source, maxIter=None, eLevel=None):
467 """An implementation of the successive over relaxation (SOR) method.
469 A numerical method for solving a system of linear equations
470 with faster convergence than the Gauss-Seidel method.
472 Parameters
473 ----------
474 source : `numpy.ndarray`, (N, N)
475 The input array.
476 maxIter : `int`, optional
477 Maximum number of iterations to attempt before aborting.
478 eLevel : `float`, optional
479 The target error level at which we deem convergence to have
480 occurred.
482 Returns
483 -------
484 output : `numpy.ndarray`, (N, N)
485 The solution.
486 """
487 if not maxIter:
488 maxIter = self.config.maxIterSuccessiveOverRelaxation
489 if not eLevel:
490 eLevel = self.config.eLevelSuccessiveOverRelaxation
492 assert source.shape[0] == source.shape[1], "Input array must be square"
493 # initialize, and set boundary conditions
494 func = np.zeros([source.shape[0] + 2, source.shape[1] + 2])
495 resid = np.zeros([source.shape[0] + 2, source.shape[1] + 2])
496 rhoSpe = np.cos(np.pi/source.shape[0]) # Here a square grid is assumed
498 # Calculate the initial error
499 for i in range(1, func.shape[0] - 1):
500 for j in range(1, func.shape[1] - 1):
501 resid[i, j] = (func[i, j - 1] + func[i, j + 1] + func[i - 1, j]
502 + func[i + 1, j] - 4*func[i, j] - source[i - 1, j - 1])
503 inError = np.sum(np.abs(resid))
505 # Iterate until convergence
506 # We perform two sweeps per cycle,
507 # updating 'odd' and 'even' points separately
508 nIter = 0
509 omega = 1.0
510 dx = 1.0
511 while nIter < maxIter*2:
512 outError = 0
513 if nIter%2 == 0:
514 for i in range(1, func.shape[0] - 1, 2):
515 for j in range(1, func.shape[1] - 1, 2):
516 resid[i, j] = float(func[i, j-1] + func[i, j + 1] + func[i - 1, j]
517 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
518 func[i, j] += omega*resid[i, j]*.25
519 for i in range(2, func.shape[0] - 1, 2):
520 for j in range(2, func.shape[1] - 1, 2):
521 resid[i, j] = float(func[i, j - 1] + func[i, j + 1] + func[i - 1, j]
522 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
523 func[i, j] += omega*resid[i, j]*.25
524 else:
525 for i in range(1, func.shape[0] - 1, 2):
526 for j in range(2, func.shape[1] - 1, 2):
527 resid[i, j] = float(func[i, j - 1] + func[i, j + 1] + func[i - 1, j]
528 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
529 func[i, j] += omega*resid[i, j]*.25
530 for i in range(2, func.shape[0] - 1, 2):
531 for j in range(1, func.shape[1] - 1, 2):
532 resid[i, j] = float(func[i, j - 1] + func[i, j + 1] + func[i - 1, j]
533 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
534 func[i, j] += omega*resid[i, j]*.25
535 outError = np.sum(np.abs(resid))
536 if outError < inError*eLevel:
537 break
538 if nIter == 0:
539 omega = 1.0/(1 - rhoSpe*rhoSpe/2.0)
540 else:
541 omega = 1.0/(1 - rhoSpe*rhoSpe*omega/4.0)
542 nIter += 1
544 if nIter >= maxIter*2:
545 self.log.warning("Failure: SuccessiveOverRelaxation did not converge in %s iterations."
546 "\noutError: %s, inError: %s,", nIter//2, outError, inError*eLevel)
547 else:
548 self.log.info("Success: SuccessiveOverRelaxation converged in %s iterations."
549 "\noutError: %s, inError: %s", nIter//2, outError, inError*eLevel)
550 return func[1: -1, 1: -1]