Coverage for python/lsst/cp/pipe/makeBrighterFatterKernel.py: 12%
215 statements
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-20 02:19 -0700
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-20 02:19 -0700
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22"""Calculation of brighter-fatter effect correlations and kernels."""
24__all__ = ['BrighterFatterKernelSolveTask',
25 'BrighterFatterKernelSolveConfig']
27import numpy as np
29import lsst.afw.math as afwMath
30import lsst.pex.config as pexConfig
31import lsst.pipe.base as pipeBase
32import lsst.pipe.base.connectionTypes as cT
34from lsst.ip.isr import (BrighterFatterKernel)
35from .utils import (funcPolynomial, irlsFit)
36from ._lookupStaticCalibration import lookupStaticCalibration
39class BrighterFatterKernelSolveConnections(pipeBase.PipelineTaskConnections,
40 dimensions=("instrument", "exposure", "detector")):
41 dummy = cT.Input(
42 name="raw",
43 doc="Dummy exposure.",
44 storageClass='Exposure',
45 dimensions=("instrument", "exposure", "detector"),
46 multiple=True,
47 deferLoad=True,
48 )
49 camera = cT.PrerequisiteInput(
50 name="camera",
51 doc="Camera associated with this data.",
52 storageClass="Camera",
53 dimensions=("instrument", ),
54 isCalibration=True,
55 lookupFunction=lookupStaticCalibration,
56 )
57 inputPtc = cT.PrerequisiteInput(
58 name="ptc",
59 doc="Photon transfer curve dataset.",
60 storageClass="PhotonTransferCurveDataset",
61 dimensions=("instrument", "detector"),
62 isCalibration=True,
63 )
65 outputBFK = cT.Output(
66 name="brighterFatterKernel",
67 doc="Output measured brighter-fatter kernel.",
68 storageClass="BrighterFatterKernel",
69 dimensions=("instrument", "detector"),
70 isCalibration=True,
71 )
74class BrighterFatterKernelSolveConfig(pipeBase.PipelineTaskConfig,
75 pipelineConnections=BrighterFatterKernelSolveConnections):
76 level = pexConfig.ChoiceField(
77 doc="The level at which to calculate the brighter-fatter kernels",
78 dtype=str,
79 default="AMP",
80 allowed={
81 "AMP": "Every amplifier treated separately",
82 "DETECTOR": "One kernel per detector",
83 }
84 )
85 ignoreAmpsForAveraging = pexConfig.ListField(
86 dtype=str,
87 doc="List of amp names to ignore when averaging the amplifier kernels into the detector"
88 " kernel. Only relevant for level = DETECTOR",
89 default=[]
90 )
91 xcorrCheckRejectLevel = pexConfig.Field(
92 dtype=float,
93 doc="Rejection level for the sum of the input cross-correlations. Arrays which "
94 "sum to greater than this are discarded before the clipped mean is calculated.",
95 default=2.0
96 )
97 nSigmaClip = pexConfig.Field(
98 dtype=float,
99 doc="Number of sigma to clip when calculating means for the cross-correlation",
100 default=5
101 )
102 forceZeroSum = pexConfig.Field(
103 dtype=bool,
104 doc="Force the correlation matrix to have zero sum by adjusting the (0,0) value?",
105 default=False,
106 )
107 useAmatrix = pexConfig.Field(
108 dtype=bool,
109 doc="Use the PTC 'a' matrix (Astier et al. 2019 equation 20) "
110 "instead of the average of measured covariances?",
111 default=False,
112 )
114 maxIterSuccessiveOverRelaxation = pexConfig.Field(
115 dtype=int,
116 doc="The maximum number of iterations allowed for the successive over-relaxation method",
117 default=10000
118 )
119 eLevelSuccessiveOverRelaxation = pexConfig.Field(
120 dtype=float,
121 doc="The target residual error for the successive over-relaxation method",
122 default=5.0e-14
123 )
125 correlationQuadraticFit = pexConfig.Field(
126 dtype=bool,
127 doc="Use a quadratic fit to find the correlations instead of simple averaging?",
128 default=False,
129 )
130 correlationModelRadius = pexConfig.Field(
131 dtype=int,
132 doc="Build a model of the correlation coefficients for radii larger than this value in pixels?",
133 default=100,
134 )
135 correlationModelSlope = pexConfig.Field(
136 dtype=float,
137 doc="Slope of the correlation model for radii larger than correlationModelRadius",
138 default=-1.35,
139 )
142class BrighterFatterKernelSolveTask(pipeBase.PipelineTask):
143 """Measure appropriate Brighter-Fatter Kernel from the PTC dataset.
144 """
146 ConfigClass = BrighterFatterKernelSolveConfig
147 _DefaultName = 'cpBfkMeasure'
149 def runQuantum(self, butlerQC, inputRefs, outputRefs):
150 """Ensure that the input and output dimensions are passed along.
152 Parameters
153 ----------
154 butlerQC : `lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
155 Butler to operate on.
156 inputRefs : `lsst.pipe.base.connections.InputQuantizedConnection`
157 Input data refs to load.
158 ouptutRefs : `lsst.pipe.base.connections.OutputQuantizedConnection`
159 Output data refs to persist.
160 """
161 inputs = butlerQC.get(inputRefs)
163 # Use the dimensions to set calib/provenance information.
164 inputs['inputDims'] = inputRefs.inputPtc.dataId.byName()
166 outputs = self.run(**inputs)
167 butlerQC.put(outputs, outputRefs)
169 def run(self, inputPtc, dummy, camera, inputDims):
170 """Combine covariance information from PTC into brighter-fatter
171 kernels.
173 Parameters
174 ----------
175 inputPtc : `lsst.ip.isr.PhotonTransferCurveDataset`
176 PTC data containing per-amplifier covariance measurements.
177 dummy : `lsst.afw.image.Exposure`
178 The exposure used to select the appropriate PTC dataset.
179 In almost all circumstances, one of the input exposures
180 used to generate the PTC dataset is the best option.
181 camera : `lsst.afw.cameraGeom.Camera`
182 Camera to use for camera geometry information.
183 inputDims : `lsst.daf.butler.DataCoordinate` or `dict`
184 DataIds to use to populate the output calibration.
186 Returns
187 -------
188 results : `lsst.pipe.base.Struct`
189 The resulst struct containing:
191 ``outputBfk``
192 Resulting Brighter-Fatter Kernel
193 (`lsst.ip.isr.BrighterFatterKernel`).
194 """
195 if len(dummy) == 0:
196 self.log.warning("No dummy exposure found.")
198 detector = camera[inputDims['detector']]
199 detName = detector.getName()
201 if self.config.level == 'DETECTOR':
202 detectorCorrList = list()
203 detectorFluxes = list()
205 bfk = BrighterFatterKernel(camera=camera, detectorId=detector.getId(), level=self.config.level)
206 bfk.rawMeans = inputPtc.rawMeans # ADU
207 bfk.rawVariances = inputPtc.rawVars # ADU^2
208 bfk.expIdMask = inputPtc.expIdMask
210 # Use the PTC covariances as the cross-correlations. These
211 # are scaled before the kernel is generated, which performs
212 # the conversion. The input covariances are in (x, y) index
213 # ordering, as is the aMatrix.
214 bfk.rawXcorrs = inputPtc.covariances # ADU^2
215 bfk.badAmps = inputPtc.badAmps
216 bfk.shape = (inputPtc.covMatrixSide*2 + 1, inputPtc.covMatrixSide*2 + 1)
217 bfk.gain = inputPtc.gain
218 bfk.noise = inputPtc.noise
219 bfk.meanXcorrs = dict()
220 bfk.valid = dict()
221 bfk.updateMetadataFromExposures([inputPtc])
223 for amp in detector:
224 ampName = amp.getName()
225 gain = bfk.gain[ampName]
226 mask = inputPtc.expIdMask[ampName]
227 if gain <= 0:
228 # We've received very bad data.
229 self.log.warning("Impossible gain recieved from PTC for %s: %f. Skipping bad amplifier.",
230 ampName, gain)
231 bfk.meanXcorrs[ampName] = np.zeros(bfk.shape)
232 bfk.ampKernels[ampName] = np.zeros(bfk.shape)
233 bfk.rawXcorrs[ampName] = np.zeros((len(mask), inputPtc.covMatrixSide, inputPtc.covMatrixSide))
234 bfk.valid[ampName] = False
235 continue
237 # Use inputPtc.expIdMask to get the means, variances, and
238 # covariances that were not masked after PTC. The
239 # covariances may now have the mask already applied.
240 fluxes = np.array(bfk.rawMeans[ampName])[mask]
241 variances = np.array(bfk.rawVariances[ampName])[mask]
242 xCorrList = np.array([np.array(xcorr) for xcorr in bfk.rawXcorrs[ampName]])
243 if np.sum(mask) < len(xCorrList):
244 # Only apply the mask if needed.
245 xCorrList = xCorrList[mask]
247 fluxes = np.array([flux*gain for flux in fluxes]) # Now in e^-
248 variances = np.array([variance*gain*gain for variance in variances]) # Now in e^2-
250 # This should duplicate Coulton et al. 2017 Equation 22-29
251 # (arxiv:1711.06273)
252 scaledCorrList = list()
253 corrList = list()
254 truncatedFluxes = list()
255 for xcorrNum, (xcorr, flux, var) in enumerate(zip(xCorrList, fluxes, variances), 1):
256 q = np.array(xcorr) * gain * gain # xcorr now in e^-
257 q *= 2.0 # Remove factor of 1/2 applied in PTC.
258 self.log.info("Amp: %s %d/%d Flux: %f Var: %f Q(0,0): %g Q(1,0): %g Q(0,1): %g",
259 ampName, xcorrNum, len(xCorrList), flux, var, q[0][0], q[1][0], q[0][1])
261 # Normalize by the flux, which removes the (0,0)
262 # component attributable to Poisson noise. This
263 # contains the two "t I delta(x - x')" terms in
264 # Coulton et al. 2017 equation 29
265 q[0][0] -= 2.0*(flux)
267 if q[0][0] > 0.0:
268 self.log.warning("Amp: %s %d skipped due to value of (variance-mean)=%f",
269 ampName, xcorrNum, q[0][0])
270 # If we drop an element of ``scaledCorrList``
271 # (which is what this does), we need to ensure we
272 # drop the flux entry as well.
273 continue
275 # This removes the "t (I_a^2 + I_b^2)" factor in
276 # Coulton et al. 2017 equation 29.
277 # The quadratic fit option needs the correlations unscaled
278 q /= -2.0
279 unscaled = self._tileArray(q)
280 q /= flux**2
281 scaled = self._tileArray(q)
282 xcorrCheck = np.abs(np.sum(scaled))/np.sum(np.abs(scaled))
283 if (xcorrCheck > self.config.xcorrCheckRejectLevel) or not (np.isfinite(xcorrCheck)):
284 self.log.warning("Amp: %s %d skipped due to value of triangle-inequality sum %f",
285 ampName, xcorrNum, xcorrCheck)
286 continue
288 scaledCorrList.append(scaled)
289 corrList.append(unscaled)
290 truncatedFluxes.append(flux)
291 self.log.info("Amp: %s %d/%d Final: %g XcorrCheck: %f",
292 ampName, xcorrNum, len(xCorrList), q[0][0], xcorrCheck)
294 fluxes = np.array(truncatedFluxes)
296 if len(scaledCorrList) == 0:
297 self.log.warning("Amp: %s All inputs rejected for amp!", ampName)
298 bfk.meanXcorrs[ampName] = np.zeros(bfk.shape)
299 bfk.ampKernels[ampName] = np.zeros(bfk.shape)
300 bfk.valid[ampName] = False
301 continue
303 if self.config.useAmatrix:
304 # Use the aMatrix, ignoring the meanXcorr generated above.
305 preKernel = np.pad(self._tileArray(-1.0 * np.array(inputPtc.aMatrix[ampName])), ((1, 1)))
306 elif self.config.correlationQuadraticFit:
307 # Use a quadratic fit to the correlations as a
308 # function of flux.
309 preKernel = self.quadraticCorrelations(corrList, fluxes, f"Amp: {ampName}")
310 else:
311 # Use a simple average of the measured correlations.
312 preKernel = self.averageCorrelations(scaledCorrList, f"Amp: {ampName}")
314 center = int((bfk.shape[0] - 1) / 2)
316 if self.config.forceZeroSum:
317 totalSum = np.sum(preKernel)
319 if self.config.correlationModelRadius < (preKernel.shape[0] - 1) / 2:
320 # Assume a correlation model of
321 # Corr(r) = -preFactor * r^(2 * slope)
322 preFactor = np.sqrt(preKernel[center, center + 1] * preKernel[center + 1, center])
323 slopeFactor = 2.0 * np.abs(self.config.correlationModelSlope)
324 totalSum += 2.0*np.pi*(preFactor / (slopeFactor*(center + 0.5))**slopeFactor)
326 preKernel[center, center] -= totalSum
327 self.log.info("%s Zero-Sum Scale: %g", ampName, totalSum)
329 finalSum = np.sum(preKernel)
330 bfk.meanXcorrs[ampName] = preKernel
332 postKernel = self.successiveOverRelax(preKernel)
333 bfk.ampKernels[ampName] = postKernel
334 if self.config.level == 'DETECTOR' and ampName not in self.config.ignoreAmpsForAveraging:
335 detectorCorrList.extend(scaledCorrList)
336 detectorFluxes.extend(fluxes)
337 bfk.valid[ampName] = True
338 self.log.info("Amp: %s Sum: %g Center Info Pre: %g Post: %g",
339 ampName, finalSum, preKernel[center, center], postKernel[center, center])
341 # Assemble a detector kernel?
342 if self.config.level == 'DETECTOR':
343 if self.config.correlationQuadraticFit:
344 preKernel = self.quadraticCorrelations(detectorCorrList, detectorFluxes, f"Amp: {ampName}")
345 else:
346 preKernel = self.averageCorrelations(detectorCorrList, f"Det: {detName}")
347 finalSum = np.sum(preKernel)
348 center = int((bfk.shape[0] - 1) / 2)
350 postKernel = self.successiveOverRelax(preKernel)
351 bfk.detKernels[detName] = postKernel
352 self.log.info("Det: %s Sum: %g Center Info Pre: %g Post: %g",
353 detName, finalSum, preKernel[center, center], postKernel[center, center])
355 return pipeBase.Struct(
356 outputBFK=bfk,
357 )
359 def averageCorrelations(self, xCorrList, name):
360 """Average input correlations.
362 Parameters
363 ----------
364 xCorrList : `list` [`numpy.array`]
365 List of cross-correlations. These are expected to be
366 square arrays.
367 name : `str`
368 Name for log messages.
370 Returns
371 -------
372 meanXcorr : `numpy.array`, (N, N)
373 The averaged cross-correlation.
374 """
375 meanXcorr = np.zeros_like(xCorrList[0])
376 xCorrList = np.array(xCorrList)
378 sctrl = afwMath.StatisticsControl()
379 sctrl.setNumSigmaClip(self.config.nSigmaClip)
380 for i in range(np.shape(meanXcorr)[0]):
381 for j in range(np.shape(meanXcorr)[1]):
382 meanXcorr[i, j] = afwMath.makeStatistics(xCorrList[:, i, j],
383 afwMath.MEANCLIP, sctrl).getValue()
385 # To match previous definitions, pad by one element.
386 meanXcorr = np.pad(meanXcorr, ((1, 1)))
388 return meanXcorr
390 def quadraticCorrelations(self, xCorrList, fluxList, name):
391 """Measure a quadratic correlation model.
393 Parameters
394 ----------
395 xCorrList : `list` [`numpy.array`]
396 List of cross-correlations. These are expected to be
397 square arrays.
398 fluxList : `numpy.array`, (Nflux,)
399 Associated list of fluxes.
400 name : `str`
401 Name for log messages.
403 Returns
404 -------
405 meanXcorr : `numpy.array`, (N, N)
406 The averaged cross-correlation.
407 """
408 meanXcorr = np.zeros_like(xCorrList[0])
409 fluxList = np.square(fluxList)
410 xCorrList = np.array(xCorrList)
412 for i in range(np.shape(meanXcorr)[0]):
413 for j in range(np.shape(meanXcorr)[1]):
414 # Fit corrlation_i(x, y) = a0 + a1 * (flux_i)^2 We do
415 # not want to transpose, so use (i, j) without
416 # inversion.
417 linearFit, linearFitErr, chiSq, weights = irlsFit([0.0, 1e-4], fluxList,
418 xCorrList[:, i, j], funcPolynomial,
419 scaleResidual=False)
420 meanXcorr[i, j] = linearFit[1] # Discard the intercept.
421 self.log.info("Quad fit meanXcorr[%d,%d] = %g", i, j, linearFit[1])
423 # To match previous definitions, pad by one element.
424 meanXcorr = np.pad(meanXcorr, ((1, 1)))
426 return meanXcorr
428 @staticmethod
429 def _tileArray(in_array):
430 """Given an input quarter-image, tile/mirror it and return full image.
432 Given a square input of side-length n, of the form
434 input = array([[1, 2, 3],
435 [4, 5, 6],
436 [7, 8, 9]])
438 return an array of size 2n-1 as
440 output = array([[ 9, 8, 7, 8, 9],
441 [ 6, 5, 4, 5, 6],
442 [ 3, 2, 1, 2, 3],
443 [ 6, 5, 4, 5, 6],
444 [ 9, 8, 7, 8, 9]])
446 Parameters
447 ----------
448 input : `np.array`, (N, N)
449 The square input quarter-array
451 Returns
452 -------
453 output : `np.array`, (2*N + 1, 2*N + 1)
454 The full, tiled array
455 """
456 assert(in_array.shape[0] == in_array.shape[1])
457 length = in_array.shape[0] - 1
458 output = np.zeros((2*length + 1, 2*length + 1))
460 for i in range(length + 1):
461 for j in range(length + 1):
462 output[i + length, j + length] = in_array[i, j]
463 output[-i + length, j + length] = in_array[i, j]
464 output[i + length, -j + length] = in_array[i, j]
465 output[-i + length, -j + length] = in_array[i, j]
466 return output
468 def successiveOverRelax(self, source, maxIter=None, eLevel=None):
469 """An implementation of the successive over relaxation (SOR) method.
471 A numerical method for solving a system of linear equations
472 with faster convergence than the Gauss-Seidel method.
474 Parameters
475 ----------
476 source : `numpy.ndarray`, (N, N)
477 The input array.
478 maxIter : `int`, optional
479 Maximum number of iterations to attempt before aborting.
480 eLevel : `float`, optional
481 The target error level at which we deem convergence to have
482 occurred.
484 Returns
485 -------
486 output : `numpy.ndarray`, (N, N)
487 The solution.
488 """
489 if not maxIter:
490 maxIter = self.config.maxIterSuccessiveOverRelaxation
491 if not eLevel:
492 eLevel = self.config.eLevelSuccessiveOverRelaxation
494 assert source.shape[0] == source.shape[1], "Input array must be square"
495 # initialize, and set boundary conditions
496 func = np.zeros([source.shape[0] + 2, source.shape[1] + 2])
497 resid = np.zeros([source.shape[0] + 2, source.shape[1] + 2])
498 rhoSpe = np.cos(np.pi/source.shape[0]) # Here a square grid is assumed
500 # Calculate the initial error
501 for i in range(1, func.shape[0] - 1):
502 for j in range(1, func.shape[1] - 1):
503 resid[i, j] = (func[i, j - 1] + func[i, j + 1] + func[i - 1, j]
504 + func[i + 1, j] - 4*func[i, j] - source[i - 1, j - 1])
505 inError = np.sum(np.abs(resid))
507 # Iterate until convergence
508 # We perform two sweeps per cycle,
509 # updating 'odd' and 'even' points separately
510 nIter = 0
511 omega = 1.0
512 dx = 1.0
513 while nIter < maxIter*2:
514 outError = 0
515 if nIter%2 == 0:
516 for i in range(1, func.shape[0] - 1, 2):
517 for j in range(1, func.shape[1] - 1, 2):
518 resid[i, j] = float(func[i, j-1] + func[i, j + 1] + func[i - 1, j]
519 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
520 func[i, j] += omega*resid[i, j]*.25
521 for i in range(2, func.shape[0] - 1, 2):
522 for j in range(2, func.shape[1] - 1, 2):
523 resid[i, j] = float(func[i, j - 1] + func[i, j + 1] + func[i - 1, j]
524 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
525 func[i, j] += omega*resid[i, j]*.25
526 else:
527 for i in range(1, func.shape[0] - 1, 2):
528 for j in range(2, func.shape[1] - 1, 2):
529 resid[i, j] = float(func[i, j - 1] + func[i, j + 1] + func[i - 1, j]
530 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
531 func[i, j] += omega*resid[i, j]*.25
532 for i in range(2, func.shape[0] - 1, 2):
533 for j in range(1, func.shape[1] - 1, 2):
534 resid[i, j] = float(func[i, j - 1] + func[i, j + 1] + func[i - 1, j]
535 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
536 func[i, j] += omega*resid[i, j]*.25
537 outError = np.sum(np.abs(resid))
538 if outError < inError*eLevel:
539 break
540 if nIter == 0:
541 omega = 1.0/(1 - rhoSpe*rhoSpe/2.0)
542 else:
543 omega = 1.0/(1 - rhoSpe*rhoSpe*omega/4.0)
544 nIter += 1
546 if nIter >= maxIter*2:
547 self.log.warning("Failure: SuccessiveOverRelaxation did not converge in %s iterations."
548 "\noutError: %s, inError: %s,", nIter//2, outError, inError*eLevel)
549 else:
550 self.log.info("Success: SuccessiveOverRelaxation converged in %s iterations."
551 "\noutError: %s, inError: %s", nIter//2, outError, inError*eLevel)
552 return func[1: -1, 1: -1]