Coverage for python/lsst/cp/pipe/makeBrighterFatterKernel.py: 12%
214 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-15 03:29 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-15 03:29 -0800
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22"""Calculation of brighter-fatter effect correlations and kernels."""
24__all__ = ['BrighterFatterKernelSolveTask',
25 'BrighterFatterKernelSolveConfig']
27import numpy as np
29import lsst.afw.math as afwMath
30import lsst.pex.config as pexConfig
31import lsst.pipe.base as pipeBase
32import lsst.pipe.base.connectionTypes as cT
34from lsst.ip.isr import (BrighterFatterKernel)
35from .utils import (funcPolynomial, irlsFit)
36from ._lookupStaticCalibration import lookupStaticCalibration
39class BrighterFatterKernelSolveConnections(pipeBase.PipelineTaskConnections,
40 dimensions=("instrument", "exposure", "detector")):
41 dummy = cT.Input(
42 name="raw",
43 doc="Dummy exposure.",
44 storageClass='Exposure',
45 dimensions=("instrument", "exposure", "detector"),
46 multiple=True,
47 deferLoad=True,
48 )
49 camera = cT.PrerequisiteInput(
50 name="camera",
51 doc="Camera associated with this data.",
52 storageClass="Camera",
53 dimensions=("instrument", ),
54 isCalibration=True,
55 lookupFunction=lookupStaticCalibration,
56 )
57 inputPtc = cT.PrerequisiteInput(
58 name="ptc",
59 doc="Photon transfer curve dataset.",
60 storageClass="PhotonTransferCurveDataset",
61 dimensions=("instrument", "detector"),
62 isCalibration=True,
63 )
65 outputBFK = cT.Output(
66 name="brighterFatterKernel",
67 doc="Output measured brighter-fatter kernel.",
68 storageClass="BrighterFatterKernel",
69 dimensions=("instrument", "detector"),
70 isCalibration=True,
71 )
74class BrighterFatterKernelSolveConfig(pipeBase.PipelineTaskConfig,
75 pipelineConnections=BrighterFatterKernelSolveConnections):
76 level = pexConfig.ChoiceField(
77 doc="The level at which to calculate the brighter-fatter kernels",
78 dtype=str,
79 default="AMP",
80 allowed={
81 "AMP": "Every amplifier treated separately",
82 "DETECTOR": "One kernel per detector",
83 }
84 )
85 ignoreAmpsForAveraging = pexConfig.ListField(
86 dtype=str,
87 doc="List of amp names to ignore when averaging the amplifier kernels into the detector"
88 " kernel. Only relevant for level = DETECTOR",
89 default=[]
90 )
91 xcorrCheckRejectLevel = pexConfig.Field(
92 dtype=float,
93 doc="Rejection level for the sum of the input cross-correlations. Arrays which "
94 "sum to greater than this are discarded before the clipped mean is calculated.",
95 default=2.0
96 )
97 nSigmaClip = pexConfig.Field(
98 dtype=float,
99 doc="Number of sigma to clip when calculating means for the cross-correlation",
100 default=5
101 )
102 forceZeroSum = pexConfig.Field(
103 dtype=bool,
104 doc="Force the correlation matrix to have zero sum by adjusting the (0,0) value?",
105 default=False,
106 )
107 useAmatrix = pexConfig.Field(
108 dtype=bool,
109 doc="Use the PTC 'a' matrix (Astier et al. 2019 equation 20) "
110 "instead of the average of measured covariances?",
111 default=False,
112 )
114 maxIterSuccessiveOverRelaxation = pexConfig.Field(
115 dtype=int,
116 doc="The maximum number of iterations allowed for the successive over-relaxation method",
117 default=10000
118 )
119 eLevelSuccessiveOverRelaxation = pexConfig.Field(
120 dtype=float,
121 doc="The target residual error for the successive over-relaxation method",
122 default=5.0e-14
123 )
125 correlationQuadraticFit = pexConfig.Field(
126 dtype=bool,
127 doc="Use a quadratic fit to find the correlations instead of simple averaging?",
128 default=False,
129 )
130 correlationModelRadius = pexConfig.Field(
131 dtype=int,
132 doc="Build a model of the correlation coefficients for radii larger than this value in pixels?",
133 default=100,
134 )
135 correlationModelSlope = pexConfig.Field(
136 dtype=float,
137 doc="Slope of the correlation model for radii larger than correlationModelRadius",
138 default=-1.35,
139 )
142class BrighterFatterKernelSolveTask(pipeBase.PipelineTask):
143 """Measure appropriate Brighter-Fatter Kernel from the PTC dataset.
144 """
146 ConfigClass = BrighterFatterKernelSolveConfig
147 _DefaultName = 'cpBfkMeasure'
149 def runQuantum(self, butlerQC, inputRefs, outputRefs):
150 """Ensure that the input and output dimensions are passed along.
152 Parameters
153 ----------
154 butlerQC : `lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
155 Butler to operate on.
156 inputRefs : `lsst.pipe.base.connections.InputQuantizedConnection`
157 Input data refs to load.
158 ouptutRefs : `lsst.pipe.base.connections.OutputQuantizedConnection`
159 Output data refs to persist.
160 """
161 inputs = butlerQC.get(inputRefs)
163 # Use the dimensions to set calib/provenance information.
164 inputs['inputDims'] = inputRefs.inputPtc.dataId.byName()
166 outputs = self.run(**inputs)
167 butlerQC.put(outputs, outputRefs)
169 def run(self, inputPtc, dummy, camera, inputDims):
170 """Combine covariance information from PTC into brighter-fatter
171 kernels.
173 Parameters
174 ----------
175 inputPtc : `lsst.ip.isr.PhotonTransferCurveDataset`
176 PTC data containing per-amplifier covariance measurements.
177 dummy : `lsst.afw.image.Exposure`
178 The exposure used to select the appropriate PTC dataset.
179 In almost all circumstances, one of the input exposures
180 used to generate the PTC dataset is the best option.
181 camera : `lsst.afw.cameraGeom.Camera`
182 Camera to use for camera geometry information.
183 inputDims : `lsst.daf.butler.DataCoordinate` or `dict`
184 DataIds to use to populate the output calibration.
186 Returns
187 -------
188 results : `lsst.pipe.base.Struct`
189 The resulst struct containing:
191 ``outputBfk``
192 Resulting Brighter-Fatter Kernel
193 (`lsst.ip.isr.BrighterFatterKernel`).
194 """
195 if len(dummy) == 0:
196 self.log.warning("No dummy exposure found.")
198 detector = camera[inputDims['detector']]
199 detName = detector.getName()
201 if self.config.level == 'DETECTOR':
202 detectorCorrList = list()
203 detectorFluxes = list()
205 bfk = BrighterFatterKernel(camera=camera, detectorId=detector.getId(), level=self.config.level)
206 bfk.rawMeans = inputPtc.rawMeans # ADU
207 bfk.rawVariances = inputPtc.rawVars # ADU^2
208 bfk.expIdMask = inputPtc.expIdMask
210 # Use the PTC covariances as the cross-correlations. These
211 # are scaled before the kernel is generated, which performs
212 # the conversion.
213 bfk.rawXcorrs = inputPtc.covariances # ADU^2
214 bfk.badAmps = inputPtc.badAmps
215 bfk.shape = (inputPtc.covMatrixSide*2 + 1, inputPtc.covMatrixSide*2 + 1)
216 bfk.gain = inputPtc.gain
217 bfk.noise = inputPtc.noise
218 bfk.meanXcorrs = dict()
219 bfk.valid = dict()
221 for amp in detector:
222 ampName = amp.getName()
223 gain = bfk.gain[ampName]
224 mask = inputPtc.expIdMask[ampName]
225 if gain <= 0:
226 # We've received very bad data.
227 self.log.warning("Impossible gain recieved from PTC for %s: %f. Skipping bad amplifier.",
228 ampName, gain)
229 bfk.meanXcorrs[ampName] = np.zeros(bfk.shape)
230 bfk.ampKernels[ampName] = np.zeros(bfk.shape)
231 bfk.rawXcorrs[ampName] = np.zeros((len(mask), inputPtc.covMatrixSide, inputPtc.covMatrixSide))
232 bfk.valid[ampName] = False
233 continue
235 # Use inputPtc.expIdMask to get the means, variances, and
236 # covariances that were not masked after PTC. The
237 # covariances may now have the mask already applied.
238 fluxes = np.array(bfk.rawMeans[ampName])[mask]
239 variances = np.array(bfk.rawVariances[ampName])[mask]
240 xCorrList = np.array([np.array(xcorr) for xcorr in bfk.rawXcorrs[ampName]])
241 if np.sum(mask) < len(xCorrList):
242 # Only apply the mask if needed.
243 xCorrList = xCorrList[mask]
245 fluxes = np.array([flux*gain for flux in fluxes]) # Now in e^-
246 variances = np.array([variance*gain*gain for variance in variances]) # Now in e^2-
248 # This should duplicate Coulton et al. 2017 Equation 22-29
249 # (arxiv:1711.06273)
250 scaledCorrList = list()
251 corrList = list()
252 truncatedFluxes = list()
253 for xcorrNum, (xcorr, flux, var) in enumerate(zip(xCorrList, fluxes, variances), 1):
254 q = np.array(xcorr) * gain * gain # xcorr now in e^-
255 q *= 2.0 # Remove factor of 1/2 applied in PTC.
256 self.log.info("Amp: %s %d/%d Flux: %f Var: %f Q(0,0): %g Q(1,0): %g Q(0,1): %g",
257 ampName, xcorrNum, len(xCorrList), flux, var, q[0][0], q[1][0], q[0][1])
259 # Normalize by the flux, which removes the (0,0)
260 # component attributable to Poisson noise. This
261 # contains the two "t I delta(x - x')" terms in
262 # Coulton et al. 2017 equation 29
263 q[0][0] -= 2.0*(flux)
265 if q[0][0] > 0.0:
266 self.log.warning("Amp: %s %d skipped due to value of (variance-mean)=%f",
267 ampName, xcorrNum, q[0][0])
268 # If we drop an element of ``scaledCorrList``
269 # (which is what this does), we need to ensure we
270 # drop the flux entry as well.
271 continue
273 # This removes the "t (I_a^2 + I_b^2)" factor in
274 # Coulton et al. 2017 equation 29.
275 # The quadratic fit option needs the correlations unscaled
276 q /= -2.0
277 unscaled = self._tileArray(q)
278 q /= flux**2
279 scaled = self._tileArray(q)
280 xcorrCheck = np.abs(np.sum(scaled))/np.sum(np.abs(scaled))
281 if (xcorrCheck > self.config.xcorrCheckRejectLevel) or not (np.isfinite(xcorrCheck)):
282 self.log.warning("Amp: %s %d skipped due to value of triangle-inequality sum %f",
283 ampName, xcorrNum, xcorrCheck)
284 continue
286 scaledCorrList.append(scaled)
287 corrList.append(unscaled)
288 truncatedFluxes.append(flux)
289 self.log.info("Amp: %s %d/%d Final: %g XcorrCheck: %f",
290 ampName, xcorrNum, len(xCorrList), q[0][0], xcorrCheck)
292 fluxes = np.array(truncatedFluxes)
294 if len(scaledCorrList) == 0:
295 self.log.warning("Amp: %s All inputs rejected for amp!", ampName)
296 bfk.meanXcorrs[ampName] = np.zeros(bfk.shape)
297 bfk.ampKernels[ampName] = np.zeros(bfk.shape)
298 bfk.valid[ampName] = False
299 continue
301 if self.config.useAmatrix:
302 # Use the aMatrix, ignoring the meanXcorr generated above.
303 preKernel = np.pad(self._tileArray(np.array(inputPtc.aMatrix[ampName])), ((1, 1)))
304 elif self.config.correlationQuadraticFit:
305 # Use a quadratic fit to the correlations as a
306 # function of flux.
307 preKernel = self.quadraticCorrelations(corrList, fluxes, f"Amp: {ampName}")
308 else:
309 # Use a simple average of the measured correlations.
310 preKernel = self.averageCorrelations(scaledCorrList, f"Amp: {ampName}")
312 center = int((bfk.shape[0] - 1) / 2)
314 if self.config.forceZeroSum:
315 totalSum = np.sum(preKernel)
317 if self.config.correlationModelRadius < (preKernel.shape[0] - 1) / 2:
318 # Assume a correlation model of
319 # Corr(r) = -preFactor * r^(2 * slope)
320 preFactor = np.sqrt(preKernel[center, center + 1] * preKernel[center + 1, center])
321 slopeFactor = 2.0 * np.abs(self.config.correlationModelSlope)
322 totalSum += 2.0*np.pi*(preFactor / (slopeFactor*(center + 0.5))**slopeFactor)
324 preKernel[center, center] -= totalSum
325 self.log.info("%s Zero-Sum Scale: %g", ampName, totalSum)
327 finalSum = np.sum(preKernel)
328 bfk.meanXcorrs[ampName] = preKernel
330 postKernel = self.successiveOverRelax(preKernel)
331 bfk.ampKernels[ampName] = postKernel
332 if self.config.level == 'DETECTOR' and ampName not in self.config.ignoreAmpsForAveraging:
333 detectorCorrList.extend(scaledCorrList)
334 detectorFluxes.extend(fluxes)
335 bfk.valid[ampName] = True
336 self.log.info("Amp: %s Sum: %g Center Info Pre: %g Post: %g",
337 ampName, finalSum, preKernel[center, center], postKernel[center, center])
339 # Assemble a detector kernel?
340 if self.config.level == 'DETECTOR':
341 if self.config.correlationQuadraticFit:
342 preKernel = self.quadraticCorrelations(detectorCorrList, detectorFluxes, f"Amp: {ampName}")
343 else:
344 preKernel = self.averageCorrelations(detectorCorrList, f"Det: {detName}")
345 finalSum = np.sum(preKernel)
346 center = int((bfk.shape[0] - 1) / 2)
348 postKernel = self.successiveOverRelax(preKernel)
349 bfk.detKernels[detName] = postKernel
350 self.log.info("Det: %s Sum: %g Center Info Pre: %g Post: %g",
351 detName, finalSum, preKernel[center, center], postKernel[center, center])
353 return pipeBase.Struct(
354 outputBFK=bfk,
355 )
357 def averageCorrelations(self, xCorrList, name):
358 """Average input correlations.
360 Parameters
361 ----------
362 xCorrList : `list` [`numpy.array`]
363 List of cross-correlations. These are expected to be
364 square arrays.
365 name : `str`
366 Name for log messages.
368 Returns
369 -------
370 meanXcorr : `numpy.array`, (N, N)
371 The averaged cross-correlation.
372 """
373 meanXcorr = np.zeros_like(xCorrList[0])
374 xCorrList = np.transpose(xCorrList)
375 sctrl = afwMath.StatisticsControl()
376 sctrl.setNumSigmaClip(self.config.nSigmaClip)
377 for i in range(np.shape(meanXcorr)[0]):
378 for j in range(np.shape(meanXcorr)[1]):
379 meanXcorr[i, j] = afwMath.makeStatistics(xCorrList[i, j],
380 afwMath.MEANCLIP, sctrl).getValue()
382 # To match previous definitions, pad by one element.
383 meanXcorr = np.pad(meanXcorr, ((1, 1)))
385 return meanXcorr
387 def quadraticCorrelations(self, xCorrList, fluxList, name):
388 """Measure a quadratic correlation model.
390 Parameters
391 ----------
392 xCorrList : `list` [`numpy.array`]
393 List of cross-correlations. These are expected to be
394 square arrays.
395 fluxList : `numpy.array`, (Nflux,)
396 Associated list of fluxes.
397 name : `str`
398 Name for log messages.
400 Returns
401 -------
402 meanXcorr : `numpy.array`, (N, N)
403 The averaged cross-correlation.
404 """
405 meanXcorr = np.zeros_like(xCorrList[0])
406 fluxList = np.square(fluxList)
407 xCorrList = np.array(xCorrList)
409 for i in range(np.shape(meanXcorr)[0]):
410 for j in range(np.shape(meanXcorr)[1]):
411 # Fit corrlation_i(x, y) = a0 + a1 * (flux_i)^2 The
412 # i,j indices are inverted to apply the transposition,
413 # as is done in the averaging case.
414 linearFit, linearFitErr, chiSq, weights = irlsFit([0.0, 1e-4], fluxList,
415 xCorrList[:, j, i], funcPolynomial,
416 scaleResidual=False)
417 meanXcorr[i, j] = linearFit[1] # Discard the intercept.
418 self.log.info("Quad fit meanXcorr[%d,%d] = %g", i, j, linearFit[1])
420 # To match previous definitions, pad by one element.
421 meanXcorr = np.pad(meanXcorr, ((1, 1)))
423 return meanXcorr
425 @staticmethod
426 def _tileArray(in_array):
427 """Given an input quarter-image, tile/mirror it and return full image.
429 Given a square input of side-length n, of the form
431 input = array([[1, 2, 3],
432 [4, 5, 6],
433 [7, 8, 9]])
435 return an array of size 2n-1 as
437 output = array([[ 9, 8, 7, 8, 9],
438 [ 6, 5, 4, 5, 6],
439 [ 3, 2, 1, 2, 3],
440 [ 6, 5, 4, 5, 6],
441 [ 9, 8, 7, 8, 9]])
443 Parameters
444 ----------
445 input : `np.array`, (N, N)
446 The square input quarter-array
448 Returns
449 -------
450 output : `np.array`, (2*N + 1, 2*N + 1)
451 The full, tiled array
452 """
453 assert(in_array.shape[0] == in_array.shape[1])
454 length = in_array.shape[0] - 1
455 output = np.zeros((2*length + 1, 2*length + 1))
457 for i in range(length + 1):
458 for j in range(length + 1):
459 output[i + length, j + length] = in_array[i, j]
460 output[-i + length, j + length] = in_array[i, j]
461 output[i + length, -j + length] = in_array[i, j]
462 output[-i + length, -j + length] = in_array[i, j]
463 return output
465 def successiveOverRelax(self, source, maxIter=None, eLevel=None):
466 """An implementation of the successive over relaxation (SOR) method.
468 A numerical method for solving a system of linear equations
469 with faster convergence than the Gauss-Seidel method.
471 Parameters
472 ----------
473 source : `numpy.ndarray`, (N, N)
474 The input array.
475 maxIter : `int`, optional
476 Maximum number of iterations to attempt before aborting.
477 eLevel : `float`, optional
478 The target error level at which we deem convergence to have
479 occurred.
481 Returns
482 -------
483 output : `numpy.ndarray`, (N, N)
484 The solution.
485 """
486 if not maxIter:
487 maxIter = self.config.maxIterSuccessiveOverRelaxation
488 if not eLevel:
489 eLevel = self.config.eLevelSuccessiveOverRelaxation
491 assert source.shape[0] == source.shape[1], "Input array must be square"
492 # initialize, and set boundary conditions
493 func = np.zeros([source.shape[0] + 2, source.shape[1] + 2])
494 resid = np.zeros([source.shape[0] + 2, source.shape[1] + 2])
495 rhoSpe = np.cos(np.pi/source.shape[0]) # Here a square grid is assumed
497 # Calculate the initial error
498 for i in range(1, func.shape[0] - 1):
499 for j in range(1, func.shape[1] - 1):
500 resid[i, j] = (func[i, j - 1] + func[i, j + 1] + func[i - 1, j]
501 + func[i + 1, j] - 4*func[i, j] - source[i - 1, j - 1])
502 inError = np.sum(np.abs(resid))
504 # Iterate until convergence
505 # We perform two sweeps per cycle,
506 # updating 'odd' and 'even' points separately
507 nIter = 0
508 omega = 1.0
509 dx = 1.0
510 while nIter < maxIter*2:
511 outError = 0
512 if nIter%2 == 0:
513 for i in range(1, func.shape[0] - 1, 2):
514 for j in range(1, func.shape[1] - 1, 2):
515 resid[i, j] = float(func[i, j-1] + func[i, j + 1] + func[i - 1, j]
516 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
517 func[i, j] += omega*resid[i, j]*.25
518 for i in range(2, func.shape[0] - 1, 2):
519 for j in range(2, func.shape[1] - 1, 2):
520 resid[i, j] = float(func[i, j - 1] + func[i, j + 1] + func[i - 1, j]
521 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
522 func[i, j] += omega*resid[i, j]*.25
523 else:
524 for i in range(1, func.shape[0] - 1, 2):
525 for j in range(2, func.shape[1] - 1, 2):
526 resid[i, j] = float(func[i, j - 1] + func[i, j + 1] + func[i - 1, j]
527 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
528 func[i, j] += omega*resid[i, j]*.25
529 for i in range(2, func.shape[0] - 1, 2):
530 for j in range(1, func.shape[1] - 1, 2):
531 resid[i, j] = float(func[i, j - 1] + func[i, j + 1] + func[i - 1, j]
532 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
533 func[i, j] += omega*resid[i, j]*.25
534 outError = np.sum(np.abs(resid))
535 if outError < inError*eLevel:
536 break
537 if nIter == 0:
538 omega = 1.0/(1 - rhoSpe*rhoSpe/2.0)
539 else:
540 omega = 1.0/(1 - rhoSpe*rhoSpe*omega/4.0)
541 nIter += 1
543 if nIter >= maxIter*2:
544 self.log.warning("Failure: SuccessiveOverRelaxation did not converge in %s iterations."
545 "\noutError: %s, inError: %s,", nIter//2, outError, inError*eLevel)
546 else:
547 self.log.info("Success: SuccessiveOverRelaxation converged in %s iterations."
548 "\noutError: %s, inError: %s", nIter//2, outError, inError*eLevel)
549 return func[1: -1, 1: -1]