22 """Calculation of brighter-fatter effect correlations and kernels."""
24 __all__ = [
'BrighterFatterKernelSolveTask',
25 'BrighterFatterKernelSolveConfig']
35 from .utils
import (funcPolynomial, irlsFit)
36 from ._lookupStaticCalibration
import lookupStaticCalibration
40 dimensions=(
"instrument",
"exposure",
"detector")):
43 doc=
"Dummy exposure.",
44 storageClass=
'Exposure',
45 dimensions=(
"instrument",
"exposure",
"detector"),
49 camera = cT.PrerequisiteInput(
51 doc=
"Camera associated with this data.",
52 storageClass=
"Camera",
53 dimensions=(
"instrument", ),
55 lookupFunction=lookupStaticCalibration,
57 inputPtc = cT.PrerequisiteInput(
59 doc=
"Photon transfer curve dataset.",
60 storageClass=
"PhotonTransferCurveDataset",
61 dimensions=(
"instrument",
"detector"),
65 outputBFK = cT.Output(
66 name=
"brighterFatterKernel",
67 doc=
"Output measured brighter-fatter kernel.",
68 storageClass=
"BrighterFatterKernel",
69 dimensions=(
"instrument",
"detector"),
75 pipelineConnections=BrighterFatterKernelSolveConnections):
76 level = pexConfig.ChoiceField(
77 doc=
"The level at which to calculate the brighter-fatter kernels",
81 "AMP":
"Every amplifier treated separately",
82 "DETECTOR":
"One kernel per detector",
85 ignoreAmpsForAveraging = pexConfig.ListField(
87 doc=
"List of amp names to ignore when averaging the amplifier kernels into the detector"
88 " kernel. Only relevant for level = DETECTOR",
91 xcorrCheckRejectLevel = pexConfig.Field(
93 doc=
"Rejection level for the sum of the input cross-correlations. Arrays which "
94 "sum to greater than this are discarded before the clipped mean is calculated.",
97 nSigmaClip = pexConfig.Field(
99 doc=
"Number of sigma to clip when calculating means for the cross-correlation",
102 forceZeroSum = pexConfig.Field(
104 doc=
"Force the correlation matrix to have zero sum by adjusting the (0,0) value?",
107 useAmatrix = pexConfig.Field(
109 doc=
"Use the PTC 'a' matrix (Astier et al. 2019 equation 20) "
110 "instead of the average of measured covariances?",
114 maxIterSuccessiveOverRelaxation = pexConfig.Field(
116 doc=
"The maximum number of iterations allowed for the successive over-relaxation method",
119 eLevelSuccessiveOverRelaxation = pexConfig.Field(
121 doc=
"The target residual error for the successive over-relaxation method",
125 correlationQuadraticFit = pexConfig.Field(
127 doc=
"Use a quadratic fit to find the correlations instead of simple averaging?",
130 correlationModelRadius = pexConfig.Field(
132 doc=
"Build a model of the correlation coefficients for radii larger than this value in pixels?",
135 correlationModelSlope = pexConfig.Field(
137 doc=
"Slope of the correlation model for radii larger than correlationModelRadius",
143 """Measure appropriate Brighter-Fatter Kernel from the PTC dataset.
146 ConfigClass = BrighterFatterKernelSolveConfig
147 _DefaultName =
'cpBfkMeasure'
150 """Ensure that the input and output dimensions are passed along.
154 butlerQC : `lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
155 Butler to operate on.
156 inputRefs : `lsst.pipe.base.connections.InputQuantizedConnection`
157 Input data refs to load.
158 ouptutRefs : `lsst.pipe.base.connections.OutputQuantizedConnection`
159 Output data refs to persist.
161 inputs = butlerQC.get(inputRefs)
164 inputs[
'inputDims'] = inputRefs.inputPtc.dataId.byName()
166 outputs = self.
runrun(**inputs)
167 butlerQC.put(outputs, outputRefs)
169 def run(self, inputPtc, dummy, camera, inputDims):
170 """Combine covariance information from PTC into brighter-fatter kernels.
174 inputPtc : `lsst.ip.isr.PhotonTransferCurveDataset`
175 PTC data containing per-amplifier covariance measurements.
176 dummy : `lsst.afw.image.Exposure
177 The exposure used to select the appropriate PTC dataset.
178 camera : `lsst.afw.cameraGeom.Camera`
179 Camera to use for camera geometry information.
180 inputDims : `lsst.daf.butler.DataCoordinate` or `dict`
181 DataIds to use to populate the output calibration.
185 results : `lsst.pipe.base.Struct`
186 The resulst struct containing:
188 ``outputBfk`` : `lsst.ip.isr.BrighterFatterKernel`
189 Resulting Brighter-Fatter Kernel.
192 self.log.warn(
"No dummy exposure found.")
194 detector = camera[inputDims[
'detector']]
195 detName = detector.getName()
197 if self.config.level ==
'DETECTOR':
198 detectorCorrList = list()
200 bfk = BrighterFatterKernel(camera=camera, detectorId=detector.getId(), level=self.config.level)
201 bfk.means = inputPtc.finalMeans
202 bfk.variances = inputPtc.finalVars
206 bfk.rawXcorrs = inputPtc.covariances
207 bfk.badAmps = inputPtc.badAmps
208 bfk.shape = (inputPtc.covMatrixSide*2 + 1, inputPtc.covMatrixSide*2 + 1)
209 bfk.gain = inputPtc.gain
210 bfk.noise = inputPtc.noise
211 bfk.meanXcorrs = dict()
215 ampName = amp.getName()
216 mask = np.array(inputPtc.expIdMask[ampName], dtype=bool)
218 gain = bfk.gain[ampName]
219 fluxes = np.array(bfk.means[ampName])[mask]
220 variances = np.array(bfk.variances[ampName])[mask]
221 xCorrList = [np.array(xcorr)
for xcorr
in bfk.rawXcorrs[ampName]]
222 xCorrList = np.array(xCorrList)[mask]
226 self.log.warn(
"Impossible gain recieved from PTC for %s: %f. Skipping amplifier.",
228 bfk.meanXcorrs[ampName] = np.zeros(bfk.shape)
229 bfk.ampKernels[ampName] = np.zeros(bfk.shape)
230 bfk.valid[ampName] =
False
233 fluxes = np.array([flux*gain
for flux
in fluxes])
234 variances = np.array([variance*gain*gain
for variance
in variances])
237 scaledCorrList = list()
238 for xcorrNum, (xcorr, flux, var)
in enumerate(zip(xCorrList, fluxes, variances), 1):
239 q = np.array(xcorr) * gain * gain
241 self.log.info(
"Amp: %s %d/%d Flux: %f Var: %f Q(0,0): %g Q(1,0): %g Q(0,1): %g",
242 ampName, xcorrNum, len(xCorrList), flux, var, q[0][0], q[1][0], q[0][1])
247 q[0][0] -= 2.0*(flux)
250 self.log.warn(
"Amp: %s %d skipped due to value of (variance-mean)=%f",
251 ampName, xcorrNum, q[0][0])
258 xcorrCheck = np.abs(np.sum(scaled))/np.sum(np.abs(scaled))
259 if (xcorrCheck > self.config.xcorrCheckRejectLevel)
or not (np.isfinite(xcorrCheck)):
260 self.log.warn(
"Amp: %s %d skipped due to value of triangle-inequality sum %f",
261 ampName, xcorrNum, xcorrCheck)
264 scaledCorrList.append(scaled)
265 self.log.info(
"Amp: %s %d/%d Final: %g XcorrCheck: %f",
266 ampName, xcorrNum, len(xCorrList), q[0][0], xcorrCheck)
268 if len(scaledCorrList) == 0:
269 self.log.warn(
"Amp: %s All inputs rejected for amp!", ampName)
270 bfk.meanXcorrs[ampName] = np.zeros(bfk.shape)
271 bfk.ampKernels[ampName] = np.zeros(bfk.shape)
272 bfk.valid[ampName] =
False
275 if self.config.useAmatrix:
277 preKernel = np.pad(self.
_tileArray_tileArray(np.array(inputPtc.aMatrix[ampName])), ((1, 1)))
278 elif self.config.correlationQuadraticFit:
280 preKernel = self.
quadraticCorrelationsquadraticCorrelations(scaledCorrList, fluxes, f
"Amp: {ampName}")
285 center = int((bfk.shape[0] - 1) / 2)
287 if self.config.forceZeroSum:
288 totalSum = np.sum(preKernel)
290 if self.config.correlationModelRadius < (preKernel.shape[0] - 1) / 2:
292 preFactor = np.sqrt(preKernel[center, center + 1] * preKernel[center + 1, center])
293 slopeFactor = 2.0 * np.abs(self.config.correlationModelSlope)
294 totalSum += 2.0*np.pi*(preFactor / (slopeFactor*(center + 0.5))**slopeFactor)
296 preKernel[center, center] -= totalSum
297 self.log.info(
"%s Zero-Sum Scale: %g", ampName, totalSum)
299 finalSum = np.sum(preKernel)
300 bfk.meanXcorrs[ampName] = preKernel
303 bfk.ampKernels[ampName] = postKernel
304 if self.config.level ==
'DETECTOR':
305 detectorCorrList.extend(scaledCorrList)
306 bfk.valid[ampName] =
True
307 self.log.info(
"Amp: %s Sum: %g Center Info Pre: %g Post: %g",
308 ampName, finalSum, preKernel[center, center], postKernel[center, center])
311 if self.config.level ==
'DETECTOR':
312 preKernel = self.
averageCorrelationsaverageCorrelations(detectorCorrList, f
"Det: {detName}")
313 finalSum = np.sum(preKernel)
314 center = int((bfk.shape[0] - 1) / 2)
317 bfk.detKernels[detName] = postKernel
318 self.log.info(
"Det: %s Sum: %g Center Info Pre: %g Post: %g",
319 detName, finalSum, preKernel[center, center], postKernel[center, center])
321 return pipeBase.Struct(
326 """Average input correlations.
330 xCorrList : `list` [`numpy.array`]
331 List of cross-correlations.
333 Name for log messages.
337 meanXcorr : `numpy.array`
338 The averaged cross-correlation.
340 meanXcorr = np.zeros_like(xCorrList[0])
341 xCorrList = np.transpose(xCorrList)
342 sctrl = afwMath.StatisticsControl()
343 sctrl.setNumSigmaClip(self.config.nSigmaClip)
344 for i
in range(np.shape(meanXcorr)[0]):
345 for j
in range(np.shape(meanXcorr)[1]):
346 meanXcorr[i, j] = afwMath.makeStatistics(xCorrList[i, j],
347 afwMath.MEANCLIP, sctrl).getValue()
350 meanXcorr = np.pad(meanXcorr, ((1, 1)))
355 """Measure a quadratic correlation model.
359 xCorrList : `list` [`numpy.array`]
360 List of cross-correlations.
361 fluxList : `numpy.array`
362 Associated list of fluxes.
364 Name for log messages.
368 meanXcorr : `numpy.array`
369 The averaged cross-correlation.
371 meanXcorr = np.zeros_like(xCorrList[0])
372 fluxList = np.square(fluxList)
373 xCorrList = np.array(xCorrList)
375 for i
in range(np.shape(meanXcorr)[0]):
376 for j
in range(np.shape(meanXcorr)[1]):
380 linearFit, linearFitErr, chiSq, weights =
irlsFit([0.0, 1e-4], fluxList,
381 xCorrList[:, j, i], funcPolynomial)
382 meanXcorr[i, j] = linearFit[1]
383 self.log.debug(
"Quad fit meanXcorr[%d,%d] = %g", i, j, linearFit[1])
386 meanXcorr = np.pad(meanXcorr, ((1, 1)))
391 def _tileArray(in_array):
392 """Given an input quarter-image, tile/mirror it and return full image.
394 Given a square input of side-length n, of the form
396 input = array([[1, 2, 3],
400 return an array of size 2n-1 as
402 output = array([[ 9, 8, 7, 8, 9],
411 The square input quarter-array
416 The full, tiled array
418 assert(in_array.shape[0] == in_array.shape[1])
419 length = in_array.shape[0] - 1
420 output = np.zeros((2*length + 1, 2*length + 1))
422 for i
in range(length + 1):
423 for j
in range(length + 1):
424 output[i + length, j + length] = in_array[i, j]
425 output[-i + length, j + length] = in_array[i, j]
426 output[i + length, -j + length] = in_array[i, j]
427 output[-i + length, -j + length] = in_array[i, j]
431 """An implementation of the successive over relaxation (SOR) method.
433 A numerical method for solving a system of linear equations
434 with faster convergence than the Gauss-Seidel method.
438 source : `numpy.ndarray`
440 maxIter : `int`, optional
441 Maximum number of iterations to attempt before aborting.
442 eLevel : `float`, optional
443 The target error level at which we deem convergence to have
448 output : `numpy.ndarray`
452 maxIter = self.config.maxIterSuccessiveOverRelaxation
454 eLevel = self.config.eLevelSuccessiveOverRelaxation
456 assert source.shape[0] == source.shape[1],
"Input array must be square"
458 func = np.zeros([source.shape[0] + 2, source.shape[1] + 2])
459 resid = np.zeros([source.shape[0] + 2, source.shape[1] + 2])
460 rhoSpe = np.cos(np.pi/source.shape[0])
463 for i
in range(1, func.shape[0] - 1):
464 for j
in range(1, func.shape[1] - 1):
465 resid[i, j] = (func[i, j - 1] + func[i, j + 1] + func[i - 1, j]
466 + func[i + 1, j] - 4*func[i, j] - source[i - 1, j - 1])
467 inError = np.sum(np.abs(resid))
475 while nIter < maxIter*2:
478 for i
in range(1, func.shape[0] - 1, 2):
479 for j
in range(1, func.shape[1] - 1, 2):
480 resid[i, j] = float(func[i, j-1] + func[i, j + 1] + func[i - 1, j]
481 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
482 func[i, j] += omega*resid[i, j]*.25
483 for i
in range(2, func.shape[0] - 1, 2):
484 for j
in range(2, func.shape[1] - 1, 2):
485 resid[i, j] = float(func[i, j - 1] + func[i, j + 1] + func[i - 1, j]
486 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
487 func[i, j] += omega*resid[i, j]*.25
489 for i
in range(1, func.shape[0] - 1, 2):
490 for j
in range(2, func.shape[1] - 1, 2):
491 resid[i, j] = float(func[i, j - 1] + func[i, j + 1] + func[i - 1, j]
492 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
493 func[i, j] += omega*resid[i, j]*.25
494 for i
in range(2, func.shape[0] - 1, 2):
495 for j
in range(1, func.shape[1] - 1, 2):
496 resid[i, j] = float(func[i, j - 1] + func[i, j + 1] + func[i - 1, j]
497 + func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
498 func[i, j] += omega*resid[i, j]*.25
499 outError = np.sum(np.abs(resid))
500 if outError < inError*eLevel:
503 omega = 1.0/(1 - rhoSpe*rhoSpe/2.0)
505 omega = 1.0/(1 - rhoSpe*rhoSpe*omega/4.0)
508 if nIter >= maxIter*2:
509 self.log.warn(
"Failure: SuccessiveOverRelaxation did not converge in %s iterations."
510 "\noutError: %s, inError: %s," % (nIter//2, outError, inError*eLevel))
512 self.log.info(
"Success: SuccessiveOverRelaxation converged in %s iterations."
513 "\noutError: %s, inError: %s", nIter//2, outError, inError*eLevel)
514 return func[1: -1, 1: -1]
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def averageCorrelations(self, xCorrList, name)
def run(self, inputPtc, dummy, camera, inputDims)
def quadraticCorrelations(self, xCorrList, fluxList, name)
def successiveOverRelax(self, source, maxIter=None, eLevel=None)
def irlsFit(initialParams, dataX, dataY, function, weightsY=None)