lsst.ip.diffim g5706f010af+3c6b2f0ce6
imageDecorrelation.py
Go to the documentation of this file.
2# LSST Data Management System
3# Copyright 2016 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22
23import numpy as np
24
25import lsst.afw.image as afwImage
26import lsst.afw.math as afwMath
27import lsst.geom as geom
28import lsst.log
29import lsst.meas.algorithms as measAlg
30import lsst.pex.config as pexConfig
31import lsst.pipe.base as pipeBase
32from lsst.utils.timer import timeMethod
33
34from .imageMapReduce import (ImageMapReduceConfig, ImageMapReduceTask,
35 ImageMapper)
36
37__all__ = ("DecorrelateALKernelTask", "DecorrelateALKernelConfig",
38 "DecorrelateALKernelMapper", "DecorrelateALKernelMapReduceConfig",
39 "DecorrelateALKernelSpatialConfig", "DecorrelateALKernelSpatialTask")
40
41
42class DecorrelateALKernelConfig(pexConfig.Config):
43 """Configuration parameters for the DecorrelateALKernelTask
44 """
45
46 ignoreMaskPlanes = pexConfig.ListField(
47 dtype=str,
48 doc="""Mask planes to ignore for sigma-clipped statistics""",
49 default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE")
50 )
51 completeVarPlanePropagation = pexConfig.Field(
52 dtype=bool,
53 default=False,
54 doc="Compute the full effect of the decorrelated matching kernel on the variance plane."
55 " Otherwise use a model weighed sum of the input variances."
56 )
57
58
59class DecorrelateALKernelTask(pipeBase.Task):
60 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference
61
62 Notes
63 -----
64
65 Pipe-task that removes the neighboring-pixel covariance in an
66 image difference that are added when the template image is
67 convolved with the Alard-Lupton PSF matching kernel.
68
69 The image differencing pipeline task @link
70 ip.diffim.psfMatch.PsfMatchTask PSFMatchTask@endlink and @link
71 ip.diffim.psfMatch.PsfMatchConfigAL PSFMatchConfigAL@endlink uses
72 the Alard and Lupton (1998) method for matching the PSFs of the
73 template and science exposures prior to subtraction. The
74 Alard-Lupton method identifies a matching kernel, which is then
75 (typically) convolved with the template image to perform PSF
76 matching. This convolution has the effect of adding covariance
77 between neighboring pixels in the template image, which is then
78 added to the image difference by subtraction.
79
80 The pixel covariance may be corrected by whitening the noise of
81 the image difference. This task performs such a decorrelation by
82 computing a decorrelation kernel (based upon the A&L matching
83 kernel and variances in the template and science images) and
84 convolving the image difference with it. This process is described
85 in detail in [DMTN-021](http://dmtn-021.lsst.io).
86
87 This task has no standalone example, however it is applied as a
88 subtask of pipe.tasks.imageDifference.ImageDifferenceTask.
89 """
90 ConfigClass = DecorrelateALKernelConfig
91 _DefaultName = "ip_diffim_decorrelateALKernel"
92
93 def __init__(self, *args, **kwargs):
94 """Create the image decorrelation Task
95
96 Parameters
97 ----------
98 args :
99 arguments to be passed to ``lsst.pipe.base.task.Task.__init__``
100 kwargs :
101 keyword arguments to be passed to ``lsst.pipe.base.task.Task.__init__``
102 """
103 pipeBase.Task.__init__(self, *args, **kwargs)
104
106 self.statsControlstatsControl.setNumSigmaClip(3.)
107 self.statsControlstatsControl.setNumIter(3)
108 self.statsControlstatsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes))
109
110 def computeVarianceMean(self, exposure):
111 statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(),
112 exposure.getMaskedImage().getMask(),
113 afwMath.MEANCLIP, self.statsControlstatsControl)
114 var = statObj.getValue(afwMath.MEANCLIP)
115 return var
116
117 @timeMethod
118 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel,
119 preConvKernel=None, xcen=None, ycen=None, svar=None, tvar=None,
120 templateMatched=True, preConvMode=False, **kwargs):
121 """Perform decorrelation of an image difference or of a score difference exposure.
122
123 Corrects the difference or score image due to the convolution of the
124 templateExposure with the A&L PSF matching kernel.
125 See [DMTN-021, Equation 1](http://dmtn-021.lsst.io/#equation-1) and
126 [DMTN-179](http://dmtn-179.lsst.io/) for details.
127
128 Parameters
129 ----------
130 scienceExposure : `lsst.afw.image.Exposure`
131 The original science exposure (before pre-convolution, if ``preConvMode==True``).
132 templateExposure : `lsst.afw.image.Exposure`
133 The original template exposure warped into the science exposure dimensions.
134 subtractedExposure : `lsst.afw.image.Exposure`
135 the subtracted exposure produced by
136 `ip_diffim.ImagePsfMatchTask.subtractExposures()`. The `subtractedExposure` must
137 inherit its PSF from `exposure`, see notes below.
138 psfMatchingKernel : `lsst.afw.detection.Psf`
139 An (optionally spatially-varying) PSF matching kernel produced
140 by `ip_diffim.ImagePsfMatchTask.subtractExposures()`.
141 preConvKernel : `lsst.afw.math.Kernel`, optional
142 If not `None`, then the `scienceExposure` was pre-convolved with (the reflection of)
143 this kernel. Must be normalized to sum to 1.
144 Allowed only if ``templateMatched==True`` and ``preConvMode==True``.
145 Defaults to the PSF of the science exposure at the image center.
146 xcen : `float`, optional
147 X-pixel coordinate to use for computing constant matching kernel to use
148 If `None` (default), then use the center of the image.
149 ycen : `float`, optional
150 Y-pixel coordinate to use for computing constant matching kernel to use
151 If `None` (default), then use the center of the image.
152 svar : `float`, optional
153 Image variance for science image
154 If `None` (default) then compute the variance over the entire input science image.
155 tvar : `float`, optional
156 Image variance for template image
157 If `None` (default) then compute the variance over the entire input template image.
158 templateMatched : `bool`, optional
159 If True, the template exposure was matched (convolved) to the science exposure.
160 See also notes below.
161 preConvMode : `bool`, optional
162 If True, ``subtractedExposure`` is assumed to be a likelihood difference image
163 and will be noise corrected as a likelihood image.
164 **kwargs
165 Additional keyword arguments propagated from DecorrelateALKernelSpatialTask.
166
167 Returns
168 -------
169 result : `lsst.pipe.base.Struct`
170 - ``correctedExposure`` : the decorrelated diffim
171
172 Notes
173 -----
174 If ``preConvMode==True``, ``subtractedExposure`` is assumed to be a
175 score image and the noise correction for likelihood images
176 is applied. The resulting image is an optimal detection likelihood image
177 when the templateExposure has noise. (See DMTN-179) If ``preConvKernel`` is
178 not specified, the PSF of ``scienceExposure`` is assumed as pre-convolution kernel.
179
180 The ``subtractedExposure`` is NOT updated. The returned ``correctedExposure``
181 has an updated but spatially fixed PSF. It is calculated as the center of
182 image PSF corrected by the center of image matching kernel.
183
184 If ``templateMatched==True``, the templateExposure was matched (convolved)
185 to the ``scienceExposure`` by ``psfMatchingKernel``. Otherwise the ``scienceExposure``
186 was matched (convolved) by ``psfMatchingKernel``.
187
188 This task discards the variance plane of ``subtractedExposure`` and re-computes
189 it from the variance planes of ``scienceExposure`` and ``templateExposure``.
190 The image plane of ``subtractedExposure`` must be at the photometric level
191 set by the AL PSF matching in `ImagePsfMatchTask.subtractExposures`.
192 The assumptions about the photometric level are controlled by the
193 `templateMatched` option in this task.
194
195 Here we currently convert a spatially-varying matching kernel into a constant kernel,
196 just by computing it at the center of the image (tickets DM-6243, DM-6244).
197
198 We are also using a constant accross-the-image measure of sigma (sqrt(variance)) to compute
199 the decorrelation kernel.
200
201 TODO DM-23857 As part of the spatially varying correction implementation
202 consider whether returning a Struct is still necessary.
203 """
204 if preConvKernel is not None and not (templateMatched and preConvMode):
205 raise ValueError("Pre-convolution kernel is allowed only if "
206 "preConvMode==True and templateMatched==True.")
207
208 spatialKernel = psfMatchingKernel
209 kimg = afwImage.ImageD(spatialKernel.getDimensions())
210 bbox = subtractedExposure.getBBox()
211 if xcen is None:
212 xcen = (bbox.getBeginX() + bbox.getEndX()) / 2.
213 if ycen is None:
214 ycen = (bbox.getBeginY() + bbox.getEndY()) / 2.
215 self.log.info("Using matching kernel computed at (%d, %d)", xcen, ycen)
216 spatialKernel.computeImage(kimg, False, xcen, ycen)
217
218 preConvImg = None
219 if preConvMode:
220 if preConvKernel is None:
221 preConvKernel = scienceExposure.getPsf().getLocalKernel() # at average position
222 preConvImg = afwImage.ImageD(preConvKernel.getDimensions())
223 preConvKernel.computeImage(preConvImg, True)
224
225 if svar is None:
226 svar = self.computeVarianceMeancomputeVarianceMean(scienceExposure)
227 if tvar is None:
228 tvar = self.computeVarianceMeancomputeVarianceMean(templateExposure)
229 self.log.info("Original variance plane means. Science:%.5e, warped template:%.5e)",
230 svar, tvar)
231
232 if templateMatched:
233 # Regular subtraction, we convolved the template
234 self.log.info("Decorrelation after template image convolution")
235 expVar = svar
236 matchedVar = tvar
237 exposure = scienceExposure
238 matchedExposure = templateExposure
239 else:
240 # We convolved the science image
241 self.log.info("Decorrelation after science image convolution")
242 expVar = tvar
243 matchedVar = svar
244 exposure = templateExposure
245 matchedExposure = scienceExposure
246
247 # Should not happen unless entire image has been masked, which could happen
248 # if this is a small subimage of the main exposure. In this case, just return a full NaN
249 # exposure
250 if np.isnan(expVar) or np.isnan(matchedVar):
251 # Double check that one of the exposures is all NaNs
252 if (np.all(np.isnan(exposure.image.array))
253 or np.all(np.isnan(matchedExposure.image.array))):
254 self.log.warning('Template or science image is entirely NaNs: skipping decorrelation.')
255 outExposure = subtractedExposure.clone()
256 return pipeBase.Struct(correctedExposure=outExposure, )
257
258 # The maximal correction value converges to sqrt(matchedVar/expVar).
259 # Correction divergence warning if the correction exceeds 4 orders of magnitude.
260 mOverExpVar = matchedVar/expVar
261 if mOverExpVar > 1e8:
262 self.log.warning("Diverging correction: matched image variance is "
263 " much larger than the unconvolved one's"
264 ", matchedVar/expVar:%.2e", mOverExpVar)
265
266 oldVarMean = self.computeVarianceMeancomputeVarianceMean(subtractedExposure)
267 self.log.info("Variance plane mean of uncorrected diffim: %f", oldVarMean)
268
269 kArr = kimg.array
270 diffExpArr = subtractedExposure.image.array
271 psfImg = subtractedExposure.getPsf().computeKernelImage(geom.Point2D(xcen, ycen))
272 psfDim = psfImg.getDimensions()
273 psfArr = psfImg.array
274
275 # Determine the common shape
276 kSum = np.sum(kArr)
277 kSumSq = kSum*kSum
278 self.log.debug("Matching kernel sum: %.3e", kSum)
279
280 if preConvMode:
281 self.log.info("Decorrelation of likelihood image")
282 self.computeCommonShapecomputeCommonShape(preConvImg.array.shape, kArr.shape,
283 psfArr.shape, diffExpArr.shape)
284 corr = self.computeScoreCorrectioncomputeScoreCorrection(kArr, expVar, matchedVar, preConvImg.array)
285 else:
286 self.log.info("Decorrelation of difference image")
287 self.computeCommonShapecomputeCommonShape(kArr.shape, psfArr.shape, diffExpArr.shape)
288 corr = self.computeDiffimCorrectioncomputeDiffimCorrection(kArr, expVar, matchedVar)
289
290 diffExpArr = self.computeCorrectedImagecomputeCorrectedImage(corr.corrft, diffExpArr)
291
292 corrPsfArr = self.computeCorrectedDiffimPsfcomputeCorrectedDiffimPsf(corr.corrft, psfArr)
293 psfcI = afwImage.ImageD(psfDim)
294 psfcI.array = corrPsfArr
295 psfcK = afwMath.FixedKernel(psfcI)
296 psfNew = measAlg.KernelPsf(psfcK)
297
298 correctedExposure = subtractedExposure.clone()
299 correctedExposure.image.array[...] = diffExpArr # Allow for numpy type casting
300 # The subtracted exposure variance plane is already correlated, we cannot propagate
301 # it through another convolution; instead we need to use the uncorrelated originals
302 # The whitening should scale it to expVar + matchedVar on average
303 if self.config.completeVarPlanePropagation:
304 self.log.debug("Using full variance plane calculation in decorrelation")
305 newVarArr = self.calculateVariancePlanecalculateVariancePlane(
306 exposure.variance.array, matchedExposure.variance.array,
307 expVar, matchedVar, corr.cnft, corr.crft)
308 else:
309 self.log.debug("Using estimated variance plane calculation in decorrelation")
310 newVarArr = self.estimateVariancePlaneestimateVariancePlane(
311 exposure.variance.array, matchedExposure.variance.array,
312 corr.cnft, corr.crft)
313
314 corrExpVarArr = correctedExposure.variance.array
315 corrExpVarArr[...] = newVarArr # Allow for numpy type casting
316
317 if not templateMatched:
318 # ImagePsfMatch.subtractExposures re-scales the difference in
319 # the science image convolution mode
320 corrExpVarArr /= kSumSq
321 correctedExposure.setPsf(psfNew)
322
323 newVarMean = self.computeVarianceMeancomputeVarianceMean(correctedExposure)
324 self.log.info("Variance plane mean of corrected diffim: %.5e", newVarMean)
325
326 # TODO DM-23857 As part of the spatially varying correction implementation
327 # consider whether returning a Struct is still necessary.
328 return pipeBase.Struct(correctedExposure=correctedExposure, )
329
330 def computeCommonShape(self, *shapes):
331 """Calculate the common shape for FFT operations. Set `self.freqSpaceShape`
332 internally.
333
334 Parameters
335 ----------
336 shapes : one or more `tuple` of `int`
337 Shapes of the arrays. All must have the same dimensionality.
338 At least one shape must be provided.
339
340 Returns
341 -------
342 None.
343
344 Notes
345 -----
346 For each dimension, gets the smallest even number greater than or equal to
347 `N1+N2-1` where `N1` and `N2` are the two largest values.
348 In case of only one shape given, rounds up to even each dimension value.
349 """
350 S = np.array(shapes, dtype=int)
351 if len(shapes) > 2:
352 S.sort(axis=0)
353 S = S[-2:]
354 if len(shapes) > 1:
355 commonShape = np.sum(S, axis=0) - 1
356 else:
357 commonShape = S[0]
358 commonShape[commonShape % 2 != 0] += 1
359 self.freqSpaceShapefreqSpaceShape = tuple(commonShape)
360 self.log.info("Common frequency space shape %s", self.freqSpaceShapefreqSpaceShape)
361
362 @staticmethod
363 def padCenterOriginArray(A, newShape: tuple, useInverse=False):
364 """Zero pad an image where the origin is at the center and replace the
365 origin to the corner as required by the periodic input of FFT. Implement also
366 the inverse operation, crop the padding and re-center data.
367
368 Parameters
369 ----------
370 A : `numpy.ndarray`
371 An array to copy from.
372 newShape : `tuple` of `int`
373 The dimensions of the resulting array. For padding, the resulting array
374 must be larger than A in each dimension. For the inverse operation this
375 must be the original, before padding size of the array.
376 useInverse : bool, optional
377 Selector of forward, add padding, operation (False)
378 or its inverse, crop padding, operation (True).
379
380 Returns
381 -------
382 R : `numpy.ndarray`
383 The padded or unpadded array with shape of `newShape` and the same dtype as A.
384
385 Notes
386 -----
387 For odd dimensions, the splitting is rounded to
388 put the center pixel into the new corner origin (0,0). This is to be consistent
389 e.g. for a dirac delta kernel that is originally located at the center pixel.
390 """
391
392 # The forward and inverse operations should round odd dimension halves at the opposite
393 # sides to get the pixels back to their original positions.
394 if not useInverse:
395 # Forward operation: First and second halves with respect to the axes of A.
396 firstHalves = [x//2 for x in A.shape]
397 secondHalves = [x-y for x, y in zip(A.shape, firstHalves)]
398 else:
399 # Inverse operation: Opposite rounding
400 secondHalves = [x//2 for x in newShape]
401 firstHalves = [x-y for x, y in zip(newShape, secondHalves)]
402
403 R = np.zeros_like(A, shape=newShape)
404 R[-firstHalves[0]:, -firstHalves[1]:] = A[:firstHalves[0], :firstHalves[1]]
405 R[:secondHalves[0], -firstHalves[1]:] = A[-secondHalves[0]:, :firstHalves[1]]
406 R[:secondHalves[0], :secondHalves[1]] = A[-secondHalves[0]:, -secondHalves[1]:]
407 R[-firstHalves[0]:, :secondHalves[1]] = A[:firstHalves[0], -secondHalves[1]:]
408 return R
409
410 def computeDiffimCorrection(self, kappa, svar, tvar):
411 """Compute the Lupton decorrelation post-convolution kernel for decorrelating an
412 image difference, based on the PSF-matching kernel.
413
414 Parameters
415 ----------
416 kappa : `numpy.ndarray` of `float`
417 A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching.
418 svar : `float` > 0.
419 Average variance of science image used for PSF matching.
420 tvar : `float` > 0.
421 Average variance of the template (matched) image used for PSF matching.
422
423 Returns
424 -------
425 corrft : `numpy.ndarray` of `float`
426 The frequency space representation of the correction. The array is real (dtype float).
427 Shape is `self.freqSpaceShapefreqSpaceShape`.
428
429 cnft, crft : `numpy.ndarray` of `complex`
430 The overall convolution (pre-conv, PSF matching, noise correction) kernel
431 for the science and template images, respectively for the variance plane
432 calculations. These are intermediate results in frequency space.
433
434 Notes
435 -----
436 The maximum correction factor converges to `sqrt(tvar/svar)` towards high frequencies.
437 This should be a plausible value.
438 """
439 kSum = np.sum(kappa) # We scale the decorrelation to preserve fluxes
440 kappa = self.padCenterOriginArraypadCenterOriginArray(kappa, self.freqSpaceShapefreqSpaceShape)
441 kft = np.fft.fft2(kappa)
442 kftAbsSq = np.real(np.conj(kft) * kft)
443
444 denom = svar + tvar * kftAbsSq
445 corrft = np.sqrt((svar + tvar * kSum*kSum) / denom)
446 cnft = corrft
447 crft = kft*corrft
448 return pipeBase.Struct(corrft=corrft, cnft=cnft, crft=crft)
449
450 def computeScoreCorrection(self, kappa, svar, tvar, preConvArr):
451 """Compute the correction kernel for a score image.
452
453 Parameters
454 ----------
455 kappa : `numpy.ndarray`
456 A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching.
457 svar : `float`
458 Average variance of science image used for PSF matching (before pre-convolution).
459 tvar : `float`
460 Average variance of the template (matched) image used for PSF matching.
461 preConvArr : `numpy.ndarray`
462 The pre-convolution kernel of the science image. It should be the PSF
463 of the science image or an approximation of it. It must be normed to sum 1.
464
465 Returns
466 -------
467 corrft : `numpy.ndarray` of `float`
468 The frequency space representation of the correction. The array is real (dtype float).
469 Shape is `self.freqSpaceShapefreqSpaceShape`.
470 cnft, crft : `numpy.ndarray` of `complex`
471 The overall convolution (pre-conv, PSF matching, noise correction) kernel
472 for the science and template images, respectively for the variance plane
473 calculations. These are intermediate results in frequency space.
474
475 Notes
476 -----
477 To be precise, the science image should be _correlated_ by ``preConvArray`` but this
478 does not matter for this calculation.
479
480 ``cnft``, ``crft`` contain the scaling factor as well.
481
482 """
483 kSum = np.sum(kappa)
484 kappa = self.padCenterOriginArraypadCenterOriginArray(kappa, self.freqSpaceShapefreqSpaceShape)
485 kft = np.fft.fft2(kappa)
486 preConvArr = self.padCenterOriginArraypadCenterOriginArray(preConvArr, self.freqSpaceShapefreqSpaceShape)
487 preFt = np.fft.fft2(preConvArr)
488 preFtAbsSq = np.real(np.conj(preFt) * preFt)
489 kftAbsSq = np.real(np.conj(kft) * kft)
490 # Avoid zero division, though we don't normally approach `tiny`.
491 # We have numerical noise instead.
492 tiny = np.finfo(preFtAbsSq.dtype).tiny * 1000.
493 flt = preFtAbsSq < tiny
494 # If we pre-convolve to avoid deconvolution in AL, then kftAbsSq / preFtAbsSq
495 # theoretically expected to diverge to +inf. But we don't care about the convergence
496 # properties here, S goes to 0 at these frequencies anyway.
497 preFtAbsSq[flt] = tiny
498 denom = svar + tvar * kftAbsSq / preFtAbsSq
499 corrft = (svar + tvar * kSum*kSum) / denom
500 cnft = np.conj(preFt)*corrft
501 crft = kft*corrft
502 return pipeBase.Struct(corrft=corrft, cnft=cnft, crft=crft)
503
504 @staticmethod
505 def estimateVariancePlane(vplane1, vplane2, c1ft, c2ft):
506 """Estimate the variance planes.
507
508 The estimation assumes that around each pixel the surrounding
509 pixels' sigmas within the convolution kernel are the same.
510
511 Parameters
512 ----------
513 vplane1, vplane2 : `numpy.ndarray` of `float`
514 Variance planes of the original (before pre-convolution or matching)
515 exposures.
516 c1ft, c2ft : `numpy.ndarray` of `complex`
517 The overall convolution that includes the matching and the
518 afterburner in frequency space. The result of either
519 ``computeScoreCorrection`` or ``computeDiffimCorrection``.
520
521 Returns
522 -------
523 vplaneD : `numpy.ndarray` of `float`
524 The estimated variance plane of the difference/score image
525 as a weighted sum of the input variances.
526
527 Notes
528 ------
529 See DMTN-179 Section 5 about the variance plane calculations.
530 """
531 w1 = np.sum(np.real(np.conj(c1ft)*c1ft)) / c1ft.size
532 w2 = np.sum(np.real(np.conj(c2ft)*c2ft)) / c2ft.size
533 # w1, w2: the frequency space sum of abs(c1)^2 is the same as in image
534 # space.
535 return vplane1*w1 + vplane2*w2
536
537 def calculateVariancePlane(self, vplane1, vplane2, varMean1, varMean2, c1ft, c2ft):
538 """Full propagation of the variance planes of the original exposures.
539
540 The original variance planes of independent pixels are convolved with the
541 image space square of the overall kernels.
542
543 Parameters
544 ----------
545 vplane1, vplane2 : `numpy.ndarray` of `float`
546 Variance planes of the original (before pre-convolution or matching)
547 exposures.
548 varMean1, varMean2 : `float`
549 Replacement average values for non-finite ``vplane1`` and ``vplane2`` values respectively.
550
551 c1ft, c2ft : `numpy.ndarray` of `complex`
552 The overall convolution that includes the matching and the
553 afterburner in frequency space. The result of either
554 ``computeScoreCorrection`` or ``computeDiffimCorrection``.
555
556 Returns
557 -------
558 vplaneD : `numpy.ndarray` of `float`
559 The variance plane of the difference/score images.
560
561 Notes
562 ------
563 See DMTN-179 Section 5 about the variance plane calculations.
564
565 Infs and NaNs are allowed and kept in the returned array.
566 """
567 D = np.real(np.fft.ifft2(c1ft))
568 c1SqFt = np.fft.fft2(D*D)
569
570 v1shape = vplane1.shape
571 filtInf = np.isinf(vplane1)
572 filtNan = np.isnan(vplane1)
573 # This copy could be eliminated if inf/nan handling were go into padCenterOriginArray
574 vplane1 = np.copy(vplane1)
575 vplane1[filtInf | filtNan] = varMean1
576 D = self.padCenterOriginArraypadCenterOriginArray(vplane1, self.freqSpaceShapefreqSpaceShape)
577 v1 = np.real(np.fft.ifft2(np.fft.fft2(D) * c1SqFt))
578 v1 = self.padCenterOriginArraypadCenterOriginArray(v1, v1shape, useInverse=True)
579 v1[filtNan] = np.nan
580 v1[filtInf] = np.inf
581
582 D = np.real(np.fft.ifft2(c2ft))
583 c2ft = np.fft.fft2(D*D)
584
585 v2shape = vplane2.shape
586 filtInf = np.isinf(vplane2)
587 filtNan = np.isnan(vplane2)
588 vplane2 = np.copy(vplane2)
589 vplane2[filtInf | filtNan] = varMean2
590 D = self.padCenterOriginArraypadCenterOriginArray(vplane2, self.freqSpaceShapefreqSpaceShape)
591 v2 = np.real(np.fft.ifft2(np.fft.fft2(D) * c2ft))
592 v2 = self.padCenterOriginArraypadCenterOriginArray(v2, v2shape, useInverse=True)
593 v2[filtNan] = np.nan
594 v2[filtInf] = np.inf
595
596 return v1 + v2
597
598 def computeCorrectedDiffimPsf(self, corrft, psfOld):
599 """Compute the (decorrelated) difference image's new PSF.
600
601 Parameters
602 ----------
603 corrft : `numpy.ndarray`
604 The frequency space representation of the correction calculated by
605 `computeCorrection`. Shape must be `self.freqSpaceShapefreqSpaceShape`.
606 psfOld : `numpy.ndarray`
607 The psf of the difference image to be corrected.
608
609 Returns
610 -------
611 psfNew : `numpy.ndarray`
612 The corrected psf, same shape as `psfOld`, sum normed to 1.
613
614 Notes
615 -----
616 There is no algorithmic guarantee that the corrected psf can
617 meaningfully fit to the same size as the original one.
618 """
619 psfShape = psfOld.shape
620 psfNew = self.padCenterOriginArraypadCenterOriginArray(psfOld, self.freqSpaceShapefreqSpaceShape)
621 psfNew = np.fft.fft2(psfNew)
622 psfNew *= corrft
623 psfNew = np.fft.ifft2(psfNew)
624 psfNew = psfNew.real
625 psfNew = self.padCenterOriginArraypadCenterOriginArray(psfNew, psfShape, useInverse=True)
626 psfNew = psfNew/psfNew.sum()
627 return psfNew
628
629 def computeCorrectedImage(self, corrft, imgOld):
630 """Compute the decorrelated difference image.
631
632 Parameters
633 ----------
634 corrft : `numpy.ndarray`
635 The frequency space representation of the correction calculated by
636 `computeCorrection`. Shape must be `self.freqSpaceShapefreqSpaceShape`.
637 imgOld : `numpy.ndarray`
638 The difference image to be corrected.
639
640 Returns
641 -------
642 imgNew : `numpy.ndarray`
643 The corrected image, same size as the input.
644 """
645 expShape = imgOld.shape
646 imgNew = np.copy(imgOld)
647 filtInf = np.isinf(imgNew)
648 filtNan = np.isnan(imgNew)
649 imgNew[filtInf] = np.nan
650 imgNew[filtInf | filtNan] = np.nanmean(imgNew)
651 imgNew = self.padCenterOriginArraypadCenterOriginArray(imgNew, self.freqSpaceShapefreqSpaceShape)
652 imgNew = np.fft.fft2(imgNew)
653 imgNew *= corrft
654 imgNew = np.fft.ifft2(imgNew)
655 imgNew = imgNew.real
656 imgNew = self.padCenterOriginArraypadCenterOriginArray(imgNew, expShape, useInverse=True)
657 imgNew[filtNan] = np.nan
658 imgNew[filtInf] = np.inf
659 return imgNew
660
661
663 """Task to be used as an ImageMapper for performing
664 A&L decorrelation on subimages on a grid across a A&L difference image.
665
666 This task subclasses DecorrelateALKernelTask in order to implement
667 all of that task's configuration parameters, as well as its `run` method.
668 """
669
670 ConfigClass = DecorrelateALKernelConfig
671 _DefaultName = 'ip_diffim_decorrelateALKernelMapper'
672
673 def __init__(self, *args, **kwargs):
674 DecorrelateALKernelTask.__init__(self, *args, **kwargs)
675
676 def run(self, subExposure, expandedSubExposure, fullBBox,
677 template, science, alTaskResult=None, psfMatchingKernel=None,
678 preConvKernel=None, **kwargs):
679 """Perform decorrelation operation on `subExposure`, using
680 `expandedSubExposure` to allow for invalid edge pixels arising from
681 convolutions.
682
683 This method performs A&L decorrelation on `subExposure` using
684 local measures for image variances and PSF. `subExposure` is a
685 sub-exposure of the non-decorrelated A&L diffim. It also
686 requires the corresponding sub-exposures of the template
687 (`template`) and science (`science`) exposures.
688
689 Parameters
690 ----------
691 subExposure : `lsst.afw.image.Exposure`
692 the sub-exposure of the diffim
693 expandedSubExposure : `lsst.afw.image.Exposure`
694 the expanded sub-exposure upon which to operate
695 fullBBox : `lsst.geom.Box2I`
696 the bounding box of the original exposure
697 template : `lsst.afw.image.Exposure`
698 the corresponding sub-exposure of the template exposure
699 science : `lsst.afw.image.Exposure`
700 the corresponding sub-exposure of the science exposure
701 alTaskResult : `lsst.pipe.base.Struct`
702 the result of A&L image differencing on `science` and
703 `template`, importantly containing the resulting
704 `psfMatchingKernel`. Can be `None`, only if
705 `psfMatchingKernel` is not `None`.
706 psfMatchingKernel : Alternative parameter for passing the
707 A&L `psfMatchingKernel` directly.
708 preConvKernel : If not None, then pre-filtering was applied
709 to science exposure, and this is the pre-convolution
710 kernel.
711 kwargs :
712 additional keyword arguments propagated from
713 `ImageMapReduceTask.run`.
714
715 Returns
716 -------
717 A `pipeBase.Struct` containing:
718
719 - ``subExposure`` : the result of the `subExposure` processing.
720 - ``decorrelationKernel`` : the decorrelation kernel, currently
721 not used.
722
723 Notes
724 -----
725 This `run` method accepts parameters identical to those of
726 `ImageMapper.run`, since it is called from the
727 `ImageMapperTask`. See that class for more information.
728 """
729 templateExposure = template # input template
730 scienceExposure = science # input science image
731 if alTaskResult is None and psfMatchingKernel is None:
732 raise RuntimeError('Both alTaskResult and psfMatchingKernel cannot be None')
733 psfMatchingKernel = alTaskResult.psfMatchingKernel if alTaskResult is not None else psfMatchingKernel
734
735 # subExp and expandedSubExp are subimages of the (un-decorrelated) diffim!
736 # So here we compute corresponding subimages of templateExposure and scienceExposure
737 subExp2 = scienceExposure.Factory(scienceExposure, expandedSubExposure.getBBox())
738 subExp1 = templateExposure.Factory(templateExposure, expandedSubExposure.getBBox())
739
740 # Prevent too much log INFO verbosity from DecorrelateALKernelTask.run
741 logLevel = self.log.getLevel()
742 self.log.setLevel(lsst.log.WARN)
743 res = DecorrelateALKernelTask.run(self, subExp2, subExp1, expandedSubExposure,
744 psfMatchingKernel, preConvKernel, **kwargs)
745 self.log.setLevel(logLevel) # reset the log level
746
747 diffim = res.correctedExposure.Factory(res.correctedExposure, subExposure.getBBox())
748 out = pipeBase.Struct(subExposure=diffim, )
749 return out
750
751
753 """Configuration parameters for the ImageMapReduceTask to direct it to use
754 DecorrelateALKernelMapper as its mapper for A&L decorrelation.
755 """
756 mapper = pexConfig.ConfigurableField(
757 doc='A&L decorrelation task to run on each sub-image',
758 target=DecorrelateALKernelMapper
759 )
760
761
762class DecorrelateALKernelSpatialConfig(pexConfig.Config):
763 """Configuration parameters for the DecorrelateALKernelSpatialTask.
764 """
765 decorrelateConfig = pexConfig.ConfigField(
766 dtype=DecorrelateALKernelConfig,
767 doc='DecorrelateALKernel config to use when running on complete exposure (non spatially-varying)',
768 )
769
770 decorrelateMapReduceConfig = pexConfig.ConfigField(
771 dtype=DecorrelateALKernelMapReduceConfig,
772 doc='DecorrelateALKernelMapReduce config to use when running on each sub-image (spatially-varying)',
773 )
774
775 ignoreMaskPlanes = pexConfig.ListField(
776 dtype=str,
777 doc="""Mask planes to ignore for sigma-clipped statistics""",
778 default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE")
779 )
780
781 def setDefaults(self):
782 self.decorrelateMapReduceConfigdecorrelateMapReduceConfig.gridStepX = self.decorrelateMapReduceConfigdecorrelateMapReduceConfig.gridStepY = 40
783 self.decorrelateMapReduceConfigdecorrelateMapReduceConfig.cellSizeX = self.decorrelateMapReduceConfigdecorrelateMapReduceConfig.cellSizeY = 41
784 self.decorrelateMapReduceConfigdecorrelateMapReduceConfig.borderSizeX = self.decorrelateMapReduceConfigdecorrelateMapReduceConfig.borderSizeY = 8
785 self.decorrelateMapReduceConfigdecorrelateMapReduceConfig.reducer.reduceOperation = 'average'
786
787
789 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference
790
791 Notes
792 -----
793
794 Pipe-task that removes the neighboring-pixel covariance in an
795 image difference that are added when the template image is
796 convolved with the Alard-Lupton PSF matching kernel.
797
798 This task is a simple wrapper around @ref DecorrelateALKernelTask,
799 which takes a `spatiallyVarying` parameter in its `run` method. If
800 it is `False`, then it simply calls the `run` method of @ref
801 DecorrelateALKernelTask. If it is True, then it uses the @ref
802 ImageMapReduceTask framework to break the exposures into
803 subExposures on a grid, and performs the `run` method of @ref
804 DecorrelateALKernelTask on each subExposure. This enables it to
805 account for spatially-varying PSFs and noise in the exposures when
806 performing the decorrelation.
807
808 This task has no standalone example, however it is applied as a
809 subtask of pipe.tasks.imageDifference.ImageDifferenceTask.
810 There is also an example of its use in `tests/testImageDecorrelation.py`.
811 """
812 ConfigClass = DecorrelateALKernelSpatialConfig
813 _DefaultName = "ip_diffim_decorrelateALKernelSpatial"
814
815 def __init__(self, *args, **kwargs):
816 """Create the image decorrelation Task
817
818 Parameters
819 ----------
820 args :
821 arguments to be passed to
822 `lsst.pipe.base.task.Task.__init__`
823 kwargs :
824 additional keyword arguments to be passed to
825 `lsst.pipe.base.task.Task.__init__`
826 """
827 pipeBase.Task.__init__(self, *args, **kwargs)
828
830 self.statsControlstatsControl.setNumSigmaClip(3.)
831 self.statsControlstatsControl.setNumIter(3)
832 self.statsControlstatsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes))
833
834 def computeVarianceMean(self, exposure):
835 """Compute the mean of the variance plane of `exposure`.
836 """
837 statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(),
838 exposure.getMaskedImage().getMask(),
839 afwMath.MEANCLIP, self.statsControlstatsControl)
840 var = statObj.getValue(afwMath.MEANCLIP)
841 return var
842
843 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel,
844 spatiallyVarying=True, preConvKernel=None, templateMatched=True, preConvMode=False):
845 """Perform decorrelation of an image difference exposure.
846
847 Decorrelates the diffim due to the convolution of the
848 templateExposure with the A&L psfMatchingKernel. If
849 `spatiallyVarying` is True, it utilizes the spatially varying
850 matching kernel via the `imageMapReduce` framework to perform
851 spatially-varying decorrelation on a grid of subExposures.
852
853 Parameters
854 ----------
855 scienceExposure : `lsst.afw.image.Exposure`
856 the science Exposure used for PSF matching
857 templateExposure : `lsst.afw.image.Exposure`
858 the template Exposure used for PSF matching
859 subtractedExposure : `lsst.afw.image.Exposure`
860 the subtracted Exposure produced by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
861 psfMatchingKernel : an (optionally spatially-varying) PSF matching kernel produced
862 by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
863 spatiallyVarying : `bool`
864 if True, perform the spatially-varying operation
865 preConvKernel : `lsst.meas.algorithms.Psf`
866 if not none, the scienceExposure has been pre-filtered with this kernel. (Currently
867 this option is experimental.)
868 templateMatched : `bool`, optional
869 If True, the template exposure was matched (convolved) to the science exposure.
870 preConvMode : `bool`, optional
871 If True, ``subtractedExposure`` is assumed to be a likelihood difference image
872 and will be noise corrected as a likelihood image.
873
874 Returns
875 -------
876 results : `lsst.pipe.base.Struct`
877 a structure containing:
878 - ``correctedExposure`` : the decorrelated diffim
879 """
880 self.log.info('Running A&L decorrelation: spatiallyVarying=%r', spatiallyVarying)
881
882 svar = self.computeVarianceMeancomputeVarianceMean(scienceExposure)
883 tvar = self.computeVarianceMeancomputeVarianceMean(templateExposure)
884 if np.isnan(svar) or np.isnan(tvar): # Should not happen unless entire image has been masked.
885 # Double check that one of the exposures is all NaNs
886 if (np.all(np.isnan(scienceExposure.image.array))
887 or np.all(np.isnan(templateExposure.image.array))):
888 self.log.warning('Template or science image is entirely NaNs: skipping decorrelation.')
889 if np.isnan(svar):
890 svar = 1e-9
891 if np.isnan(tvar):
892 tvar = 1e-9
893
894 var = self.computeVarianceMeancomputeVarianceMean(subtractedExposure)
895
896 if spatiallyVarying:
897 self.log.info("Variance (science, template): (%f, %f)", svar, tvar)
898 self.log.info("Variance (uncorrected diffim): %f", var)
899 config = self.config.decorrelateMapReduceConfig
900 task = ImageMapReduceTask(config=config)
901 results = task.run(subtractedExposure, science=scienceExposure,
902 template=templateExposure, psfMatchingKernel=psfMatchingKernel,
903 preConvKernel=preConvKernel, forceEvenSized=True,
904 templateMatched=templateMatched, preConvMode=preConvMode)
905 results.correctedExposure = results.exposure
906
907 # Make sure masks of input image are propagated to diffim
908 def gm(exp):
909 return exp.getMaskedImage().getMask()
910 gm(results.correctedExposure)[:, :] = gm(subtractedExposure)
911
912 var = self.computeVarianceMeancomputeVarianceMean(results.correctedExposure)
913 self.log.info("Variance (corrected diffim): %f", var)
914
915 else:
916 config = self.config.decorrelateConfig
917 task = DecorrelateALKernelTask(config=config)
918 results = task.run(scienceExposure, templateExposure,
919 subtractedExposure, psfMatchingKernel, preConvKernel=preConvKernel,
920 templateMatched=templateMatched, preConvMode=preConvMode)
921
922 return results
def run(self, subExposure, expandedSubExposure, fullBBox, template, science, alTaskResult=None, psfMatchingKernel=None, preConvKernel=None, **kwargs)
def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel, spatiallyVarying=True, preConvKernel=None, templateMatched=True, preConvMode=False)
def computeScoreCorrection(self, kappa, svar, tvar, preConvArr)
def calculateVariancePlane(self, vplane1, vplane2, varMean1, varMean2, c1ft, c2ft)
def padCenterOriginArray(A, tuple newShape, useInverse=False)
def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel, preConvKernel=None, xcen=None, ycen=None, svar=None, tvar=None, templateMatched=True, preConvMode=False, **kwargs)
Statistics makeStatistics(lsst::afw::image::Image< Pixel > const &img, lsst::afw::image::Mask< image::MaskPixel > const &msk, int const flags, StatisticsControl const &sctrl=StatisticsControl())