Coverage for python/lsst/ip/diffim/imageDecorrelation.py: 17%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2016 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
23import numpy as np
25import lsst.afw.image as afwImage
26import lsst.afw.math as afwMath
27import lsst.geom as geom
28import lsst.meas.algorithms as measAlg
29import lsst.pex.config as pexConfig
30import lsst.pipe.base as pipeBase
31from lsst.utils.timer import timeMethod
33from .imageMapReduce import (ImageMapReduceConfig, ImageMapReduceTask,
34 ImageMapper)
36__all__ = ("DecorrelateALKernelTask", "DecorrelateALKernelConfig",
37 "DecorrelateALKernelMapper", "DecorrelateALKernelMapReduceConfig",
38 "DecorrelateALKernelSpatialConfig", "DecorrelateALKernelSpatialTask")
41class DecorrelateALKernelConfig(pexConfig.Config):
42 """Configuration parameters for the DecorrelateALKernelTask
43 """
45 ignoreMaskPlanes = pexConfig.ListField(
46 dtype=str,
47 doc="""Mask planes to ignore for sigma-clipped statistics""",
48 default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE")
49 )
50 completeVarPlanePropagation = pexConfig.Field(
51 dtype=bool,
52 default=False,
53 doc="Compute the full effect of the decorrelated matching kernel on the variance plane."
54 " Otherwise use a model weighed sum of the input variances."
55 )
58class DecorrelateALKernelTask(pipeBase.Task):
59 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference
61 Notes
62 -----
64 Pipe-task that removes the neighboring-pixel covariance in an
65 image difference that are added when the template image is
66 convolved with the Alard-Lupton PSF matching kernel.
68 The image differencing pipeline task @link
69 ip.diffim.psfMatch.PsfMatchTask PSFMatchTask@endlink and @link
70 ip.diffim.psfMatch.PsfMatchConfigAL PSFMatchConfigAL@endlink uses
71 the Alard and Lupton (1998) method for matching the PSFs of the
72 template and science exposures prior to subtraction. The
73 Alard-Lupton method identifies a matching kernel, which is then
74 (typically) convolved with the template image to perform PSF
75 matching. This convolution has the effect of adding covariance
76 between neighboring pixels in the template image, which is then
77 added to the image difference by subtraction.
79 The pixel covariance may be corrected by whitening the noise of
80 the image difference. This task performs such a decorrelation by
81 computing a decorrelation kernel (based upon the A&L matching
82 kernel and variances in the template and science images) and
83 convolving the image difference with it. This process is described
84 in detail in [DMTN-021](http://dmtn-021.lsst.io).
86 This task has no standalone example, however it is applied as a
87 subtask of pipe.tasks.imageDifference.ImageDifferenceTask.
88 """
89 ConfigClass = DecorrelateALKernelConfig
90 _DefaultName = "ip_diffim_decorrelateALKernel"
92 def __init__(self, *args, **kwargs):
93 """Create the image decorrelation Task
95 Parameters
96 ----------
97 args :
98 arguments to be passed to ``lsst.pipe.base.task.Task.__init__``
99 kwargs :
100 keyword arguments to be passed to ``lsst.pipe.base.task.Task.__init__``
101 """
102 pipeBase.Task.__init__(self, *args, **kwargs)
104 self.statsControl = afwMath.StatisticsControl()
105 self.statsControl.setNumSigmaClip(3.)
106 self.statsControl.setNumIter(3)
107 self.statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes))
109 def computeVarianceMean(self, exposure):
110 statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(),
111 exposure.getMaskedImage().getMask(),
112 afwMath.MEANCLIP, self.statsControl)
113 var = statObj.getValue(afwMath.MEANCLIP)
114 return var
116 @timeMethod
117 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel,
118 preConvKernel=None, xcen=None, ycen=None, svar=None, tvar=None,
119 templateMatched=True, preConvMode=False, **kwargs):
120 """Perform decorrelation of an image difference or of a score difference exposure.
122 Corrects the difference or score image due to the convolution of the
123 templateExposure with the A&L PSF matching kernel.
124 See [DMTN-021, Equation 1](http://dmtn-021.lsst.io/#equation-1) and
125 [DMTN-179](http://dmtn-179.lsst.io/) for details.
127 Parameters
128 ----------
129 scienceExposure : `lsst.afw.image.Exposure`
130 The original science exposure (before pre-convolution, if ``preConvMode==True``).
131 templateExposure : `lsst.afw.image.Exposure`
132 The original template exposure warped into the science exposure dimensions.
133 subtractedExposure : `lsst.afw.image.Exposure`
134 the subtracted exposure produced by
135 `ip_diffim.ImagePsfMatchTask.subtractExposures()`. The `subtractedExposure` must
136 inherit its PSF from `exposure`, see notes below.
137 psfMatchingKernel : `lsst.afw.detection.Psf`
138 An (optionally spatially-varying) PSF matching kernel produced
139 by `ip_diffim.ImagePsfMatchTask.subtractExposures()`.
140 preConvKernel : `lsst.afw.math.Kernel`, optional
141 If not `None`, then the `scienceExposure` was pre-convolved with (the reflection of)
142 this kernel. Must be normalized to sum to 1.
143 Allowed only if ``templateMatched==True`` and ``preConvMode==True``.
144 Defaults to the PSF of the science exposure at the image center.
145 xcen : `float`, optional
146 X-pixel coordinate to use for computing constant matching kernel to use
147 If `None` (default), then use the center of the image.
148 ycen : `float`, optional
149 Y-pixel coordinate to use for computing constant matching kernel to use
150 If `None` (default), then use the center of the image.
151 svar : `float`, optional
152 Image variance for science image
153 If `None` (default) then compute the variance over the entire input science image.
154 tvar : `float`, optional
155 Image variance for template image
156 If `None` (default) then compute the variance over the entire input template image.
157 templateMatched : `bool`, optional
158 If True, the template exposure was matched (convolved) to the science exposure.
159 See also notes below.
160 preConvMode : `bool`, optional
161 If True, ``subtractedExposure`` is assumed to be a likelihood difference image
162 and will be noise corrected as a likelihood image.
163 **kwargs
164 Additional keyword arguments propagated from DecorrelateALKernelSpatialTask.
166 Returns
167 -------
168 result : `lsst.pipe.base.Struct`
169 - ``correctedExposure`` : the decorrelated diffim
171 Notes
172 -----
173 If ``preConvMode==True``, ``subtractedExposure`` is assumed to be a
174 score image and the noise correction for likelihood images
175 is applied. The resulting image is an optimal detection likelihood image
176 when the templateExposure has noise. (See DMTN-179) If ``preConvKernel`` is
177 not specified, the PSF of ``scienceExposure`` is assumed as pre-convolution kernel.
179 The ``subtractedExposure`` is NOT updated. The returned ``correctedExposure``
180 has an updated but spatially fixed PSF. It is calculated as the center of
181 image PSF corrected by the center of image matching kernel.
183 If ``templateMatched==True``, the templateExposure was matched (convolved)
184 to the ``scienceExposure`` by ``psfMatchingKernel``. Otherwise the ``scienceExposure``
185 was matched (convolved) by ``psfMatchingKernel``.
187 This task discards the variance plane of ``subtractedExposure`` and re-computes
188 it from the variance planes of ``scienceExposure`` and ``templateExposure``.
189 The image plane of ``subtractedExposure`` must be at the photometric level
190 set by the AL PSF matching in `ImagePsfMatchTask.subtractExposures`.
191 The assumptions about the photometric level are controlled by the
192 `templateMatched` option in this task.
194 Here we currently convert a spatially-varying matching kernel into a constant kernel,
195 just by computing it at the center of the image (tickets DM-6243, DM-6244).
197 We are also using a constant accross-the-image measure of sigma (sqrt(variance)) to compute
198 the decorrelation kernel.
200 TODO DM-23857 As part of the spatially varying correction implementation
201 consider whether returning a Struct is still necessary.
202 """
203 if preConvKernel is not None and not (templateMatched and preConvMode):
204 raise ValueError("Pre-convolution kernel is allowed only if "
205 "preConvMode==True and templateMatched==True.")
207 spatialKernel = psfMatchingKernel
208 kimg = afwImage.ImageD(spatialKernel.getDimensions())
209 bbox = subtractedExposure.getBBox()
210 if xcen is None:
211 xcen = (bbox.getBeginX() + bbox.getEndX()) / 2.
212 if ycen is None:
213 ycen = (bbox.getBeginY() + bbox.getEndY()) / 2.
214 self.log.info("Using matching kernel computed at (%d, %d)", xcen, ycen)
215 spatialKernel.computeImage(kimg, False, xcen, ycen)
217 preConvImg = None
218 if preConvMode:
219 if preConvKernel is None:
220 preConvKernel = scienceExposure.getPsf().getLocalKernel() # at average position
221 preConvImg = afwImage.ImageD(preConvKernel.getDimensions())
222 preConvKernel.computeImage(preConvImg, True)
224 if svar is None:
225 svar = self.computeVarianceMean(scienceExposure)
226 if tvar is None:
227 tvar = self.computeVarianceMean(templateExposure)
228 self.log.info("Original variance plane means. Science:%.5e, warped template:%.5e)",
229 svar, tvar)
231 if templateMatched:
232 # Regular subtraction, we convolved the template
233 self.log.info("Decorrelation after template image convolution")
234 expVar = svar
235 matchedVar = tvar
236 exposure = scienceExposure
237 matchedExposure = templateExposure
238 else:
239 # We convolved the science image
240 self.log.info("Decorrelation after science image convolution")
241 expVar = tvar
242 matchedVar = svar
243 exposure = templateExposure
244 matchedExposure = scienceExposure
246 # Should not happen unless entire image has been masked, which could happen
247 # if this is a small subimage of the main exposure. In this case, just return a full NaN
248 # exposure
249 if np.isnan(expVar) or np.isnan(matchedVar):
250 # Double check that one of the exposures is all NaNs
251 if (np.all(np.isnan(exposure.image.array))
252 or np.all(np.isnan(matchedExposure.image.array))):
253 self.log.warning('Template or science image is entirely NaNs: skipping decorrelation.')
254 outExposure = subtractedExposure.clone()
255 return pipeBase.Struct(correctedExposure=outExposure, )
257 # The maximal correction value converges to sqrt(matchedVar/expVar).
258 # Correction divergence warning if the correction exceeds 4 orders of magnitude.
259 mOverExpVar = matchedVar/expVar
260 if mOverExpVar > 1e8:
261 self.log.warning("Diverging correction: matched image variance is "
262 " much larger than the unconvolved one's"
263 ", matchedVar/expVar:%.2e", mOverExpVar)
265 oldVarMean = self.computeVarianceMean(subtractedExposure)
266 self.log.info("Variance plane mean of uncorrected diffim: %f", oldVarMean)
268 kArr = kimg.array
269 diffExpArr = subtractedExposure.image.array
270 psfImg = subtractedExposure.getPsf().computeKernelImage(geom.Point2D(xcen, ycen))
271 psfDim = psfImg.getDimensions()
272 psfArr = psfImg.array
274 # Determine the common shape
275 kSum = np.sum(kArr)
276 kSumSq = kSum*kSum
277 self.log.debug("Matching kernel sum: %.3e", kSum)
279 if preConvMode:
280 self.log.info("Decorrelation of likelihood image")
281 self.computeCommonShape(preConvImg.array.shape, kArr.shape,
282 psfArr.shape, diffExpArr.shape)
283 corr = self.computeScoreCorrection(kArr, expVar, matchedVar, preConvImg.array)
284 else:
285 self.log.info("Decorrelation of difference image")
286 self.computeCommonShape(kArr.shape, psfArr.shape, diffExpArr.shape)
287 corr = self.computeDiffimCorrection(kArr, expVar, matchedVar)
289 diffExpArr = self.computeCorrectedImage(corr.corrft, diffExpArr)
291 corrPsfArr = self.computeCorrectedDiffimPsf(corr.corrft, psfArr)
292 psfcI = afwImage.ImageD(psfDim)
293 psfcI.array = corrPsfArr
294 psfcK = afwMath.FixedKernel(psfcI)
295 psfNew = measAlg.KernelPsf(psfcK)
297 correctedExposure = subtractedExposure.clone()
298 correctedExposure.image.array[...] = diffExpArr # Allow for numpy type casting
299 # The subtracted exposure variance plane is already correlated, we cannot propagate
300 # it through another convolution; instead we need to use the uncorrelated originals
301 # The whitening should scale it to expVar + matchedVar on average
302 if self.config.completeVarPlanePropagation:
303 self.log.debug("Using full variance plane calculation in decorrelation")
304 newVarArr = self.calculateVariancePlane(
305 exposure.variance.array, matchedExposure.variance.array,
306 expVar, matchedVar, corr.cnft, corr.crft)
307 else:
308 self.log.debug("Using estimated variance plane calculation in decorrelation")
309 newVarArr = self.estimateVariancePlane(
310 exposure.variance.array, matchedExposure.variance.array,
311 corr.cnft, corr.crft)
313 corrExpVarArr = correctedExposure.variance.array
314 corrExpVarArr[...] = newVarArr # Allow for numpy type casting
316 if not templateMatched:
317 # ImagePsfMatch.subtractExposures re-scales the difference in
318 # the science image convolution mode
319 corrExpVarArr /= kSumSq
320 correctedExposure.setPsf(psfNew)
322 newVarMean = self.computeVarianceMean(correctedExposure)
323 self.log.info("Variance plane mean of corrected diffim: %.5e", newVarMean)
325 # TODO DM-23857 As part of the spatially varying correction implementation
326 # consider whether returning a Struct is still necessary.
327 return pipeBase.Struct(correctedExposure=correctedExposure, )
329 def computeCommonShape(self, *shapes):
330 """Calculate the common shape for FFT operations. Set `self.freqSpaceShape`
331 internally.
333 Parameters
334 ----------
335 shapes : one or more `tuple` of `int`
336 Shapes of the arrays. All must have the same dimensionality.
337 At least one shape must be provided.
339 Returns
340 -------
341 None.
343 Notes
344 -----
345 For each dimension, gets the smallest even number greater than or equal to
346 `N1+N2-1` where `N1` and `N2` are the two largest values.
347 In case of only one shape given, rounds up to even each dimension value.
348 """
349 S = np.array(shapes, dtype=int)
350 if len(shapes) > 2:
351 S.sort(axis=0)
352 S = S[-2:]
353 if len(shapes) > 1:
354 commonShape = np.sum(S, axis=0) - 1
355 else:
356 commonShape = S[0]
357 commonShape[commonShape % 2 != 0] += 1
358 self.freqSpaceShape = tuple(commonShape)
359 self.log.info("Common frequency space shape %s", self.freqSpaceShape)
361 @staticmethod
362 def padCenterOriginArray(A, newShape: tuple, useInverse=False):
363 """Zero pad an image where the origin is at the center and replace the
364 origin to the corner as required by the periodic input of FFT. Implement also
365 the inverse operation, crop the padding and re-center data.
367 Parameters
368 ----------
369 A : `numpy.ndarray`
370 An array to copy from.
371 newShape : `tuple` of `int`
372 The dimensions of the resulting array. For padding, the resulting array
373 must be larger than A in each dimension. For the inverse operation this
374 must be the original, before padding size of the array.
375 useInverse : bool, optional
376 Selector of forward, add padding, operation (False)
377 or its inverse, crop padding, operation (True).
379 Returns
380 -------
381 R : `numpy.ndarray`
382 The padded or unpadded array with shape of `newShape` and the same dtype as A.
384 Notes
385 -----
386 For odd dimensions, the splitting is rounded to
387 put the center pixel into the new corner origin (0,0). This is to be consistent
388 e.g. for a dirac delta kernel that is originally located at the center pixel.
389 """
391 # The forward and inverse operations should round odd dimension halves at the opposite
392 # sides to get the pixels back to their original positions.
393 if not useInverse:
394 # Forward operation: First and second halves with respect to the axes of A.
395 firstHalves = [x//2 for x in A.shape]
396 secondHalves = [x-y for x, y in zip(A.shape, firstHalves)]
397 else:
398 # Inverse operation: Opposite rounding
399 secondHalves = [x//2 for x in newShape]
400 firstHalves = [x-y for x, y in zip(newShape, secondHalves)]
402 R = np.zeros_like(A, shape=newShape)
403 R[-firstHalves[0]:, -firstHalves[1]:] = A[:firstHalves[0], :firstHalves[1]]
404 R[:secondHalves[0], -firstHalves[1]:] = A[-secondHalves[0]:, :firstHalves[1]]
405 R[:secondHalves[0], :secondHalves[1]] = A[-secondHalves[0]:, -secondHalves[1]:]
406 R[-firstHalves[0]:, :secondHalves[1]] = A[:firstHalves[0], -secondHalves[1]:]
407 return R
409 def computeDiffimCorrection(self, kappa, svar, tvar):
410 """Compute the Lupton decorrelation post-convolution kernel for decorrelating an
411 image difference, based on the PSF-matching kernel.
413 Parameters
414 ----------
415 kappa : `numpy.ndarray` of `float`
416 A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching.
417 svar : `float` > 0.
418 Average variance of science image used for PSF matching.
419 tvar : `float` > 0.
420 Average variance of the template (matched) image used for PSF matching.
422 Returns
423 -------
424 corrft : `numpy.ndarray` of `float`
425 The frequency space representation of the correction. The array is real (dtype float).
426 Shape is `self.freqSpaceShape`.
428 cnft, crft : `numpy.ndarray` of `complex`
429 The overall convolution (pre-conv, PSF matching, noise correction) kernel
430 for the science and template images, respectively for the variance plane
431 calculations. These are intermediate results in frequency space.
433 Notes
434 -----
435 The maximum correction factor converges to `sqrt(tvar/svar)` towards high frequencies.
436 This should be a plausible value.
437 """
438 kSum = np.sum(kappa) # We scale the decorrelation to preserve fluxes
439 kappa = self.padCenterOriginArray(kappa, self.freqSpaceShape)
440 kft = np.fft.fft2(kappa)
441 kftAbsSq = np.real(np.conj(kft) * kft)
443 denom = svar + tvar * kftAbsSq
444 corrft = np.sqrt((svar + tvar * kSum*kSum) / denom)
445 cnft = corrft
446 crft = kft*corrft
447 return pipeBase.Struct(corrft=corrft, cnft=cnft, crft=crft)
449 def computeScoreCorrection(self, kappa, svar, tvar, preConvArr):
450 """Compute the correction kernel for a score image.
452 Parameters
453 ----------
454 kappa : `numpy.ndarray`
455 A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching.
456 svar : `float`
457 Average variance of science image used for PSF matching (before pre-convolution).
458 tvar : `float`
459 Average variance of the template (matched) image used for PSF matching.
460 preConvArr : `numpy.ndarray`
461 The pre-convolution kernel of the science image. It should be the PSF
462 of the science image or an approximation of it. It must be normed to sum 1.
464 Returns
465 -------
466 corrft : `numpy.ndarray` of `float`
467 The frequency space representation of the correction. The array is real (dtype float).
468 Shape is `self.freqSpaceShape`.
469 cnft, crft : `numpy.ndarray` of `complex`
470 The overall convolution (pre-conv, PSF matching, noise correction) kernel
471 for the science and template images, respectively for the variance plane
472 calculations. These are intermediate results in frequency space.
474 Notes
475 -----
476 To be precise, the science image should be _correlated_ by ``preConvArray`` but this
477 does not matter for this calculation.
479 ``cnft``, ``crft`` contain the scaling factor as well.
481 """
482 kSum = np.sum(kappa)
483 kappa = self.padCenterOriginArray(kappa, self.freqSpaceShape)
484 kft = np.fft.fft2(kappa)
485 preConvArr = self.padCenterOriginArray(preConvArr, self.freqSpaceShape)
486 preFt = np.fft.fft2(preConvArr)
487 preFtAbsSq = np.real(np.conj(preFt) * preFt)
488 kftAbsSq = np.real(np.conj(kft) * kft)
489 # Avoid zero division, though we don't normally approach `tiny`.
490 # We have numerical noise instead.
491 tiny = np.finfo(preFtAbsSq.dtype).tiny * 1000.
492 flt = preFtAbsSq < tiny
493 # If we pre-convolve to avoid deconvolution in AL, then kftAbsSq / preFtAbsSq
494 # theoretically expected to diverge to +inf. But we don't care about the convergence
495 # properties here, S goes to 0 at these frequencies anyway.
496 preFtAbsSq[flt] = tiny
497 denom = svar + tvar * kftAbsSq / preFtAbsSq
498 corrft = (svar + tvar * kSum*kSum) / denom
499 cnft = np.conj(preFt)*corrft
500 crft = kft*corrft
501 return pipeBase.Struct(corrft=corrft, cnft=cnft, crft=crft)
503 @staticmethod
504 def estimateVariancePlane(vplane1, vplane2, c1ft, c2ft):
505 """Estimate the variance planes.
507 The estimation assumes that around each pixel the surrounding
508 pixels' sigmas within the convolution kernel are the same.
510 Parameters
511 ----------
512 vplane1, vplane2 : `numpy.ndarray` of `float`
513 Variance planes of the original (before pre-convolution or matching)
514 exposures.
515 c1ft, c2ft : `numpy.ndarray` of `complex`
516 The overall convolution that includes the matching and the
517 afterburner in frequency space. The result of either
518 ``computeScoreCorrection`` or ``computeDiffimCorrection``.
520 Returns
521 -------
522 vplaneD : `numpy.ndarray` of `float`
523 The estimated variance plane of the difference/score image
524 as a weighted sum of the input variances.
526 Notes
527 ------
528 See DMTN-179 Section 5 about the variance plane calculations.
529 """
530 w1 = np.sum(np.real(np.conj(c1ft)*c1ft)) / c1ft.size
531 w2 = np.sum(np.real(np.conj(c2ft)*c2ft)) / c2ft.size
532 # w1, w2: the frequency space sum of abs(c1)^2 is the same as in image
533 # space.
534 return vplane1*w1 + vplane2*w2
536 def calculateVariancePlane(self, vplane1, vplane2, varMean1, varMean2, c1ft, c2ft):
537 """Full propagation of the variance planes of the original exposures.
539 The original variance planes of independent pixels are convolved with the
540 image space square of the overall kernels.
542 Parameters
543 ----------
544 vplane1, vplane2 : `numpy.ndarray` of `float`
545 Variance planes of the original (before pre-convolution or matching)
546 exposures.
547 varMean1, varMean2 : `float`
548 Replacement average values for non-finite ``vplane1`` and ``vplane2`` values respectively.
550 c1ft, c2ft : `numpy.ndarray` of `complex`
551 The overall convolution that includes the matching and the
552 afterburner in frequency space. The result of either
553 ``computeScoreCorrection`` or ``computeDiffimCorrection``.
555 Returns
556 -------
557 vplaneD : `numpy.ndarray` of `float`
558 The variance plane of the difference/score images.
560 Notes
561 ------
562 See DMTN-179 Section 5 about the variance plane calculations.
564 Infs and NaNs are allowed and kept in the returned array.
565 """
566 D = np.real(np.fft.ifft2(c1ft))
567 c1SqFt = np.fft.fft2(D*D)
569 v1shape = vplane1.shape
570 filtInf = np.isinf(vplane1)
571 filtNan = np.isnan(vplane1)
572 # This copy could be eliminated if inf/nan handling were go into padCenterOriginArray
573 vplane1 = np.copy(vplane1)
574 vplane1[filtInf | filtNan] = varMean1
575 D = self.padCenterOriginArray(vplane1, self.freqSpaceShape)
576 v1 = np.real(np.fft.ifft2(np.fft.fft2(D) * c1SqFt))
577 v1 = self.padCenterOriginArray(v1, v1shape, useInverse=True)
578 v1[filtNan] = np.nan
579 v1[filtInf] = np.inf
581 D = np.real(np.fft.ifft2(c2ft))
582 c2ft = np.fft.fft2(D*D)
584 v2shape = vplane2.shape
585 filtInf = np.isinf(vplane2)
586 filtNan = np.isnan(vplane2)
587 vplane2 = np.copy(vplane2)
588 vplane2[filtInf | filtNan] = varMean2
589 D = self.padCenterOriginArray(vplane2, self.freqSpaceShape)
590 v2 = np.real(np.fft.ifft2(np.fft.fft2(D) * c2ft))
591 v2 = self.padCenterOriginArray(v2, v2shape, useInverse=True)
592 v2[filtNan] = np.nan
593 v2[filtInf] = np.inf
595 return v1 + v2
597 def computeCorrectedDiffimPsf(self, corrft, psfOld):
598 """Compute the (decorrelated) difference image's new PSF.
600 Parameters
601 ----------
602 corrft : `numpy.ndarray`
603 The frequency space representation of the correction calculated by
604 `computeCorrection`. Shape must be `self.freqSpaceShape`.
605 psfOld : `numpy.ndarray`
606 The psf of the difference image to be corrected.
608 Returns
609 -------
610 psfNew : `numpy.ndarray`
611 The corrected psf, same shape as `psfOld`, sum normed to 1.
613 Notes
614 -----
615 There is no algorithmic guarantee that the corrected psf can
616 meaningfully fit to the same size as the original one.
617 """
618 psfShape = psfOld.shape
619 psfNew = self.padCenterOriginArray(psfOld, self.freqSpaceShape)
620 psfNew = np.fft.fft2(psfNew)
621 psfNew *= corrft
622 psfNew = np.fft.ifft2(psfNew)
623 psfNew = psfNew.real
624 psfNew = self.padCenterOriginArray(psfNew, psfShape, useInverse=True)
625 psfNew = psfNew/psfNew.sum()
626 return psfNew
628 def computeCorrectedImage(self, corrft, imgOld):
629 """Compute the decorrelated difference image.
631 Parameters
632 ----------
633 corrft : `numpy.ndarray`
634 The frequency space representation of the correction calculated by
635 `computeCorrection`. Shape must be `self.freqSpaceShape`.
636 imgOld : `numpy.ndarray`
637 The difference image to be corrected.
639 Returns
640 -------
641 imgNew : `numpy.ndarray`
642 The corrected image, same size as the input.
643 """
644 expShape = imgOld.shape
645 imgNew = np.copy(imgOld)
646 filtInf = np.isinf(imgNew)
647 filtNan = np.isnan(imgNew)
648 imgNew[filtInf] = np.nan
649 imgNew[filtInf | filtNan] = np.nanmean(imgNew)
650 imgNew = self.padCenterOriginArray(imgNew, self.freqSpaceShape)
651 imgNew = np.fft.fft2(imgNew)
652 imgNew *= corrft
653 imgNew = np.fft.ifft2(imgNew)
654 imgNew = imgNew.real
655 imgNew = self.padCenterOriginArray(imgNew, expShape, useInverse=True)
656 imgNew[filtNan] = np.nan
657 imgNew[filtInf] = np.inf
658 return imgNew
661class DecorrelateALKernelMapper(DecorrelateALKernelTask, ImageMapper):
662 """Task to be used as an ImageMapper for performing
663 A&L decorrelation on subimages on a grid across a A&L difference image.
665 This task subclasses DecorrelateALKernelTask in order to implement
666 all of that task's configuration parameters, as well as its `run` method.
667 """
669 ConfigClass = DecorrelateALKernelConfig
670 _DefaultName = 'ip_diffim_decorrelateALKernelMapper'
672 def __init__(self, *args, **kwargs):
673 DecorrelateALKernelTask.__init__(self, *args, **kwargs)
675 def run(self, subExposure, expandedSubExposure, fullBBox,
676 template, science, alTaskResult=None, psfMatchingKernel=None,
677 preConvKernel=None, **kwargs):
678 """Perform decorrelation operation on `subExposure`, using
679 `expandedSubExposure` to allow for invalid edge pixels arising from
680 convolutions.
682 This method performs A&L decorrelation on `subExposure` using
683 local measures for image variances and PSF. `subExposure` is a
684 sub-exposure of the non-decorrelated A&L diffim. It also
685 requires the corresponding sub-exposures of the template
686 (`template`) and science (`science`) exposures.
688 Parameters
689 ----------
690 subExposure : `lsst.afw.image.Exposure`
691 the sub-exposure of the diffim
692 expandedSubExposure : `lsst.afw.image.Exposure`
693 the expanded sub-exposure upon which to operate
694 fullBBox : `lsst.geom.Box2I`
695 the bounding box of the original exposure
696 template : `lsst.afw.image.Exposure`
697 the corresponding sub-exposure of the template exposure
698 science : `lsst.afw.image.Exposure`
699 the corresponding sub-exposure of the science exposure
700 alTaskResult : `lsst.pipe.base.Struct`
701 the result of A&L image differencing on `science` and
702 `template`, importantly containing the resulting
703 `psfMatchingKernel`. Can be `None`, only if
704 `psfMatchingKernel` is not `None`.
705 psfMatchingKernel : Alternative parameter for passing the
706 A&L `psfMatchingKernel` directly.
707 preConvKernel : If not None, then pre-filtering was applied
708 to science exposure, and this is the pre-convolution
709 kernel.
710 kwargs :
711 additional keyword arguments propagated from
712 `ImageMapReduceTask.run`.
714 Returns
715 -------
716 A `pipeBase.Struct` containing:
718 - ``subExposure`` : the result of the `subExposure` processing.
719 - ``decorrelationKernel`` : the decorrelation kernel, currently
720 not used.
722 Notes
723 -----
724 This `run` method accepts parameters identical to those of
725 `ImageMapper.run`, since it is called from the
726 `ImageMapperTask`. See that class for more information.
727 """
728 templateExposure = template # input template
729 scienceExposure = science # input science image
730 if alTaskResult is None and psfMatchingKernel is None:
731 raise RuntimeError('Both alTaskResult and psfMatchingKernel cannot be None')
732 psfMatchingKernel = alTaskResult.psfMatchingKernel if alTaskResult is not None else psfMatchingKernel
734 # subExp and expandedSubExp are subimages of the (un-decorrelated) diffim!
735 # So here we compute corresponding subimages of templateExposure and scienceExposure
736 subExp2 = scienceExposure.Factory(scienceExposure, expandedSubExposure.getBBox())
737 subExp1 = templateExposure.Factory(templateExposure, expandedSubExposure.getBBox())
739 # Prevent too much log INFO verbosity from DecorrelateALKernelTask.run
740 logLevel = self.log.getLevel()
741 self.log.setLevel(self.log.WARNING)
742 res = DecorrelateALKernelTask.run(self, subExp2, subExp1, expandedSubExposure,
743 psfMatchingKernel, preConvKernel, **kwargs)
744 self.log.setLevel(logLevel) # reset the log level
746 diffim = res.correctedExposure.Factory(res.correctedExposure, subExposure.getBBox())
747 out = pipeBase.Struct(subExposure=diffim, )
748 return out
751class DecorrelateALKernelMapReduceConfig(ImageMapReduceConfig):
752 """Configuration parameters for the ImageMapReduceTask to direct it to use
753 DecorrelateALKernelMapper as its mapper for A&L decorrelation.
754 """
755 mapper = pexConfig.ConfigurableField(
756 doc='A&L decorrelation task to run on each sub-image',
757 target=DecorrelateALKernelMapper
758 )
761class DecorrelateALKernelSpatialConfig(pexConfig.Config):
762 """Configuration parameters for the DecorrelateALKernelSpatialTask.
763 """
764 decorrelateConfig = pexConfig.ConfigField(
765 dtype=DecorrelateALKernelConfig,
766 doc='DecorrelateALKernel config to use when running on complete exposure (non spatially-varying)',
767 )
769 decorrelateMapReduceConfig = pexConfig.ConfigField(
770 dtype=DecorrelateALKernelMapReduceConfig,
771 doc='DecorrelateALKernelMapReduce config to use when running on each sub-image (spatially-varying)',
772 )
774 ignoreMaskPlanes = pexConfig.ListField(
775 dtype=str,
776 doc="""Mask planes to ignore for sigma-clipped statistics""",
777 default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE")
778 )
780 def setDefaults(self):
781 self.decorrelateMapReduceConfig.gridStepX = self.decorrelateMapReduceConfig.gridStepY = 40
782 self.decorrelateMapReduceConfig.cellSizeX = self.decorrelateMapReduceConfig.cellSizeY = 41
783 self.decorrelateMapReduceConfig.borderSizeX = self.decorrelateMapReduceConfig.borderSizeY = 8
784 self.decorrelateMapReduceConfig.reducer.reduceOperation = 'average'
787class DecorrelateALKernelSpatialTask(pipeBase.Task):
788 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference
790 Notes
791 -----
793 Pipe-task that removes the neighboring-pixel covariance in an
794 image difference that are added when the template image is
795 convolved with the Alard-Lupton PSF matching kernel.
797 This task is a simple wrapper around @ref DecorrelateALKernelTask,
798 which takes a `spatiallyVarying` parameter in its `run` method. If
799 it is `False`, then it simply calls the `run` method of @ref
800 DecorrelateALKernelTask. If it is True, then it uses the @ref
801 ImageMapReduceTask framework to break the exposures into
802 subExposures on a grid, and performs the `run` method of @ref
803 DecorrelateALKernelTask on each subExposure. This enables it to
804 account for spatially-varying PSFs and noise in the exposures when
805 performing the decorrelation.
807 This task has no standalone example, however it is applied as a
808 subtask of pipe.tasks.imageDifference.ImageDifferenceTask.
809 There is also an example of its use in `tests/testImageDecorrelation.py`.
810 """
811 ConfigClass = DecorrelateALKernelSpatialConfig
812 _DefaultName = "ip_diffim_decorrelateALKernelSpatial"
814 def __init__(self, *args, **kwargs):
815 """Create the image decorrelation Task
817 Parameters
818 ----------
819 args :
820 arguments to be passed to
821 `lsst.pipe.base.task.Task.__init__`
822 kwargs :
823 additional keyword arguments to be passed to
824 `lsst.pipe.base.task.Task.__init__`
825 """
826 pipeBase.Task.__init__(self, *args, **kwargs)
828 self.statsControl = afwMath.StatisticsControl()
829 self.statsControl.setNumSigmaClip(3.)
830 self.statsControl.setNumIter(3)
831 self.statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes))
833 def computeVarianceMean(self, exposure):
834 """Compute the mean of the variance plane of `exposure`.
835 """
836 statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(),
837 exposure.getMaskedImage().getMask(),
838 afwMath.MEANCLIP, self.statsControl)
839 var = statObj.getValue(afwMath.MEANCLIP)
840 return var
842 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel,
843 spatiallyVarying=True, preConvKernel=None, templateMatched=True, preConvMode=False):
844 """Perform decorrelation of an image difference exposure.
846 Decorrelates the diffim due to the convolution of the
847 templateExposure with the A&L psfMatchingKernel. If
848 `spatiallyVarying` is True, it utilizes the spatially varying
849 matching kernel via the `imageMapReduce` framework to perform
850 spatially-varying decorrelation on a grid of subExposures.
852 Parameters
853 ----------
854 scienceExposure : `lsst.afw.image.Exposure`
855 the science Exposure used for PSF matching
856 templateExposure : `lsst.afw.image.Exposure`
857 the template Exposure used for PSF matching
858 subtractedExposure : `lsst.afw.image.Exposure`
859 the subtracted Exposure produced by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
860 psfMatchingKernel : an (optionally spatially-varying) PSF matching kernel produced
861 by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
862 spatiallyVarying : `bool`
863 if True, perform the spatially-varying operation
864 preConvKernel : `lsst.meas.algorithms.Psf`
865 if not none, the scienceExposure has been pre-filtered with this kernel. (Currently
866 this option is experimental.)
867 templateMatched : `bool`, optional
868 If True, the template exposure was matched (convolved) to the science exposure.
869 preConvMode : `bool`, optional
870 If True, ``subtractedExposure`` is assumed to be a likelihood difference image
871 and will be noise corrected as a likelihood image.
873 Returns
874 -------
875 results : `lsst.pipe.base.Struct`
876 a structure containing:
877 - ``correctedExposure`` : the decorrelated diffim
878 """
879 self.log.info('Running A&L decorrelation: spatiallyVarying=%r', spatiallyVarying)
881 svar = self.computeVarianceMean(scienceExposure)
882 tvar = self.computeVarianceMean(templateExposure)
883 if np.isnan(svar) or np.isnan(tvar): # Should not happen unless entire image has been masked.
884 # Double check that one of the exposures is all NaNs
885 if (np.all(np.isnan(scienceExposure.image.array))
886 or np.all(np.isnan(templateExposure.image.array))):
887 self.log.warning('Template or science image is entirely NaNs: skipping decorrelation.')
888 if np.isnan(svar):
889 svar = 1e-9
890 if np.isnan(tvar):
891 tvar = 1e-9
893 var = self.computeVarianceMean(subtractedExposure)
895 if spatiallyVarying:
896 self.log.info("Variance (science, template): (%f, %f)", svar, tvar)
897 self.log.info("Variance (uncorrected diffim): %f", var)
898 config = self.config.decorrelateMapReduceConfig
899 task = ImageMapReduceTask(config=config)
900 results = task.run(subtractedExposure, science=scienceExposure,
901 template=templateExposure, psfMatchingKernel=psfMatchingKernel,
902 preConvKernel=preConvKernel, forceEvenSized=True,
903 templateMatched=templateMatched, preConvMode=preConvMode)
904 results.correctedExposure = results.exposure
906 # Make sure masks of input image are propagated to diffim
907 def gm(exp):
908 return exp.getMaskedImage().getMask()
909 gm(results.correctedExposure)[:, :] = gm(subtractedExposure)
911 var = self.computeVarianceMean(results.correctedExposure)
912 self.log.info("Variance (corrected diffim): %f", var)
914 else:
915 config = self.config.decorrelateConfig
916 task = DecorrelateALKernelTask(config=config)
917 results = task.run(scienceExposure, templateExposure,
918 subtractedExposure, psfMatchingKernel, preConvKernel=preConvKernel,
919 templateMatched=templateMatched, preConvMode=preConvMode)
921 return results