lsst.ip.diffim g8c31ad8cb1+5cfcba9665
imageDecorrelation.py
Go to the documentation of this file.
2# LSST Data Management System
3# Copyright 2016 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22
23import numpy as np
24
25import lsst.afw.image as afwImage
26import lsst.afw.math as afwMath
27import lsst.geom as geom
28import lsst.meas.algorithms as measAlg
29import lsst.pex.config as pexConfig
30import lsst.pipe.base as pipeBase
31from lsst.utils.timer import timeMethod
32
33from .imageMapReduce import (ImageMapReduceConfig, ImageMapReduceTask,
34 ImageMapper)
35
36__all__ = ("DecorrelateALKernelTask", "DecorrelateALKernelConfig",
37 "DecorrelateALKernelMapper", "DecorrelateALKernelMapReduceConfig",
38 "DecorrelateALKernelSpatialConfig", "DecorrelateALKernelSpatialTask")
39
40
41class DecorrelateALKernelConfig(pexConfig.Config):
42 """Configuration parameters for the DecorrelateALKernelTask
43 """
44
45 ignoreMaskPlanes = pexConfig.ListField(
46 dtype=str,
47 doc="""Mask planes to ignore for sigma-clipped statistics""",
48 default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE")
49 )
50 completeVarPlanePropagation = pexConfig.Field(
51 dtype=bool,
52 default=False,
53 doc="Compute the full effect of the decorrelated matching kernel on the variance plane."
54 " Otherwise use a model weighed sum of the input variances."
55 )
56
57
58class DecorrelateALKernelTask(pipeBase.Task):
59 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference
60
61 Notes
62 -----
63
64 Pipe-task that removes the neighboring-pixel covariance in an
65 image difference that are added when the template image is
66 convolved with the Alard-Lupton PSF matching kernel.
67
68 The image differencing pipeline task @link
69 ip.diffim.psfMatch.PsfMatchTask PSFMatchTask@endlink and @link
70 ip.diffim.psfMatch.PsfMatchConfigAL PSFMatchConfigAL@endlink uses
71 the Alard and Lupton (1998) method for matching the PSFs of the
72 template and science exposures prior to subtraction. The
73 Alard-Lupton method identifies a matching kernel, which is then
74 (typically) convolved with the template image to perform PSF
75 matching. This convolution has the effect of adding covariance
76 between neighboring pixels in the template image, which is then
77 added to the image difference by subtraction.
78
79 The pixel covariance may be corrected by whitening the noise of
80 the image difference. This task performs such a decorrelation by
81 computing a decorrelation kernel (based upon the A&L matching
82 kernel and variances in the template and science images) and
83 convolving the image difference with it. This process is described
84 in detail in [DMTN-021](http://dmtn-021.lsst.io).
85
86 This task has no standalone example, however it is applied as a
87 subtask of pipe.tasks.imageDifference.ImageDifferenceTask.
88 """
89 ConfigClass = DecorrelateALKernelConfig
90 _DefaultName = "ip_diffim_decorrelateALKernel"
91
92 def __init__(self, *args, **kwargs):
93 """Create the image decorrelation Task
94
95 Parameters
96 ----------
97 args :
98 arguments to be passed to ``lsst.pipe.base.task.Task.__init__``
99 kwargs :
100 keyword arguments to be passed to ``lsst.pipe.base.task.Task.__init__``
101 """
102 pipeBase.Task.__init__(self, *args, **kwargs)
103
105 self.statsControl.setNumSigmaClip(3.)
106 self.statsControl.setNumIter(3)
107 self.statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes))
108
109 def computeVarianceMean(self, exposure):
110 statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(),
111 exposure.getMaskedImage().getMask(),
112 afwMath.MEANCLIP, self.statsControl)
113 var = statObj.getValue(afwMath.MEANCLIP)
114 return var
115
116 @timeMethod
117 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel,
118 preConvKernel=None, xcen=None, ycen=None, svar=None, tvar=None,
119 templateMatched=True, preConvMode=False, **kwargs):
120 """Perform decorrelation of an image difference or of a score difference exposure.
121
122 Corrects the difference or score image due to the convolution of the
123 templateExposure with the A&L PSF matching kernel.
124 See [DMTN-021, Equation 1](http://dmtn-021.lsst.io/#equation-1) and
125 [DMTN-179](http://dmtn-179.lsst.io/) for details.
126
127 Parameters
128 ----------
129 scienceExposure : `lsst.afw.image.Exposure`
130 The original science exposure (before pre-convolution, if ``preConvMode==True``).
131 templateExposure : `lsst.afw.image.Exposure`
132 The original template exposure warped, but not psf-matched, to the science exposure.
133 subtractedExposure : `lsst.afw.image.Exposure`
134 the subtracted exposure produced by
135 `ip_diffim.ImagePsfMatchTask.subtractExposures()`. The `subtractedExposure` must
136 inherit its PSF from `exposure`, see notes below.
137 psfMatchingKernel : `lsst.afw.detection.Psf`
138 An (optionally spatially-varying) PSF matching kernel produced
139 by `ip_diffim.ImagePsfMatchTask.subtractExposures()`.
140 preConvKernel : `lsst.afw.math.Kernel`, optional
141 If not `None`, then the `scienceExposure` was pre-convolved with (the reflection of)
142 this kernel. Must be normalized to sum to 1.
143 Allowed only if ``templateMatched==True`` and ``preConvMode==True``.
144 Defaults to the PSF of the science exposure at the image center.
145 xcen : `float`, optional
146 X-pixel coordinate to use for computing constant matching kernel to use
147 If `None` (default), then use the center of the image.
148 ycen : `float`, optional
149 Y-pixel coordinate to use for computing constant matching kernel to use
150 If `None` (default), then use the center of the image.
151 svar : `float`, optional
152 Image variance for science image
153 If `None` (default) then compute the variance over the entire input science image.
154 tvar : `float`, optional
155 Image variance for template image
156 If `None` (default) then compute the variance over the entire input template image.
157 templateMatched : `bool`, optional
158 If True, the template exposure was matched (convolved) to the science exposure.
159 See also notes below.
160 preConvMode : `bool`, optional
161 If True, ``subtractedExposure`` is assumed to be a likelihood difference image
162 and will be noise corrected as a likelihood image.
163 **kwargs
164 Additional keyword arguments propagated from DecorrelateALKernelSpatialTask.
165
166 Returns
167 -------
168 result : `lsst.pipe.base.Struct`
169 - ``correctedExposure`` : the decorrelated diffim
170
171 Notes
172 -----
173 If ``preConvMode==True``, ``subtractedExposure`` is assumed to be a
174 score image and the noise correction for likelihood images
175 is applied. The resulting image is an optimal detection likelihood image
176 when the templateExposure has noise. (See DMTN-179) If ``preConvKernel`` is
177 not specified, the PSF of ``scienceExposure`` is assumed as pre-convolution kernel.
178
179 The ``subtractedExposure`` is NOT updated. The returned ``correctedExposure``
180 has an updated but spatially fixed PSF. It is calculated as the center of
181 image PSF corrected by the center of image matching kernel.
182
183 If ``templateMatched==True``, the templateExposure was matched (convolved)
184 to the ``scienceExposure`` by ``psfMatchingKernel`` during image differencing.
185 Otherwise the ``scienceExposure`` was matched (convolved) by ``psfMatchingKernel``.
186 In either case, note that the original template and science images are required,
187 not the psf-matched version.
188
189 This task discards the variance plane of ``subtractedExposure`` and re-computes
190 it from the variance planes of ``scienceExposure`` and ``templateExposure``.
191 The image plane of ``subtractedExposure`` must be at the photometric level
192 set by the AL PSF matching in `ImagePsfMatchTask.subtractExposures`.
193 The assumptions about the photometric level are controlled by the
194 `templateMatched` option in this task.
195
196 Here we currently convert a spatially-varying matching kernel into a constant kernel,
197 just by computing it at the center of the image (tickets DM-6243, DM-6244).
198
199 We are also using a constant accross-the-image measure of sigma (sqrt(variance)) to compute
200 the decorrelation kernel.
201
202 TODO DM-23857 As part of the spatially varying correction implementation
203 consider whether returning a Struct is still necessary.
204 """
205 if preConvKernel is not None and not (templateMatched and preConvMode):
206 raise ValueError("Pre-convolution kernel is allowed only if "
207 "preConvMode==True and templateMatched==True.")
208
209 spatialKernel = psfMatchingKernel
210 kimg = afwImage.ImageD(spatialKernel.getDimensions())
211 bbox = subtractedExposure.getBBox()
212 if xcen is None:
213 xcen = (bbox.getBeginX() + bbox.getEndX()) / 2.
214 if ycen is None:
215 ycen = (bbox.getBeginY() + bbox.getEndY()) / 2.
216 self.log.info("Using matching kernel computed at (%d, %d)", xcen, ycen)
217 spatialKernel.computeImage(kimg, False, xcen, ycen)
218
219 preConvImg = None
220 if preConvMode:
221 if preConvKernel is None:
222 preConvKernel = scienceExposure.getPsf().getLocalKernel() # at average position
223 preConvImg = afwImage.ImageD(preConvKernel.getDimensions())
224 preConvKernel.computeImage(preConvImg, True)
225
226 if svar is None:
227 svar = self.computeVarianceMean(scienceExposure)
228 if tvar is None:
229 tvar = self.computeVarianceMean(templateExposure)
230 self.log.info("Original variance plane means. Science:%.5e, warped template:%.5e)",
231 svar, tvar)
232
233 # Should not happen unless entire image has been masked, which could happen
234 # if this is a small subimage of the main exposure. In this case, just return a full NaN
235 # exposure
236 if np.isnan(svar) or np.isnan(tvar):
237 # Double check that one of the exposures is all NaNs
238 if (np.all(np.isnan(scienceExposure.image.array))
239 or np.all(np.isnan(templateExposure.image.array))):
240 self.log.warning('Template or science image is entirely NaNs: skipping decorrelation.')
241 outExposure = subtractedExposure.clone()
242 return pipeBase.Struct(correctedExposure=outExposure, )
243
244 if templateMatched:
245 # Regular subtraction, we convolved the template
246 self.log.info("Decorrelation after template image convolution")
247 varianceMean = svar
248 targetVarianceMean = tvar
249 # variance plane of the image that is not convolved
250 variance = scienceExposure.variance.array
251 # Variance plane of the convolved image, before convolution.
252 targetVariance = templateExposure.variance.array
253 else:
254 # We convolved the science image
255 self.log.info("Decorrelation after science image convolution")
256 varianceMean = tvar
257 targetVarianceMean = svar
258 # variance plane of the image that is not convolved
259 variance = templateExposure.variance.array
260 # Variance plane of the convolved image, before convolution.
261 targetVariance = scienceExposure.variance.array
262
263 # The maximal correction value converges to sqrt(targetVarianceMean/varianceMean).
264 # Correction divergence warning if the correction exceeds 4 orders of magnitude.
265 mOverExpVar = targetVarianceMean/varianceMean
266 if mOverExpVar > 1e8:
267 self.log.warning("Diverging correction: matched image variance is "
268 " much larger than the unconvolved one's"
269 ", targetVarianceMean/varianceMean:%.2e", mOverExpVar)
270
271 oldVarMean = self.computeVarianceMean(subtractedExposure)
272 self.log.info("Variance plane mean of uncorrected diffim: %f", oldVarMean)
273
274 kArr = kimg.array
275 diffimShape = subtractedExposure.image.array.shape
276 psfImg = subtractedExposure.getPsf().computeKernelImage(geom.Point2D(xcen, ycen))
277 psfShape = psfImg.array.shape
278
279 if preConvMode:
280 self.log.info("Decorrelation of likelihood image")
281 self.computeCommonShape(preConvImg.array.shape, kArr.shape,
282 psfShape, diffimShape)
283 corr = self.computeScoreCorrection(kArr, varianceMean, targetVarianceMean, preConvImg.array)
284 else:
285 self.log.info("Decorrelation of difference image")
286 self.computeCommonShape(kArr.shape, psfShape, diffimShape)
287 corr = self.computeDiffimCorrection(kArr, varianceMean, targetVarianceMean)
288
289 correctedImage = self.computeCorrectedImage(corr.corrft, subtractedExposure.image.array)
290 correctedPsf = self.computeCorrectedDiffimPsf(corr.corrft, psfImg.array)
291
292 # The subtracted exposure variance plane is already correlated, we cannot propagate
293 # it through another convolution; instead we need to use the uncorrelated originals
294 # The whitening should scale it to varianceMean + targetVarianceMean on average
295 if self.config.completeVarPlanePropagation:
296 self.log.debug("Using full variance plane calculation in decorrelation")
297 correctedVariance = self.calculateVariancePlane(
298 variance, targetVariance,
299 varianceMean, targetVarianceMean, corr.cnft, corr.crft)
300 else:
301 self.log.debug("Using estimated variance plane calculation in decorrelation")
302 correctedVariance = self.estimateVariancePlane(
303 variance, targetVariance,
304 corr.cnft, corr.crft)
305
306 # Determine the common shape
307 kSum = np.sum(kArr)
308 kSumSq = kSum*kSum
309 self.log.debug("Matching kernel sum: %.3e", kSum)
310 if not templateMatched:
311 # ImagePsfMatch.subtractExposures re-scales the difference in
312 # the science image convolution mode
313 correctedVariance /= kSumSq
314 subtractedExposure.image.array[...] = correctedImage # Allow for numpy type casting
315 subtractedExposure.variance.array[...] = correctedVariance
316 subtractedExposure.setPsf(correctedPsf)
317
318 newVarMean = self.computeVarianceMean(subtractedExposure)
319 self.log.info("Variance plane mean of corrected diffim: %.5e", newVarMean)
320
321 # TODO DM-23857 As part of the spatially varying correction implementation
322 # consider whether returning a Struct is still necessary.
323 return pipeBase.Struct(correctedExposure=subtractedExposure, )
324
325 def computeCommonShape(self, *shapes):
326 """Calculate the common shape for FFT operations. Set `self.freqSpaceShape`
327 internally.
328
329 Parameters
330 ----------
331 shapes : one or more `tuple` of `int`
332 Shapes of the arrays. All must have the same dimensionality.
333 At least one shape must be provided.
334
335 Returns
336 -------
337 None.
338
339 Notes
340 -----
341 For each dimension, gets the smallest even number greater than or equal to
342 `N1+N2-1` where `N1` and `N2` are the two largest values.
343 In case of only one shape given, rounds up to even each dimension value.
344 """
345 S = np.array(shapes, dtype=int)
346 if len(shapes) > 2:
347 S.sort(axis=0)
348 S = S[-2:]
349 if len(shapes) > 1:
350 commonShape = np.sum(S, axis=0) - 1
351 else:
352 commonShape = S[0]
353 commonShape[commonShape % 2 != 0] += 1
354 self.freqSpaceShape = tuple(commonShape)
355 self.log.info("Common frequency space shape %s", self.freqSpaceShape)
356
357 @staticmethod
358 def padCenterOriginArray(A, newShape: tuple, useInverse=False):
359 """Zero pad an image where the origin is at the center and replace the
360 origin to the corner as required by the periodic input of FFT. Implement also
361 the inverse operation, crop the padding and re-center data.
362
363 Parameters
364 ----------
365 A : `numpy.ndarray`
366 An array to copy from.
367 newShape : `tuple` of `int`
368 The dimensions of the resulting array. For padding, the resulting array
369 must be larger than A in each dimension. For the inverse operation this
370 must be the original, before padding size of the array.
371 useInverse : bool, optional
372 Selector of forward, add padding, operation (False)
373 or its inverse, crop padding, operation (True).
374
375 Returns
376 -------
377 R : `numpy.ndarray`
378 The padded or unpadded array with shape of `newShape` and the same dtype as A.
379
380 Notes
381 -----
382 For odd dimensions, the splitting is rounded to
383 put the center pixel into the new corner origin (0,0). This is to be consistent
384 e.g. for a dirac delta kernel that is originally located at the center pixel.
385 """
386
387 # The forward and inverse operations should round odd dimension halves at the opposite
388 # sides to get the pixels back to their original positions.
389 if not useInverse:
390 # Forward operation: First and second halves with respect to the axes of A.
391 firstHalves = [x//2 for x in A.shape]
392 secondHalves = [x-y for x, y in zip(A.shape, firstHalves)]
393 else:
394 # Inverse operation: Opposite rounding
395 secondHalves = [x//2 for x in newShape]
396 firstHalves = [x-y for x, y in zip(newShape, secondHalves)]
397
398 R = np.zeros_like(A, shape=newShape)
399 R[-firstHalves[0]:, -firstHalves[1]:] = A[:firstHalves[0], :firstHalves[1]]
400 R[:secondHalves[0], -firstHalves[1]:] = A[-secondHalves[0]:, :firstHalves[1]]
401 R[:secondHalves[0], :secondHalves[1]] = A[-secondHalves[0]:, -secondHalves[1]:]
402 R[-firstHalves[0]:, :secondHalves[1]] = A[:firstHalves[0], -secondHalves[1]:]
403 return R
404
405 def computeDiffimCorrection(self, kappa, svar, tvar):
406 """Compute the Lupton decorrelation post-convolution kernel for decorrelating an
407 image difference, based on the PSF-matching kernel.
408
409 Parameters
410 ----------
411 kappa : `numpy.ndarray` of `float`
412 A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching.
413 svar : `float` > 0.
414 Average variance of science image used for PSF matching.
415 tvar : `float` > 0.
416 Average variance of the template (matched) image used for PSF matching.
417
418 Returns
419 -------
420 corrft : `numpy.ndarray` of `float`
421 The frequency space representation of the correction. The array is real (dtype float).
422 Shape is `self.freqSpaceShape`.
423
424 cnft, crft : `numpy.ndarray` of `complex`
425 The overall convolution (pre-conv, PSF matching, noise correction) kernel
426 for the science and template images, respectively for the variance plane
427 calculations. These are intermediate results in frequency space.
428
429 Notes
430 -----
431 The maximum correction factor converges to `sqrt(tvar/svar)` towards high frequencies.
432 This should be a plausible value.
433 """
434 kSum = np.sum(kappa) # We scale the decorrelation to preserve fluxes
435 kappa = self.padCenterOriginArray(kappa, self.freqSpaceShape)
436 kft = np.fft.fft2(kappa)
437 kftAbsSq = np.real(np.conj(kft) * kft)
438
439 denom = svar + tvar * kftAbsSq
440 corrft = np.sqrt((svar + tvar * kSum*kSum) / denom)
441 cnft = corrft
442 crft = kft*corrft
443 return pipeBase.Struct(corrft=corrft, cnft=cnft, crft=crft)
444
445 def computeScoreCorrection(self, kappa, svar, tvar, preConvArr):
446 """Compute the correction kernel for a score image.
447
448 Parameters
449 ----------
450 kappa : `numpy.ndarray`
451 A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching.
452 svar : `float`
453 Average variance of science image used for PSF matching (before pre-convolution).
454 tvar : `float`
455 Average variance of the template (matched) image used for PSF matching.
456 preConvArr : `numpy.ndarray`
457 The pre-convolution kernel of the science image. It should be the PSF
458 of the science image or an approximation of it. It must be normed to sum 1.
459
460 Returns
461 -------
462 corrft : `numpy.ndarray` of `float`
463 The frequency space representation of the correction. The array is real (dtype float).
464 Shape is `self.freqSpaceShape`.
465 cnft, crft : `numpy.ndarray` of `complex`
466 The overall convolution (pre-conv, PSF matching, noise correction) kernel
467 for the science and template images, respectively for the variance plane
468 calculations. These are intermediate results in frequency space.
469
470 Notes
471 -----
472 To be precise, the science image should be _correlated_ by ``preConvArray`` but this
473 does not matter for this calculation.
474
475 ``cnft``, ``crft`` contain the scaling factor as well.
476
477 """
478 kSum = np.sum(kappa)
479 kappa = self.padCenterOriginArray(kappa, self.freqSpaceShape)
480 kft = np.fft.fft2(kappa)
481 preConvArr = self.padCenterOriginArray(preConvArr, self.freqSpaceShape)
482 preFt = np.fft.fft2(preConvArr)
483 preFtAbsSq = np.real(np.conj(preFt) * preFt)
484 kftAbsSq = np.real(np.conj(kft) * kft)
485 # Avoid zero division, though we don't normally approach `tiny`.
486 # We have numerical noise instead.
487 tiny = np.finfo(preFtAbsSq.dtype).tiny * 1000.
488 flt = preFtAbsSq < tiny
489 # If we pre-convolve to avoid deconvolution in AL, then kftAbsSq / preFtAbsSq
490 # theoretically expected to diverge to +inf. But we don't care about the convergence
491 # properties here, S goes to 0 at these frequencies anyway.
492 preFtAbsSq[flt] = tiny
493 denom = svar + tvar * kftAbsSq / preFtAbsSq
494 corrft = (svar + tvar * kSum*kSum) / denom
495 cnft = np.conj(preFt)*corrft
496 crft = kft*corrft
497 return pipeBase.Struct(corrft=corrft, cnft=cnft, crft=crft)
498
499 @staticmethod
500 def estimateVariancePlane(vplane1, vplane2, c1ft, c2ft):
501 """Estimate the variance planes.
502
503 The estimation assumes that around each pixel the surrounding
504 pixels' sigmas within the convolution kernel are the same.
505
506 Parameters
507 ----------
508 vplane1, vplane2 : `numpy.ndarray` of `float`
509 Variance planes of the original (before pre-convolution or matching)
510 exposures.
511 c1ft, c2ft : `numpy.ndarray` of `complex`
512 The overall convolution that includes the matching and the
513 afterburner in frequency space. The result of either
514 ``computeScoreCorrection`` or ``computeDiffimCorrection``.
515
516 Returns
517 -------
518 vplaneD : `numpy.ndarray` of `float`
519 The estimated variance plane of the difference/score image
520 as a weighted sum of the input variances.
521
522 Notes
523 ------
524 See DMTN-179 Section 5 about the variance plane calculations.
525 """
526 w1 = np.sum(np.real(np.conj(c1ft)*c1ft)) / c1ft.size
527 w2 = np.sum(np.real(np.conj(c2ft)*c2ft)) / c2ft.size
528 # w1, w2: the frequency space sum of abs(c1)^2 is the same as in image
529 # space.
530 return vplane1*w1 + vplane2*w2
531
532 def calculateVariancePlane(self, vplane1, vplane2, varMean1, varMean2, c1ft, c2ft):
533 """Full propagation of the variance planes of the original exposures.
534
535 The original variance planes of independent pixels are convolved with the
536 image space square of the overall kernels.
537
538 Parameters
539 ----------
540 vplane1, vplane2 : `numpy.ndarray` of `float`
541 Variance planes of the original (before pre-convolution or matching)
542 exposures.
543 varMean1, varMean2 : `float`
544 Replacement average values for non-finite ``vplane1`` and ``vplane2`` values respectively.
545
546 c1ft, c2ft : `numpy.ndarray` of `complex`
547 The overall convolution that includes the matching and the
548 afterburner in frequency space. The result of either
549 ``computeScoreCorrection`` or ``computeDiffimCorrection``.
550
551 Returns
552 -------
553 vplaneD : `numpy.ndarray` of `float`
554 The variance plane of the difference/score images.
555
556 Notes
557 ------
558 See DMTN-179 Section 5 about the variance plane calculations.
559
560 Infs and NaNs are allowed and kept in the returned array.
561 """
562 D = np.real(np.fft.ifft2(c1ft))
563 c1SqFt = np.fft.fft2(D*D)
564
565 v1shape = vplane1.shape
566 filtInf = np.isinf(vplane1)
567 filtNan = np.isnan(vplane1)
568 # This copy could be eliminated if inf/nan handling were go into padCenterOriginArray
569 vplane1 = np.copy(vplane1)
570 vplane1[filtInf | filtNan] = varMean1
571 D = self.padCenterOriginArray(vplane1, self.freqSpaceShape)
572 v1 = np.real(np.fft.ifft2(np.fft.fft2(D) * c1SqFt))
573 v1 = self.padCenterOriginArray(v1, v1shape, useInverse=True)
574 v1[filtNan] = np.nan
575 v1[filtInf] = np.inf
576
577 D = np.real(np.fft.ifft2(c2ft))
578 c2SqFt = np.fft.fft2(D*D)
579
580 v2shape = vplane2.shape
581 filtInf = np.isinf(vplane2)
582 filtNan = np.isnan(vplane2)
583 vplane2 = np.copy(vplane2)
584 vplane2[filtInf | filtNan] = varMean2
585 D = self.padCenterOriginArray(vplane2, self.freqSpaceShape)
586 v2 = np.real(np.fft.ifft2(np.fft.fft2(D) * c2SqFt))
587 v2 = self.padCenterOriginArray(v2, v2shape, useInverse=True)
588 v2[filtNan] = np.nan
589 v2[filtInf] = np.inf
590
591 return v1 + v2
592
593 def computeCorrectedDiffimPsf(self, corrft, psfOld):
594 """Compute the (decorrelated) difference image's new PSF.
595
596 Parameters
597 ----------
598 corrft : `numpy.ndarray`
599 The frequency space representation of the correction calculated by
600 `computeCorrection`. Shape must be `self.freqSpaceShape`.
601 psfOld : `numpy.ndarray`
602 The psf of the difference image to be corrected.
603
604 Returns
605 -------
606 correctedPsf : `lsst.meas.algorithms.KernelPsf`
607 The corrected psf, same shape as `psfOld`, sum normed to 1.
608
609 Notes
610 -----
611 There is no algorithmic guarantee that the corrected psf can
612 meaningfully fit to the same size as the original one.
613 """
614 psfShape = psfOld.shape
615 psfNew = self.padCenterOriginArray(psfOld, self.freqSpaceShape)
616 psfNew = np.fft.fft2(psfNew)
617 psfNew *= corrft
618 psfNew = np.fft.ifft2(psfNew)
619 psfNew = psfNew.real
620 psfNew = self.padCenterOriginArray(psfNew, psfShape, useInverse=True)
621 psfNew = psfNew/psfNew.sum()
622
623 psfcI = afwImage.ImageD(geom.Extent2I(psfShape[1], psfShape[0]))
624 psfcI.array = psfNew
625 psfcK = afwMath.FixedKernel(psfcI)
626 correctedPsf = measAlg.KernelPsf(psfcK)
627 return correctedPsf
628
629 def computeCorrectedImage(self, corrft, imgOld):
630 """Compute the decorrelated difference image.
631
632 Parameters
633 ----------
634 corrft : `numpy.ndarray`
635 The frequency space representation of the correction calculated by
636 `computeCorrection`. Shape must be `self.freqSpaceShape`.
637 imgOld : `numpy.ndarray`
638 The difference image to be corrected.
639
640 Returns
641 -------
642 imgNew : `numpy.ndarray`
643 The corrected image, same size as the input.
644 """
645 expShape = imgOld.shape
646 imgNew = np.copy(imgOld)
647 filtInf = np.isinf(imgNew)
648 filtNan = np.isnan(imgNew)
649 imgNew[filtInf] = np.nan
650 imgNew[filtInf | filtNan] = np.nanmean(imgNew)
651 imgNew = self.padCenterOriginArray(imgNew, self.freqSpaceShape)
652 imgNew = np.fft.fft2(imgNew)
653 imgNew *= corrft
654 imgNew = np.fft.ifft2(imgNew)
655 imgNew = imgNew.real
656 imgNew = self.padCenterOriginArray(imgNew, expShape, useInverse=True)
657 imgNew[filtNan] = np.nan
658 imgNew[filtInf] = np.inf
659 return imgNew
660
661
663 """Task to be used as an ImageMapper for performing
664 A&L decorrelation on subimages on a grid across a A&L difference image.
665
666 This task subclasses DecorrelateALKernelTask in order to implement
667 all of that task's configuration parameters, as well as its `run` method.
668 """
669
670 ConfigClass = DecorrelateALKernelConfig
671 _DefaultName = 'ip_diffim_decorrelateALKernelMapper'
672
673 def __init__(self, *args, **kwargs):
674 DecorrelateALKernelTask.__init__(self, *args, **kwargs)
675
676 def run(self, subExposure, expandedSubExposure, fullBBox,
677 template, science, alTaskResult=None, psfMatchingKernel=None,
678 preConvKernel=None, **kwargs):
679 """Perform decorrelation operation on `subExposure`, using
680 `expandedSubExposure` to allow for invalid edge pixels arising from
681 convolutions.
682
683 This method performs A&L decorrelation on `subExposure` using
684 local measures for image variances and PSF. `subExposure` is a
685 sub-exposure of the non-decorrelated A&L diffim. It also
686 requires the corresponding sub-exposures of the template
687 (`template`) and science (`science`) exposures.
688
689 Parameters
690 ----------
691 subExposure : `lsst.afw.image.Exposure`
692 the sub-exposure of the diffim
693 expandedSubExposure : `lsst.afw.image.Exposure`
694 the expanded sub-exposure upon which to operate
695 fullBBox : `lsst.geom.Box2I`
696 the bounding box of the original exposure
697 template : `lsst.afw.image.Exposure`
698 the corresponding sub-exposure of the template exposure
699 science : `lsst.afw.image.Exposure`
700 the corresponding sub-exposure of the science exposure
701 alTaskResult : `lsst.pipe.base.Struct`
702 the result of A&L image differencing on `science` and
703 `template`, importantly containing the resulting
704 `psfMatchingKernel`. Can be `None`, only if
705 `psfMatchingKernel` is not `None`.
706 psfMatchingKernel : Alternative parameter for passing the
707 A&L `psfMatchingKernel` directly.
708 preConvKernel : If not None, then pre-filtering was applied
709 to science exposure, and this is the pre-convolution
710 kernel.
711 kwargs :
712 additional keyword arguments propagated from
713 `ImageMapReduceTask.run`.
714
715 Returns
716 -------
717 A `pipeBase.Struct` containing:
718
719 - ``subExposure`` : the result of the `subExposure` processing.
720 - ``decorrelationKernel`` : the decorrelation kernel, currently
721 not used.
722
723 Notes
724 -----
725 This `run` method accepts parameters identical to those of
726 `ImageMapper.run`, since it is called from the
727 `ImageMapperTask`. See that class for more information.
728 """
729 templateExposure = template # input template
730 scienceExposure = science # input science image
731 if alTaskResult is None and psfMatchingKernel is None:
732 raise RuntimeError('Both alTaskResult and psfMatchingKernel cannot be None')
733 psfMatchingKernel = alTaskResult.psfMatchingKernel if alTaskResult is not None else psfMatchingKernel
734
735 # subExp and expandedSubExp are subimages of the (un-decorrelated) diffim!
736 # So here we compute corresponding subimages of templateExposure and scienceExposure
737 subExp2 = scienceExposure.Factory(scienceExposure, expandedSubExposure.getBBox())
738 subExp1 = templateExposure.Factory(templateExposure, expandedSubExposure.getBBox())
739
740 # Prevent too much log INFO verbosity from DecorrelateALKernelTask.run
741 logLevel = self.log.level
742 self.log.setLevel(self.log.WARNING)
743 res = DecorrelateALKernelTask.run(self, subExp2, subExp1, expandedSubExposure,
744 psfMatchingKernel, preConvKernel, **kwargs)
745 self.log.setLevel(logLevel) # reset the log level
746
747 diffim = res.correctedExposure.Factory(res.correctedExposure, subExposure.getBBox())
748 out = pipeBase.Struct(subExposure=diffim, )
749 return out
750
751
753 """Configuration parameters for the ImageMapReduceTask to direct it to use
754 DecorrelateALKernelMapper as its mapper for A&L decorrelation.
755 """
756 mapper = pexConfig.ConfigurableField(
757 doc='A&L decorrelation task to run on each sub-image',
758 target=DecorrelateALKernelMapper
759 )
760
761
762class DecorrelateALKernelSpatialConfig(pexConfig.Config):
763 """Configuration parameters for the DecorrelateALKernelSpatialTask.
764 """
765 decorrelateConfig = pexConfig.ConfigField(
766 dtype=DecorrelateALKernelConfig,
767 doc='DecorrelateALKernel config to use when running on complete exposure (non spatially-varying)',
768 )
769
770 decorrelateMapReduceConfig = pexConfig.ConfigField(
771 dtype=DecorrelateALKernelMapReduceConfig,
772 doc='DecorrelateALKernelMapReduce config to use when running on each sub-image (spatially-varying)',
773 )
774
775 ignoreMaskPlanes = pexConfig.ListField(
776 dtype=str,
777 doc="""Mask planes to ignore for sigma-clipped statistics""",
778 default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE")
779 )
780
781 def setDefaults(self):
782 self.decorrelateMapReduceConfig.gridStepX = self.decorrelateMapReduceConfig.gridStepY = 40
783 self.decorrelateMapReduceConfig.cellSizeX = self.decorrelateMapReduceConfig.cellSizeY = 41
784 self.decorrelateMapReduceConfig.borderSizeX = self.decorrelateMapReduceConfig.borderSizeY = 8
785 self.decorrelateMapReduceConfig.reducer.reduceOperation = 'average'
786
787
789 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference
790
791 Notes
792 -----
793
794 Pipe-task that removes the neighboring-pixel covariance in an
795 image difference that are added when the template image is
796 convolved with the Alard-Lupton PSF matching kernel.
797
798 This task is a simple wrapper around @ref DecorrelateALKernelTask,
799 which takes a `spatiallyVarying` parameter in its `run` method. If
800 it is `False`, then it simply calls the `run` method of @ref
801 DecorrelateALKernelTask. If it is True, then it uses the @ref
802 ImageMapReduceTask framework to break the exposures into
803 subExposures on a grid, and performs the `run` method of @ref
804 DecorrelateALKernelTask on each subExposure. This enables it to
805 account for spatially-varying PSFs and noise in the exposures when
806 performing the decorrelation.
807
808 This task has no standalone example, however it is applied as a
809 subtask of pipe.tasks.imageDifference.ImageDifferenceTask.
810 There is also an example of its use in `tests/testImageDecorrelation.py`.
811 """
812 ConfigClass = DecorrelateALKernelSpatialConfig
813 _DefaultName = "ip_diffim_decorrelateALKernelSpatial"
814
815 def __init__(self, *args, **kwargs):
816 """Create the image decorrelation Task
817
818 Parameters
819 ----------
820 args :
821 arguments to be passed to
822 `lsst.pipe.base.task.Task.__init__`
823 kwargs :
824 additional keyword arguments to be passed to
825 `lsst.pipe.base.task.Task.__init__`
826 """
827 pipeBase.Task.__init__(self, *args, **kwargs)
828
830 self.statsControl.setNumSigmaClip(3.)
831 self.statsControl.setNumIter(3)
832 self.statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes))
833
834 def computeVarianceMean(self, exposure):
835 """Compute the mean of the variance plane of `exposure`.
836 """
837 statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(),
838 exposure.getMaskedImage().getMask(),
839 afwMath.MEANCLIP, self.statsControl)
840 var = statObj.getValue(afwMath.MEANCLIP)
841 return var
842
843 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel,
844 spatiallyVarying=True, preConvKernel=None, templateMatched=True, preConvMode=False):
845 """Perform decorrelation of an image difference exposure.
846
847 Decorrelates the diffim due to the convolution of the
848 templateExposure with the A&L psfMatchingKernel. If
849 `spatiallyVarying` is True, it utilizes the spatially varying
850 matching kernel via the `imageMapReduce` framework to perform
851 spatially-varying decorrelation on a grid of subExposures.
852
853 Parameters
854 ----------
855 scienceExposure : `lsst.afw.image.Exposure`
856 the science Exposure used for PSF matching
857 templateExposure : `lsst.afw.image.Exposure`
858 the template Exposure used for PSF matching
859 subtractedExposure : `lsst.afw.image.Exposure`
860 the subtracted Exposure produced by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
861 psfMatchingKernel : an (optionally spatially-varying) PSF matching kernel produced
862 by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
863 spatiallyVarying : `bool`
864 if True, perform the spatially-varying operation
865 preConvKernel : `lsst.meas.algorithms.Psf`
866 if not none, the scienceExposure has been pre-filtered with this kernel. (Currently
867 this option is experimental.)
868 templateMatched : `bool`, optional
869 If True, the template exposure was matched (convolved) to the science exposure.
870 preConvMode : `bool`, optional
871 If True, ``subtractedExposure`` is assumed to be a likelihood difference image
872 and will be noise corrected as a likelihood image.
873
874 Returns
875 -------
876 results : `lsst.pipe.base.Struct`
877 a structure containing:
878 - ``correctedExposure`` : the decorrelated diffim
879 """
880 self.log.info('Running A&L decorrelation: spatiallyVarying=%r', spatiallyVarying)
881
882 svar = self.computeVarianceMean(scienceExposure)
883 tvar = self.computeVarianceMean(templateExposure)
884 if np.isnan(svar) or np.isnan(tvar): # Should not happen unless entire image has been masked.
885 # Double check that one of the exposures is all NaNs
886 if (np.all(np.isnan(scienceExposure.image.array))
887 or np.all(np.isnan(templateExposure.image.array))):
888 self.log.warning('Template or science image is entirely NaNs: skipping decorrelation.')
889 if np.isnan(svar):
890 svar = 1e-9
891 if np.isnan(tvar):
892 tvar = 1e-9
893
894 var = self.computeVarianceMean(subtractedExposure)
895
896 if spatiallyVarying:
897 self.log.info("Variance (science, template): (%f, %f)", svar, tvar)
898 self.log.info("Variance (uncorrected diffim): %f", var)
899 config = self.config.decorrelateMapReduceConfig
900 task = ImageMapReduceTask(config=config)
901 results = task.run(subtractedExposure, science=scienceExposure,
902 template=templateExposure, psfMatchingKernel=psfMatchingKernel,
903 preConvKernel=preConvKernel, forceEvenSized=True,
904 templateMatched=templateMatched, preConvMode=preConvMode)
905 results.correctedExposure = results.exposure
906
907 # Make sure masks of input image are propagated to diffim
908 def gm(exp):
909 return exp.getMaskedImage().getMask()
910 gm(results.correctedExposure)[:, :] = gm(subtractedExposure)
911
912 var = self.computeVarianceMean(results.correctedExposure)
913 self.log.info("Variance (corrected diffim): %f", var)
914
915 else:
916 config = self.config.decorrelateConfig
917 task = DecorrelateALKernelTask(config=config)
918 results = task.run(scienceExposure, templateExposure,
919 subtractedExposure, psfMatchingKernel, preConvKernel=preConvKernel,
920 templateMatched=templateMatched, preConvMode=preConvMode)
921
922 return results
def run(self, subExposure, expandedSubExposure, fullBBox, template, science, alTaskResult=None, psfMatchingKernel=None, preConvKernel=None, **kwargs)
def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel, spatiallyVarying=True, preConvKernel=None, templateMatched=True, preConvMode=False)
def computeScoreCorrection(self, kappa, svar, tvar, preConvArr)
def calculateVariancePlane(self, vplane1, vplane2, varMean1, varMean2, c1ft, c2ft)
def padCenterOriginArray(A, tuple newShape, useInverse=False)
def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel, preConvKernel=None, xcen=None, ycen=None, svar=None, tvar=None, templateMatched=True, preConvMode=False, **kwargs)
Statistics makeStatistics(lsst::afw::image::Image< Pixel > const &img, lsst::afw::image::Mask< image::MaskPixel > const &msk, int const flags, StatisticsControl const &sctrl=StatisticsControl())