lsst.ip.diffim ga7308b0d61+987f831666
Loading...
Searching...
No Matches
imageDecorrelation.py
Go to the documentation of this file.
2# LSST Data Management System
3# Copyright 2016 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22
23import numpy as np
24
25import lsst.afw.image as afwImage
26import lsst.afw.math as afwMath
27import lsst.geom as geom
28import lsst.meas.algorithms as measAlg
29import lsst.pex.config as pexConfig
30import lsst.pipe.base as pipeBase
31from lsst.utils.timer import timeMethod
32
33from .imageMapReduce import (ImageMapReduceConfig, ImageMapReduceTask,
34 ImageMapper)
35
36__all__ = ("DecorrelateALKernelTask", "DecorrelateALKernelConfig",
37 "DecorrelateALKernelMapper", "DecorrelateALKernelMapReduceConfig",
38 "DecorrelateALKernelSpatialConfig", "DecorrelateALKernelSpatialTask")
39
40
41class DecorrelateALKernelConfig(pexConfig.Config):
42 """Configuration parameters for the DecorrelateALKernelTask
43 """
44
45 ignoreMaskPlanes = pexConfig.ListField(
46 dtype=str,
47 doc="""Mask planes to ignore for sigma-clipped statistics""",
48 default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE")
49 )
50 completeVarPlanePropagation = pexConfig.Field(
51 dtype=bool,
52 default=False,
53 doc="Compute the full effect of the decorrelated matching kernel on the variance plane."
54 " Otherwise use a model weighed sum of the input variances."
55 )
56
57
58class DecorrelateALKernelTask(pipeBase.Task):
59 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference
60
61 Notes
62 -----
63
64 Pipe-task that removes the neighboring-pixel covariance in an
65 image difference that are added when the template image is
66 convolved with the Alard-Lupton PSF matching kernel.
67
68 The image differencing pipeline task @link
69 ip.diffim.psfMatch.PsfMatchTask PSFMatchTask@endlink and @link
70 ip.diffim.psfMatch.PsfMatchConfigAL PSFMatchConfigAL@endlink uses
71 the Alard and Lupton (1998) method for matching the PSFs of the
72 template and science exposures prior to subtraction. The
73 Alard-Lupton method identifies a matching kernel, which is then
74 (typically) convolved with the template image to perform PSF
75 matching. This convolution has the effect of adding covariance
76 between neighboring pixels in the template image, which is then
77 added to the image difference by subtraction.
78
79 The pixel covariance may be corrected by whitening the noise of
80 the image difference. This task performs such a decorrelation by
81 computing a decorrelation kernel (based upon the A&L matching
82 kernel and variances in the template and science images) and
83 convolving the image difference with it. This process is described
84 in detail in [DMTN-021](http://dmtn-021.lsst.io).
85
86 This task has no standalone example, however it is applied as a
87 subtask of pipe.tasks.imageDifference.ImageDifferenceTask.
88 """
89 ConfigClass = DecorrelateALKernelConfig
90 _DefaultName = "ip_diffim_decorrelateALKernel"
91
92 def __init__(self, *args, **kwargs):
93 """Create the image decorrelation Task
94
95 Parameters
96 ----------
97 args :
98 arguments to be passed to ``lsst.pipe.base.task.Task.__init__``
99 kwargs :
100 keyword arguments to be passed to ``lsst.pipe.base.task.Task.__init__``
101 """
102 pipeBase.Task.__init__(self, *args, **kwargs)
103
105 self.statsControl.setNumSigmaClip(3.)
106 self.statsControl.setNumIter(3)
107 self.statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes))
108
109 def computeVarianceMean(self, exposure):
110 statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(),
111 exposure.getMaskedImage().getMask(),
112 afwMath.MEANCLIP, self.statsControl)
113 var = statObj.getValue(afwMath.MEANCLIP)
114 return var
115
116 @timeMethod
117 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel,
118 preConvKernel=None, xcen=None, ycen=None, svar=None, tvar=None,
119 templateMatched=True, preConvMode=False, **kwargs):
120 """Perform decorrelation of an image difference or of a score difference exposure.
121
122 Corrects the difference or score image due to the convolution of the
123 templateExposure with the A&L PSF matching kernel.
124 See [DMTN-021, Equation 1](http://dmtn-021.lsst.io/#equation-1) and
125 [DMTN-179](http://dmtn-179.lsst.io/) for details.
126
127 Parameters
128 ----------
129 scienceExposure : `lsst.afw.image.Exposure`
130 The original science exposure (before pre-convolution, if ``preConvMode==True``).
131 templateExposure : `lsst.afw.image.Exposure`
132 The original template exposure warped, but not psf-matched, to the science exposure.
133 subtractedExposure : `lsst.afw.image.Exposure`
134 the subtracted exposure produced by
135 `ip_diffim.ImagePsfMatchTask.subtractExposures()`. The `subtractedExposure` must
136 inherit its PSF from `exposure`, see notes below.
137 psfMatchingKernel : `lsst.afw.detection.Psf`
138 An (optionally spatially-varying) PSF matching kernel produced
139 by `ip_diffim.ImagePsfMatchTask.subtractExposures()`.
140 preConvKernel : `lsst.afw.math.Kernel`, optional
141 If not `None`, then the `scienceExposure` was pre-convolved with (the reflection of)
142 this kernel. Must be normalized to sum to 1.
143 Allowed only if ``templateMatched==True`` and ``preConvMode==True``.
144 Defaults to the PSF of the science exposure at the image center.
145 xcen : `float`, optional
146 X-pixel coordinate to use for computing constant matching kernel to use
147 If `None` (default), then use the center of the image.
148 ycen : `float`, optional
149 Y-pixel coordinate to use for computing constant matching kernel to use
150 If `None` (default), then use the center of the image.
151 svar : `float`, optional
152 Image variance for science image
153 If `None` (default) then compute the variance over the entire input science image.
154 tvar : `float`, optional
155 Image variance for template image
156 If `None` (default) then compute the variance over the entire input template image.
157 templateMatched : `bool`, optional
158 If True, the template exposure was matched (convolved) to the science exposure.
159 See also notes below.
160 preConvMode : `bool`, optional
161 If True, ``subtractedExposure`` is assumed to be a likelihood difference image
162 and will be noise corrected as a likelihood image.
163 **kwargs
164 Additional keyword arguments propagated from DecorrelateALKernelSpatialTask.
165
166 Returns
167 -------
168 result : `lsst.pipe.base.Struct`
169 - ``correctedExposure`` : the decorrelated diffim
170
171 Notes
172 -----
173 If ``preConvMode==True``, ``subtractedExposure`` is assumed to be a
174 score image and the noise correction for likelihood images
175 is applied. The resulting image is an optimal detection likelihood image
176 when the templateExposure has noise. (See DMTN-179) If ``preConvKernel`` is
177 not specified, the PSF of ``scienceExposure`` is assumed as pre-convolution kernel.
178
179 The ``subtractedExposure`` is NOT updated. The returned ``correctedExposure``
180 has an updated but spatially fixed PSF. It is calculated as the center of
181 image PSF corrected by the center of image matching kernel.
182
183 If ``templateMatched==True``, the templateExposure was matched (convolved)
184 to the ``scienceExposure`` by ``psfMatchingKernel`` during image differencing.
185 Otherwise the ``scienceExposure`` was matched (convolved) by ``psfMatchingKernel``.
186 In either case, note that the original template and science images are required,
187 not the psf-matched version.
188
189 This task discards the variance plane of ``subtractedExposure`` and re-computes
190 it from the variance planes of ``scienceExposure`` and ``templateExposure``.
191 The image plane of ``subtractedExposure`` must be at the photometric level
192 set by the AL PSF matching in `ImagePsfMatchTask.subtractExposures`.
193 The assumptions about the photometric level are controlled by the
194 `templateMatched` option in this task.
195
196 Here we currently convert a spatially-varying matching kernel into a constant kernel,
197 just by computing it at the center of the image (tickets DM-6243, DM-6244).
198
199 We are also using a constant accross-the-image measure of sigma (sqrt(variance)) to compute
200 the decorrelation kernel.
201
202 TODO DM-23857 As part of the spatially varying correction implementation
203 consider whether returning a Struct is still necessary.
204 """
205 if preConvKernel is not None and not (templateMatched and preConvMode):
206 raise ValueError("Pre-convolution kernel is allowed only if "
207 "preConvMode==True and templateMatched==True.")
208
209 spatialKernel = psfMatchingKernel
210 kimg = afwImage.ImageD(spatialKernel.getDimensions())
211 bbox = subtractedExposure.getBBox()
212 if xcen is None:
213 xcen = (bbox.getBeginX() + bbox.getEndX()) / 2.
214 if ycen is None:
215 ycen = (bbox.getBeginY() + bbox.getEndY()) / 2.
216 self.log.info("Using matching kernel computed at (%d, %d)", xcen, ycen)
217 spatialKernel.computeImage(kimg, False, xcen, ycen)
218
219 preConvImg = None
220 if preConvMode:
221 if preConvKernel is None:
222 pos = scienceExposure.getPsf().getAveragePosition()
223 preConvKernel = scienceExposure.getPsf().getLocalKernel(pos)
224 preConvImg = afwImage.ImageD(preConvKernel.getDimensions())
225 preConvKernel.computeImage(preConvImg, True)
226
227 if svar is None:
228 svar = self.computeVarianceMean(scienceExposure)
229 if tvar is None:
230 tvar = self.computeVarianceMean(templateExposure)
231 self.log.info("Original variance plane means. Science:%.5e, warped template:%.5e)",
232 svar, tvar)
233
234 # Should not happen unless entire image has been masked, which could happen
235 # if this is a small subimage of the main exposure. In this case, just return a full NaN
236 # exposure
237 if np.isnan(svar) or np.isnan(tvar):
238 # Double check that one of the exposures is all NaNs
239 if (np.all(np.isnan(scienceExposure.image.array))
240 or np.all(np.isnan(templateExposure.image.array))):
241 self.log.warning('Template or science image is entirely NaNs: skipping decorrelation.')
242 outExposure = subtractedExposure.clone()
243 return pipeBase.Struct(correctedExposure=outExposure, )
244
245 if templateMatched:
246 # Regular subtraction, we convolved the template
247 self.log.info("Decorrelation after template image convolution")
248 varianceMean = svar
249 targetVarianceMean = tvar
250 # variance plane of the image that is not convolved
251 variance = scienceExposure.variance.array
252 # Variance plane of the convolved image, before convolution.
253 targetVariance = templateExposure.variance.array
254 else:
255 # We convolved the science image
256 self.log.info("Decorrelation after science image convolution")
257 varianceMean = tvar
258 targetVarianceMean = svar
259 # variance plane of the image that is not convolved
260 variance = templateExposure.variance.array
261 # Variance plane of the convolved image, before convolution.
262 targetVariance = scienceExposure.variance.array
263
264 # The maximal correction value converges to sqrt(targetVarianceMean/varianceMean).
265 # Correction divergence warning if the correction exceeds 4 orders of magnitude.
266 mOverExpVar = targetVarianceMean/varianceMean
267 if mOverExpVar > 1e8:
268 self.log.warning("Diverging correction: matched image variance is "
269 " much larger than the unconvolved one's"
270 ", targetVarianceMean/varianceMean:%.2e", mOverExpVar)
271
272 oldVarMean = self.computeVarianceMean(subtractedExposure)
273 self.log.info("Variance plane mean of uncorrected diffim: %f", oldVarMean)
274
275 kArr = kimg.array
276 diffimShape = subtractedExposure.image.array.shape
277 psfImg = subtractedExposure.getPsf().computeKernelImage(geom.Point2D(xcen, ycen))
278 psfShape = psfImg.array.shape
279
280 if preConvMode:
281 self.log.info("Decorrelation of likelihood image")
282 self.computeCommonShape(preConvImg.array.shape, kArr.shape,
283 psfShape, diffimShape)
284 corr = self.computeScoreCorrection(kArr, varianceMean, targetVarianceMean, preConvImg.array)
285 else:
286 self.log.info("Decorrelation of difference image")
287 self.computeCommonShape(kArr.shape, psfShape, diffimShape)
288 corr = self.computeDiffimCorrection(kArr, varianceMean, targetVarianceMean)
289
290 correctedImage = self.computeCorrectedImage(corr.corrft, subtractedExposure.image.array)
291 correctedPsf = self.computeCorrectedDiffimPsf(corr.corrft, psfImg.array)
292
293 # The subtracted exposure variance plane is already correlated, we cannot propagate
294 # it through another convolution; instead we need to use the uncorrelated originals
295 # The whitening should scale it to varianceMean + targetVarianceMean on average
296 if self.config.completeVarPlanePropagation:
297 self.log.debug("Using full variance plane calculation in decorrelation")
298 correctedVariance = self.calculateVariancePlane(
299 variance, targetVariance,
300 varianceMean, targetVarianceMean, corr.cnft, corr.crft)
301 else:
302 self.log.debug("Using estimated variance plane calculation in decorrelation")
303 correctedVariance = self.estimateVariancePlane(
304 variance, targetVariance,
305 corr.cnft, corr.crft)
306
307 # Determine the common shape
308 kSum = np.sum(kArr)
309 kSumSq = kSum*kSum
310 self.log.debug("Matching kernel sum: %.3e", kSum)
311 if not templateMatched:
312 # ImagePsfMatch.subtractExposures re-scales the difference in
313 # the science image convolution mode
314 correctedVariance /= kSumSq
315 subtractedExposure.image.array[...] = correctedImage # Allow for numpy type casting
316 subtractedExposure.variance.array[...] = correctedVariance
317 subtractedExposure.setPsf(correctedPsf)
318
319 newVarMean = self.computeVarianceMean(subtractedExposure)
320 self.log.info("Variance plane mean of corrected diffim: %.5e", newVarMean)
321
322 # TODO DM-23857 As part of the spatially varying correction implementation
323 # consider whether returning a Struct is still necessary.
324 return pipeBase.Struct(correctedExposure=subtractedExposure, )
325
326 def computeCommonShape(self, *shapes):
327 """Calculate the common shape for FFT operations. Set `self.freqSpaceShape`
328 internally.
329
330 Parameters
331 ----------
332 shapes : one or more `tuple` of `int`
333 Shapes of the arrays. All must have the same dimensionality.
334 At least one shape must be provided.
335
336 Returns
337 -------
338 None.
339
340 Notes
341 -----
342 For each dimension, gets the smallest even number greater than or equal to
343 `N1+N2-1` where `N1` and `N2` are the two largest values.
344 In case of only one shape given, rounds up to even each dimension value.
345 """
346 S = np.array(shapes, dtype=int)
347 if len(shapes) > 2:
348 S.sort(axis=0)
349 S = S[-2:]
350 if len(shapes) > 1:
351 commonShape = np.sum(S, axis=0) - 1
352 else:
353 commonShape = S[0]
354 commonShape[commonShape % 2 != 0] += 1
355 self.freqSpaceShape = tuple(commonShape)
356 self.log.info("Common frequency space shape %s", self.freqSpaceShape)
357
358 @staticmethod
359 def padCenterOriginArray(A, newShape: tuple, useInverse=False):
360 """Zero pad an image where the origin is at the center and replace the
361 origin to the corner as required by the periodic input of FFT. Implement also
362 the inverse operation, crop the padding and re-center data.
363
364 Parameters
365 ----------
366 A : `numpy.ndarray`
367 An array to copy from.
368 newShape : `tuple` of `int`
369 The dimensions of the resulting array. For padding, the resulting array
370 must be larger than A in each dimension. For the inverse operation this
371 must be the original, before padding size of the array.
372 useInverse : bool, optional
373 Selector of forward, add padding, operation (False)
374 or its inverse, crop padding, operation (True).
375
376 Returns
377 -------
378 R : `numpy.ndarray`
379 The padded or unpadded array with shape of `newShape` and the same dtype as A.
380
381 Notes
382 -----
383 For odd dimensions, the splitting is rounded to
384 put the center pixel into the new corner origin (0,0). This is to be consistent
385 e.g. for a dirac delta kernel that is originally located at the center pixel.
386 """
387
388 # The forward and inverse operations should round odd dimension halves at the opposite
389 # sides to get the pixels back to their original positions.
390 if not useInverse:
391 # Forward operation: First and second halves with respect to the axes of A.
392 firstHalves = [x//2 for x in A.shape]
393 secondHalves = [x-y for x, y in zip(A.shape, firstHalves)]
394 else:
395 # Inverse operation: Opposite rounding
396 secondHalves = [x//2 for x in newShape]
397 firstHalves = [x-y for x, y in zip(newShape, secondHalves)]
398
399 R = np.zeros_like(A, shape=newShape)
400 R[-firstHalves[0]:, -firstHalves[1]:] = A[:firstHalves[0], :firstHalves[1]]
401 R[:secondHalves[0], -firstHalves[1]:] = A[-secondHalves[0]:, :firstHalves[1]]
402 R[:secondHalves[0], :secondHalves[1]] = A[-secondHalves[0]:, -secondHalves[1]:]
403 R[-firstHalves[0]:, :secondHalves[1]] = A[:firstHalves[0], -secondHalves[1]:]
404 return R
405
406 def computeDiffimCorrection(self, kappa, svar, tvar):
407 """Compute the Lupton decorrelation post-convolution kernel for decorrelating an
408 image difference, based on the PSF-matching kernel.
409
410 Parameters
411 ----------
412 kappa : `numpy.ndarray` of `float`
413 A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching.
414 svar : `float` > 0.
415 Average variance of science image used for PSF matching.
416 tvar : `float` > 0.
417 Average variance of the template (matched) image used for PSF matching.
418
419 Returns
420 -------
421 corrft : `numpy.ndarray` of `float`
422 The frequency space representation of the correction. The array is real (dtype float).
423 Shape is `self.freqSpaceShape`.
424
425 cnft, crft : `numpy.ndarray` of `complex`
426 The overall convolution (pre-conv, PSF matching, noise correction) kernel
427 for the science and template images, respectively for the variance plane
428 calculations. These are intermediate results in frequency space.
429
430 Notes
431 -----
432 The maximum correction factor converges to `sqrt(tvar/svar)` towards high frequencies.
433 This should be a plausible value.
434 """
435 kSum = np.sum(kappa) # We scale the decorrelation to preserve fluxes
436 kappa = self.padCenterOriginArray(kappa, self.freqSpaceShape)
437 kft = np.fft.fft2(kappa)
438 kftAbsSq = np.real(np.conj(kft) * kft)
439
440 denom = svar + tvar * kftAbsSq
441 corrft = np.sqrt((svar + tvar * kSum*kSum) / denom)
442 cnft = corrft
443 crft = kft*corrft
444 return pipeBase.Struct(corrft=corrft, cnft=cnft, crft=crft)
445
446 def computeScoreCorrection(self, kappa, svar, tvar, preConvArr):
447 """Compute the correction kernel for a score image.
448
449 Parameters
450 ----------
451 kappa : `numpy.ndarray`
452 A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching.
453 svar : `float`
454 Average variance of science image used for PSF matching (before pre-convolution).
455 tvar : `float`
456 Average variance of the template (matched) image used for PSF matching.
457 preConvArr : `numpy.ndarray`
458 The pre-convolution kernel of the science image. It should be the PSF
459 of the science image or an approximation of it. It must be normed to sum 1.
460
461 Returns
462 -------
463 corrft : `numpy.ndarray` of `float`
464 The frequency space representation of the correction. The array is real (dtype float).
465 Shape is `self.freqSpaceShape`.
466 cnft, crft : `numpy.ndarray` of `complex`
467 The overall convolution (pre-conv, PSF matching, noise correction) kernel
468 for the science and template images, respectively for the variance plane
469 calculations. These are intermediate results in frequency space.
470
471 Notes
472 -----
473 To be precise, the science image should be _correlated_ by ``preConvArray`` but this
474 does not matter for this calculation.
475
476 ``cnft``, ``crft`` contain the scaling factor as well.
477
478 """
479 kSum = np.sum(kappa)
480 kappa = self.padCenterOriginArray(kappa, self.freqSpaceShape)
481 kft = np.fft.fft2(kappa)
482 preConvArr = self.padCenterOriginArray(preConvArr, self.freqSpaceShape)
483 preFt = np.fft.fft2(preConvArr)
484 preFtAbsSq = np.real(np.conj(preFt) * preFt)
485 kftAbsSq = np.real(np.conj(kft) * kft)
486 # Avoid zero division, though we don't normally approach `tiny`.
487 # We have numerical noise instead.
488 tiny = np.finfo(preFtAbsSq.dtype).tiny * 1000.
489 flt = preFtAbsSq < tiny
490 # If we pre-convolve to avoid deconvolution in AL, then kftAbsSq / preFtAbsSq
491 # theoretically expected to diverge to +inf. But we don't care about the convergence
492 # properties here, S goes to 0 at these frequencies anyway.
493 preFtAbsSq[flt] = tiny
494 denom = svar + tvar * kftAbsSq / preFtAbsSq
495 corrft = (svar + tvar * kSum*kSum) / denom
496 cnft = np.conj(preFt)*corrft
497 crft = kft*corrft
498 return pipeBase.Struct(corrft=corrft, cnft=cnft, crft=crft)
499
500 @staticmethod
501 def estimateVariancePlane(vplane1, vplane2, c1ft, c2ft):
502 """Estimate the variance planes.
503
504 The estimation assumes that around each pixel the surrounding
505 pixels' sigmas within the convolution kernel are the same.
506
507 Parameters
508 ----------
509 vplane1, vplane2 : `numpy.ndarray` of `float`
510 Variance planes of the original (before pre-convolution or matching)
511 exposures.
512 c1ft, c2ft : `numpy.ndarray` of `complex`
513 The overall convolution that includes the matching and the
514 afterburner in frequency space. The result of either
515 ``computeScoreCorrection`` or ``computeDiffimCorrection``.
516
517 Returns
518 -------
519 vplaneD : `numpy.ndarray` of `float`
520 The estimated variance plane of the difference/score image
521 as a weighted sum of the input variances.
522
523 Notes
524 ------
525 See DMTN-179 Section 5 about the variance plane calculations.
526 """
527 w1 = np.sum(np.real(np.conj(c1ft)*c1ft)) / c1ft.size
528 w2 = np.sum(np.real(np.conj(c2ft)*c2ft)) / c2ft.size
529 # w1, w2: the frequency space sum of abs(c1)^2 is the same as in image
530 # space.
531 return vplane1*w1 + vplane2*w2
532
533 def calculateVariancePlane(self, vplane1, vplane2, varMean1, varMean2, c1ft, c2ft):
534 """Full propagation of the variance planes of the original exposures.
535
536 The original variance planes of independent pixels are convolved with the
537 image space square of the overall kernels.
538
539 Parameters
540 ----------
541 vplane1, vplane2 : `numpy.ndarray` of `float`
542 Variance planes of the original (before pre-convolution or matching)
543 exposures.
544 varMean1, varMean2 : `float`
545 Replacement average values for non-finite ``vplane1`` and ``vplane2`` values respectively.
546
547 c1ft, c2ft : `numpy.ndarray` of `complex`
548 The overall convolution that includes the matching and the
549 afterburner in frequency space. The result of either
550 ``computeScoreCorrection`` or ``computeDiffimCorrection``.
551
552 Returns
553 -------
554 vplaneD : `numpy.ndarray` of `float`
555 The variance plane of the difference/score images.
556
557 Notes
558 ------
559 See DMTN-179 Section 5 about the variance plane calculations.
560
561 Infs and NaNs are allowed and kept in the returned array.
562 """
563 D = np.real(np.fft.ifft2(c1ft))
564 c1SqFt = np.fft.fft2(D*D)
565
566 v1shape = vplane1.shape
567 filtInf = np.isinf(vplane1)
568 filtNan = np.isnan(vplane1)
569 # This copy could be eliminated if inf/nan handling were go into padCenterOriginArray
570 vplane1 = np.copy(vplane1)
571 vplane1[filtInf | filtNan] = varMean1
572 D = self.padCenterOriginArray(vplane1, self.freqSpaceShape)
573 v1 = np.real(np.fft.ifft2(np.fft.fft2(D) * c1SqFt))
574 v1 = self.padCenterOriginArray(v1, v1shape, useInverse=True)
575 v1[filtNan] = np.nan
576 v1[filtInf] = np.inf
577
578 D = np.real(np.fft.ifft2(c2ft))
579 c2SqFt = np.fft.fft2(D*D)
580
581 v2shape = vplane2.shape
582 filtInf = np.isinf(vplane2)
583 filtNan = np.isnan(vplane2)
584 vplane2 = np.copy(vplane2)
585 vplane2[filtInf | filtNan] = varMean2
586 D = self.padCenterOriginArray(vplane2, self.freqSpaceShape)
587 v2 = np.real(np.fft.ifft2(np.fft.fft2(D) * c2SqFt))
588 v2 = self.padCenterOriginArray(v2, v2shape, useInverse=True)
589 v2[filtNan] = np.nan
590 v2[filtInf] = np.inf
591
592 return v1 + v2
593
594 def computeCorrectedDiffimPsf(self, corrft, psfOld):
595 """Compute the (decorrelated) difference image's new PSF.
596
597 Parameters
598 ----------
599 corrft : `numpy.ndarray`
600 The frequency space representation of the correction calculated by
601 `computeCorrection`. Shape must be `self.freqSpaceShape`.
602 psfOld : `numpy.ndarray`
603 The psf of the difference image to be corrected.
604
605 Returns
606 -------
607 correctedPsf : `lsst.meas.algorithms.KernelPsf`
608 The corrected psf, same shape as `psfOld`, sum normed to 1.
609
610 Notes
611 -----
612 There is no algorithmic guarantee that the corrected psf can
613 meaningfully fit to the same size as the original one.
614 """
615 psfShape = psfOld.shape
616 psfNew = self.padCenterOriginArray(psfOld, self.freqSpaceShape)
617 psfNew = np.fft.fft2(psfNew)
618 psfNew *= corrft
619 psfNew = np.fft.ifft2(psfNew)
620 psfNew = psfNew.real
621 psfNew = self.padCenterOriginArray(psfNew, psfShape, useInverse=True)
622 psfNew = psfNew/psfNew.sum()
623
624 psfcI = afwImage.ImageD(geom.Extent2I(psfShape[1], psfShape[0]))
625 psfcI.array = psfNew
626 psfcK = afwMath.FixedKernel(psfcI)
627 correctedPsf = measAlg.KernelPsf(psfcK)
628 return correctedPsf
629
630 def computeCorrectedImage(self, corrft, imgOld):
631 """Compute the decorrelated difference image.
632
633 Parameters
634 ----------
635 corrft : `numpy.ndarray`
636 The frequency space representation of the correction calculated by
637 `computeCorrection`. Shape must be `self.freqSpaceShape`.
638 imgOld : `numpy.ndarray`
639 The difference image to be corrected.
640
641 Returns
642 -------
643 imgNew : `numpy.ndarray`
644 The corrected image, same size as the input.
645 """
646 expShape = imgOld.shape
647 imgNew = np.copy(imgOld)
648 filtInf = np.isinf(imgNew)
649 filtNan = np.isnan(imgNew)
650 imgNew[filtInf] = np.nan
651 imgNew[filtInf | filtNan] = np.nanmean(imgNew)
652 imgNew = self.padCenterOriginArray(imgNew, self.freqSpaceShape)
653 imgNew = np.fft.fft2(imgNew)
654 imgNew *= corrft
655 imgNew = np.fft.ifft2(imgNew)
656 imgNew = imgNew.real
657 imgNew = self.padCenterOriginArray(imgNew, expShape, useInverse=True)
658 imgNew[filtNan] = np.nan
659 imgNew[filtInf] = np.inf
660 return imgNew
661
662
664 """Task to be used as an ImageMapper for performing
665 A&L decorrelation on subimages on a grid across a A&L difference image.
666
667 This task subclasses DecorrelateALKernelTask in order to implement
668 all of that task's configuration parameters, as well as its `run` method.
669 """
670
671 ConfigClass = DecorrelateALKernelConfig
672 _DefaultName = 'ip_diffim_decorrelateALKernelMapper'
673
674 def __init__(self, *args, **kwargs):
675 DecorrelateALKernelTask.__init__(self, *args, **kwargs)
676
677 def run(self, subExposure, expandedSubExposure, fullBBox,
678 template, science, alTaskResult=None, psfMatchingKernel=None,
679 preConvKernel=None, **kwargs):
680 """Perform decorrelation operation on `subExposure`, using
681 `expandedSubExposure` to allow for invalid edge pixels arising from
682 convolutions.
683
684 This method performs A&L decorrelation on `subExposure` using
685 local measures for image variances and PSF. `subExposure` is a
686 sub-exposure of the non-decorrelated A&L diffim. It also
687 requires the corresponding sub-exposures of the template
688 (`template`) and science (`science`) exposures.
689
690 Parameters
691 ----------
692 subExposure : `lsst.afw.image.Exposure`
693 the sub-exposure of the diffim
694 expandedSubExposure : `lsst.afw.image.Exposure`
695 the expanded sub-exposure upon which to operate
696 fullBBox : `lsst.geom.Box2I`
697 the bounding box of the original exposure
698 template : `lsst.afw.image.Exposure`
699 the corresponding sub-exposure of the template exposure
700 science : `lsst.afw.image.Exposure`
701 the corresponding sub-exposure of the science exposure
702 alTaskResult : `lsst.pipe.base.Struct`
703 the result of A&L image differencing on `science` and
704 `template`, importantly containing the resulting
705 `psfMatchingKernel`. Can be `None`, only if
706 `psfMatchingKernel` is not `None`.
707 psfMatchingKernel : Alternative parameter for passing the
708 A&L `psfMatchingKernel` directly.
709 preConvKernel : If not None, then pre-filtering was applied
710 to science exposure, and this is the pre-convolution
711 kernel.
712 kwargs :
713 additional keyword arguments propagated from
714 `ImageMapReduceTask.run`.
715
716 Returns
717 -------
718 A `pipeBase.Struct` containing:
719
720 - ``subExposure`` : the result of the `subExposure` processing.
721 - ``decorrelationKernel`` : the decorrelation kernel, currently
722 not used.
723
724 Notes
725 -----
726 This `run` method accepts parameters identical to those of
727 `ImageMapper.run`, since it is called from the
728 `ImageMapperTask`. See that class for more information.
729 """
730 templateExposure = template # input template
731 scienceExposure = science # input science image
732 if alTaskResult is None and psfMatchingKernel is None:
733 raise RuntimeError('Both alTaskResult and psfMatchingKernel cannot be None')
734 psfMatchingKernel = alTaskResult.psfMatchingKernel if alTaskResult is not None else psfMatchingKernel
735
736 # subExp and expandedSubExp are subimages of the (un-decorrelated) diffim!
737 # So here we compute corresponding subimages of templateExposure and scienceExposure
738 subExp2 = scienceExposure.Factory(scienceExposure, expandedSubExposure.getBBox())
739 subExp1 = templateExposure.Factory(templateExposure, expandedSubExposure.getBBox())
740
741 # Prevent too much log INFO verbosity from DecorrelateALKernelTask.run
742 logLevel = self.log.level
743 self.log.setLevel(self.log.WARNING)
744 res = DecorrelateALKernelTask.run(self, subExp2, subExp1, expandedSubExposure,
745 psfMatchingKernel, preConvKernel, **kwargs)
746 self.log.setLevel(logLevel) # reset the log level
747
748 diffim = res.correctedExposure.Factory(res.correctedExposure, subExposure.getBBox())
749 out = pipeBase.Struct(subExposure=diffim, )
750 return out
751
752
754 """Configuration parameters for the ImageMapReduceTask to direct it to use
755 DecorrelateALKernelMapper as its mapper for A&L decorrelation.
756 """
757 mapper = pexConfig.ConfigurableField(
758 doc='A&L decorrelation task to run on each sub-image',
759 target=DecorrelateALKernelMapper
760 )
761
762
763class DecorrelateALKernelSpatialConfig(pexConfig.Config):
764 """Configuration parameters for the DecorrelateALKernelSpatialTask.
765 """
766 decorrelateConfig = pexConfig.ConfigField(
767 dtype=DecorrelateALKernelConfig,
768 doc='DecorrelateALKernel config to use when running on complete exposure (non spatially-varying)',
769 )
770
771 decorrelateMapReduceConfig = pexConfig.ConfigField(
772 dtype=DecorrelateALKernelMapReduceConfig,
773 doc='DecorrelateALKernelMapReduce config to use when running on each sub-image (spatially-varying)',
774 )
775
776 ignoreMaskPlanes = pexConfig.ListField(
777 dtype=str,
778 doc="""Mask planes to ignore for sigma-clipped statistics""",
779 default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE")
780 )
781
782 def setDefaults(self):
783 self.decorrelateMapReduceConfig.gridStepX = self.decorrelateMapReduceConfig.gridStepY = 40
784 self.decorrelateMapReduceConfig.cellSizeX = self.decorrelateMapReduceConfig.cellSizeY = 41
785 self.decorrelateMapReduceConfig.borderSizeX = self.decorrelateMapReduceConfig.borderSizeY = 8
786 self.decorrelateMapReduceConfig.reducer.reduceOperation = 'average'
787
788
790 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference
791
792 Notes
793 -----
794
795 Pipe-task that removes the neighboring-pixel covariance in an
796 image difference that are added when the template image is
797 convolved with the Alard-Lupton PSF matching kernel.
798
799 This task is a simple wrapper around @ref DecorrelateALKernelTask,
800 which takes a `spatiallyVarying` parameter in its `run` method. If
801 it is `False`, then it simply calls the `run` method of @ref
802 DecorrelateALKernelTask. If it is True, then it uses the @ref
803 ImageMapReduceTask framework to break the exposures into
804 subExposures on a grid, and performs the `run` method of @ref
805 DecorrelateALKernelTask on each subExposure. This enables it to
806 account for spatially-varying PSFs and noise in the exposures when
807 performing the decorrelation.
808
809 This task has no standalone example, however it is applied as a
810 subtask of pipe.tasks.imageDifference.ImageDifferenceTask.
811 There is also an example of its use in `tests/testImageDecorrelation.py`.
812 """
813 ConfigClass = DecorrelateALKernelSpatialConfig
814 _DefaultName = "ip_diffim_decorrelateALKernelSpatial"
815
816 def __init__(self, *args, **kwargs):
817 """Create the image decorrelation Task
818
819 Parameters
820 ----------
821 args :
822 arguments to be passed to
823 `lsst.pipe.base.task.Task.__init__`
824 kwargs :
825 additional keyword arguments to be passed to
826 `lsst.pipe.base.task.Task.__init__`
827 """
828 pipeBase.Task.__init__(self, *args, **kwargs)
829
831 self.statsControl.setNumSigmaClip(3.)
832 self.statsControl.setNumIter(3)
833 self.statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes))
834
835 def computeVarianceMean(self, exposure):
836 """Compute the mean of the variance plane of `exposure`.
837 """
838 statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(),
839 exposure.getMaskedImage().getMask(),
840 afwMath.MEANCLIP, self.statsControl)
841 var = statObj.getValue(afwMath.MEANCLIP)
842 return var
843
844 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel,
845 spatiallyVarying=True, preConvKernel=None, templateMatched=True, preConvMode=False):
846 """Perform decorrelation of an image difference exposure.
847
848 Decorrelates the diffim due to the convolution of the
849 templateExposure with the A&L psfMatchingKernel. If
850 `spatiallyVarying` is True, it utilizes the spatially varying
851 matching kernel via the `imageMapReduce` framework to perform
852 spatially-varying decorrelation on a grid of subExposures.
853
854 Parameters
855 ----------
856 scienceExposure : `lsst.afw.image.Exposure`
857 the science Exposure used for PSF matching
858 templateExposure : `lsst.afw.image.Exposure`
859 the template Exposure used for PSF matching
860 subtractedExposure : `lsst.afw.image.Exposure`
861 the subtracted Exposure produced by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
862 psfMatchingKernel : an (optionally spatially-varying) PSF matching kernel produced
863 by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
864 spatiallyVarying : `bool`
865 if True, perform the spatially-varying operation
866 preConvKernel : `lsst.meas.algorithms.Psf`
867 if not none, the scienceExposure has been pre-filtered with this kernel. (Currently
868 this option is experimental.)
869 templateMatched : `bool`, optional
870 If True, the template exposure was matched (convolved) to the science exposure.
871 preConvMode : `bool`, optional
872 If True, ``subtractedExposure`` is assumed to be a likelihood difference image
873 and will be noise corrected as a likelihood image.
874
875 Returns
876 -------
877 results : `lsst.pipe.base.Struct`
878 a structure containing:
879 - ``correctedExposure`` : the decorrelated diffim
880 """
881 self.log.info('Running A&L decorrelation: spatiallyVarying=%r', spatiallyVarying)
882
883 svar = self.computeVarianceMean(scienceExposure)
884 tvar = self.computeVarianceMean(templateExposure)
885 if np.isnan(svar) or np.isnan(tvar): # Should not happen unless entire image has been masked.
886 # Double check that one of the exposures is all NaNs
887 if (np.all(np.isnan(scienceExposure.image.array))
888 or np.all(np.isnan(templateExposure.image.array))):
889 self.log.warning('Template or science image is entirely NaNs: skipping decorrelation.')
890 if np.isnan(svar):
891 svar = 1e-9
892 if np.isnan(tvar):
893 tvar = 1e-9
894
895 var = self.computeVarianceMean(subtractedExposure)
896
897 if spatiallyVarying:
898 self.log.info("Variance (science, template): (%f, %f)", svar, tvar)
899 self.log.info("Variance (uncorrected diffim): %f", var)
900 config = self.config.decorrelateMapReduceConfig
901 task = ImageMapReduceTask(config=config)
902 results = task.run(subtractedExposure, science=scienceExposure,
903 template=templateExposure, psfMatchingKernel=psfMatchingKernel,
904 preConvKernel=preConvKernel, forceEvenSized=True,
905 templateMatched=templateMatched, preConvMode=preConvMode)
906 results.correctedExposure = results.exposure
907
908 # Make sure masks of input image are propagated to diffim
909 def gm(exp):
910 return exp.getMaskedImage().getMask()
911 gm(results.correctedExposure)[:, :] = gm(subtractedExposure)
912
913 var = self.computeVarianceMean(results.correctedExposure)
914 self.log.info("Variance (corrected diffim): %f", var)
915
916 else:
917 config = self.config.decorrelateConfig
918 task = DecorrelateALKernelTask(config=config)
919 results = task.run(scienceExposure, templateExposure,
920 subtractedExposure, psfMatchingKernel, preConvKernel=preConvKernel,
921 templateMatched=templateMatched, preConvMode=preConvMode)
922
923 return results
def run(self, subExposure, expandedSubExposure, fullBBox, template, science, alTaskResult=None, psfMatchingKernel=None, preConvKernel=None, **kwargs)
def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel, spatiallyVarying=True, preConvKernel=None, templateMatched=True, preConvMode=False)
def computeScoreCorrection(self, kappa, svar, tvar, preConvArr)
def calculateVariancePlane(self, vplane1, vplane2, varMean1, varMean2, c1ft, c2ft)
def padCenterOriginArray(A, tuple newShape, useInverse=False)
def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel, preConvKernel=None, xcen=None, ycen=None, svar=None, tvar=None, templateMatched=True, preConvMode=False, **kwargs)
Statistics makeStatistics(lsst::afw::image::Image< Pixel > const &img, lsst::afw::image::Mask< image::MaskPixel > const &msk, int const flags, StatisticsControl const &sctrl=StatisticsControl())