lsst.ip.diffim g4d7a404405+5e3fc3b9aa
imageDecorrelation.py
Go to the documentation of this file.
2# LSST Data Management System
3# Copyright 2016 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22
23import numpy as np
24
25import lsst.afw.image as afwImage
26import lsst.afw.math as afwMath
27import lsst.geom as geom
28import lsst.meas.algorithms as measAlg
29import lsst.pex.config as pexConfig
30import lsst.pipe.base as pipeBase
31from lsst.utils.timer import timeMethod
32
33from .imageMapReduce import (ImageMapReduceConfig, ImageMapReduceTask,
34 ImageMapper)
35
36__all__ = ("DecorrelateALKernelTask", "DecorrelateALKernelConfig",
37 "DecorrelateALKernelMapper", "DecorrelateALKernelMapReduceConfig",
38 "DecorrelateALKernelSpatialConfig", "DecorrelateALKernelSpatialTask")
39
40
41class DecorrelateALKernelConfig(pexConfig.Config):
42 """Configuration parameters for the DecorrelateALKernelTask
43 """
44
45 ignoreMaskPlanes = pexConfig.ListField(
46 dtype=str,
47 doc="""Mask planes to ignore for sigma-clipped statistics""",
48 default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE")
49 )
50 completeVarPlanePropagation = pexConfig.Field(
51 dtype=bool,
52 default=False,
53 doc="Compute the full effect of the decorrelated matching kernel on the variance plane."
54 " Otherwise use a model weighed sum of the input variances."
55 )
56
57
58class DecorrelateALKernelTask(pipeBase.Task):
59 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference
60
61 Notes
62 -----
63
64 Pipe-task that removes the neighboring-pixel covariance in an
65 image difference that are added when the template image is
66 convolved with the Alard-Lupton PSF matching kernel.
67
68 The image differencing pipeline task @link
69 ip.diffim.psfMatch.PsfMatchTask PSFMatchTask@endlink and @link
70 ip.diffim.psfMatch.PsfMatchConfigAL PSFMatchConfigAL@endlink uses
71 the Alard and Lupton (1998) method for matching the PSFs of the
72 template and science exposures prior to subtraction. The
73 Alard-Lupton method identifies a matching kernel, which is then
74 (typically) convolved with the template image to perform PSF
75 matching. This convolution has the effect of adding covariance
76 between neighboring pixels in the template image, which is then
77 added to the image difference by subtraction.
78
79 The pixel covariance may be corrected by whitening the noise of
80 the image difference. This task performs such a decorrelation by
81 computing a decorrelation kernel (based upon the A&L matching
82 kernel and variances in the template and science images) and
83 convolving the image difference with it. This process is described
84 in detail in [DMTN-021](http://dmtn-021.lsst.io).
85
86 This task has no standalone example, however it is applied as a
87 subtask of pipe.tasks.imageDifference.ImageDifferenceTask.
88 """
89 ConfigClass = DecorrelateALKernelConfig
90 _DefaultName = "ip_diffim_decorrelateALKernel"
91
92 def __init__(self, *args, **kwargs):
93 """Create the image decorrelation Task
94
95 Parameters
96 ----------
97 args :
98 arguments to be passed to ``lsst.pipe.base.task.Task.__init__``
99 kwargs :
100 keyword arguments to be passed to ``lsst.pipe.base.task.Task.__init__``
101 """
102 pipeBase.Task.__init__(self, *args, **kwargs)
103
105 self.statsControlstatsControl.setNumSigmaClip(3.)
106 self.statsControlstatsControl.setNumIter(3)
107 self.statsControlstatsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes))
108
109 def computeVarianceMean(self, exposure):
110 statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(),
111 exposure.getMaskedImage().getMask(),
112 afwMath.MEANCLIP, self.statsControlstatsControl)
113 var = statObj.getValue(afwMath.MEANCLIP)
114 return var
115
116 @timeMethod
117 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel,
118 preConvKernel=None, xcen=None, ycen=None, svar=None, tvar=None,
119 templateMatched=True, preConvMode=False, **kwargs):
120 """Perform decorrelation of an image difference or of a score difference exposure.
121
122 Corrects the difference or score image due to the convolution of the
123 templateExposure with the A&L PSF matching kernel.
124 See [DMTN-021, Equation 1](http://dmtn-021.lsst.io/#equation-1) and
125 [DMTN-179](http://dmtn-179.lsst.io/) for details.
126
127 Parameters
128 ----------
129 scienceExposure : `lsst.afw.image.Exposure`
130 The original science exposure (before pre-convolution, if ``preConvMode==True``).
131 templateExposure : `lsst.afw.image.Exposure`
132 The original template exposure warped into the science exposure dimensions.
133 subtractedExposure : `lsst.afw.image.Exposure`
134 the subtracted exposure produced by
135 `ip_diffim.ImagePsfMatchTask.subtractExposures()`. The `subtractedExposure` must
136 inherit its PSF from `exposure`, see notes below.
137 psfMatchingKernel : `lsst.afw.detection.Psf`
138 An (optionally spatially-varying) PSF matching kernel produced
139 by `ip_diffim.ImagePsfMatchTask.subtractExposures()`.
140 preConvKernel : `lsst.afw.math.Kernel`, optional
141 If not `None`, then the `scienceExposure` was pre-convolved with (the reflection of)
142 this kernel. Must be normalized to sum to 1.
143 Allowed only if ``templateMatched==True`` and ``preConvMode==True``.
144 Defaults to the PSF of the science exposure at the image center.
145 xcen : `float`, optional
146 X-pixel coordinate to use for computing constant matching kernel to use
147 If `None` (default), then use the center of the image.
148 ycen : `float`, optional
149 Y-pixel coordinate to use for computing constant matching kernel to use
150 If `None` (default), then use the center of the image.
151 svar : `float`, optional
152 Image variance for science image
153 If `None` (default) then compute the variance over the entire input science image.
154 tvar : `float`, optional
155 Image variance for template image
156 If `None` (default) then compute the variance over the entire input template image.
157 templateMatched : `bool`, optional
158 If True, the template exposure was matched (convolved) to the science exposure.
159 See also notes below.
160 preConvMode : `bool`, optional
161 If True, ``subtractedExposure`` is assumed to be a likelihood difference image
162 and will be noise corrected as a likelihood image.
163 **kwargs
164 Additional keyword arguments propagated from DecorrelateALKernelSpatialTask.
165
166 Returns
167 -------
168 result : `lsst.pipe.base.Struct`
169 - ``correctedExposure`` : the decorrelated diffim
170
171 Notes
172 -----
173 If ``preConvMode==True``, ``subtractedExposure`` is assumed to be a
174 score image and the noise correction for likelihood images
175 is applied. The resulting image is an optimal detection likelihood image
176 when the templateExposure has noise. (See DMTN-179) If ``preConvKernel`` is
177 not specified, the PSF of ``scienceExposure`` is assumed as pre-convolution kernel.
178
179 The ``subtractedExposure`` is NOT updated. The returned ``correctedExposure``
180 has an updated but spatially fixed PSF. It is calculated as the center of
181 image PSF corrected by the center of image matching kernel.
182
183 If ``templateMatched==True``, the templateExposure was matched (convolved)
184 to the ``scienceExposure`` by ``psfMatchingKernel``. Otherwise the ``scienceExposure``
185 was matched (convolved) by ``psfMatchingKernel``.
186
187 This task discards the variance plane of ``subtractedExposure`` and re-computes
188 it from the variance planes of ``scienceExposure`` and ``templateExposure``.
189 The image plane of ``subtractedExposure`` must be at the photometric level
190 set by the AL PSF matching in `ImagePsfMatchTask.subtractExposures`.
191 The assumptions about the photometric level are controlled by the
192 `templateMatched` option in this task.
193
194 Here we currently convert a spatially-varying matching kernel into a constant kernel,
195 just by computing it at the center of the image (tickets DM-6243, DM-6244).
196
197 We are also using a constant accross-the-image measure of sigma (sqrt(variance)) to compute
198 the decorrelation kernel.
199
200 TODO DM-23857 As part of the spatially varying correction implementation
201 consider whether returning a Struct is still necessary.
202 """
203 if preConvKernel is not None and not (templateMatched and preConvMode):
204 raise ValueError("Pre-convolution kernel is allowed only if "
205 "preConvMode==True and templateMatched==True.")
206
207 spatialKernel = psfMatchingKernel
208 kimg = afwImage.ImageD(spatialKernel.getDimensions())
209 bbox = subtractedExposure.getBBox()
210 if xcen is None:
211 xcen = (bbox.getBeginX() + bbox.getEndX()) / 2.
212 if ycen is None:
213 ycen = (bbox.getBeginY() + bbox.getEndY()) / 2.
214 self.log.info("Using matching kernel computed at (%d, %d)", xcen, ycen)
215 spatialKernel.computeImage(kimg, False, xcen, ycen)
216
217 preConvImg = None
218 if preConvMode:
219 if preConvKernel is None:
220 preConvKernel = scienceExposure.getPsf().getLocalKernel() # at average position
221 preConvImg = afwImage.ImageD(preConvKernel.getDimensions())
222 preConvKernel.computeImage(preConvImg, True)
223
224 if svar is None:
225 svar = self.computeVarianceMeancomputeVarianceMean(scienceExposure)
226 if tvar is None:
227 tvar = self.computeVarianceMeancomputeVarianceMean(templateExposure)
228 self.log.info("Original variance plane means. Science:%.5e, warped template:%.5e)",
229 svar, tvar)
230
231 if templateMatched:
232 # Regular subtraction, we convolved the template
233 self.log.info("Decorrelation after template image convolution")
234 expVar = svar
235 matchedVar = tvar
236 exposure = scienceExposure
237 matchedExposure = templateExposure
238 else:
239 # We convolved the science image
240 self.log.info("Decorrelation after science image convolution")
241 expVar = tvar
242 matchedVar = svar
243 exposure = templateExposure
244 matchedExposure = scienceExposure
245
246 # Should not happen unless entire image has been masked, which could happen
247 # if this is a small subimage of the main exposure. In this case, just return a full NaN
248 # exposure
249 if np.isnan(expVar) or np.isnan(matchedVar):
250 # Double check that one of the exposures is all NaNs
251 if (np.all(np.isnan(exposure.image.array))
252 or np.all(np.isnan(matchedExposure.image.array))):
253 self.log.warning('Template or science image is entirely NaNs: skipping decorrelation.')
254 outExposure = subtractedExposure.clone()
255 return pipeBase.Struct(correctedExposure=outExposure, )
256
257 # The maximal correction value converges to sqrt(matchedVar/expVar).
258 # Correction divergence warning if the correction exceeds 4 orders of magnitude.
259 mOverExpVar = matchedVar/expVar
260 if mOverExpVar > 1e8:
261 self.log.warning("Diverging correction: matched image variance is "
262 " much larger than the unconvolved one's"
263 ", matchedVar/expVar:%.2e", mOverExpVar)
264
265 oldVarMean = self.computeVarianceMeancomputeVarianceMean(subtractedExposure)
266 self.log.info("Variance plane mean of uncorrected diffim: %f", oldVarMean)
267
268 kArr = kimg.array
269 diffExpArr = subtractedExposure.image.array
270 psfImg = subtractedExposure.getPsf().computeKernelImage(geom.Point2D(xcen, ycen))
271 psfDim = psfImg.getDimensions()
272 psfArr = psfImg.array
273
274 # Determine the common shape
275 kSum = np.sum(kArr)
276 kSumSq = kSum*kSum
277 self.log.debug("Matching kernel sum: %.3e", kSum)
278
279 if preConvMode:
280 self.log.info("Decorrelation of likelihood image")
281 self.computeCommonShapecomputeCommonShape(preConvImg.array.shape, kArr.shape,
282 psfArr.shape, diffExpArr.shape)
283 corr = self.computeScoreCorrectioncomputeScoreCorrection(kArr, expVar, matchedVar, preConvImg.array)
284 else:
285 self.log.info("Decorrelation of difference image")
286 self.computeCommonShapecomputeCommonShape(kArr.shape, psfArr.shape, diffExpArr.shape)
287 corr = self.computeDiffimCorrectioncomputeDiffimCorrection(kArr, expVar, matchedVar)
288
289 diffExpArr = self.computeCorrectedImagecomputeCorrectedImage(corr.corrft, diffExpArr)
290
291 corrPsfArr = self.computeCorrectedDiffimPsfcomputeCorrectedDiffimPsf(corr.corrft, psfArr)
292 psfcI = afwImage.ImageD(psfDim)
293 psfcI.array = corrPsfArr
294 psfcK = afwMath.FixedKernel(psfcI)
295 psfNew = measAlg.KernelPsf(psfcK)
296
297 correctedExposure = subtractedExposure.clone()
298 correctedExposure.image.array[...] = diffExpArr # Allow for numpy type casting
299 # The subtracted exposure variance plane is already correlated, we cannot propagate
300 # it through another convolution; instead we need to use the uncorrelated originals
301 # The whitening should scale it to expVar + matchedVar on average
302 if self.config.completeVarPlanePropagation:
303 self.log.debug("Using full variance plane calculation in decorrelation")
304 newVarArr = self.calculateVariancePlanecalculateVariancePlane(
305 exposure.variance.array, matchedExposure.variance.array,
306 expVar, matchedVar, corr.cnft, corr.crft)
307 else:
308 self.log.debug("Using estimated variance plane calculation in decorrelation")
309 newVarArr = self.estimateVariancePlaneestimateVariancePlane(
310 exposure.variance.array, matchedExposure.variance.array,
311 corr.cnft, corr.crft)
312
313 corrExpVarArr = correctedExposure.variance.array
314 corrExpVarArr[...] = newVarArr # Allow for numpy type casting
315
316 if not templateMatched:
317 # ImagePsfMatch.subtractExposures re-scales the difference in
318 # the science image convolution mode
319 corrExpVarArr /= kSumSq
320 correctedExposure.setPsf(psfNew)
321
322 newVarMean = self.computeVarianceMeancomputeVarianceMean(correctedExposure)
323 self.log.info("Variance plane mean of corrected diffim: %.5e", newVarMean)
324
325 # TODO DM-23857 As part of the spatially varying correction implementation
326 # consider whether returning a Struct is still necessary.
327 return pipeBase.Struct(correctedExposure=correctedExposure, )
328
329 def computeCommonShape(self, *shapes):
330 """Calculate the common shape for FFT operations. Set `self.freqSpaceShape`
331 internally.
332
333 Parameters
334 ----------
335 shapes : one or more `tuple` of `int`
336 Shapes of the arrays. All must have the same dimensionality.
337 At least one shape must be provided.
338
339 Returns
340 -------
341 None.
342
343 Notes
344 -----
345 For each dimension, gets the smallest even number greater than or equal to
346 `N1+N2-1` where `N1` and `N2` are the two largest values.
347 In case of only one shape given, rounds up to even each dimension value.
348 """
349 S = np.array(shapes, dtype=int)
350 if len(shapes) > 2:
351 S.sort(axis=0)
352 S = S[-2:]
353 if len(shapes) > 1:
354 commonShape = np.sum(S, axis=0) - 1
355 else:
356 commonShape = S[0]
357 commonShape[commonShape % 2 != 0] += 1
358 self.freqSpaceShapefreqSpaceShape = tuple(commonShape)
359 self.log.info("Common frequency space shape %s", self.freqSpaceShapefreqSpaceShape)
360
361 @staticmethod
362 def padCenterOriginArray(A, newShape: tuple, useInverse=False):
363 """Zero pad an image where the origin is at the center and replace the
364 origin to the corner as required by the periodic input of FFT. Implement also
365 the inverse operation, crop the padding and re-center data.
366
367 Parameters
368 ----------
369 A : `numpy.ndarray`
370 An array to copy from.
371 newShape : `tuple` of `int`
372 The dimensions of the resulting array. For padding, the resulting array
373 must be larger than A in each dimension. For the inverse operation this
374 must be the original, before padding size of the array.
375 useInverse : bool, optional
376 Selector of forward, add padding, operation (False)
377 or its inverse, crop padding, operation (True).
378
379 Returns
380 -------
381 R : `numpy.ndarray`
382 The padded or unpadded array with shape of `newShape` and the same dtype as A.
383
384 Notes
385 -----
386 For odd dimensions, the splitting is rounded to
387 put the center pixel into the new corner origin (0,0). This is to be consistent
388 e.g. for a dirac delta kernel that is originally located at the center pixel.
389 """
390
391 # The forward and inverse operations should round odd dimension halves at the opposite
392 # sides to get the pixels back to their original positions.
393 if not useInverse:
394 # Forward operation: First and second halves with respect to the axes of A.
395 firstHalves = [x//2 for x in A.shape]
396 secondHalves = [x-y for x, y in zip(A.shape, firstHalves)]
397 else:
398 # Inverse operation: Opposite rounding
399 secondHalves = [x//2 for x in newShape]
400 firstHalves = [x-y for x, y in zip(newShape, secondHalves)]
401
402 R = np.zeros_like(A, shape=newShape)
403 R[-firstHalves[0]:, -firstHalves[1]:] = A[:firstHalves[0], :firstHalves[1]]
404 R[:secondHalves[0], -firstHalves[1]:] = A[-secondHalves[0]:, :firstHalves[1]]
405 R[:secondHalves[0], :secondHalves[1]] = A[-secondHalves[0]:, -secondHalves[1]:]
406 R[-firstHalves[0]:, :secondHalves[1]] = A[:firstHalves[0], -secondHalves[1]:]
407 return R
408
409 def computeDiffimCorrection(self, kappa, svar, tvar):
410 """Compute the Lupton decorrelation post-convolution kernel for decorrelating an
411 image difference, based on the PSF-matching kernel.
412
413 Parameters
414 ----------
415 kappa : `numpy.ndarray` of `float`
416 A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching.
417 svar : `float` > 0.
418 Average variance of science image used for PSF matching.
419 tvar : `float` > 0.
420 Average variance of the template (matched) image used for PSF matching.
421
422 Returns
423 -------
424 corrft : `numpy.ndarray` of `float`
425 The frequency space representation of the correction. The array is real (dtype float).
426 Shape is `self.freqSpaceShapefreqSpaceShape`.
427
428 cnft, crft : `numpy.ndarray` of `complex`
429 The overall convolution (pre-conv, PSF matching, noise correction) kernel
430 for the science and template images, respectively for the variance plane
431 calculations. These are intermediate results in frequency space.
432
433 Notes
434 -----
435 The maximum correction factor converges to `sqrt(tvar/svar)` towards high frequencies.
436 This should be a plausible value.
437 """
438 kSum = np.sum(kappa) # We scale the decorrelation to preserve fluxes
439 kappa = self.padCenterOriginArraypadCenterOriginArray(kappa, self.freqSpaceShapefreqSpaceShape)
440 kft = np.fft.fft2(kappa)
441 kftAbsSq = np.real(np.conj(kft) * kft)
442
443 denom = svar + tvar * kftAbsSq
444 corrft = np.sqrt((svar + tvar * kSum*kSum) / denom)
445 cnft = corrft
446 crft = kft*corrft
447 return pipeBase.Struct(corrft=corrft, cnft=cnft, crft=crft)
448
449 def computeScoreCorrection(self, kappa, svar, tvar, preConvArr):
450 """Compute the correction kernel for a score image.
451
452 Parameters
453 ----------
454 kappa : `numpy.ndarray`
455 A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching.
456 svar : `float`
457 Average variance of science image used for PSF matching (before pre-convolution).
458 tvar : `float`
459 Average variance of the template (matched) image used for PSF matching.
460 preConvArr : `numpy.ndarray`
461 The pre-convolution kernel of the science image. It should be the PSF
462 of the science image or an approximation of it. It must be normed to sum 1.
463
464 Returns
465 -------
466 corrft : `numpy.ndarray` of `float`
467 The frequency space representation of the correction. The array is real (dtype float).
468 Shape is `self.freqSpaceShapefreqSpaceShape`.
469 cnft, crft : `numpy.ndarray` of `complex`
470 The overall convolution (pre-conv, PSF matching, noise correction) kernel
471 for the science and template images, respectively for the variance plane
472 calculations. These are intermediate results in frequency space.
473
474 Notes
475 -----
476 To be precise, the science image should be _correlated_ by ``preConvArray`` but this
477 does not matter for this calculation.
478
479 ``cnft``, ``crft`` contain the scaling factor as well.
480
481 """
482 kSum = np.sum(kappa)
483 kappa = self.padCenterOriginArraypadCenterOriginArray(kappa, self.freqSpaceShapefreqSpaceShape)
484 kft = np.fft.fft2(kappa)
485 preConvArr = self.padCenterOriginArraypadCenterOriginArray(preConvArr, self.freqSpaceShapefreqSpaceShape)
486 preFt = np.fft.fft2(preConvArr)
487 preFtAbsSq = np.real(np.conj(preFt) * preFt)
488 kftAbsSq = np.real(np.conj(kft) * kft)
489 # Avoid zero division, though we don't normally approach `tiny`.
490 # We have numerical noise instead.
491 tiny = np.finfo(preFtAbsSq.dtype).tiny * 1000.
492 flt = preFtAbsSq < tiny
493 # If we pre-convolve to avoid deconvolution in AL, then kftAbsSq / preFtAbsSq
494 # theoretically expected to diverge to +inf. But we don't care about the convergence
495 # properties here, S goes to 0 at these frequencies anyway.
496 preFtAbsSq[flt] = tiny
497 denom = svar + tvar * kftAbsSq / preFtAbsSq
498 corrft = (svar + tvar * kSum*kSum) / denom
499 cnft = np.conj(preFt)*corrft
500 crft = kft*corrft
501 return pipeBase.Struct(corrft=corrft, cnft=cnft, crft=crft)
502
503 @staticmethod
504 def estimateVariancePlane(vplane1, vplane2, c1ft, c2ft):
505 """Estimate the variance planes.
506
507 The estimation assumes that around each pixel the surrounding
508 pixels' sigmas within the convolution kernel are the same.
509
510 Parameters
511 ----------
512 vplane1, vplane2 : `numpy.ndarray` of `float`
513 Variance planes of the original (before pre-convolution or matching)
514 exposures.
515 c1ft, c2ft : `numpy.ndarray` of `complex`
516 The overall convolution that includes the matching and the
517 afterburner in frequency space. The result of either
518 ``computeScoreCorrection`` or ``computeDiffimCorrection``.
519
520 Returns
521 -------
522 vplaneD : `numpy.ndarray` of `float`
523 The estimated variance plane of the difference/score image
524 as a weighted sum of the input variances.
525
526 Notes
527 ------
528 See DMTN-179 Section 5 about the variance plane calculations.
529 """
530 w1 = np.sum(np.real(np.conj(c1ft)*c1ft)) / c1ft.size
531 w2 = np.sum(np.real(np.conj(c2ft)*c2ft)) / c2ft.size
532 # w1, w2: the frequency space sum of abs(c1)^2 is the same as in image
533 # space.
534 return vplane1*w1 + vplane2*w2
535
536 def calculateVariancePlane(self, vplane1, vplane2, varMean1, varMean2, c1ft, c2ft):
537 """Full propagation of the variance planes of the original exposures.
538
539 The original variance planes of independent pixels are convolved with the
540 image space square of the overall kernels.
541
542 Parameters
543 ----------
544 vplane1, vplane2 : `numpy.ndarray` of `float`
545 Variance planes of the original (before pre-convolution or matching)
546 exposures.
547 varMean1, varMean2 : `float`
548 Replacement average values for non-finite ``vplane1`` and ``vplane2`` values respectively.
549
550 c1ft, c2ft : `numpy.ndarray` of `complex`
551 The overall convolution that includes the matching and the
552 afterburner in frequency space. The result of either
553 ``computeScoreCorrection`` or ``computeDiffimCorrection``.
554
555 Returns
556 -------
557 vplaneD : `numpy.ndarray` of `float`
558 The variance plane of the difference/score images.
559
560 Notes
561 ------
562 See DMTN-179 Section 5 about the variance plane calculations.
563
564 Infs and NaNs are allowed and kept in the returned array.
565 """
566 D = np.real(np.fft.ifft2(c1ft))
567 c1SqFt = np.fft.fft2(D*D)
568
569 v1shape = vplane1.shape
570 filtInf = np.isinf(vplane1)
571 filtNan = np.isnan(vplane1)
572 # This copy could be eliminated if inf/nan handling were go into padCenterOriginArray
573 vplane1 = np.copy(vplane1)
574 vplane1[filtInf | filtNan] = varMean1
575 D = self.padCenterOriginArraypadCenterOriginArray(vplane1, self.freqSpaceShapefreqSpaceShape)
576 v1 = np.real(np.fft.ifft2(np.fft.fft2(D) * c1SqFt))
577 v1 = self.padCenterOriginArraypadCenterOriginArray(v1, v1shape, useInverse=True)
578 v1[filtNan] = np.nan
579 v1[filtInf] = np.inf
580
581 D = np.real(np.fft.ifft2(c2ft))
582 c2ft = np.fft.fft2(D*D)
583
584 v2shape = vplane2.shape
585 filtInf = np.isinf(vplane2)
586 filtNan = np.isnan(vplane2)
587 vplane2 = np.copy(vplane2)
588 vplane2[filtInf | filtNan] = varMean2
589 D = self.padCenterOriginArraypadCenterOriginArray(vplane2, self.freqSpaceShapefreqSpaceShape)
590 v2 = np.real(np.fft.ifft2(np.fft.fft2(D) * c2ft))
591 v2 = self.padCenterOriginArraypadCenterOriginArray(v2, v2shape, useInverse=True)
592 v2[filtNan] = np.nan
593 v2[filtInf] = np.inf
594
595 return v1 + v2
596
597 def computeCorrectedDiffimPsf(self, corrft, psfOld):
598 """Compute the (decorrelated) difference image's new PSF.
599
600 Parameters
601 ----------
602 corrft : `numpy.ndarray`
603 The frequency space representation of the correction calculated by
604 `computeCorrection`. Shape must be `self.freqSpaceShapefreqSpaceShape`.
605 psfOld : `numpy.ndarray`
606 The psf of the difference image to be corrected.
607
608 Returns
609 -------
610 psfNew : `numpy.ndarray`
611 The corrected psf, same shape as `psfOld`, sum normed to 1.
612
613 Notes
614 -----
615 There is no algorithmic guarantee that the corrected psf can
616 meaningfully fit to the same size as the original one.
617 """
618 psfShape = psfOld.shape
619 psfNew = self.padCenterOriginArraypadCenterOriginArray(psfOld, self.freqSpaceShapefreqSpaceShape)
620 psfNew = np.fft.fft2(psfNew)
621 psfNew *= corrft
622 psfNew = np.fft.ifft2(psfNew)
623 psfNew = psfNew.real
624 psfNew = self.padCenterOriginArraypadCenterOriginArray(psfNew, psfShape, useInverse=True)
625 psfNew = psfNew/psfNew.sum()
626 return psfNew
627
628 def computeCorrectedImage(self, corrft, imgOld):
629 """Compute the decorrelated difference image.
630
631 Parameters
632 ----------
633 corrft : `numpy.ndarray`
634 The frequency space representation of the correction calculated by
635 `computeCorrection`. Shape must be `self.freqSpaceShapefreqSpaceShape`.
636 imgOld : `numpy.ndarray`
637 The difference image to be corrected.
638
639 Returns
640 -------
641 imgNew : `numpy.ndarray`
642 The corrected image, same size as the input.
643 """
644 expShape = imgOld.shape
645 imgNew = np.copy(imgOld)
646 filtInf = np.isinf(imgNew)
647 filtNan = np.isnan(imgNew)
648 imgNew[filtInf] = np.nan
649 imgNew[filtInf | filtNan] = np.nanmean(imgNew)
650 imgNew = self.padCenterOriginArraypadCenterOriginArray(imgNew, self.freqSpaceShapefreqSpaceShape)
651 imgNew = np.fft.fft2(imgNew)
652 imgNew *= corrft
653 imgNew = np.fft.ifft2(imgNew)
654 imgNew = imgNew.real
655 imgNew = self.padCenterOriginArraypadCenterOriginArray(imgNew, expShape, useInverse=True)
656 imgNew[filtNan] = np.nan
657 imgNew[filtInf] = np.inf
658 return imgNew
659
660
662 """Task to be used as an ImageMapper for performing
663 A&L decorrelation on subimages on a grid across a A&L difference image.
664
665 This task subclasses DecorrelateALKernelTask in order to implement
666 all of that task's configuration parameters, as well as its `run` method.
667 """
668
669 ConfigClass = DecorrelateALKernelConfig
670 _DefaultName = 'ip_diffim_decorrelateALKernelMapper'
671
672 def __init__(self, *args, **kwargs):
673 DecorrelateALKernelTask.__init__(self, *args, **kwargs)
674
675 def run(self, subExposure, expandedSubExposure, fullBBox,
676 template, science, alTaskResult=None, psfMatchingKernel=None,
677 preConvKernel=None, **kwargs):
678 """Perform decorrelation operation on `subExposure`, using
679 `expandedSubExposure` to allow for invalid edge pixels arising from
680 convolutions.
681
682 This method performs A&L decorrelation on `subExposure` using
683 local measures for image variances and PSF. `subExposure` is a
684 sub-exposure of the non-decorrelated A&L diffim. It also
685 requires the corresponding sub-exposures of the template
686 (`template`) and science (`science`) exposures.
687
688 Parameters
689 ----------
690 subExposure : `lsst.afw.image.Exposure`
691 the sub-exposure of the diffim
692 expandedSubExposure : `lsst.afw.image.Exposure`
693 the expanded sub-exposure upon which to operate
694 fullBBox : `lsst.geom.Box2I`
695 the bounding box of the original exposure
696 template : `lsst.afw.image.Exposure`
697 the corresponding sub-exposure of the template exposure
698 science : `lsst.afw.image.Exposure`
699 the corresponding sub-exposure of the science exposure
700 alTaskResult : `lsst.pipe.base.Struct`
701 the result of A&L image differencing on `science` and
702 `template`, importantly containing the resulting
703 `psfMatchingKernel`. Can be `None`, only if
704 `psfMatchingKernel` is not `None`.
705 psfMatchingKernel : Alternative parameter for passing the
706 A&L `psfMatchingKernel` directly.
707 preConvKernel : If not None, then pre-filtering was applied
708 to science exposure, and this is the pre-convolution
709 kernel.
710 kwargs :
711 additional keyword arguments propagated from
712 `ImageMapReduceTask.run`.
713
714 Returns
715 -------
716 A `pipeBase.Struct` containing:
717
718 - ``subExposure`` : the result of the `subExposure` processing.
719 - ``decorrelationKernel`` : the decorrelation kernel, currently
720 not used.
721
722 Notes
723 -----
724 This `run` method accepts parameters identical to those of
725 `ImageMapper.run`, since it is called from the
726 `ImageMapperTask`. See that class for more information.
727 """
728 templateExposure = template # input template
729 scienceExposure = science # input science image
730 if alTaskResult is None and psfMatchingKernel is None:
731 raise RuntimeError('Both alTaskResult and psfMatchingKernel cannot be None')
732 psfMatchingKernel = alTaskResult.psfMatchingKernel if alTaskResult is not None else psfMatchingKernel
733
734 # subExp and expandedSubExp are subimages of the (un-decorrelated) diffim!
735 # So here we compute corresponding subimages of templateExposure and scienceExposure
736 subExp2 = scienceExposure.Factory(scienceExposure, expandedSubExposure.getBBox())
737 subExp1 = templateExposure.Factory(templateExposure, expandedSubExposure.getBBox())
738
739 # Prevent too much log INFO verbosity from DecorrelateALKernelTask.run
740 logLevel = self.log.getLevel()
741 self.log.setLevel(self.log.WARNING)
742 res = DecorrelateALKernelTask.run(self, subExp2, subExp1, expandedSubExposure,
743 psfMatchingKernel, preConvKernel, **kwargs)
744 self.log.setLevel(logLevel) # reset the log level
745
746 diffim = res.correctedExposure.Factory(res.correctedExposure, subExposure.getBBox())
747 out = pipeBase.Struct(subExposure=diffim, )
748 return out
749
750
752 """Configuration parameters for the ImageMapReduceTask to direct it to use
753 DecorrelateALKernelMapper as its mapper for A&L decorrelation.
754 """
755 mapper = pexConfig.ConfigurableField(
756 doc='A&L decorrelation task to run on each sub-image',
757 target=DecorrelateALKernelMapper
758 )
759
760
761class DecorrelateALKernelSpatialConfig(pexConfig.Config):
762 """Configuration parameters for the DecorrelateALKernelSpatialTask.
763 """
764 decorrelateConfig = pexConfig.ConfigField(
765 dtype=DecorrelateALKernelConfig,
766 doc='DecorrelateALKernel config to use when running on complete exposure (non spatially-varying)',
767 )
768
769 decorrelateMapReduceConfig = pexConfig.ConfigField(
770 dtype=DecorrelateALKernelMapReduceConfig,
771 doc='DecorrelateALKernelMapReduce config to use when running on each sub-image (spatially-varying)',
772 )
773
774 ignoreMaskPlanes = pexConfig.ListField(
775 dtype=str,
776 doc="""Mask planes to ignore for sigma-clipped statistics""",
777 default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE")
778 )
779
780 def setDefaults(self):
781 self.decorrelateMapReduceConfigdecorrelateMapReduceConfig.gridStepX = self.decorrelateMapReduceConfigdecorrelateMapReduceConfig.gridStepY = 40
782 self.decorrelateMapReduceConfigdecorrelateMapReduceConfig.cellSizeX = self.decorrelateMapReduceConfigdecorrelateMapReduceConfig.cellSizeY = 41
783 self.decorrelateMapReduceConfigdecorrelateMapReduceConfig.borderSizeX = self.decorrelateMapReduceConfigdecorrelateMapReduceConfig.borderSizeY = 8
784 self.decorrelateMapReduceConfigdecorrelateMapReduceConfig.reducer.reduceOperation = 'average'
785
786
788 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference
789
790 Notes
791 -----
792
793 Pipe-task that removes the neighboring-pixel covariance in an
794 image difference that are added when the template image is
795 convolved with the Alard-Lupton PSF matching kernel.
796
797 This task is a simple wrapper around @ref DecorrelateALKernelTask,
798 which takes a `spatiallyVarying` parameter in its `run` method. If
799 it is `False`, then it simply calls the `run` method of @ref
800 DecorrelateALKernelTask. If it is True, then it uses the @ref
801 ImageMapReduceTask framework to break the exposures into
802 subExposures on a grid, and performs the `run` method of @ref
803 DecorrelateALKernelTask on each subExposure. This enables it to
804 account for spatially-varying PSFs and noise in the exposures when
805 performing the decorrelation.
806
807 This task has no standalone example, however it is applied as a
808 subtask of pipe.tasks.imageDifference.ImageDifferenceTask.
809 There is also an example of its use in `tests/testImageDecorrelation.py`.
810 """
811 ConfigClass = DecorrelateALKernelSpatialConfig
812 _DefaultName = "ip_diffim_decorrelateALKernelSpatial"
813
814 def __init__(self, *args, **kwargs):
815 """Create the image decorrelation Task
816
817 Parameters
818 ----------
819 args :
820 arguments to be passed to
821 `lsst.pipe.base.task.Task.__init__`
822 kwargs :
823 additional keyword arguments to be passed to
824 `lsst.pipe.base.task.Task.__init__`
825 """
826 pipeBase.Task.__init__(self, *args, **kwargs)
827
829 self.statsControlstatsControl.setNumSigmaClip(3.)
830 self.statsControlstatsControl.setNumIter(3)
831 self.statsControlstatsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes))
832
833 def computeVarianceMean(self, exposure):
834 """Compute the mean of the variance plane of `exposure`.
835 """
836 statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(),
837 exposure.getMaskedImage().getMask(),
838 afwMath.MEANCLIP, self.statsControlstatsControl)
839 var = statObj.getValue(afwMath.MEANCLIP)
840 return var
841
842 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel,
843 spatiallyVarying=True, preConvKernel=None, templateMatched=True, preConvMode=False):
844 """Perform decorrelation of an image difference exposure.
845
846 Decorrelates the diffim due to the convolution of the
847 templateExposure with the A&L psfMatchingKernel. If
848 `spatiallyVarying` is True, it utilizes the spatially varying
849 matching kernel via the `imageMapReduce` framework to perform
850 spatially-varying decorrelation on a grid of subExposures.
851
852 Parameters
853 ----------
854 scienceExposure : `lsst.afw.image.Exposure`
855 the science Exposure used for PSF matching
856 templateExposure : `lsst.afw.image.Exposure`
857 the template Exposure used for PSF matching
858 subtractedExposure : `lsst.afw.image.Exposure`
859 the subtracted Exposure produced by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
860 psfMatchingKernel : an (optionally spatially-varying) PSF matching kernel produced
861 by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
862 spatiallyVarying : `bool`
863 if True, perform the spatially-varying operation
864 preConvKernel : `lsst.meas.algorithms.Psf`
865 if not none, the scienceExposure has been pre-filtered with this kernel. (Currently
866 this option is experimental.)
867 templateMatched : `bool`, optional
868 If True, the template exposure was matched (convolved) to the science exposure.
869 preConvMode : `bool`, optional
870 If True, ``subtractedExposure`` is assumed to be a likelihood difference image
871 and will be noise corrected as a likelihood image.
872
873 Returns
874 -------
875 results : `lsst.pipe.base.Struct`
876 a structure containing:
877 - ``correctedExposure`` : the decorrelated diffim
878 """
879 self.log.info('Running A&L decorrelation: spatiallyVarying=%r', spatiallyVarying)
880
881 svar = self.computeVarianceMeancomputeVarianceMean(scienceExposure)
882 tvar = self.computeVarianceMeancomputeVarianceMean(templateExposure)
883 if np.isnan(svar) or np.isnan(tvar): # Should not happen unless entire image has been masked.
884 # Double check that one of the exposures is all NaNs
885 if (np.all(np.isnan(scienceExposure.image.array))
886 or np.all(np.isnan(templateExposure.image.array))):
887 self.log.warning('Template or science image is entirely NaNs: skipping decorrelation.')
888 if np.isnan(svar):
889 svar = 1e-9
890 if np.isnan(tvar):
891 tvar = 1e-9
892
893 var = self.computeVarianceMeancomputeVarianceMean(subtractedExposure)
894
895 if spatiallyVarying:
896 self.log.info("Variance (science, template): (%f, %f)", svar, tvar)
897 self.log.info("Variance (uncorrected diffim): %f", var)
898 config = self.config.decorrelateMapReduceConfig
899 task = ImageMapReduceTask(config=config)
900 results = task.run(subtractedExposure, science=scienceExposure,
901 template=templateExposure, psfMatchingKernel=psfMatchingKernel,
902 preConvKernel=preConvKernel, forceEvenSized=True,
903 templateMatched=templateMatched, preConvMode=preConvMode)
904 results.correctedExposure = results.exposure
905
906 # Make sure masks of input image are propagated to diffim
907 def gm(exp):
908 return exp.getMaskedImage().getMask()
909 gm(results.correctedExposure)[:, :] = gm(subtractedExposure)
910
911 var = self.computeVarianceMeancomputeVarianceMean(results.correctedExposure)
912 self.log.info("Variance (corrected diffim): %f", var)
913
914 else:
915 config = self.config.decorrelateConfig
916 task = DecorrelateALKernelTask(config=config)
917 results = task.run(scienceExposure, templateExposure,
918 subtractedExposure, psfMatchingKernel, preConvKernel=preConvKernel,
919 templateMatched=templateMatched, preConvMode=preConvMode)
920
921 return results
def run(self, subExposure, expandedSubExposure, fullBBox, template, science, alTaskResult=None, psfMatchingKernel=None, preConvKernel=None, **kwargs)
def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel, spatiallyVarying=True, preConvKernel=None, templateMatched=True, preConvMode=False)
def computeScoreCorrection(self, kappa, svar, tvar, preConvArr)
def calculateVariancePlane(self, vplane1, vplane2, varMean1, varMean2, c1ft, c2ft)
def padCenterOriginArray(A, tuple newShape, useInverse=False)
def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel, preConvKernel=None, xcen=None, ycen=None, svar=None, tvar=None, templateMatched=True, preConvMode=False, **kwargs)
Statistics makeStatistics(lsst::afw::image::Image< Pixel > const &img, lsst::afw::image::Mask< image::MaskPixel > const &msk, int const flags, StatisticsControl const &sctrl=StatisticsControl())