lsst.ip.diffim g2959009a90+beea512472
Loading...
Searching...
No Matches
imageDecorrelation.py
Go to the documentation of this file.
2# LSST Data Management System
3# Copyright 2016 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22
23import numpy as np
24
25import lsst.afw.image as afwImage
26import lsst.afw.math as afwMath
27import lsst.geom as geom
28import lsst.meas.algorithms as measAlg
29import lsst.pex.config as pexConfig
30import lsst.pipe.base as pipeBase
31from lsst.utils.timer import timeMethod
32
33from .imageMapReduce import (ImageMapReduceConfig, ImageMapReduceTask,
34 ImageMapper)
35
36__all__ = ("DecorrelateALKernelTask", "DecorrelateALKernelConfig",
37 "DecorrelateALKernelMapper", "DecorrelateALKernelMapReduceConfig",
38 "DecorrelateALKernelSpatialConfig", "DecorrelateALKernelSpatialTask")
39
40
41class DecorrelateALKernelConfig(pexConfig.Config):
42 """Configuration parameters for the DecorrelateALKernelTask
43 """
44
45 ignoreMaskPlanes = pexConfig.ListField(
46 dtype=str,
47 doc="""Mask planes to ignore for sigma-clipped statistics""",
48 default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE")
49 )
50 completeVarPlanePropagation = pexConfig.Field(
51 dtype=bool,
52 default=False,
53 doc="Compute the full effect of the decorrelated matching kernel on the variance plane."
54 " Otherwise use a model weighed sum of the input variances."
55 )
56
57
58class DecorrelateALKernelTask(pipeBase.Task):
59 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference
60
61 """
62 ConfigClass = DecorrelateALKernelConfig
63 _DefaultName = "ip_diffim_decorrelateALKernel"
64
65 def __init__(self, *args, **kwargs):
66 """Create the image decorrelation Task
67
68 Parameters
69 ----------
70 args :
71 arguments to be passed to ``lsst.pipe.base.task.Task.__init__``
72 kwargs :
73 keyword arguments to be passed to ``lsst.pipe.base.task.Task.__init__``
74 """
75 pipeBase.Task.__init__(self, *args, **kwargs)
76
78 self.statsControl.setNumSigmaClip(3.)
79 self.statsControl.setNumIter(3)
80 self.statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes))
81
82 def computeVarianceMean(self, exposure):
83 statObj = afwMath.makeStatistics(exposure.variance,
84 exposure.mask,
85 afwMath.MEANCLIP, self.statsControl)
86 var = statObj.getValue(afwMath.MEANCLIP)
87 return var
88
89 @timeMethod
90 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel,
91 preConvKernel=None, xcen=None, ycen=None, svar=None, tvar=None,
92 templateMatched=True, preConvMode=False, **kwargs):
93 """Perform decorrelation of an image difference or of a score difference exposure.
94
95 Corrects the difference or score image due to the convolution of the
96 templateExposure with the A&L PSF matching kernel.
97 See [DMTN-021, Equation 1](http://dmtn-021.lsst.io/#equation-1) and
98 [DMTN-179](http://dmtn-179.lsst.io/) for details.
99
100 Parameters
101 ----------
102 scienceExposure : `lsst.afw.image.Exposure`
103 The original science exposure (before pre-convolution, if ``preConvMode==True``).
104 templateExposure : `lsst.afw.image.Exposure`
105 The original template exposure warped, but not psf-matched, to the science exposure.
106 subtractedExposure : `lsst.afw.image.Exposure`
107 the subtracted exposure produced by
108 `ip_diffim.ImagePsfMatchTask.subtractExposures()`. The `subtractedExposure` must
109 inherit its PSF from `exposure`, see notes below.
110 psfMatchingKernel : `lsst.afw.detection.Psf`
111 An (optionally spatially-varying) PSF matching kernel produced
112 by `ip_diffim.ImagePsfMatchTask.subtractExposures()`.
113 preConvKernel : `lsst.afw.math.Kernel`, optional
114 If not `None`, then the `scienceExposure` was pre-convolved with (the reflection of)
115 this kernel. Must be normalized to sum to 1.
116 Allowed only if ``templateMatched==True`` and ``preConvMode==True``.
117 Defaults to the PSF of the science exposure at the image center.
118 xcen : `float`, optional
119 X-pixel coordinate to use for computing constant matching kernel to use
120 If `None` (default), then use the center of the image.
121 ycen : `float`, optional
122 Y-pixel coordinate to use for computing constant matching kernel to use
123 If `None` (default), then use the center of the image.
124 svar : `float`, optional
125 Image variance for science image
126 If `None` (default) then compute the variance over the entire input science image.
127 tvar : `float`, optional
128 Image variance for template image
129 If `None` (default) then compute the variance over the entire input template image.
130 templateMatched : `bool`, optional
131 If True, the template exposure was matched (convolved) to the science exposure.
132 See also notes below.
133 preConvMode : `bool`, optional
134 If True, ``subtractedExposure`` is assumed to be a likelihood difference image
135 and will be noise corrected as a likelihood image.
136 **kwargs
137 Additional keyword arguments propagated from DecorrelateALKernelSpatialTask.
138
139 Returns
140 -------
141 result : `lsst.pipe.base.Struct`
142 - ``correctedExposure`` : the decorrelated diffim
143
144 Notes
145 -----
146 If ``preConvMode==True``, ``subtractedExposure`` is assumed to be a
147 score image and the noise correction for likelihood images
148 is applied. The resulting image is an optimal detection likelihood image
149 when the templateExposure has noise. (See DMTN-179) If ``preConvKernel`` is
150 not specified, the PSF of ``scienceExposure`` is assumed as pre-convolution kernel.
151
152 The ``subtractedExposure`` is NOT updated. The returned ``correctedExposure``
153 has an updated but spatially fixed PSF. It is calculated as the center of
154 image PSF corrected by the center of image matching kernel.
155
156 If ``templateMatched==True``, the templateExposure was matched (convolved)
157 to the ``scienceExposure`` by ``psfMatchingKernel`` during image differencing.
158 Otherwise the ``scienceExposure`` was matched (convolved) by ``psfMatchingKernel``.
159 In either case, note that the original template and science images are required,
160 not the psf-matched version.
161
162 This task discards the variance plane of ``subtractedExposure`` and re-computes
163 it from the variance planes of ``scienceExposure`` and ``templateExposure``.
164 The image plane of ``subtractedExposure`` must be at the photometric level
165 set by the AL PSF matching in `ImagePsfMatchTask.subtractExposures`.
166 The assumptions about the photometric level are controlled by the
167 `templateMatched` option in this task.
168
169 Here we currently convert a spatially-varying matching kernel into a constant kernel,
170 just by computing it at the center of the image (tickets DM-6243, DM-6244).
171
172 We are also using a constant accross-the-image measure of sigma (sqrt(variance)) to compute
173 the decorrelation kernel.
174
175 TODO DM-23857 As part of the spatially varying correction implementation
176 consider whether returning a Struct is still necessary.
177 """
178 if preConvKernel is not None and not (templateMatched and preConvMode):
179 raise ValueError("Pre-convolution kernel is allowed only if "
180 "preConvMode==True and templateMatched==True.")
181
182 spatialKernel = psfMatchingKernel
183 kimg = afwImage.ImageD(spatialKernel.getDimensions())
184 bbox = subtractedExposure.getBBox()
185 if xcen is None:
186 xcen = (bbox.getBeginX() + bbox.getEndX()) / 2.
187 if ycen is None:
188 ycen = (bbox.getBeginY() + bbox.getEndY()) / 2.
189 self.log.info("Using matching kernel computed at (%d, %d)", xcen, ycen)
190 spatialKernel.computeImage(kimg, False, xcen, ycen)
191
192 preConvImg = None
193 if preConvMode:
194 if preConvKernel is None:
195 pos = scienceExposure.getPsf().getAveragePosition()
196 preConvKernel = scienceExposure.getPsf().getLocalKernel(pos)
197 preConvImg = afwImage.ImageD(preConvKernel.getDimensions())
198 preConvKernel.computeImage(preConvImg, True)
199
200 if svar is None:
201 svar = self.computeVarianceMean(scienceExposure)
202 if tvar is None:
203 tvar = self.computeVarianceMean(templateExposure)
204 self.log.info("Original variance plane means. Science:%.5e, warped template:%.5e)",
205 svar, tvar)
206
207 # Should not happen unless entire image has been masked, which could happen
208 # if this is a small subimage of the main exposure. In this case, just return a full NaN
209 # exposure
210 if np.isnan(svar) or np.isnan(tvar):
211 # Double check that one of the exposures is all NaNs
212 if (np.all(np.isnan(scienceExposure.image.array))
213 or np.all(np.isnan(templateExposure.image.array))):
214 self.log.warning('Template or science image is entirely NaNs: skipping decorrelation.')
215 outExposure = subtractedExposure.clone()
216 return pipeBase.Struct(correctedExposure=outExposure, )
217
218 if templateMatched:
219 # Regular subtraction, we convolved the template
220 self.log.info("Decorrelation after template image convolution")
221 varianceMean = svar
222 targetVarianceMean = tvar
223 # variance plane of the image that is not convolved
224 variance = scienceExposure.variance.array
225 # Variance plane of the convolved image, before convolution.
226 targetVariance = templateExposure.variance.array
227 else:
228 # We convolved the science image
229 self.log.info("Decorrelation after science image convolution")
230 varianceMean = tvar
231 targetVarianceMean = svar
232 # variance plane of the image that is not convolved
233 variance = templateExposure.variance.array
234 # Variance plane of the convolved image, before convolution.
235 targetVariance = scienceExposure.variance.array
236
237 # The maximal correction value converges to sqrt(targetVarianceMean/varianceMean).
238 # Correction divergence warning if the correction exceeds 4 orders of magnitude.
239 mOverExpVar = targetVarianceMean/varianceMean
240 if mOverExpVar > 1e8:
241 self.log.warning("Diverging correction: matched image variance is "
242 " much larger than the unconvolved one's"
243 ", targetVarianceMean/varianceMean:%.2e", mOverExpVar)
244
245 oldVarMean = self.computeVarianceMean(subtractedExposure)
246 self.log.info("Variance plane mean of uncorrected diffim: %f", oldVarMean)
247
248 kArr = kimg.array
249 diffimShape = subtractedExposure.image.array.shape
250 psfImg = subtractedExposure.getPsf().computeKernelImage(geom.Point2D(xcen, ycen))
251 psfShape = psfImg.array.shape
252
253 if preConvMode:
254 self.log.info("Decorrelation of likelihood image")
255 self.computeCommonShape(preConvImg.array.shape, kArr.shape,
256 psfShape, diffimShape)
257 corr = self.computeScoreCorrection(kArr, varianceMean, targetVarianceMean, preConvImg.array)
258 else:
259 self.log.info("Decorrelation of difference image")
260 self.computeCommonShape(kArr.shape, psfShape, diffimShape)
261 corr = self.computeDiffimCorrection(kArr, varianceMean, targetVarianceMean)
262
263 correctedImage = self.computeCorrectedImage(corr.corrft, subtractedExposure.image.array)
264 correctedPsf = self.computeCorrectedDiffimPsf(corr.corrft, psfImg.array)
265
266 # The subtracted exposure variance plane is already correlated, we cannot propagate
267 # it through another convolution; instead we need to use the uncorrelated originals
268 # The whitening should scale it to varianceMean + targetVarianceMean on average
269 if self.config.completeVarPlanePropagation:
270 self.log.debug("Using full variance plane calculation in decorrelation")
271 correctedVariance = self.calculateVariancePlane(
272 variance, targetVariance,
273 varianceMean, targetVarianceMean, corr.cnft, corr.crft)
274 else:
275 self.log.debug("Using estimated variance plane calculation in decorrelation")
276 correctedVariance = self.estimateVariancePlane(
277 variance, targetVariance,
278 corr.cnft, corr.crft)
279
280 # Determine the common shape
281 kSum = np.sum(kArr)
282 kSumSq = kSum*kSum
283 self.log.debug("Matching kernel sum: %.3e", kSum)
284 if not templateMatched:
285 # ImagePsfMatch.subtractExposures re-scales the difference in
286 # the science image convolution mode
287 correctedVariance /= kSumSq
288 subtractedExposure.image.array[...] = correctedImage # Allow for numpy type casting
289 subtractedExposure.variance.array[...] = correctedVariance
290 subtractedExposure.setPsf(correctedPsf)
291
292 newVarMean = self.computeVarianceMean(subtractedExposure)
293 self.log.info("Variance plane mean of corrected diffim: %.5e", newVarMean)
294
295 # TODO DM-23857 As part of the spatially varying correction implementation
296 # consider whether returning a Struct is still necessary.
297 return pipeBase.Struct(correctedExposure=subtractedExposure, )
298
299 def computeCommonShape(self, *shapes):
300 """Calculate the common shape for FFT operations. Set `self.freqSpaceShape`
301 internally.
302
303 Parameters
304 ----------
305 shapes : one or more `tuple` of `int`
306 Shapes of the arrays. All must have the same dimensionality.
307 At least one shape must be provided.
308
309 Returns
310 -------
311 None.
312
313 Notes
314 -----
315 For each dimension, gets the smallest even number greater than or equal to
316 `N1+N2-1` where `N1` and `N2` are the two largest values.
317 In case of only one shape given, rounds up to even each dimension value.
318 """
319 S = np.array(shapes, dtype=int)
320 if len(shapes) > 2:
321 S.sort(axis=0)
322 S = S[-2:]
323 if len(shapes) > 1:
324 commonShape = np.sum(S, axis=0) - 1
325 else:
326 commonShape = S[0]
327 commonShape[commonShape % 2 != 0] += 1
328 self.freqSpaceShape = tuple(commonShape)
329 self.log.info("Common frequency space shape %s", self.freqSpaceShape)
330
331 @staticmethod
332 def padCenterOriginArray(A, newShape: tuple, useInverse=False):
333 """Zero pad an image where the origin is at the center and replace the
334 origin to the corner as required by the periodic input of FFT. Implement also
335 the inverse operation, crop the padding and re-center data.
336
337 Parameters
338 ----------
339 A : `numpy.ndarray`
340 An array to copy from.
341 newShape : `tuple` of `int`
342 The dimensions of the resulting array. For padding, the resulting array
343 must be larger than A in each dimension. For the inverse operation this
344 must be the original, before padding size of the array.
345 useInverse : bool, optional
346 Selector of forward, add padding, operation (False)
347 or its inverse, crop padding, operation (True).
348
349 Returns
350 -------
351 R : `numpy.ndarray`
352 The padded or unpadded array with shape of `newShape` and the same dtype as A.
353
354 Notes
355 -----
356 For odd dimensions, the splitting is rounded to
357 put the center pixel into the new corner origin (0,0). This is to be consistent
358 e.g. for a dirac delta kernel that is originally located at the center pixel.
359 """
360
361 # The forward and inverse operations should round odd dimension halves at the opposite
362 # sides to get the pixels back to their original positions.
363 if not useInverse:
364 # Forward operation: First and second halves with respect to the axes of A.
365 firstHalves = [x//2 for x in A.shape]
366 secondHalves = [x-y for x, y in zip(A.shape, firstHalves)]
367 else:
368 # Inverse operation: Opposite rounding
369 secondHalves = [x//2 for x in newShape]
370 firstHalves = [x-y for x, y in zip(newShape, secondHalves)]
371
372 R = np.zeros_like(A, shape=newShape)
373 R[-firstHalves[0]:, -firstHalves[1]:] = A[:firstHalves[0], :firstHalves[1]]
374 R[:secondHalves[0], -firstHalves[1]:] = A[-secondHalves[0]:, :firstHalves[1]]
375 R[:secondHalves[0], :secondHalves[1]] = A[-secondHalves[0]:, -secondHalves[1]:]
376 R[-firstHalves[0]:, :secondHalves[1]] = A[:firstHalves[0], -secondHalves[1]:]
377 return R
378
379 def computeDiffimCorrection(self, kappa, svar, tvar):
380 """Compute the Lupton decorrelation post-convolution kernel for decorrelating an
381 image difference, based on the PSF-matching kernel.
382
383 Parameters
384 ----------
385 kappa : `numpy.ndarray` of `float`
386 A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching.
387 svar : `float` > 0.
388 Average variance of science image used for PSF matching.
389 tvar : `float` > 0.
390 Average variance of the template (matched) image used for PSF matching.
391
392 Returns
393 -------
394 corrft : `numpy.ndarray` of `float`
395 The frequency space representation of the correction. The array is real (dtype float).
396 Shape is `self.freqSpaceShape`.
397
398 cnft, crft : `numpy.ndarray` of `complex`
399 The overall convolution (pre-conv, PSF matching, noise correction) kernel
400 for the science and template images, respectively for the variance plane
401 calculations. These are intermediate results in frequency space.
402
403 Notes
404 -----
405 The maximum correction factor converges to `sqrt(tvar/svar)` towards high frequencies.
406 This should be a plausible value.
407 """
408 kSum = np.sum(kappa) # We scale the decorrelation to preserve fluxes
409 kappa = self.padCenterOriginArray(kappa, self.freqSpaceShape)
410 kft = np.fft.fft2(kappa)
411 kftAbsSq = np.real(np.conj(kft) * kft)
412
413 denom = svar + tvar * kftAbsSq
414 corrft = np.sqrt((svar + tvar * kSum*kSum) / denom)
415 cnft = corrft
416 crft = kft*corrft
417 return pipeBase.Struct(corrft=corrft, cnft=cnft, crft=crft)
418
419 def computeScoreCorrection(self, kappa, svar, tvar, preConvArr):
420 """Compute the correction kernel for a score image.
421
422 Parameters
423 ----------
424 kappa : `numpy.ndarray`
425 A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching.
426 svar : `float`
427 Average variance of science image used for PSF matching (before pre-convolution).
428 tvar : `float`
429 Average variance of the template (matched) image used for PSF matching.
430 preConvArr : `numpy.ndarray`
431 The pre-convolution kernel of the science image. It should be the PSF
432 of the science image or an approximation of it. It must be normed to sum 1.
433
434 Returns
435 -------
436 corrft : `numpy.ndarray` of `float`
437 The frequency space representation of the correction. The array is real (dtype float).
438 Shape is `self.freqSpaceShape`.
439 cnft, crft : `numpy.ndarray` of `complex`
440 The overall convolution (pre-conv, PSF matching, noise correction) kernel
441 for the science and template images, respectively for the variance plane
442 calculations. These are intermediate results in frequency space.
443
444 Notes
445 -----
446 To be precise, the science image should be _correlated_ by ``preConvArray`` but this
447 does not matter for this calculation.
448
449 ``cnft``, ``crft`` contain the scaling factor as well.
450
451 """
452 kSum = np.sum(kappa)
453 kappa = self.padCenterOriginArray(kappa, self.freqSpaceShape)
454 kft = np.fft.fft2(kappa)
455 preConvArr = self.padCenterOriginArray(preConvArr, self.freqSpaceShape)
456 preFt = np.fft.fft2(preConvArr)
457 preFtAbsSq = np.real(np.conj(preFt) * preFt)
458 kftAbsSq = np.real(np.conj(kft) * kft)
459 # Avoid zero division, though we don't normally approach `tiny`.
460 # We have numerical noise instead.
461 tiny = np.finfo(preFtAbsSq.dtype).tiny * 1000.
462 flt = preFtAbsSq < tiny
463 # If we pre-convolve to avoid deconvolution in AL, then kftAbsSq / preFtAbsSq
464 # theoretically expected to diverge to +inf. But we don't care about the convergence
465 # properties here, S goes to 0 at these frequencies anyway.
466 preFtAbsSq[flt] = tiny
467 denom = svar + tvar * kftAbsSq / preFtAbsSq
468 corrft = (svar + tvar * kSum*kSum) / denom
469 cnft = np.conj(preFt)*corrft
470 crft = kft*corrft
471 return pipeBase.Struct(corrft=corrft, cnft=cnft, crft=crft)
472
473 @staticmethod
474 def estimateVariancePlane(vplane1, vplane2, c1ft, c2ft):
475 """Estimate the variance planes.
476
477 The estimation assumes that around each pixel the surrounding
478 pixels' sigmas within the convolution kernel are the same.
479
480 Parameters
481 ----------
482 vplane1, vplane2 : `numpy.ndarray` of `float`
483 Variance planes of the original (before pre-convolution or matching)
484 exposures.
485 c1ft, c2ft : `numpy.ndarray` of `complex`
486 The overall convolution that includes the matching and the
487 afterburner in frequency space. The result of either
488 ``computeScoreCorrection`` or ``computeDiffimCorrection``.
489
490 Returns
491 -------
492 vplaneD : `numpy.ndarray` of `float`
493 The estimated variance plane of the difference/score image
494 as a weighted sum of the input variances.
495
496 Notes
497 ------
498 See DMTN-179 Section 5 about the variance plane calculations.
499 """
500 w1 = np.sum(np.real(np.conj(c1ft)*c1ft)) / c1ft.size
501 w2 = np.sum(np.real(np.conj(c2ft)*c2ft)) / c2ft.size
502 # w1, w2: the frequency space sum of abs(c1)^2 is the same as in image
503 # space.
504 return vplane1*w1 + vplane2*w2
505
506 def calculateVariancePlane(self, vplane1, vplane2, varMean1, varMean2, c1ft, c2ft):
507 """Full propagation of the variance planes of the original exposures.
508
509 The original variance planes of independent pixels are convolved with the
510 image space square of the overall kernels.
511
512 Parameters
513 ----------
514 vplane1, vplane2 : `numpy.ndarray` of `float`
515 Variance planes of the original (before pre-convolution or matching)
516 exposures.
517 varMean1, varMean2 : `float`
518 Replacement average values for non-finite ``vplane1`` and ``vplane2`` values respectively.
519
520 c1ft, c2ft : `numpy.ndarray` of `complex`
521 The overall convolution that includes the matching and the
522 afterburner in frequency space. The result of either
523 ``computeScoreCorrection`` or ``computeDiffimCorrection``.
524
525 Returns
526 -------
527 vplaneD : `numpy.ndarray` of `float`
528 The variance plane of the difference/score images.
529
530 Notes
531 ------
532 See DMTN-179 Section 5 about the variance plane calculations.
533
534 Infs and NaNs are allowed and kept in the returned array.
535 """
536 D = np.real(np.fft.ifft2(c1ft))
537 c1SqFt = np.fft.fft2(D*D)
538
539 v1shape = vplane1.shape
540 filtInf = np.isinf(vplane1)
541 filtNan = np.isnan(vplane1)
542 # This copy could be eliminated if inf/nan handling were go into padCenterOriginArray
543 vplane1 = np.copy(vplane1)
544 vplane1[filtInf | filtNan] = varMean1
545 D = self.padCenterOriginArray(vplane1, self.freqSpaceShape)
546 v1 = np.real(np.fft.ifft2(np.fft.fft2(D) * c1SqFt))
547 v1 = self.padCenterOriginArray(v1, v1shape, useInverse=True)
548 v1[filtNan] = np.nan
549 v1[filtInf] = np.inf
550
551 D = np.real(np.fft.ifft2(c2ft))
552 c2SqFt = np.fft.fft2(D*D)
553
554 v2shape = vplane2.shape
555 filtInf = np.isinf(vplane2)
556 filtNan = np.isnan(vplane2)
557 vplane2 = np.copy(vplane2)
558 vplane2[filtInf | filtNan] = varMean2
559 D = self.padCenterOriginArray(vplane2, self.freqSpaceShape)
560 v2 = np.real(np.fft.ifft2(np.fft.fft2(D) * c2SqFt))
561 v2 = self.padCenterOriginArray(v2, v2shape, useInverse=True)
562 v2[filtNan] = np.nan
563 v2[filtInf] = np.inf
564
565 return v1 + v2
566
567 def computeCorrectedDiffimPsf(self, corrft, psfOld):
568 """Compute the (decorrelated) difference image's new PSF.
569
570 Parameters
571 ----------
572 corrft : `numpy.ndarray`
573 The frequency space representation of the correction calculated by
574 `computeCorrection`. Shape must be `self.freqSpaceShape`.
575 psfOld : `numpy.ndarray`
576 The psf of the difference image to be corrected.
577
578 Returns
579 -------
580 correctedPsf : `lsst.meas.algorithms.KernelPsf`
581 The corrected psf, same shape as `psfOld`, sum normed to 1.
582
583 Notes
584 -----
585 There is no algorithmic guarantee that the corrected psf can
586 meaningfully fit to the same size as the original one.
587 """
588 psfShape = psfOld.shape
589 psfNew = self.padCenterOriginArray(psfOld, self.freqSpaceShape)
590 psfNew = np.fft.fft2(psfNew)
591 psfNew *= corrft
592 psfNew = np.fft.ifft2(psfNew)
593 psfNew = psfNew.real
594 psfNew = self.padCenterOriginArray(psfNew, psfShape, useInverse=True)
595 psfNew = psfNew/psfNew.sum()
596
597 psfcI = afwImage.ImageD(geom.Extent2I(psfShape[1], psfShape[0]))
598 psfcI.array = psfNew
599 psfcK = afwMath.FixedKernel(psfcI)
600 correctedPsf = measAlg.KernelPsf(psfcK)
601 return correctedPsf
602
603 def computeCorrectedImage(self, corrft, imgOld):
604 """Compute the decorrelated difference image.
605
606 Parameters
607 ----------
608 corrft : `numpy.ndarray`
609 The frequency space representation of the correction calculated by
610 `computeCorrection`. Shape must be `self.freqSpaceShape`.
611 imgOld : `numpy.ndarray`
612 The difference image to be corrected.
613
614 Returns
615 -------
616 imgNew : `numpy.ndarray`
617 The corrected image, same size as the input.
618 """
619 expShape = imgOld.shape
620 imgNew = np.copy(imgOld)
621 filtInf = np.isinf(imgNew)
622 filtNan = np.isnan(imgNew)
623 imgNew[filtInf] = np.nan
624 imgNew[filtInf | filtNan] = np.nanmean(imgNew)
625 imgNew = self.padCenterOriginArray(imgNew, self.freqSpaceShape)
626 imgNew = np.fft.fft2(imgNew)
627 imgNew *= corrft
628 imgNew = np.fft.ifft2(imgNew)
629 imgNew = imgNew.real
630 imgNew = self.padCenterOriginArray(imgNew, expShape, useInverse=True)
631 imgNew[filtNan] = np.nan
632 imgNew[filtInf] = np.inf
633 return imgNew
634
635
637 """Task to be used as an ImageMapper for performing
638 A&L decorrelation on subimages on a grid across a A&L difference image.
639
640 This task subclasses DecorrelateALKernelTask in order to implement
641 all of that task's configuration parameters, as well as its `run` method.
642 """
643
644 ConfigClass = DecorrelateALKernelConfig
645 _DefaultName = 'ip_diffim_decorrelateALKernelMapper'
646
647 def __init__(self, *args, **kwargs):
648 DecorrelateALKernelTask.__init__(self, *args, **kwargs)
649
650 def run(self, subExposure, expandedSubExposure, fullBBox,
651 template, science, alTaskResult=None, psfMatchingKernel=None,
652 preConvKernel=None, **kwargs):
653 """Perform decorrelation operation on `subExposure`, using
654 `expandedSubExposure` to allow for invalid edge pixels arising from
655 convolutions.
656
657 This method performs A&L decorrelation on `subExposure` using
658 local measures for image variances and PSF. `subExposure` is a
659 sub-exposure of the non-decorrelated A&L diffim. It also
660 requires the corresponding sub-exposures of the template
661 (`template`) and science (`science`) exposures.
662
663 Parameters
664 ----------
665 subExposure : `lsst.afw.image.Exposure`
666 the sub-exposure of the diffim
667 expandedSubExposure : `lsst.afw.image.Exposure`
668 the expanded sub-exposure upon which to operate
669 fullBBox : `lsst.geom.Box2I`
670 the bounding box of the original exposure
671 template : `lsst.afw.image.Exposure`
672 the corresponding sub-exposure of the template exposure
673 science : `lsst.afw.image.Exposure`
674 the corresponding sub-exposure of the science exposure
675 alTaskResult : `lsst.pipe.base.Struct`
676 the result of A&L image differencing on `science` and
677 `template`, importantly containing the resulting
678 `psfMatchingKernel`. Can be `None`, only if
679 `psfMatchingKernel` is not `None`.
680 psfMatchingKernel : Alternative parameter for passing the
681 A&L `psfMatchingKernel` directly.
682 preConvKernel : If not None, then pre-filtering was applied
683 to science exposure, and this is the pre-convolution
684 kernel.
685 kwargs :
686 additional keyword arguments propagated from
687 `ImageMapReduceTask.run`.
688
689 Returns
690 -------
691 A `pipeBase.Struct` containing:
692
693 - ``subExposure`` : the result of the `subExposure` processing.
694 - ``decorrelationKernel`` : the decorrelation kernel, currently
695 not used.
696
697 Notes
698 -----
699 This `run` method accepts parameters identical to those of
700 `ImageMapper.run`, since it is called from the
701 `ImageMapperTask`. See that class for more information.
702 """
703 templateExposure = template # input template
704 scienceExposure = science # input science image
705 if alTaskResult is None and psfMatchingKernel is None:
706 raise RuntimeError('Both alTaskResult and psfMatchingKernel cannot be None')
707 psfMatchingKernel = alTaskResult.psfMatchingKernel if alTaskResult is not None else psfMatchingKernel
708
709 # subExp and expandedSubExp are subimages of the (un-decorrelated) diffim!
710 # So here we compute corresponding subimages of templateExposure and scienceExposure
711 subExp2 = scienceExposure.Factory(scienceExposure, expandedSubExposure.getBBox())
712 subExp1 = templateExposure.Factory(templateExposure, expandedSubExposure.getBBox())
713
714 # Prevent too much log INFO verbosity from DecorrelateALKernelTask.run
715 logLevel = self.log.level
716 self.log.setLevel(self.log.WARNING)
717 res = DecorrelateALKernelTask.run(self, subExp2, subExp1, expandedSubExposure,
718 psfMatchingKernel, preConvKernel, **kwargs)
719 self.log.setLevel(logLevel) # reset the log level
720
721 diffim = res.correctedExposure.Factory(res.correctedExposure, subExposure.getBBox())
722 out = pipeBase.Struct(subExposure=diffim, )
723 return out
724
725
727 """Configuration parameters for the ImageMapReduceTask to direct it to use
728 DecorrelateALKernelMapper as its mapper for A&L decorrelation.
729 """
730 mapper = pexConfig.ConfigurableField(
731 doc='A&L decorrelation task to run on each sub-image',
732 target=DecorrelateALKernelMapper
733 )
734
735
736class DecorrelateALKernelSpatialConfig(pexConfig.Config):
737 """Configuration parameters for the DecorrelateALKernelSpatialTask.
738 """
739 decorrelateConfig = pexConfig.ConfigField(
740 dtype=DecorrelateALKernelConfig,
741 doc='DecorrelateALKernel config to use when running on complete exposure (non spatially-varying)',
742 )
743
744 decorrelateMapReduceConfig = pexConfig.ConfigField(
745 dtype=DecorrelateALKernelMapReduceConfig,
746 doc='DecorrelateALKernelMapReduce config to use when running on each sub-image (spatially-varying)',
747 )
748
749 ignoreMaskPlanes = pexConfig.ListField(
750 dtype=str,
751 doc="""Mask planes to ignore for sigma-clipped statistics""",
752 default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE")
753 )
754
755 def setDefaults(self):
756 self.decorrelateMapReduceConfig.gridStepX = self.decorrelateMapReduceConfig.gridStepY = 40
757 self.decorrelateMapReduceConfig.cellSizeX = self.decorrelateMapReduceConfig.cellSizeY = 41
758 self.decorrelateMapReduceConfig.borderSizeX = self.decorrelateMapReduceConfig.borderSizeY = 8
759 self.decorrelateMapReduceConfig.reducer.reduceOperation = 'average'
760
761
763 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference
764
765 """
766 ConfigClass = DecorrelateALKernelSpatialConfig
767 _DefaultName = "ip_diffim_decorrelateALKernelSpatial"
768
769 def __init__(self, *args, **kwargs):
770 """Create the image decorrelation Task
771
772 Parameters
773 ----------
774 args :
775 arguments to be passed to
776 `lsst.pipe.base.task.Task.__init__`
777 kwargs :
778 additional keyword arguments to be passed to
779 `lsst.pipe.base.task.Task.__init__`
780 """
781 pipeBase.Task.__init__(self, *args, **kwargs)
782
784 self.statsControl.setNumSigmaClip(3.)
785 self.statsControl.setNumIter(3)
786 self.statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes))
787
788 def computeVarianceMean(self, exposure):
789 """Compute the mean of the variance plane of `exposure`.
790 """
791 statObj = afwMath.makeStatistics(exposure.variance,
792 exposure.mask,
793 afwMath.MEANCLIP, self.statsControl)
794 var = statObj.getValue(afwMath.MEANCLIP)
795 return var
796
797 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel,
798 spatiallyVarying=True, preConvKernel=None, templateMatched=True, preConvMode=False):
799 """Perform decorrelation of an image difference exposure.
800
801 Decorrelates the diffim due to the convolution of the
802 templateExposure with the A&L psfMatchingKernel. If
803 `spatiallyVarying` is True, it utilizes the spatially varying
804 matching kernel via the `imageMapReduce` framework to perform
805 spatially-varying decorrelation on a grid of subExposures.
806
807 Parameters
808 ----------
809 scienceExposure : `lsst.afw.image.Exposure`
810 the science Exposure used for PSF matching
811 templateExposure : `lsst.afw.image.Exposure`
812 the template Exposure used for PSF matching
813 subtractedExposure : `lsst.afw.image.Exposure`
814 the subtracted Exposure produced by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
815 psfMatchingKernel : an (optionally spatially-varying) PSF matching kernel produced
816 by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
817 spatiallyVarying : `bool`
818 if True, perform the spatially-varying operation
819 preConvKernel : `lsst.meas.algorithms.Psf`
820 if not none, the scienceExposure has been pre-filtered with this kernel. (Currently
821 this option is experimental.)
822 templateMatched : `bool`, optional
823 If True, the template exposure was matched (convolved) to the science exposure.
824 preConvMode : `bool`, optional
825 If True, ``subtractedExposure`` is assumed to be a likelihood difference image
826 and will be noise corrected as a likelihood image.
827
828 Returns
829 -------
830 results : `lsst.pipe.base.Struct`
831 a structure containing:
832 - ``correctedExposure`` : the decorrelated diffim
833 """
834 self.log.info('Running A&L decorrelation: spatiallyVarying=%r', spatiallyVarying)
835
836 svar = self.computeVarianceMean(scienceExposure)
837 tvar = self.computeVarianceMean(templateExposure)
838 if np.isnan(svar) or np.isnan(tvar): # Should not happen unless entire image has been masked.
839 # Double check that one of the exposures is all NaNs
840 if (np.all(np.isnan(scienceExposure.image.array))
841 or np.all(np.isnan(templateExposure.image.array))):
842 self.log.warning('Template or science image is entirely NaNs: skipping decorrelation.')
843 if np.isnan(svar):
844 svar = 1e-9
845 if np.isnan(tvar):
846 tvar = 1e-9
847
848 var = self.computeVarianceMean(subtractedExposure)
849
850 if spatiallyVarying:
851 self.log.info("Variance (science, template): (%f, %f)", svar, tvar)
852 self.log.info("Variance (uncorrected diffim): %f", var)
853 config = self.config.decorrelateMapReduceConfig
854 task = ImageMapReduceTask(config=config)
855 results = task.run(subtractedExposure, science=scienceExposure,
856 template=templateExposure, psfMatchingKernel=psfMatchingKernel,
857 preConvKernel=preConvKernel, forceEvenSized=True,
858 templateMatched=templateMatched, preConvMode=preConvMode)
859 results.correctedExposure = results.exposure
860
861 # Make sure masks of input image are propagated to diffim
862 def gm(exp):
863 return exp.mask
864 gm(results.correctedExposure)[:, :] = gm(subtractedExposure)
865
866 var = self.computeVarianceMean(results.correctedExposure)
867 self.log.info("Variance (corrected diffim): %f", var)
868
869 else:
870 config = self.config.decorrelateConfig
871 task = DecorrelateALKernelTask(config=config)
872 results = task.run(scienceExposure, templateExposure,
873 subtractedExposure, psfMatchingKernel, preConvKernel=preConvKernel,
874 templateMatched=templateMatched, preConvMode=preConvMode)
875
876 return results
def run(self, subExposure, expandedSubExposure, fullBBox, template, science, alTaskResult=None, psfMatchingKernel=None, preConvKernel=None, **kwargs)
def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel, spatiallyVarying=True, preConvKernel=None, templateMatched=True, preConvMode=False)
def computeScoreCorrection(self, kappa, svar, tvar, preConvArr)
def calculateVariancePlane(self, vplane1, vplane2, varMean1, varMean2, c1ft, c2ft)
def padCenterOriginArray(A, tuple newShape, useInverse=False)
def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel, preConvKernel=None, xcen=None, ycen=None, svar=None, tvar=None, templateMatched=True, preConvMode=False, **kwargs)
Statistics makeStatistics(lsst::afw::image::Image< Pixel > const &img, lsst::afw::image::Mask< image::MaskPixel > const &msk, int const flags, StatisticsControl const &sctrl=StatisticsControl())