Coverage for python/lsst/ip/diffim/imageDecorrelation.py : 14%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2016 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
23import numpy as np
25import lsst.afw.image as afwImage
26import lsst.afw.math as afwMath
27import lsst.geom as geom
28import lsst.log
29import lsst.meas.algorithms as measAlg
30import lsst.pex.config as pexConfig
31import lsst.pipe.base as pipeBase
34from .imageMapReduce import (ImageMapReduceConfig, ImageMapReduceTask,
35 ImageMapper)
37__all__ = ("DecorrelateALKernelTask", "DecorrelateALKernelConfig",
38 "DecorrelateALKernelMapper", "DecorrelateALKernelMapReduceConfig",
39 "DecorrelateALKernelSpatialConfig", "DecorrelateALKernelSpatialTask")
42class DecorrelateALKernelConfig(pexConfig.Config):
43 """Configuration parameters for the DecorrelateALKernelTask
44 """
46 ignoreMaskPlanes = pexConfig.ListField(
47 dtype=str,
48 doc="""Mask planes to ignore for sigma-clipped statistics""",
49 default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE")
50 )
53class DecorrelateALKernelTask(pipeBase.Task):
54 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference
56 Notes
57 -----
59 Pipe-task that removes the neighboring-pixel covariance in an
60 image difference that are added when the template image is
61 convolved with the Alard-Lupton PSF matching kernel.
63 The image differencing pipeline task @link
64 ip.diffim.psfMatch.PsfMatchTask PSFMatchTask@endlink and @link
65 ip.diffim.psfMatch.PsfMatchConfigAL PSFMatchConfigAL@endlink uses
66 the Alard and Lupton (1998) method for matching the PSFs of the
67 template and science exposures prior to subtraction. The
68 Alard-Lupton method identifies a matching kernel, which is then
69 (typically) convolved with the template image to perform PSF
70 matching. This convolution has the effect of adding covariance
71 between neighboring pixels in the template image, which is then
72 added to the image difference by subtraction.
74 The pixel covariance may be corrected by whitening the noise of
75 the image difference. This task performs such a decorrelation by
76 computing a decorrelation kernel (based upon the A&L matching
77 kernel and variances in the template and science images) and
78 convolving the image difference with it. This process is described
79 in detail in [DMTN-021](http://dmtn-021.lsst.io).
81 This task has no standalone example, however it is applied as a
82 subtask of pipe.tasks.imageDifference.ImageDifferenceTask.
83 """
84 ConfigClass = DecorrelateALKernelConfig
85 _DefaultName = "ip_diffim_decorrelateALKernel"
87 def __init__(self, *args, **kwargs):
88 """Create the image decorrelation Task
90 Parameters
91 ----------
92 args :
93 arguments to be passed to ``lsst.pipe.base.task.Task.__init__``
94 kwargs :
95 keyword arguments to be passed to ``lsst.pipe.base.task.Task.__init__``
96 """
97 pipeBase.Task.__init__(self, *args, **kwargs)
99 self.statsControl = afwMath.StatisticsControl()
100 self.statsControl.setNumSigmaClip(3.)
101 self.statsControl.setNumIter(3)
102 self.statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes))
104 def computeVarianceMean(self, exposure):
105 statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(),
106 exposure.getMaskedImage().getMask(),
107 afwMath.MEANCLIP, self.statsControl)
108 var = statObj.getValue(afwMath.MEANCLIP)
109 return var
111 @pipeBase.timeMethod
112 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel,
113 preConvKernel=None, xcen=None, ycen=None, svar=None, tvar=None, templateMatched=True):
114 """Perform decorrelation of an image difference exposure.
116 Decorrelates the diffim due to the convolution of the templateExposure with the
117 A&L PSF matching kernel. Currently can accept a spatially varying matching kernel but in
118 this case it simply uses a static kernel from the center of the exposure. The decorrelation
119 is described in [DMTN-021, Equation 1](http://dmtn-021.lsst.io/#equation-1), where
120 `exposure` is I_1; templateExposure is I_2; `subtractedExposure` is D(k);
121 `psfMatchingKernel` is kappa; and svar and tvar are their respective
122 variances (see below).
124 Parameters
125 ----------
126 scienceExposure : `lsst.afw.image.Exposure`
127 The original science exposure (before `preConvKernel` applied).
128 templateExposure : `lsst.afw.image.Exposure`
129 The original template exposure warped into the science exposure dimensions.
130 subtractedExposure : `lsst.afw.iamge.Exposure`
131 the subtracted exposure produced by
132 `ip_diffim.ImagePsfMatchTask.subtractExposures()`. The `subtractedExposure` must
133 inherit its PSF from `exposure`, see notes below.
134 psfMatchingKernel : `lsst.afw.detection.Psf`
135 An (optionally spatially-varying) PSF matching kernel produced
136 by `ip_diffim.ImagePsfMatchTask.subtractExposures()`.
137 preConvKernel : `lsst.afw.math.Kernel`, optional
138 if not None, then the `scienceExposure` was pre-convolved with this kernel.
139 Allowed only if ``templateMatched==True``.
140 xcen : `float`, optional
141 X-pixel coordinate to use for computing constant matching kernel to use
142 If `None` (default), then use the center of the image.
143 ycen : `float`, optional
144 Y-pixel coordinate to use for computing constant matching kernel to use
145 If `None` (default), then use the center of the image.
146 svar : `float`, optional
147 Image variance for science image
148 If `None` (default) then compute the variance over the entire input science image.
149 tvar : `float`, optional
150 Image variance for template image
151 If `None` (default) then compute the variance over the entire input template image.
152 templateMatched : `bool`, optional
153 If True, the template exposure was matched (convolved) to the science exposure.
154 See also notes below.
156 Returns
157 -------
158 result : `lsst.pipe.base.Struct`
159 - ``correctedExposure`` : the decorrelated diffim
161 Notes
162 -----
163 The `subtractedExposure` is NOT updated. The returned `correctedExposure` has an updated but
164 spatially fixed PSF. It is calculated as the center of image PSF corrected by the center of
165 image matching kernel.
167 If ``templateMatched==True``, the templateExposure was matched (convolved)
168 to the ``scienceExposure`` by ``psfMatchingKernel``. Otherwise the ``scienceExposure``
169 was matched (convolved) by ``psfMatchingKernel``.
171 This task discards the variance plane of ``subtractedExposure`` and re-computes
172 it from the variance planes of ``scienceExposure`` and ``templateExposure``.
173 The image plane of ``subtractedExposure`` must be at the photometric level
174 set by the AL PSF matching in `ImagePsfMatchTask.subtractExposures`.
175 The assumptions about the photometric level are controlled by the
176 `templateMatched` option in this task.
178 Here we currently convert a spatially-varying matching kernel into a constant kernel,
179 just by computing it at the center of the image (tickets DM-6243, DM-6244).
181 We are also using a constant accross-the-image measure of sigma (sqrt(variance)) to compute
182 the decorrelation kernel.
184 TODO DM-23857 As part of the spatially varying correction implementation
185 consider whether returning a Struct is still necessary.
186 """
187 if preConvKernel is not None and not templateMatched:
188 raise ValueError("Pre-convolution and the matching of the "
189 "science exposure is not supported.")
191 spatialKernel = psfMatchingKernel
192 kimg = afwImage.ImageD(spatialKernel.getDimensions())
193 bbox = subtractedExposure.getBBox()
194 if xcen is None:
195 xcen = (bbox.getBeginX() + bbox.getEndX()) / 2.
196 if ycen is None:
197 ycen = (bbox.getBeginY() + bbox.getEndY()) / 2.
198 self.log.info("Using matching kernel computed at (%d, %d)", xcen, ycen)
199 spatialKernel.computeImage(kimg, False, xcen, ycen)
201 if svar is None:
202 svar = self.computeVarianceMean(scienceExposure)
203 if tvar is None:
204 tvar = self.computeVarianceMean(templateExposure)
205 self.log.infof("Original variance plane means. Science:{:.5e}, warped template:{:.5e})",
206 svar, tvar)
208 if templateMatched:
209 # Regular subtraction, we convolved the template
210 self.log.info("Decorrelation after template image convolution")
211 expVar = svar
212 matchedVar = tvar
213 exposure = scienceExposure
214 matchedExposure = templateExposure
215 else:
216 # We convolved the science image
217 self.log.info("Decorrelation after science image convolution")
218 expVar = tvar
219 matchedVar = svar
220 exposure = templateExposure
221 matchedExposure = scienceExposure
223 # Should not happen unless entire image has been masked, which could happen
224 # if this is a small subimage of the main exposure. In this case, just return a full NaN
225 # exposure
226 if np.isnan(expVar) or np.isnan(matchedVar):
227 # Double check that one of the exposures is all NaNs
228 if (np.all(np.isnan(exposure.image.array))
229 or np.all(np.isnan(matchedExposure.image.array))):
230 self.log.warn('Template or science image is entirely NaNs: skipping decorrelation.')
231 outExposure = subtractedExposure.clone()
232 return pipeBase.Struct(correctedExposure=outExposure, )
234 # The maximal correction value converges to sqrt(matchedVar/expVar).
235 # Correction divergence warning if the correction exceeds 4 orders of magnitude.
236 mOverExpVar = matchedVar/expVar
237 if mOverExpVar > 1e8:
238 self.log.warn("Diverging correction: matched image variance is "
239 " much larger than the unconvolved one's"
240 f", matchedVar/expVar:{mOverExpVar:.2e}")
242 oldVarMean = self.computeVarianceMean(subtractedExposure)
243 self.log.info("Variance plane mean of uncorrected diffim: %f", oldVarMean)
245 if preConvKernel is not None:
246 self.log.info('Using a pre-convolution kernel as part of decorrelation correction.')
247 kimg2 = afwImage.ImageD(preConvKernel.getDimensions())
248 preConvKernel.computeImage(kimg2, False)
249 pckArr = kimg2.array
251 kArr = kimg.array
252 diffExpArr = subtractedExposure.image.array
253 psfImg = subtractedExposure.getPsf().computeKernelImage(geom.Point2D(xcen, ycen))
254 psfDim = psfImg.getDimensions()
255 psfArr = psfImg.array
257 # Determine the common shape
258 kSum = np.sum(kArr)
259 kSumSq = kSum*kSum
260 self.log.debugf("Matching kernel sum: {:.3e}", kSum)
261 preSum = 1.
262 if preConvKernel is None:
263 self.computeCommonShape(kArr.shape, psfArr.shape, diffExpArr.shape)
264 corrft = self.computeCorrection(kArr, expVar, matchedVar)
265 else:
266 preSum = np.sum(pckArr)
267 self.log.debugf("Pre-convolution kernel sum: {:.3e}", preSum)
268 self.computeCommonShape(pckArr.shape, kArr.shape,
269 psfArr.shape, diffExpArr.shape)
270 corrft = self.computeCorrection(kArr, expVar, matchedVar, preConvArr=pckArr)
272 diffExpArr = self.computeCorrectedImage(corrft, diffExpArr)
273 corrPsfArr = self.computeCorrectedDiffimPsf(corrft, psfArr)
275 psfcI = afwImage.ImageD(psfDim)
276 psfcI.array = corrPsfArr
277 psfcK = afwMath.FixedKernel(psfcI)
278 psfNew = measAlg.KernelPsf(psfcK)
280 correctedExposure = subtractedExposure.clone()
281 correctedExposure.image.array[...] = diffExpArr # Allow for numpy type casting
282 # The subtracted exposure variance plane is already correlated, we cannot propagate
283 # it through another convolution; instead we need to use the uncorrelated originals
284 # The whitening should scale it to expVar + matchedVar on average
285 varImg = correctedExposure.variance.array
286 # Allow for numpy type casting
287 varImg[...] = preSum*preSum*exposure.variance.array + kSumSq*matchedExposure.variance.array
288 if not templateMatched:
289 # ImagePsfMatch.subtractExposures re-scales the difference in
290 # the science image convolution mode
291 varImg /= kSumSq
292 correctedExposure.setPsf(psfNew)
294 newVarMean = self.computeVarianceMean(correctedExposure)
295 self.log.infof("Variance plane mean of corrected diffim: {:.5e}", newVarMean)
297 # TODO DM-23857 As part of the spatially varying correction implementation
298 # consider whether returning a Struct is still necessary.
299 return pipeBase.Struct(correctedExposure=correctedExposure, )
301 def computeCommonShape(self, *shapes):
302 """Calculate the common shape for FFT operations. Set `self.freqSpaceShape`
303 internally.
305 Parameters
306 ----------
307 shapes : one or more `tuple` of `int`
308 Shapes of the arrays. All must have the same dimensionality.
309 At least one shape must be provided.
311 Returns
312 -------
313 None.
315 Notes
316 -----
317 For each dimension, gets the smallest even number greater than or equal to
318 `N1+N2-1` where `N1` and `N2` are the two largest values.
319 In case of only one shape given, rounds up to even each dimension value.
320 """
321 S = np.array(shapes, dtype=int)
322 if len(shapes) > 2:
323 S.sort(axis=0)
324 S = S[-2:]
325 if len(shapes) > 1:
326 commonShape = np.sum(S, axis=0) - 1
327 else:
328 commonShape = S[0]
329 commonShape[commonShape % 2 != 0] += 1
330 self.freqSpaceShape = tuple(commonShape)
331 self.log.info(f"Common frequency space shape {self.freqSpaceShape}")
333 @staticmethod
334 def padCenterOriginArray(A, newShape: tuple, useInverse=False):
335 """Zero pad an image where the origin is at the center and replace the
336 origin to the corner as required by the periodic input of FFT. Implement also
337 the inverse operation, crop the padding and re-center data.
339 Parameters
340 ----------
341 A : `numpy.ndarray`
342 An array to copy from.
343 newShape : `tuple` of `int`
344 The dimensions of the resulting array. For padding, the resulting array
345 must be larger than A in each dimension. For the inverse operation this
346 must be the original, before padding size of the array.
347 useInverse : bool, optional
348 Selector of forward, add padding, operation (False)
349 or its inverse, crop padding, operation (True).
351 Returns
352 -------
353 R : `numpy.ndarray`
354 The padded or unpadded array with shape of `newShape` and the same dtype as A.
356 Notes
357 -----
358 For odd dimensions, the splitting is rounded to
359 put the center pixel into the new corner origin (0,0). This is to be consistent
360 e.g. for a dirac delta kernel that is originally located at the center pixel.
361 """
363 # The forward and inverse operations should round odd dimension halves at the opposite
364 # sides to get the pixels back to their original positions.
365 if not useInverse:
366 # Forward operation: First and second halves with respect to the axes of A.
367 firstHalves = [x//2 for x in A.shape]
368 secondHalves = [x-y for x, y in zip(A.shape, firstHalves)]
369 else:
370 # Inverse operation: Opposite rounding
371 secondHalves = [x//2 for x in newShape]
372 firstHalves = [x-y for x, y in zip(newShape, secondHalves)]
374 R = np.zeros_like(A, shape=newShape)
375 R[-firstHalves[0]:, -firstHalves[1]:] = A[:firstHalves[0], :firstHalves[1]]
376 R[:secondHalves[0], -firstHalves[1]:] = A[-secondHalves[0]:, :firstHalves[1]]
377 R[:secondHalves[0], :secondHalves[1]] = A[-secondHalves[0]:, -secondHalves[1]:]
378 R[-firstHalves[0]:, :secondHalves[1]] = A[:firstHalves[0], -secondHalves[1]:]
379 return R
381 def computeCorrection(self, kappa, svar, tvar, preConvArr=None):
382 """Compute the Lupton decorrelation post-convolution kernel for decorrelating an
383 image difference, based on the PSF-matching kernel.
385 Parameters
386 ----------
387 kappa : `numpy.ndarray`
388 A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching.
389 svar : `float`
390 Average variance of science image used for PSF matching.
391 tvar : `float`
392 Average variance of the template (matched) image used for PSF matching.
393 preConvArr : `numpy.ndarray`, optional
394 If not None, then pre-filtering was applied
395 to science exposure, and this is the pre-convolution kernel.
397 Returns
398 -------
399 corrft : `numpy.ndarray` of `float`
400 The frequency space representation of the correction. The array is real (dtype float).
401 Shape is `self.freqSpaceShape`.
403 Notes
404 -----
405 The maximum correction factor converges to `sqrt(tvar/svar)` towards high frequencies.
406 This should be a plausible value.
407 """
408 kSum = np.sum(kappa)
409 kappa = self.padCenterOriginArray(kappa, self.freqSpaceShape)
410 kft = np.fft.fft2(kappa)
411 kftAbsSq = np.real(np.conj(kft) * kft)
412 # If there is no pre-convolution kernel, use placeholder scalars
413 if preConvArr is None:
414 preSum = 1.
415 preAbsSq = 1.
416 else:
417 preSum = np.sum(preConvArr)
418 preConvArr = self.padCenterOriginArray(preConvArr, self.freqSpaceShape)
419 preK = np.fft.fft2(preConvArr)
420 preAbsSq = np.real(np.conj(preK)*preK)
422 denom = svar * preAbsSq + tvar * kftAbsSq
423 # Division by zero protection, though we don't expect to hit it
424 # (rather we'll have numerical noise)
425 tiny = np.finfo(kftAbsSq.dtype).tiny * 1000.
426 flt = denom < tiny
427 sumFlt = np.sum(flt)
428 if sumFlt > 0:
429 self.log.warnf("Avoid zero division. Skip decorrelation "
430 "at {} divergent frequencies.", sumFlt)
431 denom[flt] = 1.
432 kft = np.sqrt((svar * preSum*preSum + tvar * kSum*kSum) / denom)
433 # Don't do any correction at these frequencies
434 # the difference image should be close to zero anyway, so can't be decorrelated
435 if sumFlt > 0:
436 kft[flt] = 1.
437 return kft
439 def computeCorrectedDiffimPsf(self, corrft, psfOld):
440 """Compute the (decorrelated) difference image's new PSF.
442 Parameters
443 ----------
444 corrft : `numpy.ndarray`
445 The frequency space representation of the correction calculated by
446 `computeCorrection`. Shape must be `self.freqSpaceShape`.
447 psfOld : `numpy.ndarray`
448 The psf of the difference image to be corrected.
450 Returns
451 -------
452 psfNew : `numpy.ndarray`
453 The corrected psf, same shape as `psfOld`, sum normed to 1.
455 Notes
456 -----
457 There is no algorithmic guarantee that the corrected psf can
458 meaningfully fit to the same size as the original one.
459 """
460 psfShape = psfOld.shape
461 psfNew = self.padCenterOriginArray(psfOld, self.freqSpaceShape)
462 psfNew = np.fft.fft2(psfNew)
463 psfNew *= corrft
464 psfNew = np.fft.ifft2(psfNew)
465 psfNew = psfNew.real
466 psfNew = self.padCenterOriginArray(psfNew, psfShape, useInverse=True)
467 psfNew = psfNew/psfNew.sum()
468 return psfNew
470 def computeCorrectedImage(self, corrft, imgOld):
471 """Compute the decorrelated difference image.
473 Parameters
474 ----------
475 corrft : `numpy.ndarray`
476 The frequency space representation of the correction calculated by
477 `computeCorrection`. Shape must be `self.freqSpaceShape`.
478 imgOld : `numpy.ndarray`
479 The difference image to be corrected.
481 Returns
482 -------
483 imgNew : `numpy.ndarray`
484 The corrected image, same size as the input.
485 """
486 expShape = imgOld.shape
487 imgNew = np.copy(imgOld)
488 filtInf = np.isinf(imgNew)
489 filtNan = np.isnan(imgNew)
490 imgNew[filtInf] = np.nan
491 imgNew[filtInf | filtNan] = np.nanmean(imgNew)
492 imgNew = self.padCenterOriginArray(imgNew, self.freqSpaceShape)
493 imgNew = np.fft.fft2(imgNew)
494 imgNew *= corrft
495 imgNew = np.fft.ifft2(imgNew)
496 imgNew = imgNew.real
497 imgNew = self.padCenterOriginArray(imgNew, expShape, useInverse=True)
498 imgNew[filtNan] = np.nan
499 imgNew[filtInf] = np.inf
500 return imgNew
503class DecorrelateALKernelMapper(DecorrelateALKernelTask, ImageMapper):
504 """Task to be used as an ImageMapper for performing
505 A&L decorrelation on subimages on a grid across a A&L difference image.
507 This task subclasses DecorrelateALKernelTask in order to implement
508 all of that task's configuration parameters, as well as its `run` method.
509 """
511 ConfigClass = DecorrelateALKernelConfig
512 _DefaultName = 'ip_diffim_decorrelateALKernelMapper'
514 def __init__(self, *args, **kwargs):
515 DecorrelateALKernelTask.__init__(self, *args, **kwargs)
517 def run(self, subExposure, expandedSubExposure, fullBBox,
518 template, science, alTaskResult=None, psfMatchingKernel=None,
519 preConvKernel=None, **kwargs):
520 """Perform decorrelation operation on `subExposure`, using
521 `expandedSubExposure` to allow for invalid edge pixels arising from
522 convolutions.
524 This method performs A&L decorrelation on `subExposure` using
525 local measures for image variances and PSF. `subExposure` is a
526 sub-exposure of the non-decorrelated A&L diffim. It also
527 requires the corresponding sub-exposures of the template
528 (`template`) and science (`science`) exposures.
530 Parameters
531 ----------
532 subExposure : `lsst.afw.image.Exposure`
533 the sub-exposure of the diffim
534 expandedSubExposure : `lsst.afw.image.Exposure`
535 the expanded sub-exposure upon which to operate
536 fullBBox : `lsst.geom.Box2I`
537 the bounding box of the original exposure
538 template : `lsst.afw.image.Exposure`
539 the corresponding sub-exposure of the template exposure
540 science : `lsst.afw.image.Exposure`
541 the corresponding sub-exposure of the science exposure
542 alTaskResult : `lsst.pipe.base.Struct`
543 the result of A&L image differencing on `science` and
544 `template`, importantly containing the resulting
545 `psfMatchingKernel`. Can be `None`, only if
546 `psfMatchingKernel` is not `None`.
547 psfMatchingKernel : Alternative parameter for passing the
548 A&L `psfMatchingKernel` directly.
549 preConvKernel : If not None, then pre-filtering was applied
550 to science exposure, and this is the pre-convolution
551 kernel.
552 kwargs :
553 additional keyword arguments propagated from
554 `ImageMapReduceTask.run`.
556 Returns
557 -------
558 A `pipeBase.Struct` containing:
560 - ``subExposure`` : the result of the `subExposure` processing.
561 - ``decorrelationKernel`` : the decorrelation kernel, currently
562 not used.
564 Notes
565 -----
566 This `run` method accepts parameters identical to those of
567 `ImageMapper.run`, since it is called from the
568 `ImageMapperTask`. See that class for more information.
569 """
570 templateExposure = template # input template
571 scienceExposure = science # input science image
572 if alTaskResult is None and psfMatchingKernel is None:
573 raise RuntimeError('Both alTaskResult and psfMatchingKernel cannot be None')
574 psfMatchingKernel = alTaskResult.psfMatchingKernel if alTaskResult is not None else psfMatchingKernel
576 # subExp and expandedSubExp are subimages of the (un-decorrelated) diffim!
577 # So here we compute corresponding subimages of templateExposure and scienceExposure
578 subExp2 = scienceExposure.Factory(scienceExposure, expandedSubExposure.getBBox())
579 subExp1 = templateExposure.Factory(templateExposure, expandedSubExposure.getBBox())
581 # Prevent too much log INFO verbosity from DecorrelateALKernelTask.run
582 logLevel = self.log.getLevel()
583 self.log.setLevel(lsst.log.WARN)
584 res = DecorrelateALKernelTask.run(self, subExp2, subExp1, expandedSubExposure,
585 psfMatchingKernel, preConvKernel)
586 self.log.setLevel(logLevel) # reset the log level
588 diffim = res.correctedExposure.Factory(res.correctedExposure, subExposure.getBBox())
589 out = pipeBase.Struct(subExposure=diffim, )
590 return out
593class DecorrelateALKernelMapReduceConfig(ImageMapReduceConfig):
594 """Configuration parameters for the ImageMapReduceTask to direct it to use
595 DecorrelateALKernelMapper as its mapper for A&L decorrelation.
596 """
597 mapper = pexConfig.ConfigurableField(
598 doc='A&L decorrelation task to run on each sub-image',
599 target=DecorrelateALKernelMapper
600 )
603class DecorrelateALKernelSpatialConfig(pexConfig.Config):
604 """Configuration parameters for the DecorrelateALKernelSpatialTask.
605 """
606 decorrelateConfig = pexConfig.ConfigField(
607 dtype=DecorrelateALKernelConfig,
608 doc='DecorrelateALKernel config to use when running on complete exposure (non spatially-varying)',
609 )
611 decorrelateMapReduceConfig = pexConfig.ConfigField(
612 dtype=DecorrelateALKernelMapReduceConfig,
613 doc='DecorrelateALKernelMapReduce config to use when running on each sub-image (spatially-varying)',
614 )
616 ignoreMaskPlanes = pexConfig.ListField(
617 dtype=str,
618 doc="""Mask planes to ignore for sigma-clipped statistics""",
619 default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE")
620 )
622 def setDefaults(self):
623 self.decorrelateMapReduceConfig.gridStepX = self.decorrelateMapReduceConfig.gridStepY = 40
624 self.decorrelateMapReduceConfig.cellSizeX = self.decorrelateMapReduceConfig.cellSizeY = 41
625 self.decorrelateMapReduceConfig.borderSizeX = self.decorrelateMapReduceConfig.borderSizeY = 8
626 self.decorrelateMapReduceConfig.reducer.reduceOperation = 'average'
629class DecorrelateALKernelSpatialTask(pipeBase.Task):
630 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference
632 Notes
633 -----
635 Pipe-task that removes the neighboring-pixel covariance in an
636 image difference that are added when the template image is
637 convolved with the Alard-Lupton PSF matching kernel.
639 This task is a simple wrapper around @ref DecorrelateALKernelTask,
640 which takes a `spatiallyVarying` parameter in its `run` method. If
641 it is `False`, then it simply calls the `run` method of @ref
642 DecorrelateALKernelTask. If it is True, then it uses the @ref
643 ImageMapReduceTask framework to break the exposures into
644 subExposures on a grid, and performs the `run` method of @ref
645 DecorrelateALKernelTask on each subExposure. This enables it to
646 account for spatially-varying PSFs and noise in the exposures when
647 performing the decorrelation.
649 This task has no standalone example, however it is applied as a
650 subtask of pipe.tasks.imageDifference.ImageDifferenceTask.
651 There is also an example of its use in `tests/testImageDecorrelation.py`.
652 """
653 ConfigClass = DecorrelateALKernelSpatialConfig
654 _DefaultName = "ip_diffim_decorrelateALKernelSpatial"
656 def __init__(self, *args, **kwargs):
657 """Create the image decorrelation Task
659 Parameters
660 ----------
661 args :
662 arguments to be passed to
663 `lsst.pipe.base.task.Task.__init__`
664 kwargs :
665 additional keyword arguments to be passed to
666 `lsst.pipe.base.task.Task.__init__`
667 """
668 pipeBase.Task.__init__(self, *args, **kwargs)
670 self.statsControl = afwMath.StatisticsControl()
671 self.statsControl.setNumSigmaClip(3.)
672 self.statsControl.setNumIter(3)
673 self.statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes))
675 def computeVarianceMean(self, exposure):
676 """Compute the mean of the variance plane of `exposure`.
677 """
678 statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(),
679 exposure.getMaskedImage().getMask(),
680 afwMath.MEANCLIP, self.statsControl)
681 var = statObj.getValue(afwMath.MEANCLIP)
682 return var
684 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel,
685 spatiallyVarying=True, preConvKernel=None, templateMatched=True):
686 """Perform decorrelation of an image difference exposure.
688 Decorrelates the diffim due to the convolution of the
689 templateExposure with the A&L psfMatchingKernel. If
690 `spatiallyVarying` is True, it utilizes the spatially varying
691 matching kernel via the `imageMapReduce` framework to perform
692 spatially-varying decorrelation on a grid of subExposures.
694 Parameters
695 ----------
696 scienceExposure : `lsst.afw.image.Exposure`
697 the science Exposure used for PSF matching
698 templateExposure : `lsst.afw.image.Exposure`
699 the template Exposure used for PSF matching
700 subtractedExposure : `lsst.afw.image.Exposure`
701 the subtracted Exposure produced by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
702 psfMatchingKernel :
703 an (optionally spatially-varying) PSF matching kernel produced
704 by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
705 spatiallyVarying : `bool`
706 if True, perform the spatially-varying operation
707 preConvKernel : `lsst.meas.algorithms.Psf`
708 if not none, the scienceExposure has been pre-filtered with this kernel. (Currently
709 this option is experimental.)
710 templateMatched : `bool`, optional
711 If True, the template exposure was matched (convolved) to the science exposure.
713 Returns
714 -------
715 results : `lsst.pipe.base.Struct`
716 a structure containing:
718 - ``correctedExposure`` : the decorrelated diffim
720 """
721 self.log.info('Running A&L decorrelation: spatiallyVarying=%r' % spatiallyVarying)
723 svar = self.computeVarianceMean(scienceExposure)
724 tvar = self.computeVarianceMean(templateExposure)
725 if np.isnan(svar) or np.isnan(tvar): # Should not happen unless entire image has been masked.
726 # Double check that one of the exposures is all NaNs
727 if (np.all(np.isnan(scienceExposure.image.array))
728 or np.all(np.isnan(templateExposure.image.array))):
729 self.log.warn('Template or science image is entirely NaNs: skipping decorrelation.')
730 if np.isnan(svar):
731 svar = 1e-9
732 if np.isnan(tvar):
733 tvar = 1e-9
735 var = self.computeVarianceMean(subtractedExposure)
737 if spatiallyVarying:
738 self.log.info("Variance (science, template): (%f, %f)", svar, tvar)
739 self.log.info("Variance (uncorrected diffim): %f", var)
740 config = self.config.decorrelateMapReduceConfig
741 task = ImageMapReduceTask(config=config)
742 results = task.run(subtractedExposure, science=scienceExposure,
743 template=templateExposure, psfMatchingKernel=psfMatchingKernel,
744 preConvKernel=preConvKernel, forceEvenSized=True,
745 templateMatched=templateMatched)
746 results.correctedExposure = results.exposure
748 # Make sure masks of input image are propagated to diffim
749 def gm(exp):
750 return exp.getMaskedImage().getMask()
751 gm(results.correctedExposure)[:, :] = gm(subtractedExposure)
753 var = self.computeVarianceMean(results.correctedExposure)
754 self.log.info("Variance (corrected diffim): %f", var)
756 else:
757 config = self.config.decorrelateConfig
758 task = DecorrelateALKernelTask(config=config)
759 results = task.run(scienceExposure, templateExposure,
760 subtractedExposure, psfMatchingKernel, preConvKernel=preConvKernel,
761 templateMatched=templateMatched)
763 return results