29 import lsst.meas.algorithms
as measAlg
30 import lsst.pex.config
as pexConfig
31 import lsst.pipe.base
as pipeBase
34 from .imageMapReduce
import (ImageMapReduceConfig, ImageMapReduceTask,
37 __all__ = (
"DecorrelateALKernelTask",
"DecorrelateALKernelConfig",
38 "DecorrelateALKernelMapper",
"DecorrelateALKernelMapReduceConfig",
39 "DecorrelateALKernelSpatialConfig",
"DecorrelateALKernelSpatialTask")
43 """Configuration parameters for the DecorrelateALKernelTask
46 ignoreMaskPlanes = pexConfig.ListField(
48 doc=
"""Mask planes to ignore for sigma-clipped statistics""",
49 default=(
"INTRP",
"EDGE",
"DETECTED",
"SAT",
"CR",
"BAD",
"NO_DATA",
"DETECTED_NEGATIVE")
54 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference
59 Pipe-task that removes the neighboring-pixel covariance in an
60 image difference that are added when the template image is
61 convolved with the Alard-Lupton PSF matching kernel.
63 The image differencing pipeline task @link
64 ip.diffim.psfMatch.PsfMatchTask PSFMatchTask@endlink and @link
65 ip.diffim.psfMatch.PsfMatchConfigAL PSFMatchConfigAL@endlink uses
66 the Alard and Lupton (1998) method for matching the PSFs of the
67 template and science exposures prior to subtraction. The
68 Alard-Lupton method identifies a matching kernel, which is then
69 (typically) convolved with the template image to perform PSF
70 matching. This convolution has the effect of adding covariance
71 between neighboring pixels in the template image, which is then
72 added to the image difference by subtraction.
74 The pixel covariance may be corrected by whitening the noise of
75 the image difference. This task performs such a decorrelation by
76 computing a decorrelation kernel (based upon the A&L matching
77 kernel and variances in the template and science images) and
78 convolving the image difference with it. This process is described
79 in detail in [DMTN-021](http://dmtn-021.lsst.io).
81 This task has no standalone example, however it is applied as a
82 subtask of pipe.tasks.imageDifference.ImageDifferenceTask.
84 ConfigClass = DecorrelateALKernelConfig
85 _DefaultName =
"ip_diffim_decorrelateALKernel"
88 """Create the image decorrelation Task
93 arguments to be passed to ``lsst.pipe.base.task.Task.__init__``
95 keyword arguments to be passed to ``lsst.pipe.base.task.Task.__init__``
97 pipeBase.Task.__init__(self, *args, **kwargs)
102 self.
statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes))
106 exposure.getMaskedImage().getMask(),
108 var = statObj.getValue(afwMath.MEANCLIP)
112 def run(self, exposure, templateExposure, subtractedExposure, psfMatchingKernel,
113 preConvKernel=None, xcen=None, ycen=None, svar=None, tvar=None):
114 """Perform decorrelation of an image difference exposure.
116 Decorrelates the diffim due to the convolution of the templateExposure with the
117 A&L PSF matching kernel. Currently can accept a spatially varying matching kernel but in
118 this case it simply uses a static kernel from the center of the exposure. The decorrelation
119 is described in [DMTN-021, Equation 1](http://dmtn-021.lsst.io/#equation-1), where
120 `exposure` is I_1; templateExposure is I_2; `subtractedExposure` is D(k);
121 `psfMatchingKernel` is kappa; and svar and tvar are their respective
122 variances (see below).
126 exposure : `lsst.afw.image.Exposure`
127 The original science exposure (before `preConvKernel` applied) used for PSF matching.
128 templateExposure : `lsst.afw.image.Exposure`
129 The original template exposure (before matched to the science exposure
130 by `psfMatchingKernel`) warped into the science exposure dimensions. Always the PSF of the
131 `templateExposure` should be matched to the PSF of `exposure`, see notes below.
133 the subtracted exposure produced by
134 `ip_diffim.ImagePsfMatchTask.subtractExposures()`. The `subtractedExposure` must
135 inherit its PSF from `exposure`, see notes below.
137 An (optionally spatially-varying) PSF matching kernel produced
138 by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
140 if not None, then the `exposure` was pre-convolved with this kernel
141 xcen : `float`, optional
142 X-pixel coordinate to use for computing constant matching kernel to use
143 If `None` (default), then use the center of the image.
144 ycen : `float`, optional
145 Y-pixel coordinate to use for computing constant matching kernel to use
146 If `None` (default), then use the center of the image.
147 svar : `float`, optional
148 Image variance for science image
149 If `None` (default) then compute the variance over the entire input science image.
150 tvar : `float`, optional
151 Image variance for template image
152 If `None` (default) then compute the variance over the entire input template image.
156 result : `lsst.pipe.base.Struct`
157 - ``correctedExposure`` : the decorrelated diffim
161 The `subtractedExposure` is NOT updated. The returned `correctedExposure` has an updated but
162 spatially fixed PSF. It is calculated as the center of image PSF corrected by the center of
163 image matching kernel.
165 In this task, it is _always_ the `templateExposure` that was matched to the `exposure`
166 by `psfMatchingKernel`. Swap arguments accordingly if actually the science exposure was matched
167 to a co-added template. In this case, tvar > svar typically occurs.
169 The `templateExposure` and `exposure` image dimensions must be the same.
171 Here we currently convert a spatially-varying matching kernel into a constant kernel,
172 just by computing it at the center of the image (tickets DM-6243, DM-6244).
174 We are also using a constant accross-the-image measure of sigma (sqrt(variance)) to compute
175 the decorrelation kernel.
177 TODO DM-23857 As part of the spatially varying correction implementation
178 consider whether returning a Struct is still necessary.
180 spatialKernel = psfMatchingKernel
181 kimg = afwImage.ImageD(spatialKernel.getDimensions())
182 bbox = subtractedExposure.getBBox()
184 xcen = (bbox.getBeginX() + bbox.getEndX()) / 2.
186 ycen = (bbox.getBeginY() + bbox.getEndY()) / 2.
187 self.log.info(
"Using matching kernel computed at (%d, %d)", xcen, ycen)
188 spatialKernel.computeImage(kimg,
True, xcen, ycen)
194 self.log.info(
"Variance (science, template): (%f, %f)", svar, tvar)
199 if np.isnan(svar)
or np.isnan(tvar):
201 if (np.all(np.isnan(exposure.image.array))
202 or np.all(np.isnan(templateExposure.image.array))):
203 self.log.warn(
'Template or science image is entirely NaNs: skipping decorrelation.')
204 outExposure = subtractedExposure.clone()
205 return pipeBase.Struct(correctedExposure=outExposure, )
209 tOverSVar = tvar/svar
211 self.log.warn(
"Diverging correction: science image variance is much smaller than template"
212 f
", tvar/svar:{tOverSVar:.2e}")
215 self.log.info(
"Variance (uncorrected diffim): %f", oldVarMean)
217 if preConvKernel
is not None:
218 self.log.info(
'Using a pre-convolution kernel as part of decorrelation correction.')
219 kimg2 = afwImage.ImageD(preConvKernel.getDimensions())
220 preConvKernel.computeImage(kimg2,
False)
224 diffExpArr = subtractedExposure.image.array
225 psfImg = subtractedExposure.getPsf().computeKernelImage(
geom.Point2D(xcen, ycen))
226 psfDim = psfImg.getDimensions()
227 psfArr = psfImg.array
230 if preConvKernel
is None:
235 psfArr.shape, diffExpArr.shape)
241 psfcI = afwImage.ImageD(psfDim)
242 psfcI.array = corrPsfArr
244 psfNew = measAlg.KernelPsf(psfcK)
246 correctedExposure = subtractedExposure.clone()
247 correctedExposure.image.array = diffExpArr
251 varImg = correctedExposure.variance
252 varImg.assign(exposure.variance)
253 varImg += templateExposure.variance
254 correctedExposure.setPsf(psfNew)
257 self.log.info(f
"Variance (corrected diffim): {newVarMean:.2e}")
261 return pipeBase.Struct(correctedExposure=correctedExposure, )
264 """Calculate the common shape for FFT operations. Set `self.freqSpaceShape`
269 shapes : one or more `tuple` of `int`
270 Shapes of the arrays. All must have the same dimensionality.
271 At least one shape must be provided.
279 For each dimension, gets the smallest even number greater than or equal to
280 `N1+N2-1` where `N1` and `N2` are the two largest values.
281 In case of only one shape given, rounds up to even each dimension value.
283 S = np.array(shapes, dtype=int)
288 commonShape = np.sum(S, axis=0) - 1
291 commonShape[commonShape % 2 != 0] += 1
293 self.log.info(f
"Common frequency space shape {self.freqSpaceShape}")
297 """Zero pad an image where the origin is at the center and replace the
298 origin to the corner as required by the periodic input of FFT. Implement also
299 the inverse operation, crop the padding and re-center data.
304 An array to copy from.
305 newShape : `tuple` of `int`
306 The dimensions of the resulting array. For padding, the resulting array
307 must be larger than A in each dimension. For the inverse operation this
308 must be the original, before padding size of the array.
309 useInverse : bool, optional
310 Selector of forward, add padding, operation (False)
311 or its inverse, crop padding, operation (True).
316 The padded or unpadded array with shape of `newShape` and the same dtype as A.
320 For odd dimensions, the splitting is rounded to
321 put the center pixel into the new corner origin (0,0). This is to be consistent
322 e.g. for a dirac delta kernel that is originally located at the center pixel.
329 firstHalves = [x//2
for x
in A.shape]
330 secondHalves = [x-y
for x, y
in zip(A.shape, firstHalves)]
333 secondHalves = [x//2
for x
in newShape]
334 firstHalves = [x-y
for x, y
in zip(newShape, secondHalves)]
336 R = np.zeros_like(A, shape=newShape)
337 R[-firstHalves[0]:, -firstHalves[1]:] = A[:firstHalves[0], :firstHalves[1]]
338 R[:secondHalves[0], -firstHalves[1]:] = A[-secondHalves[0]:, :firstHalves[1]]
339 R[:secondHalves[0], :secondHalves[1]] = A[-secondHalves[0]:, -secondHalves[1]:]
340 R[-firstHalves[0]:, :secondHalves[1]] = A[:firstHalves[0], -secondHalves[1]:]
344 """Compute the Lupton decorrelation post-convolution kernel for decorrelating an
345 image difference, based on the PSF-matching kernel.
349 kappa : `numpy.ndarray`
350 A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching.
352 Average variance of science image used for PSF matching.
354 Average variance of the template (matched) image used for PSF matching.
355 preConvArr : `numpy.ndarray`, optional
356 If not None, then pre-filtering was applied
357 to science exposure, and this is the pre-convolution kernel.
361 corrft : `numpy.ndarray` of `float`
362 The frequency space representation of the correction. The array is real (dtype float).
363 Shape is `self.freqSpaceShape`.
367 The maximum correction factor converges to `sqrt(tvar/svar)` towards high frequencies.
368 This should be a plausible value.
371 kft = np.fft.fft2(kappa)
372 kft2 = np.real(np.conj(kft) * kft)
373 if preConvArr
is None:
374 denom = svar + tvar * kft2
377 mk = np.fft.fft2(preConvArr)
378 mk2 = np.real(np.conj(mk) * mk)
379 denom = svar * mk2 + tvar * kft2
380 kft = np.sqrt((svar + tvar) / denom)
384 """Compute the (decorrelated) difference image's new PSF.
388 corrft : `numpy.ndarray`
389 The frequency space representation of the correction calculated by
390 `computeCorrection`. Shape must be `self.freqSpaceShape`.
391 psfOld : `numpy.ndarray`
392 The psf of the difference image to be corrected.
396 psfNew : `numpy.ndarray`
397 The corrected psf, same shape as `psfOld`, sum normed to 1.
401 There is no algorithmic guarantee that the corrected psf can
402 meaningfully fit to the same size as the original one.
404 psfShape = psfOld.shape
406 psfNew = np.fft.fft2(psfNew)
408 psfNew = np.fft.ifft2(psfNew)
411 psfNew = psfNew/psfNew.sum()
415 """Compute the decorrelated difference image.
419 corrft : `numpy.ndarray`
420 The frequency space representation of the correction calculated by
421 `computeCorrection`. Shape must be `self.freqSpaceShape`.
422 imgOld : `numpy.ndarray`
423 The difference image to be corrected.
427 imgNew : `numpy.ndarray`
428 The corrected image, same size as the input.
430 expShape = imgOld.shape
431 imgNew = np.copy(imgOld)
432 filtInf = np.isinf(imgNew)
433 filtNan = np.isnan(imgNew)
434 imgNew[filtInf] = np.nan
435 imgNew[filtInf | filtNan] = np.nanmean(imgNew)
437 imgNew = np.fft.fft2(imgNew)
439 imgNew = np.fft.ifft2(imgNew)
442 imgNew[filtNan] = np.nan
443 imgNew[filtInf] = np.inf
448 """Task to be used as an ImageMapper for performing
449 A&L decorrelation on subimages on a grid across a A&L difference image.
451 This task subclasses DecorrelateALKernelTask in order to implement
452 all of that task's configuration parameters, as well as its `run` method.
455 ConfigClass = DecorrelateALKernelConfig
456 _DefaultName =
'ip_diffim_decorrelateALKernelMapper'
459 DecorrelateALKernelTask.__init__(self, *args, **kwargs)
461 def run(self, subExposure, expandedSubExposure, fullBBox,
462 template, science, alTaskResult=None, psfMatchingKernel=None,
463 preConvKernel=None, **kwargs):
464 """Perform decorrelation operation on `subExposure`, using
465 `expandedSubExposure` to allow for invalid edge pixels arising from
468 This method performs A&L decorrelation on `subExposure` using
469 local measures for image variances and PSF. `subExposure` is a
470 sub-exposure of the non-decorrelated A&L diffim. It also
471 requires the corresponding sub-exposures of the template
472 (`template`) and science (`science`) exposures.
476 subExposure : `lsst.afw.image.Exposure`
477 the sub-exposure of the diffim
478 expandedSubExposure : `lsst.afw.image.Exposure`
479 the expanded sub-exposure upon which to operate
480 fullBBox : `lsst.geom.Box2I`
481 the bounding box of the original exposure
482 template : `lsst.afw.image.Exposure`
483 the corresponding sub-exposure of the template exposure
484 science : `lsst.afw.image.Exposure`
485 the corresponding sub-exposure of the science exposure
486 alTaskResult : `lsst.pipe.base.Struct`
487 the result of A&L image differencing on `science` and
488 `template`, importantly containing the resulting
489 `psfMatchingKernel`. Can be `None`, only if
490 `psfMatchingKernel` is not `None`.
491 psfMatchingKernel : Alternative parameter for passing the
492 A&L `psfMatchingKernel` directly.
493 preConvKernel : If not None, then pre-filtering was applied
494 to science exposure, and this is the pre-convolution
497 additional keyword arguments propagated from
498 `ImageMapReduceTask.run`.
502 A `pipeBase.Struct` containing:
504 - ``subExposure`` : the result of the `subExposure` processing.
505 - ``decorrelationKernel`` : the decorrelation kernel, currently
510 This `run` method accepts parameters identical to those of
511 `ImageMapper.run`, since it is called from the
512 `ImageMapperTask`. See that class for more information.
514 templateExposure = template
515 scienceExposure = science
516 if alTaskResult
is None and psfMatchingKernel
is None:
517 raise RuntimeError(
'Both alTaskResult and psfMatchingKernel cannot be None')
518 psfMatchingKernel = alTaskResult.psfMatchingKernel
if alTaskResult
is not None else psfMatchingKernel
522 subExp2 = scienceExposure.Factory(scienceExposure, expandedSubExposure.getBBox())
523 subExp1 = templateExposure.Factory(templateExposure, expandedSubExposure.getBBox())
526 logLevel = self.log.getLevel()
527 self.log.setLevel(lsst.log.WARN)
528 res = DecorrelateALKernelTask.run(self, subExp2, subExp1, expandedSubExposure,
529 psfMatchingKernel, preConvKernel)
530 self.log.setLevel(logLevel)
532 diffim = res.correctedExposure.Factory(res.correctedExposure, subExposure.getBBox())
533 out = pipeBase.Struct(subExposure=diffim, )
538 """Configuration parameters for the ImageMapReduceTask to direct it to use
539 DecorrelateALKernelMapper as its mapper for A&L decorrelation.
541 mapper = pexConfig.ConfigurableField(
542 doc=
'A&L decorrelation task to run on each sub-image',
543 target=DecorrelateALKernelMapper
548 """Configuration parameters for the DecorrelateALKernelSpatialTask.
550 decorrelateConfig = pexConfig.ConfigField(
551 dtype=DecorrelateALKernelConfig,
552 doc=
'DecorrelateALKernel config to use when running on complete exposure (non spatially-varying)',
555 decorrelateMapReduceConfig = pexConfig.ConfigField(
556 dtype=DecorrelateALKernelMapReduceConfig,
557 doc=
'DecorrelateALKernelMapReduce config to use when running on each sub-image (spatially-varying)',
560 ignoreMaskPlanes = pexConfig.ListField(
562 doc=
"""Mask planes to ignore for sigma-clipped statistics""",
563 default=(
"INTRP",
"EDGE",
"DETECTED",
"SAT",
"CR",
"BAD",
"NO_DATA",
"DETECTED_NEGATIVE")
574 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference
579 Pipe-task that removes the neighboring-pixel covariance in an
580 image difference that are added when the template image is
581 convolved with the Alard-Lupton PSF matching kernel.
583 This task is a simple wrapper around @ref DecorrelateALKernelTask,
584 which takes a `spatiallyVarying` parameter in its `run` method. If
585 it is `False`, then it simply calls the `run` method of @ref
586 DecorrelateALKernelTask. If it is True, then it uses the @ref
587 ImageMapReduceTask framework to break the exposures into
588 subExposures on a grid, and performs the `run` method of @ref
589 DecorrelateALKernelTask on each subExposure. This enables it to
590 account for spatially-varying PSFs and noise in the exposures when
591 performing the decorrelation.
593 This task has no standalone example, however it is applied as a
594 subtask of pipe.tasks.imageDifference.ImageDifferenceTask.
595 There is also an example of its use in `tests/testImageDecorrelation.py`.
597 ConfigClass = DecorrelateALKernelSpatialConfig
598 _DefaultName =
"ip_diffim_decorrelateALKernelSpatial"
601 """Create the image decorrelation Task
606 arguments to be passed to
607 `lsst.pipe.base.task.Task.__init__`
609 additional keyword arguments to be passed to
610 `lsst.pipe.base.task.Task.__init__`
612 pipeBase.Task.__init__(self, *args, **kwargs)
617 self.
statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes))
620 """Compute the mean of the variance plane of `exposure`.
623 exposure.getMaskedImage().getMask(),
625 var = statObj.getValue(afwMath.MEANCLIP)
628 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel,
629 spatiallyVarying=True, preConvKernel=None):
630 """Perform decorrelation of an image difference exposure.
632 Decorrelates the diffim due to the convolution of the
633 templateExposure with the A&L psfMatchingKernel. If
634 `spatiallyVarying` is True, it utilizes the spatially varying
635 matching kernel via the `imageMapReduce` framework to perform
636 spatially-varying decorrelation on a grid of subExposures.
640 scienceExposure : `lsst.afw.image.Exposure`
641 the science Exposure used for PSF matching
642 templateExposure : `lsst.afw.image.Exposure`
643 the template Exposure used for PSF matching
644 subtractedExposure : `lsst.afw.image.Exposure`
645 the subtracted Exposure produced by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
647 an (optionally spatially-varying) PSF matching kernel produced
648 by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
649 spatiallyVarying : `bool`
650 if True, perform the spatially-varying operation
651 preConvKernel : `lsst.meas.algorithms.Psf`
652 if not none, the scienceExposure has been pre-filtered with this kernel. (Currently
653 this option is experimental.)
657 results : `lsst.pipe.base.Struct`
658 a structure containing:
660 - ``correctedExposure`` : the decorrelated diffim
663 self.log.info(
'Running A&L decorrelation: spatiallyVarying=%r' % spatiallyVarying)
667 if np.isnan(svar)
or np.isnan(tvar):
669 if (np.all(np.isnan(scienceExposure.image.array))
670 or np.all(np.isnan(templateExposure.image.array))):
671 self.log.warn(
'Template or science image is entirely NaNs: skipping decorrelation.')
680 self.log.info(
"Variance (science, template): (%f, %f)", svar, tvar)
681 self.log.info(
"Variance (uncorrected diffim): %f", var)
682 config = self.config.decorrelateMapReduceConfig
684 results = task.run(subtractedExposure, science=scienceExposure,
685 template=templateExposure, psfMatchingKernel=psfMatchingKernel,
686 preConvKernel=preConvKernel, forceEvenSized=
True)
687 results.correctedExposure = results.exposure
691 return exp.getMaskedImage().getMask()
692 gm(results.correctedExposure)[:, :] = gm(subtractedExposure)
695 self.log.info(
"Variance (corrected diffim): %f", var)
698 config = self.config.decorrelateConfig
700 results = task.run(scienceExposure, templateExposure,
701 subtractedExposure, psfMatchingKernel, preConvKernel=preConvKernel)