29 import lsst.meas.algorithms
as measAlg
31 import lsst.pipe.base
as pipeBase
34 from .imageMapReduce
import (ImageMapReduceConfig, ImageMapReduceTask,
37 __all__ = (
"DecorrelateALKernelTask",
"DecorrelateALKernelConfig",
38 "DecorrelateALKernelMapper",
"DecorrelateALKernelMapReduceConfig",
39 "DecorrelateALKernelSpatialConfig",
"DecorrelateALKernelSpatialTask")
43 """Configuration parameters for the DecorrelateALKernelTask
46 ignoreMaskPlanes = pexConfig.ListField(
48 doc=
"""Mask planes to ignore for sigma-clipped statistics""",
49 default=(
"INTRP",
"EDGE",
"DETECTED",
"SAT",
"CR",
"BAD",
"NO_DATA",
"DETECTED_NEGATIVE")
54 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference
59 Pipe-task that removes the neighboring-pixel covariance in an
60 image difference that are added when the template image is
61 convolved with the Alard-Lupton PSF matching kernel.
63 The image differencing pipeline task @link
64 ip.diffim.psfMatch.PsfMatchTask PSFMatchTask@endlink and @link
65 ip.diffim.psfMatch.PsfMatchConfigAL PSFMatchConfigAL@endlink uses
66 the Alard and Lupton (1998) method for matching the PSFs of the
67 template and science exposures prior to subtraction. The
68 Alard-Lupton method identifies a matching kernel, which is then
69 (typically) convolved with the template image to perform PSF
70 matching. This convolution has the effect of adding covariance
71 between neighboring pixels in the template image, which is then
72 added to the image difference by subtraction.
74 The pixel covariance may be corrected by whitening the noise of
75 the image difference. This task performs such a decorrelation by
76 computing a decorrelation kernel (based upon the A&L matching
77 kernel and variances in the template and science images) and
78 convolving the image difference with it. This process is described
79 in detail in [DMTN-021](http://dmtn-021.lsst.io).
81 This task has no standalone example, however it is applied as a
82 subtask of pipe.tasks.imageDifference.ImageDifferenceTask.
84 ConfigClass = DecorrelateALKernelConfig
85 _DefaultName =
"ip_diffim_decorrelateALKernel"
88 """Create the image decorrelation Task
93 arguments to be passed to ``lsst.pipe.base.task.Task.__init__``
95 keyword arguments to be passed to ``lsst.pipe.base.task.Task.__init__``
97 pipeBase.Task.__init__(self, *args, **kwargs)
102 self.
statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes))
106 exposure.getMaskedImage().getMask(),
108 var = statObj.getValue(afwMath.MEANCLIP)
112 def run(self, exposure, templateExposure, subtractedExposure, psfMatchingKernel,
113 preConvKernel=None, xcen=None, ycen=None, svar=None, tvar=None):
114 """Perform decorrelation of an image difference exposure.
116 Decorrelates the diffim due to the convolution of the templateExposure with the
117 A&L PSF matching kernel. Currently can accept a spatially varying matching kernel but in
118 this case it simply uses a static kernel from the center of the exposure. The decorrelation
119 is described in [DMTN-021, Equation 1](http://dmtn-021.lsst.io/#equation-1), where
120 `exposure` is I_1; templateExposure is I_2; `subtractedExposure` is D(k);
121 `psfMatchingKernel` is kappa; and svar and tvar are their respective
122 variances (see below).
126 exposure : `lsst.afw.image.Exposure`
127 The original science exposure (before `preConvKernel` applied) used for PSF matching.
128 templateExposure : `lsst.afw.image.Exposure`
129 The original template exposure (before matched to the science exposure
130 by `psfMatchingKernel`) warped into the science exposure dimensions. Always the PSF of the
131 `templateExposure` should be matched to the PSF of `exposure`, see notes below.
133 the subtracted exposure produced by
134 `ip_diffim.ImagePsfMatchTask.subtractExposures()`. The `subtractedExposure` must
135 inherit its PSF from `exposure`, see notes below.
137 An (optionally spatially-varying) PSF matching kernel produced
138 by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
140 if not None, then the `exposure` was pre-convolved with this kernel
141 xcen : `float`, optional
142 X-pixel coordinate to use for computing constant matching kernel to use
143 If `None` (default), then use the center of the image.
144 ycen : `float`, optional
145 Y-pixel coordinate to use for computing constant matching kernel to use
146 If `None` (default), then use the center of the image.
147 svar : `float`, optional
148 Image variance for science image
149 If `None` (default) then compute the variance over the entire input science image.
150 tvar : `float`, optional
151 Image variance for template image
152 If `None` (default) then compute the variance over the entire input template image.
156 result : `lsst.pipe.base.Struct`
157 - ``correctedExposure`` : the decorrelated diffim
161 The `subtractedExposure` is NOT updated. The returned `correctedExposure` has an updated but
162 spatially fixed PSF. It is calculated as the center of image PSF corrected by the center of
163 image matching kernel.
165 In this task, it is _always_ the `templateExposure` that was matched to the `exposure`
166 by `psfMatchingKernel`. Swap arguments accordingly if actually the science exposure was matched
167 to a co-added template. In this case, tvar > svar typically occurs.
169 The `templateExposure` and `exposure` image dimensions must be the same.
171 Here we currently convert a spatially-varying matching kernel into a constant kernel,
172 just by computing it at the center of the image (tickets DM-6243, DM-6244).
174 We are also using a constant accross-the-image measure of sigma (sqrt(variance)) to compute
175 the decorrelation kernel.
177 TODO DM-23857 As part of the spatially varying correction implementation
178 consider whether returning a Struct is still necessary.
180 spatialKernel = psfMatchingKernel
181 kimg = afwImage.ImageD(spatialKernel.getDimensions())
182 bbox = subtractedExposure.getBBox()
184 xcen = (bbox.getBeginX() + bbox.getEndX()) / 2.
186 ycen = (bbox.getBeginY() + bbox.getEndY()) / 2.
187 self.log.info(
"Using matching kernel computed at (%d, %d)", xcen, ycen)
188 spatialKernel.computeImage(kimg,
True, xcen, ycen)
194 self.log.info(
"Variance (science, template): (%f, %f)", svar, tvar)
199 if np.isnan(svar)
or np.isnan(tvar):
201 if (np.all(np.isnan(exposure.image.array))
202 or np.all(np.isnan(templateExposure.image.array))):
203 self.log.warn(
'Template or science image is entirely NaNs: skipping decorrelation.')
204 outExposure = subtractedExposure.clone()
205 return pipeBase.Struct(correctedExposure=outExposure, )
209 tOverSVar = tvar/svar
211 self.log.warn(
"Diverging correction: science image variance is much smaller than template"
212 f
", tvar/svar:{tOverSVar:.2e}")
215 self.log.info(
"Variance (uncorrected diffim): %f", oldVarMean)
217 if preConvKernel
is not None:
218 self.log.info(
'Using a pre-convolution kernel as part of decorrelation correction.')
219 kimg2 = afwImage.ImageD(preConvKernel.getDimensions())
220 preConvKernel.computeImage(kimg2,
False)
224 diffExpArr = subtractedExposure.image.array
225 psfImg = subtractedExposure.getPsf().computeKernelImage(
geom.Point2D(xcen, ycen))
226 psfDim = psfImg.getDimensions()
227 psfArr = psfImg.array
230 if preConvKernel
is None:
235 psfArr.shape, diffExpArr.shape)
241 psfcI = afwImage.ImageD(psfDim)
242 psfcI.array = corrPsfArr
244 psfNew = measAlg.KernelPsf(psfcK)
246 correctedExposure = subtractedExposure.clone()
247 correctedExposure.image.array[...] = diffExpArr
251 varImg = correctedExposure.variance.array
253 varImg[...] = exposure.variance.array + templateExposure.variance.array
254 correctedExposure.setPsf(psfNew)
257 self.log.info(f
"Variance (corrected diffim): {newVarMean:.2e}")
261 return pipeBase.Struct(correctedExposure=correctedExposure, )
264 """Calculate the common shape for FFT operations. Set `self.freqSpaceShape`
269 shapes : one or more `tuple` of `int`
270 Shapes of the arrays. All must have the same dimensionality.
271 At least one shape must be provided.
279 For each dimension, gets the smallest even number greater than or equal to
280 `N1+N2-1` where `N1` and `N2` are the two largest values.
281 In case of only one shape given, rounds up to even each dimension value.
283 S = np.array(shapes, dtype=int)
288 commonShape = np.sum(S, axis=0) - 1
291 commonShape[commonShape % 2 != 0] += 1
293 self.log.info(f
"Common frequency space shape {self.freqSpaceShape}")
297 """Zero pad an image where the origin is at the center and replace the
298 origin to the corner as required by the periodic input of FFT. Implement also
299 the inverse operation, crop the padding and re-center data.
304 An array to copy from.
305 newShape : `tuple` of `int`
306 The dimensions of the resulting array. For padding, the resulting array
307 must be larger than A in each dimension. For the inverse operation this
308 must be the original, before padding size of the array.
309 useInverse : bool, optional
310 Selector of forward, add padding, operation (False)
311 or its inverse, crop padding, operation (True).
316 The padded or unpadded array with shape of `newShape` and the same dtype as A.
320 For odd dimensions, the splitting is rounded to
321 put the center pixel into the new corner origin (0,0). This is to be consistent
322 e.g. for a dirac delta kernel that is originally located at the center pixel.
329 firstHalves = [x//2
for x
in A.shape]
330 secondHalves = [x-y
for x, y
in zip(A.shape, firstHalves)]
333 secondHalves = [x//2
for x
in newShape]
334 firstHalves = [x-y
for x, y
in zip(newShape, secondHalves)]
336 R = np.zeros_like(A, shape=newShape)
337 R[-firstHalves[0]:, -firstHalves[1]:] = A[:firstHalves[0], :firstHalves[1]]
338 R[:secondHalves[0], -firstHalves[1]:] = A[-secondHalves[0]:, :firstHalves[1]]
339 R[:secondHalves[0], :secondHalves[1]] = A[-secondHalves[0]:, -secondHalves[1]:]
340 R[-firstHalves[0]:, :secondHalves[1]] = A[:firstHalves[0], -secondHalves[1]:]
344 """Compute the Lupton decorrelation post-convolution kernel for decorrelating an
345 image difference, based on the PSF-matching kernel.
349 kappa : `numpy.ndarray`
350 A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching.
352 Average variance of science image used for PSF matching.
354 Average variance of the template (matched) image used for PSF matching.
355 preConvArr : `numpy.ndarray`, optional
356 If not None, then pre-filtering was applied
357 to science exposure, and this is the pre-convolution kernel.
361 corrft : `numpy.ndarray` of `float`
362 The frequency space representation of the correction. The array is real (dtype float).
363 Shape is `self.freqSpaceShape`.
367 The maximum correction factor converges to `sqrt(tvar/svar)` towards high frequencies.
368 This should be a plausible value.
372 kft = np.fft.fft2(kappa)
373 kftAbsSq = np.real(np.conj(kft) * kft)
375 if preConvArr
is None:
379 preSum = np.sum(preConvArr)
381 preK = np.fft.fft2(preConvArr)
382 preAbsSq = np.real(np.conj(preK)*preK)
384 denom = svar * preAbsSq + tvar * kftAbsSq
387 tiny = np.finfo(kftAbsSq.dtype).tiny * 1000.
391 self.log.warnf(
"Avoid zero division. Skip decorrelation "
392 "at {} divergent frequencies.", sumFlt)
394 kft = np.sqrt((svar * preSum*preSum + tvar * kSum*kSum) / denom)
402 """Compute the (decorrelated) difference image's new PSF.
406 corrft : `numpy.ndarray`
407 The frequency space representation of the correction calculated by
408 `computeCorrection`. Shape must be `self.freqSpaceShape`.
409 psfOld : `numpy.ndarray`
410 The psf of the difference image to be corrected.
414 psfNew : `numpy.ndarray`
415 The corrected psf, same shape as `psfOld`, sum normed to 1.
419 There is no algorithmic guarantee that the corrected psf can
420 meaningfully fit to the same size as the original one.
422 psfShape = psfOld.shape
424 psfNew = np.fft.fft2(psfNew)
426 psfNew = np.fft.ifft2(psfNew)
429 psfNew = psfNew/psfNew.sum()
433 """Compute the decorrelated difference image.
437 corrft : `numpy.ndarray`
438 The frequency space representation of the correction calculated by
439 `computeCorrection`. Shape must be `self.freqSpaceShape`.
440 imgOld : `numpy.ndarray`
441 The difference image to be corrected.
445 imgNew : `numpy.ndarray`
446 The corrected image, same size as the input.
448 expShape = imgOld.shape
449 imgNew = np.copy(imgOld)
450 filtInf = np.isinf(imgNew)
451 filtNan = np.isnan(imgNew)
452 imgNew[filtInf] = np.nan
453 imgNew[filtInf | filtNan] = np.nanmean(imgNew)
455 imgNew = np.fft.fft2(imgNew)
457 imgNew = np.fft.ifft2(imgNew)
460 imgNew[filtNan] = np.nan
461 imgNew[filtInf] = np.inf
466 """Task to be used as an ImageMapper for performing
467 A&L decorrelation on subimages on a grid across a A&L difference image.
469 This task subclasses DecorrelateALKernelTask in order to implement
470 all of that task's configuration parameters, as well as its `run` method.
473 ConfigClass = DecorrelateALKernelConfig
474 _DefaultName =
'ip_diffim_decorrelateALKernelMapper'
477 DecorrelateALKernelTask.__init__(self, *args, **kwargs)
479 def run(self, subExposure, expandedSubExposure, fullBBox,
480 template, science, alTaskResult=None, psfMatchingKernel=None,
481 preConvKernel=None, **kwargs):
482 """Perform decorrelation operation on `subExposure`, using
483 `expandedSubExposure` to allow for invalid edge pixels arising from
486 This method performs A&L decorrelation on `subExposure` using
487 local measures for image variances and PSF. `subExposure` is a
488 sub-exposure of the non-decorrelated A&L diffim. It also
489 requires the corresponding sub-exposures of the template
490 (`template`) and science (`science`) exposures.
494 subExposure : `lsst.afw.image.Exposure`
495 the sub-exposure of the diffim
496 expandedSubExposure : `lsst.afw.image.Exposure`
497 the expanded sub-exposure upon which to operate
498 fullBBox : `lsst.geom.Box2I`
499 the bounding box of the original exposure
500 template : `lsst.afw.image.Exposure`
501 the corresponding sub-exposure of the template exposure
502 science : `lsst.afw.image.Exposure`
503 the corresponding sub-exposure of the science exposure
504 alTaskResult : `lsst.pipe.base.Struct`
505 the result of A&L image differencing on `science` and
506 `template`, importantly containing the resulting
507 `psfMatchingKernel`. Can be `None`, only if
508 `psfMatchingKernel` is not `None`.
509 psfMatchingKernel : Alternative parameter for passing the
510 A&L `psfMatchingKernel` directly.
511 preConvKernel : If not None, then pre-filtering was applied
512 to science exposure, and this is the pre-convolution
515 additional keyword arguments propagated from
516 `ImageMapReduceTask.run`.
520 A `pipeBase.Struct` containing:
522 - ``subExposure`` : the result of the `subExposure` processing.
523 - ``decorrelationKernel`` : the decorrelation kernel, currently
528 This `run` method accepts parameters identical to those of
529 `ImageMapper.run`, since it is called from the
530 `ImageMapperTask`. See that class for more information.
532 templateExposure = template
533 scienceExposure = science
534 if alTaskResult
is None and psfMatchingKernel
is None:
535 raise RuntimeError(
'Both alTaskResult and psfMatchingKernel cannot be None')
536 psfMatchingKernel = alTaskResult.psfMatchingKernel
if alTaskResult
is not None else psfMatchingKernel
540 subExp2 = scienceExposure.Factory(scienceExposure, expandedSubExposure.getBBox())
541 subExp1 = templateExposure.Factory(templateExposure, expandedSubExposure.getBBox())
544 logLevel = self.log.getLevel()
545 self.log.setLevel(lsst.log.WARN)
546 res = DecorrelateALKernelTask.run(self, subExp2, subExp1, expandedSubExposure,
547 psfMatchingKernel, preConvKernel)
548 self.log.setLevel(logLevel)
550 diffim = res.correctedExposure.Factory(res.correctedExposure, subExposure.getBBox())
551 out = pipeBase.Struct(subExposure=diffim, )
556 """Configuration parameters for the ImageMapReduceTask to direct it to use
557 DecorrelateALKernelMapper as its mapper for A&L decorrelation.
559 mapper = pexConfig.ConfigurableField(
560 doc=
'A&L decorrelation task to run on each sub-image',
561 target=DecorrelateALKernelMapper
566 """Configuration parameters for the DecorrelateALKernelSpatialTask.
568 decorrelateConfig = pexConfig.ConfigField(
569 dtype=DecorrelateALKernelConfig,
570 doc=
'DecorrelateALKernel config to use when running on complete exposure (non spatially-varying)',
573 decorrelateMapReduceConfig = pexConfig.ConfigField(
574 dtype=DecorrelateALKernelMapReduceConfig,
575 doc=
'DecorrelateALKernelMapReduce config to use when running on each sub-image (spatially-varying)',
578 ignoreMaskPlanes = pexConfig.ListField(
580 doc=
"""Mask planes to ignore for sigma-clipped statistics""",
581 default=(
"INTRP",
"EDGE",
"DETECTED",
"SAT",
"CR",
"BAD",
"NO_DATA",
"DETECTED_NEGATIVE")
592 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference
597 Pipe-task that removes the neighboring-pixel covariance in an
598 image difference that are added when the template image is
599 convolved with the Alard-Lupton PSF matching kernel.
601 This task is a simple wrapper around @ref DecorrelateALKernelTask,
602 which takes a `spatiallyVarying` parameter in its `run` method. If
603 it is `False`, then it simply calls the `run` method of @ref
604 DecorrelateALKernelTask. If it is True, then it uses the @ref
605 ImageMapReduceTask framework to break the exposures into
606 subExposures on a grid, and performs the `run` method of @ref
607 DecorrelateALKernelTask on each subExposure. This enables it to
608 account for spatially-varying PSFs and noise in the exposures when
609 performing the decorrelation.
611 This task has no standalone example, however it is applied as a
612 subtask of pipe.tasks.imageDifference.ImageDifferenceTask.
613 There is also an example of its use in `tests/testImageDecorrelation.py`.
615 ConfigClass = DecorrelateALKernelSpatialConfig
616 _DefaultName =
"ip_diffim_decorrelateALKernelSpatial"
619 """Create the image decorrelation Task
624 arguments to be passed to
625 `lsst.pipe.base.task.Task.__init__`
627 additional keyword arguments to be passed to
628 `lsst.pipe.base.task.Task.__init__`
630 pipeBase.Task.__init__(self, *args, **kwargs)
635 self.
statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes))
638 """Compute the mean of the variance plane of `exposure`.
641 exposure.getMaskedImage().getMask(),
643 var = statObj.getValue(afwMath.MEANCLIP)
646 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel,
647 spatiallyVarying=True, preConvKernel=None):
648 """Perform decorrelation of an image difference exposure.
650 Decorrelates the diffim due to the convolution of the
651 templateExposure with the A&L psfMatchingKernel. If
652 `spatiallyVarying` is True, it utilizes the spatially varying
653 matching kernel via the `imageMapReduce` framework to perform
654 spatially-varying decorrelation on a grid of subExposures.
658 scienceExposure : `lsst.afw.image.Exposure`
659 the science Exposure used for PSF matching
660 templateExposure : `lsst.afw.image.Exposure`
661 the template Exposure used for PSF matching
662 subtractedExposure : `lsst.afw.image.Exposure`
663 the subtracted Exposure produced by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
665 an (optionally spatially-varying) PSF matching kernel produced
666 by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
667 spatiallyVarying : `bool`
668 if True, perform the spatially-varying operation
669 preConvKernel : `lsst.meas.algorithms.Psf`
670 if not none, the scienceExposure has been pre-filtered with this kernel. (Currently
671 this option is experimental.)
675 results : `lsst.pipe.base.Struct`
676 a structure containing:
678 - ``correctedExposure`` : the decorrelated diffim
681 self.log.info(
'Running A&L decorrelation: spatiallyVarying=%r' % spatiallyVarying)
685 if np.isnan(svar)
or np.isnan(tvar):
687 if (np.all(np.isnan(scienceExposure.image.array))
688 or np.all(np.isnan(templateExposure.image.array))):
689 self.log.warn(
'Template or science image is entirely NaNs: skipping decorrelation.')
698 self.log.info(
"Variance (science, template): (%f, %f)", svar, tvar)
699 self.log.info(
"Variance (uncorrected diffim): %f", var)
700 config = self.config.decorrelateMapReduceConfig
702 results = task.run(subtractedExposure, science=scienceExposure,
703 template=templateExposure, psfMatchingKernel=psfMatchingKernel,
704 preConvKernel=preConvKernel, forceEvenSized=
True)
705 results.correctedExposure = results.exposure
709 return exp.getMaskedImage().getMask()
710 gm(results.correctedExposure)[:, :] = gm(subtractedExposure)
713 self.log.info(
"Variance (corrected diffim): %f", var)
716 config = self.config.decorrelateConfig
718 results = task.run(scienceExposure, templateExposure,
719 subtractedExposure, psfMatchingKernel, preConvKernel=preConvKernel)