1 from __future__
import absolute_import, division, print_function
2 from future
import standard_library
3 standard_library.install_aliases()
29 import lsst.afw.image
as afwImage
30 import lsst.afw.geom
as afwGeom
31 import lsst.meas.algorithms
as measAlg
32 import lsst.afw.math
as afwMath
33 import lsst.pex.config
as pexConfig
34 import lsst.pipe.base
as pipeBase
37 from .imageMapReduce
import (ImageMapReduceConfig, ImageMapReduceTask,
40 __all__ = (
"DecorrelateALKernelTask",
"DecorrelateALKernelConfig",
41 "DecorrelateALKernelMapperSubtask",
"DecorrelateALKernelMapReduceConfig",
42 "DecorrelateALKernelSpatialConfig",
"DecorrelateALKernelSpatialTask")
47 \anchor DecorrelateALKernelConfig_
49 \brief Configuration parameters for the DecorrelateALKernelTask
52 ignoreMaskPlanes = pexConfig.ListField(
54 doc=
"""Mask planes to ignore for sigma-clipped statistics""",
55 default=(
"INTRP",
"EDGE",
"DETECTED",
"SAT",
"CR",
"BAD",
"NO_DATA",
"DETECTED_NEGATIVE")
68 \anchor DecorrelateALKernelTask_
70 \brief Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference
72 \section ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Contents Contents
74 - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Purpose
75 - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Config
76 - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Run
77 - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Debug
78 - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Example
80 \section ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Purpose Description
82 Pipe-task that removes the neighboring-pixel covariance in an
83 image difference that are added when the template image is
84 convolved with the Alard-Lupton PSF matching kernel.
86 The image differencing pipeline task \link
87 ip.diffim.psfMatch.PsfMatchTask PSFMatchTask\endlink and \link
88 ip.diffim.psfMatch.PsfMatchConfigAL PSFMatchConfigAL\endlink uses
89 the Alard and Lupton (1998) method for matching the PSFs of the
90 template and science exposures prior to subtraction. The
91 Alard-Lupton method identifies a matching kernel, which is then
92 (typically) convolved with the template image to perform PSF
93 matching. This convolution has the effect of adding covariance
94 between neighboring pixels in the template image, which is then
95 added to the image difference by subtraction.
97 The pixel covariance may be corrected by whitening the noise of
98 the image difference. This task performs such a decorrelation by
99 computing a decorrelation kernel (based upon the A&L matching
100 kernel and variances in the template and science images) and
101 convolving the image difference with it. This process is described
102 in detail in [DMTN-021](http://dmtn-021.lsst.io).
104 \section ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Initialize Task initialization
106 \copydoc \_\_init\_\_
108 \section ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Run Invoking the Task
112 \section ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Config Configuration parameters
114 See \ref DecorrelateALKernelConfig
116 \section ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Debug Debug variables
118 This task has no debug variables
120 \section ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Example Example of using DecorrelateALKernelTask
122 This task has no standalone example, however it is applied as a
123 subtask of pipe.tasks.imageDifference.ImageDifferenceTask.
125 ConfigClass = DecorrelateALKernelConfig
126 _DefaultName =
"ip_diffim_decorrelateALKernel"
129 """! Create the image decorrelation Task
130 @param *args arguments to be passed to lsst.pipe.base.task.Task.__init__
131 @param **kwargs keyword arguments to be passed to lsst.pipe.base.task.Task.__init__
133 pipeBase.Task.__init__(self, *args, **kwargs)
136 self.statsControl.setNumSigmaClip(3.)
137 self.statsControl.setNumIter(3)
138 self.statsControl.setAndMask(afwImage.Mask\
139 .getPlaneBitMask(self.config.ignoreMaskPlanes))
142 statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(),
143 exposure.getMaskedImage().getMask(),
145 var = statObj.getValue(afwMath.MEANCLIP)
149 def run(self, exposure, templateExposure, subtractedExposure, psfMatchingKernel,
150 xcen=
None, ycen=
None, svar=
None, tvar=
None):
151 """! Perform decorrelation of an image difference exposure.
153 Decorrelates the diffim due to the convolution of the templateExposure with the
154 A&L PSF matching kernel. Currently can accept a spatially varying matching kernel but in
155 this case it simply uses a static kernel from the center of the exposure. The decorrelation
156 is described in [DMTN-021, Equation 1](http://dmtn-021.lsst.io/#equation-1), where
157 `exposure` is I_1; templateExposure is I_2; `subtractedExposure` is D(k);
158 `psfMatchingKernel` is kappa; and svar and tvar are their respective
159 variances (see below).
161 @param[in] exposure the science afwImage.Exposure used for PSF matching
162 @param[in] templateExposure the template afwImage.Exposure used for PSF matching
163 @param[in] subtractedExposure the subtracted exposure produced by
164 `ip_diffim.ImagePsfMatchTask.subtractExposures()`
165 @param[in] psfMatchingKernel an (optionally spatially-varying) PSF matching kernel produced
166 by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
167 @param[in] xcen X-pixel coordinate to use for computing constant matching kernel to use
168 If `None` (default), then use the center of the image.
169 @param[in] ycen Y-pixel coordinate to use for computing constant matching kernel to use
170 If `None` (default), then use the center of the image.
171 @param[in] svar image variance for science image
172 If `None` (default) then compute the variance over the entire input science image.
173 @param[in] tvar image variance for template image
174 If `None` (default) then compute the variance over the entire input template image.
176 @return a `pipeBase.Struct` containing:
177 * `correctedExposure`: the decorrelated diffim
178 * `correctionKernel`: the decorrelation correction kernel (which may be ignored)
180 @note The `subtractedExposure` is NOT updated
181 @note The returned `correctedExposure` has an updated PSF as well.
182 @note Here we currently convert a spatially-varying matching kernel into a constant kernel,
183 just by computing it at the center of the image (tickets DM-6243, DM-6244).
184 @note We are also using a constant accross-the-image measure of sigma (sqrt(variance)) to compute
185 the decorrelation kernel.
186 @note Still TBD (ticket DM-6580): understand whether the convolution is correctly modifying
187 the variance plane of the new subtractedExposure.
189 spatialKernel = psfMatchingKernel
190 kimg = afwImage.ImageD(spatialKernel.getDimensions())
191 bbox = subtractedExposure.getBBox()
193 xcen = (bbox.getBeginX() + bbox.getEndX()) / 2.
195 ycen = (bbox.getBeginY() + bbox.getEndY()) / 2.
196 self.log.info(
"Using matching kernel computed at (%d, %d)", xcen, ycen)
197 spatialKernel.computeImage(kimg,
True, xcen, ycen)
203 self.log.info(
"Variance (science, template): (%f, %f)", svar, tvar)
206 self.log.info(
"Variance (uncorrected diffim): %f", var)
208 corrKernel = DecorrelateALKernelTask._computeDecorrelationKernel(kimg.getArray(), svar, tvar)
209 correctedExposure, corrKern = DecorrelateALKernelTask._doConvolve(subtractedExposure, corrKernel)
212 psf = subtractedExposure.getPsf().computeKernelImage(afwGeom.Point2D(xcen, ycen)).getArray()
213 psfc = DecorrelateALKernelTask.computeCorrectedDiffimPsf(corrKernel, psf, svar=svar, tvar=tvar)
214 psfcI = afwImage.ImageD(psfc.shape[0], psfc.shape[1])
215 psfcI.getArray()[:, :] = psfc
216 psfcK = afwMath.FixedKernel(psfcI)
217 psfNew = measAlg.KernelPsf(psfcK)
218 correctedExposure.setPsf(psfNew)
221 self.log.info(
"Variance (corrected diffim): %f", var)
223 return pipeBase.Struct(correctedExposure=correctedExposure, correctionKernel=corrKern)
227 """! Compute the Lupton/ZOGY post-conv. kernel for decorrelating an
228 image difference, based on the PSF-matching kernel.
229 @param kappa A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching
230 @param svar Average variance of science image used for PSF matching
231 @param tvar Average variance of template image used for PSF matching
232 @return a 2-d numpy.array containing the correction kernel
234 @note As currently implemented, kappa is a static (single, non-spatially-varying) kernel.
236 kappa = DecorrelateALKernelTask._fixOddKernel(kappa)
237 kft = scipy.fftpack.fft2(kappa)
238 kft = np.sqrt((svar + tvar) / (svar + tvar * kft**2))
239 pck = scipy.fftpack.ifft2(kft)
240 pck = scipy.fftpack.ifftshift(pck.real)
241 fkernel = DecorrelateALKernelTask._fixEvenKernel(pck)
246 fkernel = fkernel[::-1, :]
252 """! Compute the (decorrelated) difference image's new PSF.
253 new_psf = psf(k) * sqrt((svar + tvar) / (svar + tvar * kappa_ft(k)**2))
255 @param kappa A matching kernel array derived from Alard & Lupton PSF matching
256 @param psf The uncorrected psf array of the science image (and also of the diffim)
257 @param svar Average variance of science image used for PSF matching
258 @param tvar Average variance of template image used for PSF matching
259 @return a 2-d numpy.array containing the new PSF
261 def post_conv_psf_ft2(psf, kernel, svar, tvar):
264 if psf.shape[0] < kernel.shape[0]:
265 diff = (kernel.shape[0] - psf.shape[0]) // 2
266 psf = np.pad(psf, (diff, diff), mode=
'constant')
267 elif psf.shape[0] > kernel.shape[0]:
268 diff = (psf.shape[0] - kernel.shape[0]) // 2
269 kernel = np.pad(kernel, (diff, diff), mode=
'constant')
270 psf_ft = scipy.fftpack.fft2(psf)
271 kft = scipy.fftpack.fft2(kernel)
272 out = psf_ft * np.sqrt((svar + tvar) / (svar + tvar * kft**2))
275 def post_conv_psf(psf, kernel, svar, tvar):
276 kft = post_conv_psf_ft2(psf, kernel, svar, tvar)
277 out = scipy.fftpack.ifft2(kft)
280 pcf = post_conv_psf(psf=psf, kernel=kappa, svar=svar, tvar=tvar)
281 pcf = pcf.real / pcf.real.sum()
286 """! Take a kernel with odd dimensions and make them even for FFT
288 @param kernel a numpy.array
289 @return a fixed kernel numpy.array. Returns a copy if the dimensions needed to change;
290 otherwise just return the input kernel.
295 if (out.shape[0] % 2) == 1:
296 out = np.pad(out, ((1, 0), (0, 0)), mode=
'constant')
298 if (out.shape[1] % 2) == 1:
299 out = np.pad(out, ((0, 0), (1, 0)), mode=
'constant')
302 out *= (np.mean(kernel) / np.mean(out))
307 """! Take a kernel with even dimensions and make them odd, centered correctly.
308 @param kernel a numpy.array
309 @return a fixed kernel numpy.array
312 maxloc = np.unravel_index(np.argmax(kernel), kernel.shape)
313 out = np.roll(kernel, kernel.shape[0]//2 - maxloc[0], axis=0)
314 out = np.roll(out, out.shape[1]//2 - maxloc[1], axis=1)
316 if (out.shape[0] % 2) == 0:
317 maxloc = np.unravel_index(np.argmax(out), out.shape)
318 if out.shape[0] - maxloc[0] > maxloc[0]:
322 if out.shape[1] - maxloc[1] > maxloc[1]:
330 """! Convolve an Exposure with a decorrelation convolution kernel.
331 @param exposure Input afw.image.Exposure to be convolved.
332 @param kernel Input 2-d numpy.array to convolve the image with
333 @return a new Exposure with the convolved pixels and the (possibly
336 @note We use afwMath.convolve() but keep scipy.convolve for debugging.
337 @note We re-center the kernel if necessary and return the possibly re-centered kernel
339 kernelImg = afwImage.ImageD(kernel.shape[0], kernel.shape[1])
340 kernelImg.getArray()[:, :] = kernel
341 kern = afwMath.FixedKernel(kernelImg)
342 maxloc = np.unravel_index(np.argmax(kernel), kernel.shape)
343 kern.setCtrX(maxloc[0])
344 kern.setCtrY(maxloc[1])
345 outExp = exposure.clone()
346 convCntrl = afwMath.ConvolutionControl(
False,
True, 0)
347 afwMath.convolve(outExp.getMaskedImage(), exposure.getMaskedImage(), kern, convCntrl)
353 """Task to be used as an ImageMapperSubtask for performing
354 A&L decorrelation on subimages on a grid across a A&L difference image.
356 This task subclasses DecorrelateALKernelTask in order to implement
357 all of that task's configuration parameters, as well as its `run` method.
359 ConfigClass = DecorrelateALKernelConfig
360 _DefaultName =
'ip_diffim_decorrelateALKernelMapper'
363 DecorrelateALKernelTask.__init__(self, *args, **kwargs)
365 def run(self, subExposure, expandedSubExposure, fullBBox,
366 template, science, alTaskResult=
None, psfMatchingKernel=
None,
367 preConvKernel=
None, **kwargs):
368 """Perform decorrelation operation on `subExposure`, using
369 `expandedSubExposure` to allow for invalid edge pixels arising from
372 This method performs A&L decorrelation on `subExposure` using
373 local measures for image variances and PSF. `subExposure` is a
374 sub-exposure of the non-decorrelated A&L diffim. It also
375 requires the corresponding sub-exposures of the template
376 (`template`) and science (`science`) exposures.
380 subExposure : lsst.afw.image.Exposure
381 the sub-exposure of the diffim
382 expandedSubExposure : lsst.afw.image.Exposure
383 the expanded sub-exposure upon which to operate
384 fullBBox : afwGeom.BoundingBox
385 the bounding box of the original exposure
386 template : afw.Exposure
387 the corresponding sub-exposure of the template exposure
388 science : afw.Exposure
389 the corresponding sub-exposure of the science exposure
390 alTaskResult : pipeBase.Struct
391 the result of A&L image differencing on `science` and
392 `template`, importantly containing the resulting
393 `psfMatchingKernel`. Can be `None`, only if
394 `psfMatchingKernel` is not `None`.
395 psfMatchingKernel : Alternative parameter for passing the
396 A&L `psfMatchingKernel` directly.
397 preConvKernel : If not None, then pre-filtering was applied
398 to science exposure, and this is the pre-convolution
401 additional keyword arguments propagated from
402 `ImageMapReduceTask.run`.
406 A `pipeBase.Struct` containing:
407 * `subExposure` : the result of the `subExposure` processing.
408 * `decorrelationKernel` : the decorrelation kernel, currently
413 This `run` method accepts parameters identical to those of
414 `ImageMapperSubtask.run`, since it is called from the
415 `ImageMapperTask`. See that class for more information.
417 templateExposure = template
418 scienceExposure = science
419 if alTaskResult
is None and psfMatchingKernel
is None:
420 raise RuntimeError(
'Both alTaskResult and psfMatchingKernel cannot be None')
421 psfMatchingKernel = alTaskResult.psfMatchingKernel
if alTaskResult
is not None else psfMatchingKernel
425 subExp2 = scienceExposure.Factory(scienceExposure, expandedSubExposure.getBBox())
426 subExp1 = templateExposure.Factory(templateExposure, expandedSubExposure.getBBox())
429 logLevel = self.log.getLevel()
430 self.log.setLevel(lsst.log.WARN)
431 res = DecorrelateALKernelTask.run(self, subExp2, subExp1, expandedSubExposure,
433 self.log.setLevel(logLevel)
435 diffim = res.correctedExposure.Factory(res.correctedExposure, subExposure.getBBox())
436 out = pipeBase.Struct(subExposure=diffim, decorrelationKernel=res.correctionKernel)
441 """Configuration parameters for the ImageMapReduceTask to direct it to use
442 DecorrelateALKernelMapperSubtask as its mapperSubtask for A&L decorrelation.
444 mapperSubtask = pexConfig.ConfigurableField(
445 doc=
'A&L decorrelation subtask to run on each sub-image',
446 target=DecorrelateALKernelMapperSubtask
460 """Configuration parameters for the DecorrelateALKernelSpatialTask.
462 decorrelateConfig = pexConfig.ConfigField(
463 dtype=DecorrelateALKernelConfig,
464 doc=
'DecorrelateALKernel config to use when running on complete exposure (non spatially-varying)',
467 decorrelateMapReduceConfig = pexConfig.ConfigField(
468 dtype=DecorrelateALKernelMapReduceConfig,
469 doc=
'DecorrelateALKernelMapReduce config to use when running on each sub-image (spatially-varying)',
472 ignoreMaskPlanes = pexConfig.ListField(
474 doc=
"""Mask planes to ignore for sigma-clipped statistics""",
475 default=(
"INTRP",
"EDGE",
"DETECTED",
"SAT",
"CR",
"BAD",
"NO_DATA",
"DETECTED_NEGATIVE")
479 self.decorrelateMapReduceConfig.gridStepX = self.decorrelateMapReduceConfig.gridStepY = 19
480 self.decorrelateMapReduceConfig.cellSizeX = self.decorrelateMapReduceConfig.cellSizeY = 20
481 self.decorrelateMapReduceConfig.borderSizeX = self.decorrelateMapReduceConfig.borderSizeY = 6
482 self.decorrelateMapReduceConfig.reducerSubtask.reduceOperation =
'average'
487 \anchor DecorrelateALKernelSpatialTask_
489 \brief Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference
491 \section ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Contents Contents
493 - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Purpose
494 - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Config
495 - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Run
496 - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Debug
497 - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Example
499 \section ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Purpose Description
501 Pipe-task that removes the neighboring-pixel covariance in an
502 image difference that are added when the template image is
503 convolved with the Alard-Lupton PSF matching kernel.
505 This task is a simple wrapper around \ref DecorrelateALKernelTask,
506 which takes a `spatiallyVarying` parameter in its `run` method. If
507 it is `False`, then it simply calls the `run` method of \ref
508 DecorrelateALKernelTask. If it is True, then it uses the \ref
509 ImageMapReduceTask framework to break the exposures into
510 subExposures on a grid, and performs the `run` method of \ref
511 DecorrelateALKernelTask on each subExposure. This enables it to
512 account for spatially-varying PSFs and noise in the exposures when
513 performing the decorrelation.
515 \section ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Initialize Task initialization
517 \copydoc \_\_init\_\_
519 \section ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Run Invoking the Task
523 \section ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Config Configuration parameters
525 See \ref DecorrelateALKernelSpatialConfig
527 \section ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Debug Debug variables
529 This task has no debug variables
531 \section ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Example Example of using DecorrelateALKernelSpatialTask
533 This task has no standalone example, however it is applied as a
534 subtask of pipe.tasks.imageDifference.ImageDifferenceTask.
535 There is also an example of its use in `tests/testImageDecorrelation.py`.
537 ConfigClass = DecorrelateALKernelSpatialConfig
538 _DefaultName =
"ip_diffim_decorrelateALKernelSpatial"
541 """Create the image decorrelation Task
546 arguments to be passed to
547 `lsst.pipe.base.task.Task.__init__`
549 additional keyword arguments to be passed to
550 `lsst.pipe.base.task.Task.__init__`
552 pipeBase.Task.__init__(self, *args, **kwargs)
555 self.statsControl.setNumSigmaClip(3.)
556 self.statsControl.setNumIter(3)
557 self.statsControl.setAndMask(afwImage.Mask\
558 .getPlaneBitMask(self.config.ignoreMaskPlanes))
561 """Compute the mean of the variance plane of `exposure`.
563 statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(),
564 exposure.getMaskedImage().getMask(),
566 var = statObj.getValue(afwMath.MEANCLIP)
569 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel,
570 spatiallyVarying=
True, doPreConvolve=
False):
571 """! Perform decorrelation of an image difference exposure.
573 Decorrelates the diffim due to the convolution of the
574 templateExposure with the A&L psfMatchingKernel. If
575 `spatiallyVarying` is True, it utilizes the spatially varying
576 matching kernel via the `imageMapReduce` framework to perform
577 spatially-varying decorrelation on a grid of subExposures.
581 scienceExposure : lsst.afw.image.Exposure
582 the science Exposure used for PSF matching
583 templateExposure : lsst.afw.image.Exposure
584 the template Exposure used for PSF matching
585 subtractedExposure : lsst.afw.image.Exposure
586 the subtracted Exposure produced by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
588 an (optionally spatially-varying) PSF matching kernel produced
589 by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
590 spatiallyVarying : bool
591 if True, perform the spatially-varying operation
593 if True, the scienceExposure has been pre-filtered with its PSF. (Currently
594 this option is experimental.)
598 a `pipeBase.Struct` containing:
599 * `correctedExposure`: the decorrelated diffim
601 self.log.info(
'Running A&L decorrelation: spatiallyVarying=%r' % spatiallyVarying)
609 self.log.info(
"Variance (science, template): (%f, %f)", svar, tvar)
610 self.log.info(
"Variance (uncorrected diffim): %f", var)
611 config = self.config.decorrelateMapReduceConfig
612 task = ImageMapReduceTask(config=config)
613 results = task.run(subtractedExposure, science=scienceExposure,
614 template=templateExposure, psfMatchingKernel=psfMatchingKernel,
615 preConvKernel=
None, forceEvenSized=
True)
616 results.correctedExposure = results.exposure
620 return exp.getMaskedImage().getMask()
621 gm(results.correctedExposure)[:, :] = gm(subtractedExposure)
624 self.log.info(
"Variance (corrected diffim): %f", var)
627 config = self.config.decorrelateConfig
629 results = task.run(scienceExposure, templateExposure,
630 subtractedExposure, psfMatchingKernel)
def _fixOddKernel
Take a kernel with odd dimensions and make them even for FFT.
def run
Perform decorrelation of an image difference exposure.
def computeCorrectedDiffimPsf
Compute the (decorrelated) difference image's new PSF.
def run
Perform decorrelation of an image difference exposure.
Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference.
def _fixEvenKernel
Take a kernel with even dimensions and make them odd, centered correctly.
Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference.
Configuration parameters for the DecorrelateALKernelTask.
def _doConvolve
Convolve an Exposure with a decorrelation convolution kernel.
def _computeDecorrelationKernel
Compute the Lupton/ZOGY post-conv.
def __init__
Create the image decorrelation Task.