1 from __future__
import absolute_import, division, print_function
2 from future
import standard_library
3 standard_library.install_aliases()
29 import lsst.afw.image
as afwImage
30 import lsst.afw.geom
as afwGeom
31 import lsst.meas.algorithms
as measAlg
32 import lsst.afw.math
as afwMath
33 import lsst.pex.config
as pexConfig
34 import lsst.pipe.base
as pipeBase
37 from .imageMapReduce
import (ImageMapReduceConfig, ImageMapReduceTask,
40 __all__ = (
"DecorrelateALKernelTask",
"DecorrelateALKernelConfig",
41 "DecorrelateALKernelMapperSubtask",
"DecorrelateALKernelMapReduceConfig",
42 "DecorrelateALKernelSpatialConfig",
"DecorrelateALKernelSpatialTask")
47 \anchor DecorrelateALKernelConfig_
49 \brief Configuration parameters for the DecorrelateALKernelTask
52 ignoreMaskPlanes = pexConfig.ListField(
54 doc=
"""Mask planes to ignore for sigma-clipped statistics""",
55 default=(
"INTRP",
"EDGE",
"DETECTED",
"SAT",
"CR",
"BAD",
"NO_DATA",
"DETECTED_NEGATIVE")
68 \anchor DecorrelateALKernelTask_
70 \brief Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference
72 \section ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Contents Contents
74 - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Purpose
75 - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Config
76 - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Run
77 - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Debug
78 - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Example
80 \section ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Purpose Description
82 Pipe-task that removes the neighboring-pixel covariance in an
83 image difference that are added when the template image is
84 convolved with the Alard-Lupton PSF matching kernel.
86 The image differencing pipeline task \link
87 ip.diffim.psfMatch.PsfMatchTask PSFMatchTask\endlink and \link
88 ip.diffim.psfMatch.PsfMatchConfigAL PSFMatchConfigAL\endlink uses
89 the Alard and Lupton (1998) method for matching the PSFs of the
90 template and science exposures prior to subtraction. The
91 Alard-Lupton method identifies a matching kernel, which is then
92 (typically) convolved with the template image to perform PSF
93 matching. This convolution has the effect of adding covariance
94 between neighboring pixels in the template image, which is then
95 added to the image difference by subtraction.
97 The pixel covariance may be corrected by whitening the noise of
98 the image difference. This task performs such a decorrelation by
99 computing a decorrelation kernel (based upon the A&L matching
100 kernel and variances in the template and science images) and
101 convolving the image difference with it. This process is described
102 in detail in [DMTN-021](http://dmtn-021.lsst.io).
104 \section ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Initialize Task initialization
106 \copydoc \_\_init\_\_
108 \section ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Run Invoking the Task
112 \section ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Config Configuration parameters
114 See \ref DecorrelateALKernelConfig
116 \section ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Debug Debug variables
118 This task has no debug variables
120 \section ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Example Example of using DecorrelateALKernelTask
122 This task has no standalone example, however it is applied as a
123 subtask of pipe.tasks.imageDifference.ImageDifferenceTask.
125 ConfigClass = DecorrelateALKernelConfig
126 _DefaultName =
"ip_diffim_decorrelateALKernel"
129 """! Create the image decorrelation Task
130 @param *args arguments to be passed to lsst.pipe.base.task.Task.__init__
131 @param **kwargs keyword arguments to be passed to lsst.pipe.base.task.Task.__init__
133 pipeBase.Task.__init__(self, *args, **kwargs)
136 self.statsControl.setNumSigmaClip(3.)
137 self.statsControl.setNumIter(3)
138 self.statsControl.setAndMask(afwImage.MaskU.getPlaneBitMask(self.config.ignoreMaskPlanes))
141 statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(),
142 exposure.getMaskedImage().getMask(),
144 var = statObj.getValue(afwMath.MEANCLIP)
148 def run(self, exposure, templateExposure, subtractedExposure, psfMatchingKernel,
149 xcen=
None, ycen=
None, svar=
None, tvar=
None):
150 """! Perform decorrelation of an image difference exposure.
152 Decorrelates the diffim due to the convolution of the templateExposure with the
153 A&L PSF matching kernel. Currently can accept a spatially varying matching kernel but in
154 this case it simply uses a static kernel from the center of the exposure. The decorrelation
155 is described in [DMTN-021, Equation 1](http://dmtn-021.lsst.io/#equation-1), where
156 `exposure` is I_1; templateExposure is I_2; `subtractedExposure` is D(k);
157 `psfMatchingKernel` is kappa; and svar and tvar are their respective
158 variances (see below).
160 @param[in] exposure the science afwImage.Exposure used for PSF matching
161 @param[in] templateExposure the template afwImage.Exposure used for PSF matching
162 @param[in] subtractedExposure the subtracted exposure produced by
163 `ip_diffim.ImagePsfMatchTask.subtractExposures()`
164 @param[in] psfMatchingKernel an (optionally spatially-varying) PSF matching kernel produced
165 by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
166 @param[in] xcen X-pixel coordinate to use for computing constant matching kernel to use
167 If `None` (default), then use the center of the image.
168 @param[in] ycen Y-pixel coordinate to use for computing constant matching kernel to use
169 If `None` (default), then use the center of the image.
170 @param[in] svar image variance for science image
171 If `None` (default) then compute the variance over the entire input science image.
172 @param[in] tvar image variance for template image
173 If `None` (default) then compute the variance over the entire input template image.
175 @return a `pipeBase.Struct` containing:
176 * `correctedExposure`: the decorrelated diffim
177 * `correctionKernel`: the decorrelation correction kernel (which may be ignored)
179 @note The `subtractedExposure` is NOT updated
180 @note The returned `correctedExposure` has an updated PSF as well.
181 @note Here we currently convert a spatially-varying matching kernel into a constant kernel,
182 just by computing it at the center of the image (tickets DM-6243, DM-6244).
183 @note We are also using a constant accross-the-image measure of sigma (sqrt(variance)) to compute
184 the decorrelation kernel.
185 @note Still TBD (ticket DM-6580): understand whether the convolution is correctly modifying
186 the variance plane of the new subtractedExposure.
188 spatialKernel = psfMatchingKernel
189 kimg = afwImage.ImageD(spatialKernel.getDimensions())
190 bbox = subtractedExposure.getBBox()
192 xcen = (bbox.getBeginX() + bbox.getEndX()) / 2.
194 ycen = (bbox.getBeginY() + bbox.getEndY()) / 2.
195 self.log.info(
"Using matching kernel computed at (%d, %d)", xcen, ycen)
196 spatialKernel.computeImage(kimg,
True, xcen, ycen)
202 self.log.info(
"Variance (science, template): (%f, %f)", svar, tvar)
205 self.log.info(
"Variance (uncorrected diffim): %f", var)
207 corrKernel = DecorrelateALKernelTask._computeDecorrelationKernel(kimg.getArray(), svar, tvar)
208 correctedExposure, corrKern = DecorrelateALKernelTask._doConvolve(subtractedExposure, corrKernel)
211 psf = subtractedExposure.getPsf().computeKernelImage(afwGeom.Point2D(xcen, ycen)).getArray()
212 psfc = DecorrelateALKernelTask.computeCorrectedDiffimPsf(corrKernel, psf, svar=svar, tvar=tvar)
213 psfcI = afwImage.ImageD(psfc.shape[0], psfc.shape[1])
214 psfcI.getArray()[:, :] = psfc
215 psfcK = afwMath.FixedKernel(psfcI)
216 psfNew = measAlg.KernelPsf(psfcK)
217 correctedExposure.setPsf(psfNew)
220 self.log.info(
"Variance (corrected diffim): %f", var)
222 return pipeBase.Struct(correctedExposure=correctedExposure, correctionKernel=corrKern)
226 """! Compute the Lupton/ZOGY post-conv. kernel for decorrelating an
227 image difference, based on the PSF-matching kernel.
228 @param kappa A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching
229 @param svar Average variance of science image used for PSF matching
230 @param tvar Average variance of template image used for PSF matching
231 @return a 2-d numpy.array containing the correction kernel
233 @note As currently implemented, kappa is a static (single, non-spatially-varying) kernel.
235 kappa = DecorrelateALKernelTask._fixOddKernel(kappa)
236 kft = scipy.fftpack.fft2(kappa)
237 kft = np.sqrt((svar + tvar) / (svar + tvar * kft**2))
238 pck = scipy.fftpack.ifft2(kft)
239 pck = scipy.fftpack.ifftshift(pck.real)
240 fkernel = DecorrelateALKernelTask._fixEvenKernel(pck)
245 fkernel = fkernel[::-1, :]
251 """! Compute the (decorrelated) difference image's new PSF.
252 new_psf = psf(k) * sqrt((svar + tvar) / (svar + tvar * kappa_ft(k)**2))
254 @param kappa A matching kernel array derived from Alard & Lupton PSF matching
255 @param psf The uncorrected psf array of the science image (and also of the diffim)
256 @param svar Average variance of science image used for PSF matching
257 @param tvar Average variance of template image used for PSF matching
258 @return a 2-d numpy.array containing the new PSF
260 def post_conv_psf_ft2(psf, kernel, svar, tvar):
263 if psf.shape[0] < kernel.shape[0]:
264 diff = (kernel.shape[0] - psf.shape[0]) // 2
265 psf = np.pad(psf, (diff, diff), mode=
'constant')
266 elif psf.shape[0] > kernel.shape[0]:
267 diff = (psf.shape[0] - kernel.shape[0]) // 2
268 kernel = np.pad(kernel, (diff, diff), mode=
'constant')
269 psf_ft = scipy.fftpack.fft2(psf)
270 kft = scipy.fftpack.fft2(kernel)
271 out = psf_ft * np.sqrt((svar + tvar) / (svar + tvar * kft**2))
274 def post_conv_psf(psf, kernel, svar, tvar):
275 kft = post_conv_psf_ft2(psf, kernel, svar, tvar)
276 out = scipy.fftpack.ifft2(kft)
279 pcf = post_conv_psf(psf=psf, kernel=kappa, svar=svar, tvar=tvar)
280 pcf = pcf.real / pcf.real.sum()
285 """! Take a kernel with odd dimensions and make them even for FFT
287 @param kernel a numpy.array
288 @return a fixed kernel numpy.array. Returns a copy if the dimensions needed to change;
289 otherwise just return the input kernel.
294 if (out.shape[0] % 2) == 1:
295 out = np.pad(out, ((1, 0), (0, 0)), mode=
'constant')
297 if (out.shape[1] % 2) == 1:
298 out = np.pad(out, ((0, 0), (1, 0)), mode=
'constant')
301 out *= (np.mean(kernel) / np.mean(out))
306 """! Take a kernel with even dimensions and make them odd, centered correctly.
307 @param kernel a numpy.array
308 @return a fixed kernel numpy.array
311 maxloc = np.unravel_index(np.argmax(kernel), kernel.shape)
312 out = np.roll(kernel, kernel.shape[0]//2 - maxloc[0], axis=0)
313 out = np.roll(out, out.shape[1]//2 - maxloc[1], axis=1)
315 if (out.shape[0] % 2) == 0:
316 maxloc = np.unravel_index(np.argmax(out), out.shape)
317 if out.shape[0] - maxloc[0] > maxloc[0]:
321 if out.shape[1] - maxloc[1] > maxloc[1]:
329 """! Convolve an Exposure with a decorrelation convolution kernel.
330 @param exposure Input afw.image.Exposure to be convolved.
331 @param kernel Input 2-d numpy.array to convolve the image with
332 @return a new Exposure with the convolved pixels and the (possibly
335 @note We use afwMath.convolve() but keep scipy.convolve for debugging.
336 @note We re-center the kernel if necessary and return the possibly re-centered kernel
338 kernelImg = afwImage.ImageD(kernel.shape[0], kernel.shape[1])
339 kernelImg.getArray()[:, :] = kernel
340 kern = afwMath.FixedKernel(kernelImg)
341 maxloc = np.unravel_index(np.argmax(kernel), kernel.shape)
342 kern.setCtrX(maxloc[0])
343 kern.setCtrY(maxloc[1])
344 outExp = exposure.clone()
345 convCntrl = afwMath.ConvolutionControl(
False,
True, 0)
346 afwMath.convolve(outExp.getMaskedImage(), exposure.getMaskedImage(), kern, convCntrl)
352 """Task to be used as an ImageMapperSubtask for performing
353 A&L decorrelation on subimages on a grid across a A&L difference image.
355 This task subclasses DecorrelateALKernelTask in order to implement
356 all of that task's configuration parameters, as well as its `run` method.
358 ConfigClass = DecorrelateALKernelConfig
359 _DefaultName =
'ip_diffim_decorrelateALKernelMapper'
362 DecorrelateALKernelTask.__init__(self, *args, **kwargs)
364 def run(self, subExposure, expandedSubExposure, fullBBox,
365 template, science, alTaskResult=
None, psfMatchingKernel=
None,
366 preConvKernel=
None, **kwargs):
367 """Perform decorrelation operation on `subExposure`, using
368 `expandedSubExposure` to allow for invalid edge pixels arising from
371 This method performs A&L decorrelation on `subExposure` using
372 local measures for image variances and PSF. `subExposure` is a
373 sub-exposure of the non-decorrelated A&L diffim. It also
374 requires the corresponding sub-exposures of the template
375 (`template`) and science (`science`) exposures.
379 subExposure : lsst.afw.image.Exposure
380 the sub-exposure of the diffim
381 expandedSubExposure : lsst.afw.image.Exposure
382 the expanded sub-exposure upon which to operate
383 fullBBox : afwGeom.BoundingBox
384 the bounding box of the original exposure
385 template : afw.Exposure
386 the corresponding sub-exposure of the template exposure
387 science : afw.Exposure
388 the corresponding sub-exposure of the science exposure
389 alTaskResult : pipeBase.Struct
390 the result of A&L image differencing on `science` and
391 `template`, importantly containing the resulting
392 `psfMatchingKernel`. Can be `None`, only if
393 `psfMatchingKernel` is not `None`.
394 psfMatchingKernel : Alternative parameter for passing the
395 A&L `psfMatchingKernel` directly.
396 preConvKernel : If not None, then pre-filtering was applied
397 to science exposure, and this is the pre-convolution
400 additional keyword arguments propagated from
401 `ImageMapReduceTask.run`.
405 A `pipeBase.Struct` containing:
406 * `subExposure` : the result of the `subExposure` processing.
407 * `decorrelationKernel` : the decorrelation kernel, currently
412 This `run` method accepts parameters identical to those of
413 `ImageMapperSubtask.run`, since it is called from the
414 `ImageMapperTask`. See that class for more information.
416 templateExposure = template
417 scienceExposure = science
418 if alTaskResult
is None and psfMatchingKernel
is None:
419 raise RuntimeError(
'Both alTaskResult and psfMatchingKernel cannot be None')
420 psfMatchingKernel = alTaskResult.psfMatchingKernel
if alTaskResult
is not None else psfMatchingKernel
424 subExp2 = scienceExposure.Factory(scienceExposure, expandedSubExposure.getBBox())
425 subExp1 = templateExposure.Factory(templateExposure, expandedSubExposure.getBBox())
428 logLevel = self.log.getLevel()
429 self.log.setLevel(lsst.log.WARN)
430 res = DecorrelateALKernelTask.run(self, subExp2, subExp1, expandedSubExposure,
432 self.log.setLevel(logLevel)
434 diffim = res.correctedExposure.Factory(res.correctedExposure, subExposure.getBBox())
435 out = pipeBase.Struct(subExposure=diffim, decorrelationKernel=res.correctionKernel)
440 """Configuration parameters for the ImageMapReduceTask to direct it to use
441 DecorrelateALKernelMapperSubtask as its mapperSubtask for A&L decorrelation.
443 mapperSubtask = pexConfig.ConfigurableField(
444 doc=
'A&L decorrelation subtask to run on each sub-image',
445 target=DecorrelateALKernelMapperSubtask
459 """Configuration parameters for the DecorrelateALKernelSpatialTask.
461 decorrelateConfig = pexConfig.ConfigField(
462 dtype=DecorrelateALKernelConfig,
463 doc=
'DecorrelateALKernel config to use when running on complete exposure (non spatially-varying)',
466 decorrelateMapReduceConfig = pexConfig.ConfigField(
467 dtype=DecorrelateALKernelMapReduceConfig,
468 doc=
'DecorrelateALKernelMapReduce config to use when running on each sub-image (spatially-varying)',
471 ignoreMaskPlanes = pexConfig.ListField(
473 doc=
"""Mask planes to ignore for sigma-clipped statistics""",
474 default=(
"INTRP",
"EDGE",
"DETECTED",
"SAT",
"CR",
"BAD",
"NO_DATA",
"DETECTED_NEGATIVE")
478 self.decorrelateMapReduceConfig.gridStepX = self.decorrelateMapReduceConfig.gridStepY = 19
479 self.decorrelateMapReduceConfig.cellSizeX = self.decorrelateMapReduceConfig.cellSizeY = 20
480 self.decorrelateMapReduceConfig.borderSizeX = self.decorrelateMapReduceConfig.borderSizeY = 6
481 self.decorrelateMapReduceConfig.reducerSubtask.reduceOperation =
'average'
486 \anchor DecorrelateALKernelSpatialTask_
488 \brief Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference
490 \section ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Contents Contents
492 - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Purpose
493 - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Config
494 - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Run
495 - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Debug
496 - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Example
498 \section ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Purpose Description
500 Pipe-task that removes the neighboring-pixel covariance in an
501 image difference that are added when the template image is
502 convolved with the Alard-Lupton PSF matching kernel.
504 This task is a simple wrapper around \ref DecorrelateALKernelTask,
505 which takes a `spatiallyVarying` parameter in its `run` method. If
506 it is `False`, then it simply calls the `run` method of \ref
507 DecorrelateALKernelTask. If it is True, then it uses the \ref
508 ImageMapReduceTask framework to break the exposures into
509 subExposures on a grid, and performs the `run` method of \ref
510 DecorrelateALKernelTask on each subExposure. This enables it to
511 account for spatially-varying PSFs and noise in the exposures when
512 performing the decorrelation.
514 \section ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Initialize Task initialization
516 \copydoc \_\_init\_\_
518 \section ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Run Invoking the Task
522 \section ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Config Configuration parameters
524 See \ref DecorrelateALKernelSpatialConfig
526 \section ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Debug Debug variables
528 This task has no debug variables
530 \section ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Example Example of using DecorrelateALKernelSpatialTask
532 This task has no standalone example, however it is applied as a
533 subtask of pipe.tasks.imageDifference.ImageDifferenceTask.
534 There is also an example of its use in `tests/testImageDecorrelation.py`.
536 ConfigClass = DecorrelateALKernelSpatialConfig
537 _DefaultName =
"ip_diffim_decorrelateALKernelSpatial"
540 """Create the image decorrelation Task
545 arguments to be passed to
546 `lsst.pipe.base.task.Task.__init__`
548 additional keyword arguments to be passed to
549 `lsst.pipe.base.task.Task.__init__`
551 pipeBase.Task.__init__(self, *args, **kwargs)
554 self.statsControl.setNumSigmaClip(3.)
555 self.statsControl.setNumIter(3)
556 self.statsControl.setAndMask(afwImage.MaskU.getPlaneBitMask(self.config.ignoreMaskPlanes))
559 """Compute the mean of the variance plane of `exposure`.
561 statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(),
562 exposure.getMaskedImage().getMask(),
564 var = statObj.getValue(afwMath.MEANCLIP)
567 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel,
568 spatiallyVarying=
True, doPreConvolve=
False):
569 """! Perform decorrelation of an image difference exposure.
571 Decorrelates the diffim due to the convolution of the
572 templateExposure with the A&L psfMatchingKernel. If
573 `spatiallyVarying` is True, it utilizes the spatially varying
574 matching kernel via the `imageMapReduce` framework to perform
575 spatially-varying decorrelation on a grid of subExposures.
579 scienceExposure : lsst.afw.image.Exposure
580 the science Exposure used for PSF matching
581 templateExposure : lsst.afw.image.Exposure
582 the template Exposure used for PSF matching
583 subtractedExposure : lsst.afw.image.Exposure
584 the subtracted Exposure produced by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
586 an (optionally spatially-varying) PSF matching kernel produced
587 by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
588 spatiallyVarying : bool
589 if True, perform the spatially-varying operation
591 if True, the scienceExposure has been pre-filtered with its PSF. (Currently
592 this option is experimental.)
596 a `pipeBase.Struct` containing:
597 * `correctedExposure`: the decorrelated diffim
599 self.log.info(
'Running A&L decorrelation: spatiallyVarying=%r' % spatiallyVarying)
607 self.log.info(
"Variance (science, template): (%f, %f)", svar, tvar)
608 self.log.info(
"Variance (uncorrected diffim): %f", var)
609 config = self.config.decorrelateMapReduceConfig
610 task = ImageMapReduceTask(config=config)
611 results = task.run(subtractedExposure, science=scienceExposure,
612 template=templateExposure, psfMatchingKernel=psfMatchingKernel,
613 preConvKernel=
None, forceEvenSized=
True)
614 results.correctedExposure = results.exposure
618 return exp.getMaskedImage().getMask()
619 gm(results.correctedExposure)[:, :] = gm(subtractedExposure)
622 self.log.info(
"Variance (corrected diffim): %f", var)
625 config = self.config.decorrelateConfig
627 results = task.run(scienceExposure, templateExposure,
628 subtractedExposure, psfMatchingKernel)
def _fixOddKernel
Take a kernel with odd dimensions and make them even for FFT.
def run
Perform decorrelation of an image difference exposure.
def computeCorrectedDiffimPsf
Compute the (decorrelated) difference image's new PSF.
def run
Perform decorrelation of an image difference exposure.
Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference.
def _fixEvenKernel
Take a kernel with even dimensions and make them odd, centered correctly.
Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference.
Configuration parameters for the DecorrelateALKernelTask.
def _doConvolve
Convolve an Exposure with a decorrelation convolution kernel.
def _computeDecorrelationKernel
Compute the Lupton/ZOGY post-conv.
def __init__
Create the image decorrelation Task.