1 from __future__
import absolute_import, division, print_function
2 from future
import standard_library
3 standard_library.install_aliases()
29 import lsst.afw.image
as afwImage
30 import lsst.afw.geom
as afwGeom
31 import lsst.meas.algorithms
as measAlg
32 import lsst.afw.math
as afwMath
33 import lsst.pex.config
as pexConfig
34 import lsst.pipe.base
as pipeBase
37 from .imageMapReduce
import (ImageMapReduceConfig, ImageMapReduceTask,
40 __all__ = (
"DecorrelateALKernelTask",
"DecorrelateALKernelConfig",
41 "DecorrelateALKernelMapperSubtask",
"DecorrelateALKernelMapReduceConfig",
42 "DecorrelateALKernelSpatialConfig",
"DecorrelateALKernelSpatialTask")
47 \anchor DecorrelateALKernelConfig_ 49 \brief Configuration parameters for the DecorrelateALKernelTask 52 ignoreMaskPlanes = pexConfig.ListField(
54 doc=
"""Mask planes to ignore for sigma-clipped statistics""",
55 default=(
"INTRP",
"EDGE",
"DETECTED",
"SAT",
"CR",
"BAD",
"NO_DATA",
"DETECTED_NEGATIVE")
68 \anchor DecorrelateALKernelTask_ 70 \brief Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference 72 \section ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Contents Contents 74 - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Purpose 75 - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Config 76 - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Run 77 - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Debug 78 - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Example 80 \section ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Purpose Description 82 Pipe-task that removes the neighboring-pixel covariance in an 83 image difference that are added when the template image is 84 convolved with the Alard-Lupton PSF matching kernel. 86 The image differencing pipeline task \link 87 ip.diffim.psfMatch.PsfMatchTask PSFMatchTask\endlink and \link 88 ip.diffim.psfMatch.PsfMatchConfigAL PSFMatchConfigAL\endlink uses 89 the Alard and Lupton (1998) method for matching the PSFs of the 90 template and science exposures prior to subtraction. The 91 Alard-Lupton method identifies a matching kernel, which is then 92 (typically) convolved with the template image to perform PSF 93 matching. This convolution has the effect of adding covariance 94 between neighboring pixels in the template image, which is then 95 added to the image difference by subtraction. 97 The pixel covariance may be corrected by whitening the noise of 98 the image difference. This task performs such a decorrelation by 99 computing a decorrelation kernel (based upon the A&L matching 100 kernel and variances in the template and science images) and 101 convolving the image difference with it. This process is described 102 in detail in [DMTN-021](http://dmtn-021.lsst.io). 104 \section ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Initialize Task initialization 106 \copydoc \_\_init\_\_ 108 \section ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Run Invoking the Task 112 \section ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Config Configuration parameters 114 See \ref DecorrelateALKernelConfig 116 \section ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Debug Debug variables 118 This task has no debug variables 120 \section ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Example Example of using DecorrelateALKernelTask 122 This task has no standalone example, however it is applied as a 123 subtask of pipe.tasks.imageDifference.ImageDifferenceTask. 125 ConfigClass = DecorrelateALKernelConfig
126 _DefaultName =
"ip_diffim_decorrelateALKernel" 129 """! Create the image decorrelation Task 130 @param *args arguments to be passed to lsst.pipe.base.task.Task.__init__ 131 @param **kwargs keyword arguments to be passed to lsst.pipe.base.task.Task.__init__ 133 pipeBase.Task.__init__(self, *args, **kwargs)
139 .getPlaneBitMask(self.config.ignoreMaskPlanes))
142 statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(),
143 exposure.getMaskedImage().getMask(),
145 var = statObj.getValue(afwMath.MEANCLIP)
149 def run(self, exposure, templateExposure, subtractedExposure, psfMatchingKernel,
150 xcen=None, ycen=None, svar=None, tvar=None):
151 """! Perform decorrelation of an image difference exposure. 153 Decorrelates the diffim due to the convolution of the templateExposure with the 154 A&L PSF matching kernel. Currently can accept a spatially varying matching kernel but in 155 this case it simply uses a static kernel from the center of the exposure. The decorrelation 156 is described in [DMTN-021, Equation 1](http://dmtn-021.lsst.io/#equation-1), where 157 `exposure` is I_1; templateExposure is I_2; `subtractedExposure` is D(k); 158 `psfMatchingKernel` is kappa; and svar and tvar are their respective 159 variances (see below). 161 @param[in] exposure the science afwImage.Exposure used for PSF matching 162 @param[in] templateExposure the template afwImage.Exposure used for PSF matching 163 @param[in] subtractedExposure the subtracted exposure produced by 164 `ip_diffim.ImagePsfMatchTask.subtractExposures()` 165 @param[in] psfMatchingKernel an (optionally spatially-varying) PSF matching kernel produced 166 by `ip_diffim.ImagePsfMatchTask.subtractExposures()` 167 @param[in] xcen X-pixel coordinate to use for computing constant matching kernel to use 168 If `None` (default), then use the center of the image. 169 @param[in] ycen Y-pixel coordinate to use for computing constant matching kernel to use 170 If `None` (default), then use the center of the image. 171 @param[in] svar image variance for science image 172 If `None` (default) then compute the variance over the entire input science image. 173 @param[in] tvar image variance for template image 174 If `None` (default) then compute the variance over the entire input template image. 176 @return a `pipeBase.Struct` containing: 177 * `correctedExposure`: the decorrelated diffim 178 * `correctionKernel`: the decorrelation correction kernel (which may be ignored) 180 @note The `subtractedExposure` is NOT updated 181 @note The returned `correctedExposure` has an updated PSF as well. 182 @note Here we currently convert a spatially-varying matching kernel into a constant kernel, 183 just by computing it at the center of the image (tickets DM-6243, DM-6244). 184 @note We are also using a constant accross-the-image measure of sigma (sqrt(variance)) to compute 185 the decorrelation kernel. 186 @note Still TBD (ticket DM-6580): understand whether the convolution is correctly modifying 187 the variance plane of the new subtractedExposure. 189 spatialKernel = psfMatchingKernel
190 kimg = afwImage.ImageD(spatialKernel.getDimensions())
191 bbox = subtractedExposure.getBBox()
193 xcen = (bbox.getBeginX() + bbox.getEndX()) / 2.
195 ycen = (bbox.getBeginY() + bbox.getEndY()) / 2.
196 self.log.info(
"Using matching kernel computed at (%d, %d)", xcen, ycen)
197 spatialKernel.computeImage(kimg,
True, xcen, ycen)
203 self.log.info(
"Variance (science, template): (%f, %f)", svar, tvar)
206 self.log.info(
"Variance (uncorrected diffim): %f", var)
208 corrKernel = DecorrelateALKernelTask._computeDecorrelationKernel(kimg.getArray(), svar, tvar)
209 correctedExposure, corrKern = DecorrelateALKernelTask._doConvolve(subtractedExposure, corrKernel)
212 psf = subtractedExposure.getPsf().computeKernelImage(afwGeom.Point2D(xcen, ycen)).getArray()
213 psfc = DecorrelateALKernelTask.computeCorrectedDiffimPsf(corrKernel, psf, svar=svar, tvar=tvar)
214 psfcI = afwImage.ImageD(psfc.shape[0], psfc.shape[1])
215 psfcI.getArray()[:, :] = psfc
216 psfcK = afwMath.FixedKernel(psfcI)
217 psfNew = measAlg.KernelPsf(psfcK)
218 correctedExposure.setPsf(psfNew)
221 self.log.info(
"Variance (corrected diffim): %f", var)
223 return pipeBase.Struct(correctedExposure=correctedExposure, correctionKernel=corrKern)
227 """! Compute the Lupton/ZOGY post-conv. kernel for decorrelating an 228 image difference, based on the PSF-matching kernel. 229 @param kappa A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching 230 @param svar Average variance of science image used for PSF matching 231 @param tvar Average variance of template image used for PSF matching 232 @return a 2-d numpy.array containing the correction kernel 234 @note As currently implemented, kappa is a static (single, non-spatially-varying) kernel. 236 kappa = DecorrelateALKernelTask._fixOddKernel(kappa)
237 kft = scipy.fftpack.fft2(kappa)
238 kft = np.sqrt((svar + tvar) / (svar + tvar * kft**2))
239 pck = scipy.fftpack.ifft2(kft)
240 pck = scipy.fftpack.ifftshift(pck.real)
241 fkernel = DecorrelateALKernelTask._fixEvenKernel(pck)
246 fkernel = fkernel[::-1, :]
252 """! Compute the (decorrelated) difference image's new PSF. 253 new_psf = psf(k) * sqrt((svar + tvar) / (svar + tvar * kappa_ft(k)**2)) 255 @param kappa A matching kernel array derived from Alard & Lupton PSF matching 256 @param psf The uncorrected psf array of the science image (and also of the diffim) 257 @param svar Average variance of science image used for PSF matching 258 @param tvar Average variance of template image used for PSF matching 259 @return a 2-d numpy.array containing the new PSF 261 def post_conv_psf_ft2(psf, kernel, svar, tvar):
264 if psf.shape[0] < kernel.shape[0]:
265 diff = (kernel.shape[0] - psf.shape[0]) // 2
266 psf = np.pad(psf, (diff, diff), mode=
'constant')
267 elif psf.shape[0] > kernel.shape[0]:
268 diff = (psf.shape[0] - kernel.shape[0]) // 2
269 kernel = np.pad(kernel, (diff, diff), mode=
'constant')
270 psf_ft = scipy.fftpack.fft2(psf)
271 kft = scipy.fftpack.fft2(kernel)
272 out = psf_ft * np.sqrt((svar + tvar) / (svar + tvar * kft**2))
275 def post_conv_psf(psf, kernel, svar, tvar):
276 kft = post_conv_psf_ft2(psf, kernel, svar, tvar)
277 out = scipy.fftpack.ifft2(kft)
280 pcf = post_conv_psf(psf=psf, kernel=kappa, svar=svar, tvar=tvar)
281 pcf = pcf.real / pcf.real.sum()
286 """! Take a kernel with odd dimensions and make them even for FFT 288 @param kernel a numpy.array 289 @return a fixed kernel numpy.array. Returns a copy if the dimensions needed to change; 290 otherwise just return the input kernel. 295 if (out.shape[0] % 2) == 1:
296 out = np.pad(out, ((1, 0), (0, 0)), mode=
'constant')
298 if (out.shape[1] % 2) == 1:
299 out = np.pad(out, ((0, 0), (1, 0)), mode=
'constant')
302 out *= (np.mean(kernel) / np.mean(out))
307 """! Take a kernel with even dimensions and make them odd, centered correctly. 308 @param kernel a numpy.array 309 @return a fixed kernel numpy.array 312 maxloc = np.unravel_index(np.argmax(kernel), kernel.shape)
313 out = np.roll(kernel, kernel.shape[0]//2 - maxloc[0], axis=0)
314 out = np.roll(out, out.shape[1]//2 - maxloc[1], axis=1)
316 if (out.shape[0] % 2) == 0:
317 maxloc = np.unravel_index(np.argmax(out), out.shape)
318 if out.shape[0] - maxloc[0] > maxloc[0]:
322 if out.shape[1] - maxloc[1] > maxloc[1]:
330 """! Convolve an Exposure with a decorrelation convolution kernel. 331 @param exposure Input afw.image.Exposure to be convolved. 332 @param kernel Input 2-d numpy.array to convolve the image with 333 @return a new Exposure with the convolved pixels and the (possibly 336 @note We use afwMath.convolve() but keep scipy.convolve for debugging. 337 @note We re-center the kernel if necessary and return the possibly re-centered kernel 339 kernelImg = afwImage.ImageD(kernel.shape[0], kernel.shape[1])
340 kernelImg.getArray()[:, :] = kernel
341 kern = afwMath.FixedKernel(kernelImg)
342 maxloc = np.unravel_index(np.argmax(kernel), kernel.shape)
343 kern.setCtrX(maxloc[0])
344 kern.setCtrY(maxloc[1])
345 outExp = exposure.clone()
346 convCntrl = afwMath.ConvolutionControl(
False,
True, 0)
347 afwMath.convolve(outExp.getMaskedImage(), exposure.getMaskedImage(), kern, convCntrl)
353 """Task to be used as an ImageMapperSubtask for performing 354 A&L decorrelation on subimages on a grid across a A&L difference image. 356 This task subclasses DecorrelateALKernelTask in order to implement 357 all of that task's configuration parameters, as well as its `run` method. 359 ConfigClass = DecorrelateALKernelConfig
360 _DefaultName =
'ip_diffim_decorrelateALKernelMapper' 363 DecorrelateALKernelTask.__init__(self, *args, **kwargs)
365 def run(self, subExposure, expandedSubExposure, fullBBox,
366 template, science, alTaskResult=None, psfMatchingKernel=None,
367 preConvKernel=None, **kwargs):
368 """Perform decorrelation operation on `subExposure`, using 369 `expandedSubExposure` to allow for invalid edge pixels arising from 372 This method performs A&L decorrelation on `subExposure` using 373 local measures for image variances and PSF. `subExposure` is a 374 sub-exposure of the non-decorrelated A&L diffim. It also 375 requires the corresponding sub-exposures of the template 376 (`template`) and science (`science`) exposures. 380 subExposure : lsst.afw.image.Exposure 381 the sub-exposure of the diffim 382 expandedSubExposure : lsst.afw.image.Exposure 383 the expanded sub-exposure upon which to operate 384 fullBBox : afwGeom.BoundingBox 385 the bounding box of the original exposure 386 template : afw.Exposure 387 the corresponding sub-exposure of the template exposure 388 science : afw.Exposure 389 the corresponding sub-exposure of the science exposure 390 alTaskResult : pipeBase.Struct 391 the result of A&L image differencing on `science` and 392 `template`, importantly containing the resulting 393 `psfMatchingKernel`. Can be `None`, only if 394 `psfMatchingKernel` is not `None`. 395 psfMatchingKernel : Alternative parameter for passing the 396 A&L `psfMatchingKernel` directly. 397 preConvKernel : If not None, then pre-filtering was applied 398 to science exposure, and this is the pre-convolution 401 additional keyword arguments propagated from 402 `ImageMapReduceTask.run`. 406 A `pipeBase.Struct` containing: 407 * `subExposure` : the result of the `subExposure` processing. 408 * `decorrelationKernel` : the decorrelation kernel, currently 413 This `run` method accepts parameters identical to those of 414 `ImageMapperSubtask.run`, since it is called from the 415 `ImageMapperTask`. See that class for more information. 417 templateExposure = template
418 scienceExposure = science
419 if alTaskResult
is None and psfMatchingKernel
is None:
420 raise RuntimeError(
'Both alTaskResult and psfMatchingKernel cannot be None')
421 psfMatchingKernel = alTaskResult.psfMatchingKernel
if alTaskResult
is not None else psfMatchingKernel
425 subExp2 = scienceExposure.Factory(scienceExposure, expandedSubExposure.getBBox())
426 subExp1 = templateExposure.Factory(templateExposure, expandedSubExposure.getBBox())
429 logLevel = self.log.getLevel()
430 self.log.setLevel(lsst.log.WARN)
431 res = DecorrelateALKernelTask.run(self, subExp2, subExp1, expandedSubExposure,
433 self.log.setLevel(logLevel)
435 diffim = res.correctedExposure.Factory(res.correctedExposure, subExposure.getBBox())
436 out = pipeBase.Struct(subExposure=diffim, decorrelationKernel=res.correctionKernel)
441 """Configuration parameters for the ImageMapReduceTask to direct it to use 442 DecorrelateALKernelMapperSubtask as its mapperSubtask for A&L decorrelation. 444 mapperSubtask = pexConfig.ConfigurableField(
445 doc=
'A&L decorrelation subtask to run on each sub-image',
446 target=DecorrelateALKernelMapperSubtask
460 """Configuration parameters for the DecorrelateALKernelSpatialTask. 462 decorrelateConfig = pexConfig.ConfigField(
463 dtype=DecorrelateALKernelConfig,
464 doc=
'DecorrelateALKernel config to use when running on complete exposure (non spatially-varying)',
467 decorrelateMapReduceConfig = pexConfig.ConfigField(
468 dtype=DecorrelateALKernelMapReduceConfig,
469 doc=
'DecorrelateALKernelMapReduce config to use when running on each sub-image (spatially-varying)',
472 ignoreMaskPlanes = pexConfig.ListField(
474 doc=
"""Mask planes to ignore for sigma-clipped statistics""",
475 default=(
"INTRP",
"EDGE",
"DETECTED",
"SAT",
"CR",
"BAD",
"NO_DATA",
"DETECTED_NEGATIVE")
487 \anchor DecorrelateALKernelSpatialTask_ 489 \brief Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference 491 \section ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Contents Contents 493 - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Purpose 494 - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Config 495 - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Run 496 - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Debug 497 - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Example 499 \section ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Purpose Description 501 Pipe-task that removes the neighboring-pixel covariance in an 502 image difference that are added when the template image is 503 convolved with the Alard-Lupton PSF matching kernel. 505 This task is a simple wrapper around \ref DecorrelateALKernelTask, 506 which takes a `spatiallyVarying` parameter in its `run` method. If 507 it is `False`, then it simply calls the `run` method of \ref 508 DecorrelateALKernelTask. If it is True, then it uses the \ref 509 ImageMapReduceTask framework to break the exposures into 510 subExposures on a grid, and performs the `run` method of \ref 511 DecorrelateALKernelTask on each subExposure. This enables it to 512 account for spatially-varying PSFs and noise in the exposures when 513 performing the decorrelation. 515 \section ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Initialize Task initialization 517 \copydoc \_\_init\_\_ 519 \section ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Run Invoking the Task 523 \section ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Config Configuration parameters 525 See \ref DecorrelateALKernelSpatialConfig 527 \section ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Debug Debug variables 529 This task has no debug variables 531 \section ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Example Example of using DecorrelateALKernelSpatialTask 533 This task has no standalone example, however it is applied as a 534 subtask of pipe.tasks.imageDifference.ImageDifferenceTask. 535 There is also an example of its use in `tests/testImageDecorrelation.py`. 537 ConfigClass = DecorrelateALKernelSpatialConfig
538 _DefaultName =
"ip_diffim_decorrelateALKernelSpatial" 541 """Create the image decorrelation Task 546 arguments to be passed to 547 `lsst.pipe.base.task.Task.__init__` 549 additional keyword arguments to be passed to 550 `lsst.pipe.base.task.Task.__init__` 552 pipeBase.Task.__init__(self, *args, **kwargs)
558 .getPlaneBitMask(self.config.ignoreMaskPlanes))
561 """Compute the mean of the variance plane of `exposure`. 563 statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(),
564 exposure.getMaskedImage().getMask(),
566 var = statObj.getValue(afwMath.MEANCLIP)
569 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel,
570 spatiallyVarying=True, doPreConvolve=False):
571 """! Perform decorrelation of an image difference exposure. 573 Decorrelates the diffim due to the convolution of the 574 templateExposure with the A&L psfMatchingKernel. If 575 `spatiallyVarying` is True, it utilizes the spatially varying 576 matching kernel via the `imageMapReduce` framework to perform 577 spatially-varying decorrelation on a grid of subExposures. 581 scienceExposure : lsst.afw.image.Exposure 582 the science Exposure used for PSF matching 583 templateExposure : lsst.afw.image.Exposure 584 the template Exposure used for PSF matching 585 subtractedExposure : lsst.afw.image.Exposure 586 the subtracted Exposure produced by `ip_diffim.ImagePsfMatchTask.subtractExposures()` 588 an (optionally spatially-varying) PSF matching kernel produced 589 by `ip_diffim.ImagePsfMatchTask.subtractExposures()` 590 spatiallyVarying : bool 591 if True, perform the spatially-varying operation 593 if True, the scienceExposure has been pre-filtered with its PSF. (Currently 594 this option is experimental.) 598 a `pipeBase.Struct` containing: 599 * `correctedExposure`: the decorrelated diffim 601 self.log.info(
'Running A&L decorrelation: spatiallyVarying=%r' % spatiallyVarying)
609 self.log.info(
"Variance (science, template): (%f, %f)", svar, tvar)
610 self.log.info(
"Variance (uncorrected diffim): %f", var)
611 config = self.config.decorrelateMapReduceConfig
613 results = task.run(subtractedExposure, science=scienceExposure,
614 template=templateExposure, psfMatchingKernel=psfMatchingKernel,
615 preConvKernel=
None, forceEvenSized=
True)
616 results.correctedExposure = results.exposure
620 return exp.getMaskedImage().getMask()
621 gm(results.correctedExposure)[:, :] = gm(subtractedExposure)
624 self.log.info(
"Variance (corrected diffim): %f", var)
627 config = self.config.decorrelateConfig
629 results = task.run(scienceExposure, templateExposure,
630 subtractedExposure, psfMatchingKernel)
def __init__(self, args, kwargs)
def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel, spatiallyVarying=True, doPreConvolve=False)
Perform decorrelation of an image difference exposure.
Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference.
def _fixEvenKernel(kernel)
Take a kernel with even dimensions and make them odd, centered correctly.
def computeVarianceMean(self, exposure)
def _fixOddKernel(kernel)
Take a kernel with odd dimensions and make them even for FFT.
def computeCorrectedDiffimPsf(kappa, psf, svar=0.04, tvar=0.04)
Compute the (decorrelated) difference image's new PSF.
def _computeDecorrelationKernel(kappa, svar=0.04, tvar=0.04)
Compute the Lupton/ZOGY post-conv.
def __init__(self, args, kwargs)
Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference.
def __init__(self, args, kwargs)
Create the image decorrelation Task.
decorrelateMapReduceConfig
def computeVarianceMean(self, exposure)
def run(self, subExposure, expandedSubExposure, fullBBox, template, science, alTaskResult=None, psfMatchingKernel=None, preConvKernel=None, kwargs)
Configuration parameters for the DecorrelateALKernelTask.
def _doConvolve(exposure, kernel)
Convolve an Exposure with a decorrelation convolution kernel.
def run(self, exposure, templateExposure, subtractedExposure, psfMatchingKernel, xcen=None, ycen=None, svar=None, tvar=None)
Perform decorrelation of an image difference exposure.