29 import lsst.meas.algorithms
as measAlg
31 import lsst.pipe.base
as pipeBase
34 from .imageMapReduce
import (ImageMapReduceConfig, ImageMapReduceTask,
37 __all__ = (
"DecorrelateALKernelTask",
"DecorrelateALKernelConfig",
38 "DecorrelateALKernelMapper",
"DecorrelateALKernelMapReduceConfig",
39 "DecorrelateALKernelSpatialConfig",
"DecorrelateALKernelSpatialTask")
43 """Configuration parameters for the DecorrelateALKernelTask 46 ignoreMaskPlanes = pexConfig.ListField(
48 doc=
"""Mask planes to ignore for sigma-clipped statistics""",
49 default=(
"INTRP",
"EDGE",
"DETECTED",
"SAT",
"CR",
"BAD",
"NO_DATA",
"DETECTED_NEGATIVE")
54 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference 59 Pipe-task that removes the neighboring-pixel covariance in an 60 image difference that are added when the template image is 61 convolved with the Alard-Lupton PSF matching kernel. 63 The image differencing pipeline task @link 64 ip.diffim.psfMatch.PsfMatchTask PSFMatchTask@endlink and @link 65 ip.diffim.psfMatch.PsfMatchConfigAL PSFMatchConfigAL@endlink uses 66 the Alard and Lupton (1998) method for matching the PSFs of the 67 template and science exposures prior to subtraction. The 68 Alard-Lupton method identifies a matching kernel, which is then 69 (typically) convolved with the template image to perform PSF 70 matching. This convolution has the effect of adding covariance 71 between neighboring pixels in the template image, which is then 72 added to the image difference by subtraction. 74 The pixel covariance may be corrected by whitening the noise of 75 the image difference. This task performs such a decorrelation by 76 computing a decorrelation kernel (based upon the A&L matching 77 kernel and variances in the template and science images) and 78 convolving the image difference with it. This process is described 79 in detail in [DMTN-021](http://dmtn-021.lsst.io). 81 This task has no standalone example, however it is applied as a 82 subtask of pipe.tasks.imageDifference.ImageDifferenceTask. 84 ConfigClass = DecorrelateALKernelConfig
85 _DefaultName =
"ip_diffim_decorrelateALKernel" 88 """Create the image decorrelation Task 93 arguments to be passed to ``lsst.pipe.base.task.Task.__init__`` 95 keyword arguments to be passed to ``lsst.pipe.base.task.Task.__init__`` 97 pipeBase.Task.__init__(self, *args, **kwargs)
102 self.
statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes))
106 exposure.getMaskedImage().getMask(),
108 var = statObj.getValue(afwMath.MEANCLIP)
112 def run(self, exposure, templateExposure, subtractedExposure, psfMatchingKernel,
113 preConvKernel=None, xcen=None, ycen=None, svar=None, tvar=None):
114 """Perform decorrelation of an image difference exposure. 116 Decorrelates the diffim due to the convolution of the templateExposure with the 117 A&L PSF matching kernel. Currently can accept a spatially varying matching kernel but in 118 this case it simply uses a static kernel from the center of the exposure. The decorrelation 119 is described in [DMTN-021, Equation 1](http://dmtn-021.lsst.io/#equation-1), where 120 `exposure` is I_1; templateExposure is I_2; `subtractedExposure` is D(k); 121 `psfMatchingKernel` is kappa; and svar and tvar are their respective 122 variances (see below). 126 exposure : `lsst.afw.image.Exposure` 127 The science afwImage.Exposure used for PSF matching 128 templateExposure : `lsst.afw.image.Exposure` 129 The template exposure used for PSF matching 131 the subtracted exposure produced by 132 `ip_diffim.ImagePsfMatchTask.subtractExposures()` 134 An (optionally spatially-varying) PSF matching kernel produced 135 by `ip_diffim.ImagePsfMatchTask.subtractExposures()` 137 if not None, then the `exposure` was pre-convolved with this kernel 138 xcen : `float`, optional 139 X-pixel coordinate to use for computing constant matching kernel to use 140 If `None` (default), then use the center of the image. 141 ycen : `float`, optional 142 Y-pixel coordinate to use for computing constant matching kernel to use 143 If `None` (default), then use the center of the image. 144 svar : `float`, optional 145 image variance for science image 146 If `None` (default) then compute the variance over the entire input science image. 147 tvar : `float`, optional 148 Image variance for template image 149 If `None` (default) then compute the variance over the entire input template image. 154 a `lsst.pipe.base.Struct` containing: 156 - ``correctedExposure`` : the decorrelated diffim 157 - ``correctionKernel`` : the decorrelation correction kernel (which may be ignored) 161 The `subtractedExposure` is NOT updated 163 The returned `correctedExposure` has an updated PSF as well. 165 Here we currently convert a spatially-varying matching kernel into a constant kernel, 166 just by computing it at the center of the image (tickets DM-6243, DM-6244). 168 We are also using a constant accross-the-image measure of sigma (sqrt(variance)) to compute 169 the decorrelation kernel. 171 Still TBD (ticket DM-6580): understand whether the convolution is correctly modifying 172 the variance plane of the new subtractedExposure. 174 spatialKernel = psfMatchingKernel
175 kimg = afwImage.ImageD(spatialKernel.getDimensions())
176 bbox = subtractedExposure.getBBox()
178 xcen = (bbox.getBeginX() + bbox.getEndX()) / 2.
180 ycen = (bbox.getBeginY() + bbox.getEndY()) / 2.
181 self.log.info(
"Using matching kernel computed at (%d, %d)", xcen, ycen)
182 spatialKernel.computeImage(kimg,
True, xcen, ycen)
188 self.log.info(
"Variance (science, template): (%f, %f)", svar, tvar)
193 if np.isnan(svar)
or np.isnan(tvar):
195 if (np.all(np.isnan(exposure.getMaskedImage().getImage().getArray()))
or 196 np.all(np.isnan(templateExposure.getMaskedImage().getImage().getArray()))):
197 self.log.warn(
'Template or science image is entirely NaNs: skipping decorrelation.')
198 outExposure = subtractedExposure.clone()
199 return pipeBase.Struct(correctedExposure=outExposure, correctionKernel=
None)
202 self.log.info(
"Variance (uncorrected diffim): %f", var)
205 if preConvKernel
is not None:
206 self.log.info(
'Using a pre-convolution kernel as part of decorrelation.')
207 kimg2 = afwImage.ImageD(preConvKernel.getDimensions())
208 preConvKernel.computeImage(kimg2,
False)
209 pck = kimg2.getArray()
210 corrKernel = DecorrelateALKernelTask._computeDecorrelationKernel(kimg.getArray(), svar, tvar,
212 correctedExposure, corrKern = DecorrelateALKernelTask._doConvolve(subtractedExposure, corrKernel)
215 psf = subtractedExposure.getPsf().computeKernelImage(
geom.Point2D(xcen, ycen)).getArray()
216 psfc = DecorrelateALKernelTask.computeCorrectedDiffimPsf(corrKernel, psf, svar=svar, tvar=tvar)
217 psfcI = afwImage.ImageD(psfc.shape[0], psfc.shape[1])
218 psfcI.getArray()[:, :] = psfc
220 psfNew = measAlg.KernelPsf(psfcK)
221 correctedExposure.setPsf(psfNew)
224 self.log.info(
"Variance (corrected diffim): %f", var)
226 return pipeBase.Struct(correctedExposure=correctedExposure, correctionKernel=corrKern)
229 def _computeDecorrelationKernel(kappa, svar=0.04, tvar=0.04, preConvKernel=None):
230 """Compute the Lupton decorrelation post-conv. kernel for decorrelating an 231 image difference, based on the PSF-matching kernel. 235 kappa : `numpy.ndarray` 236 A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching 237 svar : `float`, optional 238 Average variance of science image used for PSF matching 239 tvar : `float`, optional 240 Average variance of template image used for PSF matching 241 preConvKernel If not None, then pre-filtering was applied 242 to science exposure, and this is the pre-convolution kernel. 246 fkernel : `numpy.ndarray` 247 a 2-d numpy.array containing the correction kernel 251 As currently implemented, kappa is a static (single, non-spatially-varying) kernel. 256 kappa = DecorrelateALKernelTask._fixOddKernel(kappa)
257 if preConvKernel
is not None:
258 mk = DecorrelateALKernelTask._fixOddKernel(preConvKernel)
260 if kappa.shape[0] < mk.shape[0]:
261 diff = (mk.shape[0] - kappa.shape[0]) // 2
262 kappa = np.pad(kappa, (diff, diff), mode=
'constant')
263 elif kappa.shape[0] > mk.shape[0]:
264 diff = (kappa.shape[0] - mk.shape[0]) // 2
265 mk = np.pad(mk, (diff, diff), mode=
'constant')
267 kft = np.fft.fft2(kappa)
268 kft2 = np.conj(kft) * kft
269 kft2[np.abs(kft2) < MIN_KERNEL] = MIN_KERNEL
270 denom = svar + tvar * kft2
271 if preConvKernel
is not None:
273 mk2 = np.conj(mk) * mk
274 mk2[np.abs(mk2) < MIN_KERNEL] = MIN_KERNEL
275 denom = svar * mk2 + tvar * kft2
276 denom[np.abs(denom) < MIN_KERNEL] = MIN_KERNEL
277 kft = np.sqrt((svar + tvar) / denom)
278 pck = np.fft.ifft2(kft)
279 pck = np.fft.ifftshift(pck.real)
280 fkernel = DecorrelateALKernelTask._fixEvenKernel(pck)
281 if preConvKernel
is not None:
284 fkernel[fkernel > -np.min(fkernel)] = -np.min(fkernel)
289 fkernel = fkernel[::-1, :]
295 """Compute the (decorrelated) difference image's new PSF. 296 new_psf = psf(k) * sqrt((svar + tvar) / (svar + tvar * kappa_ft(k)**2)) 300 kappa : `numpy.ndarray` 301 A matching kernel array derived from Alard & Lupton PSF matching 302 psf : `numpy.ndarray` 303 The uncorrected psf array of the science image (and also of the diffim) 304 svar : `float`, optional 305 Average variance of science image used for PSF matching 306 tvar : `float`, optional 307 Average variance of template image used for PSF matching 311 pcf : `numpy.ndarray` 312 a 2-d numpy.array containing the new PSF 314 def post_conv_psf_ft2(psf, kernel, svar, tvar):
317 if psf.shape[0] < kernel.shape[0]:
318 diff = (kernel.shape[0] - psf.shape[0]) // 2
319 psf = np.pad(psf, (diff, diff), mode=
'constant')
320 elif psf.shape[0] > kernel.shape[0]:
321 diff = (psf.shape[0] - kernel.shape[0]) // 2
322 kernel = np.pad(kernel, (diff, diff), mode=
'constant')
323 psf_ft = np.fft.fft2(psf)
324 kft = np.fft.fft2(kernel)
325 out = psf_ft * np.sqrt((svar + tvar) / (svar + tvar * kft**2))
328 def post_conv_psf(psf, kernel, svar, tvar):
329 kft = post_conv_psf_ft2(psf, kernel, svar, tvar)
330 out = np.fft.ifft2(kft)
333 pcf = post_conv_psf(psf=psf, kernel=kappa, svar=svar, tvar=tvar)
334 pcf = pcf.real / pcf.real.sum()
338 def _fixOddKernel(kernel):
339 """Take a kernel with odd dimensions and make them even for FFT 343 kernel : `numpy.array` 349 a fixed kernel numpy.array. Returns a copy if the dimensions needed to change; 350 otherwise just return the input kernel. 355 if (out.shape[0] % 2) == 1:
356 out = np.pad(out, ((1, 0), (0, 0)), mode=
'constant')
358 if (out.shape[1] % 2) == 1:
359 out = np.pad(out, ((0, 0), (1, 0)), mode=
'constant')
362 out *= (np.mean(kernel) / np.mean(out))
366 def _fixEvenKernel(kernel):
367 """Take a kernel with even dimensions and make them odd, centered correctly. 371 kernel : `numpy.array` 377 a fixed kernel numpy.array 380 maxloc = np.unravel_index(np.argmax(kernel), kernel.shape)
381 out = np.roll(kernel, kernel.shape[0]//2 - maxloc[0], axis=0)
382 out = np.roll(out, out.shape[1]//2 - maxloc[1], axis=1)
384 if (out.shape[0] % 2) == 0:
385 maxloc = np.unravel_index(np.argmax(out), out.shape)
386 if out.shape[0] - maxloc[0] > maxloc[0]:
390 if out.shape[1] - maxloc[1] > maxloc[1]:
397 def _doConvolve(exposure, kernel):
398 """Convolve an Exposure with a decorrelation convolution kernel. 402 exposure : `lsst.afw.image.Exposure` 403 Input exposure to be convolved. 404 kernel : `numpy.array` 405 Input 2-d numpy.array to convolve the image with 409 out : `lsst.afw.image.Exposure` 410 a new Exposure with the convolved pixels and the (possibly 415 We re-center the kernel if necessary and return the possibly re-centered kernel 417 kernelImg = afwImage.ImageD(kernel.shape[0], kernel.shape[1])
418 kernelImg.getArray()[:, :] = kernel
420 maxloc = np.unravel_index(np.argmax(kernel), kernel.shape)
421 kern.setCtrX(maxloc[0])
422 kern.setCtrY(maxloc[1])
423 outExp = exposure.clone()
425 afwMath.convolve(outExp.getMaskedImage(), exposure.getMaskedImage(), kern, convCntrl)
431 """Task to be used as an ImageMapper for performing 432 A&L decorrelation on subimages on a grid across a A&L difference image. 434 This task subclasses DecorrelateALKernelTask in order to implement 435 all of that task's configuration parameters, as well as its `run` method. 438 ConfigClass = DecorrelateALKernelConfig
439 _DefaultName =
'ip_diffim_decorrelateALKernelMapper' 442 DecorrelateALKernelTask.__init__(self, *args, **kwargs)
444 def run(self, subExposure, expandedSubExposure, fullBBox,
445 template, science, alTaskResult=None, psfMatchingKernel=None,
446 preConvKernel=None, **kwargs):
447 """Perform decorrelation operation on `subExposure`, using 448 `expandedSubExposure` to allow for invalid edge pixels arising from 451 This method performs A&L decorrelation on `subExposure` using 452 local measures for image variances and PSF. `subExposure` is a 453 sub-exposure of the non-decorrelated A&L diffim. It also 454 requires the corresponding sub-exposures of the template 455 (`template`) and science (`science`) exposures. 459 subExposure : `lsst.afw.image.Exposure` 460 the sub-exposure of the diffim 461 expandedSubExposure : `lsst.afw.image.Exposure` 462 the expanded sub-exposure upon which to operate 463 fullBBox : `lsst.geom.Box2I` 464 the bounding box of the original exposure 465 template : `lsst.afw.image.Exposure` 466 the corresponding sub-exposure of the template exposure 467 science : `lsst.afw.image.Exposure` 468 the corresponding sub-exposure of the science exposure 469 alTaskResult : `lsst.pipe.base.Struct` 470 the result of A&L image differencing on `science` and 471 `template`, importantly containing the resulting 472 `psfMatchingKernel`. Can be `None`, only if 473 `psfMatchingKernel` is not `None`. 474 psfMatchingKernel : Alternative parameter for passing the 475 A&L `psfMatchingKernel` directly. 476 preConvKernel : If not None, then pre-filtering was applied 477 to science exposure, and this is the pre-convolution 480 additional keyword arguments propagated from 481 `ImageMapReduceTask.run`. 485 A `pipeBase.Struct` containing: 487 - ``subExposure`` : the result of the `subExposure` processing. 488 - ``decorrelationKernel`` : the decorrelation kernel, currently 493 This `run` method accepts parameters identical to those of 494 `ImageMapper.run`, since it is called from the 495 `ImageMapperTask`. See that class for more information. 497 templateExposure = template
498 scienceExposure = science
499 if alTaskResult
is None and psfMatchingKernel
is None:
500 raise RuntimeError(
'Both alTaskResult and psfMatchingKernel cannot be None')
501 psfMatchingKernel = alTaskResult.psfMatchingKernel
if alTaskResult
is not None else psfMatchingKernel
505 subExp2 = scienceExposure.Factory(scienceExposure, expandedSubExposure.getBBox())
506 subExp1 = templateExposure.Factory(templateExposure, expandedSubExposure.getBBox())
509 logLevel = self.log.getLevel()
510 self.log.setLevel(lsst.log.WARN)
511 res = DecorrelateALKernelTask.run(self, subExp2, subExp1, expandedSubExposure,
512 psfMatchingKernel, preConvKernel)
513 self.log.setLevel(logLevel)
515 diffim = res.correctedExposure.Factory(res.correctedExposure, subExposure.getBBox())
516 out = pipeBase.Struct(subExposure=diffim, decorrelationKernel=res.correctionKernel)
521 """Configuration parameters for the ImageMapReduceTask to direct it to use 522 DecorrelateALKernelMapper as its mapper for A&L decorrelation. 524 mapper = pexConfig.ConfigurableField(
525 doc=
'A&L decorrelation task to run on each sub-image',
526 target=DecorrelateALKernelMapper
531 """Configuration parameters for the DecorrelateALKernelSpatialTask. 533 decorrelateConfig = pexConfig.ConfigField(
534 dtype=DecorrelateALKernelConfig,
535 doc=
'DecorrelateALKernel config to use when running on complete exposure (non spatially-varying)',
538 decorrelateMapReduceConfig = pexConfig.ConfigField(
539 dtype=DecorrelateALKernelMapReduceConfig,
540 doc=
'DecorrelateALKernelMapReduce config to use when running on each sub-image (spatially-varying)',
543 ignoreMaskPlanes = pexConfig.ListField(
545 doc=
"""Mask planes to ignore for sigma-clipped statistics""",
546 default=(
"INTRP",
"EDGE",
"DETECTED",
"SAT",
"CR",
"BAD",
"NO_DATA",
"DETECTED_NEGATIVE")
557 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference 562 Pipe-task that removes the neighboring-pixel covariance in an 563 image difference that are added when the template image is 564 convolved with the Alard-Lupton PSF matching kernel. 566 This task is a simple wrapper around @ref DecorrelateALKernelTask, 567 which takes a `spatiallyVarying` parameter in its `run` method. If 568 it is `False`, then it simply calls the `run` method of @ref 569 DecorrelateALKernelTask. If it is True, then it uses the @ref 570 ImageMapReduceTask framework to break the exposures into 571 subExposures on a grid, and performs the `run` method of @ref 572 DecorrelateALKernelTask on each subExposure. This enables it to 573 account for spatially-varying PSFs and noise in the exposures when 574 performing the decorrelation. 576 This task has no standalone example, however it is applied as a 577 subtask of pipe.tasks.imageDifference.ImageDifferenceTask. 578 There is also an example of its use in `tests/testImageDecorrelation.py`. 580 ConfigClass = DecorrelateALKernelSpatialConfig
581 _DefaultName =
"ip_diffim_decorrelateALKernelSpatial" 584 """Create the image decorrelation Task 589 arguments to be passed to 590 `lsst.pipe.base.task.Task.__init__` 592 additional keyword arguments to be passed to 593 `lsst.pipe.base.task.Task.__init__` 595 pipeBase.Task.__init__(self, *args, **kwargs)
600 self.
statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes))
603 """Compute the mean of the variance plane of `exposure`. 606 exposure.getMaskedImage().getMask(),
608 var = statObj.getValue(afwMath.MEANCLIP)
611 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel,
612 spatiallyVarying=True, preConvKernel=None):
613 """Perform decorrelation of an image difference exposure. 615 Decorrelates the diffim due to the convolution of the 616 templateExposure with the A&L psfMatchingKernel. If 617 `spatiallyVarying` is True, it utilizes the spatially varying 618 matching kernel via the `imageMapReduce` framework to perform 619 spatially-varying decorrelation on a grid of subExposures. 623 scienceExposure : `lsst.afw.image.Exposure` 624 the science Exposure used for PSF matching 625 templateExposure : `lsst.afw.image.Exposure` 626 the template Exposure used for PSF matching 627 subtractedExposure : `lsst.afw.image.Exposure` 628 the subtracted Exposure produced by `ip_diffim.ImagePsfMatchTask.subtractExposures()` 630 an (optionally spatially-varying) PSF matching kernel produced 631 by `ip_diffim.ImagePsfMatchTask.subtractExposures()` 632 spatiallyVarying : `bool` 633 if True, perform the spatially-varying operation 634 preConvKernel : `lsst.meas.algorithms.Psf` 635 if not none, the scienceExposure has been pre-filtered with this kernel. (Currently 636 this option is experimental.) 640 results : `lsst.pipe.base.Struct` 641 a structure containing: 643 - ``correctedExposure`` : the decorrelated diffim 646 self.log.info(
'Running A&L decorrelation: spatiallyVarying=%r' % spatiallyVarying)
650 if np.isnan(svar)
or np.isnan(tvar):
652 if (np.all(np.isnan(scienceExposure.getMaskedImage().getImage().getArray()))
or 653 np.all(np.isnan(templateExposure.getMaskedImage().getImage().getArray()))):
654 self.log.warn(
'Template or science image is entirely NaNs: skipping decorrelation.')
663 self.log.info(
"Variance (science, template): (%f, %f)", svar, tvar)
664 self.log.info(
"Variance (uncorrected diffim): %f", var)
665 config = self.config.decorrelateMapReduceConfig
667 results = task.run(subtractedExposure, science=scienceExposure,
668 template=templateExposure, psfMatchingKernel=psfMatchingKernel,
669 preConvKernel=preConvKernel, forceEvenSized=
True)
670 results.correctedExposure = results.exposure
674 return exp.getMaskedImage().getMask()
675 gm(results.correctedExposure)[:, :] = gm(subtractedExposure)
678 self.log.info(
"Variance (corrected diffim): %f", var)
681 config = self.config.decorrelateConfig
683 results = task.run(scienceExposure, templateExposure,
684 subtractedExposure, psfMatchingKernel, preConvKernel=preConvKernel)
def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel, spatiallyVarying=True, preConvKernel=None)
def __init__(self, args, kwargs)
def run(self, exposure, templateExposure, subtractedExposure, psfMatchingKernel, preConvKernel=None, xcen=None, ycen=None, svar=None, tvar=None)
def __init__(self, args, kwargs)
Statistics makeStatistics(lsst::afw::math::MaskedVector< EntryT > const &mv, std::vector< WeightPixel > const &vweights, int const flags, StatisticsControl const &sctrl=StatisticsControl())
def computeVarianceMean(self, exposure)
def computeCorrectedDiffimPsf(kappa, psf, svar=0.04, tvar=0.04)
def __init__(self, args, kwargs)
decorrelateMapReduceConfig
def computeVarianceMean(self, exposure)
void convolve(OutImageT &convolvedImage, InImageT const &inImage, KernelT const &kernel, bool doNormalize, bool doCopyEdge=false)
def run(self, subExposure, expandedSubExposure, fullBBox, template, science, alTaskResult=None, psfMatchingKernel=None, preConvKernel=None, kwargs)