lsst.ip.diffim  21.0.0-4-gd1c1571+18b81799f9
imageDecorrelation.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2016 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 import numpy as np
24 
25 import lsst.afw.image as afwImage
26 import lsst.afw.math as afwMath
27 import lsst.geom as geom
28 import lsst.log
29 import lsst.meas.algorithms as measAlg
30 import lsst.pex.config as pexConfig
31 import lsst.pipe.base as pipeBase
32 
33 
34 from .imageMapReduce import (ImageMapReduceConfig, ImageMapReduceTask,
35  ImageMapper)
36 
37 __all__ = ("DecorrelateALKernelTask", "DecorrelateALKernelConfig",
38  "DecorrelateALKernelMapper", "DecorrelateALKernelMapReduceConfig",
39  "DecorrelateALKernelSpatialConfig", "DecorrelateALKernelSpatialTask")
40 
41 
42 class DecorrelateALKernelConfig(pexConfig.Config):
43  """Configuration parameters for the DecorrelateALKernelTask
44  """
45 
46  ignoreMaskPlanes = pexConfig.ListField(
47  dtype=str,
48  doc="""Mask planes to ignore for sigma-clipped statistics""",
49  default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE")
50  )
51 
52 
53 class DecorrelateALKernelTask(pipeBase.Task):
54  """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference
55 
56  Notes
57  -----
58 
59  Pipe-task that removes the neighboring-pixel covariance in an
60  image difference that are added when the template image is
61  convolved with the Alard-Lupton PSF matching kernel.
62 
63  The image differencing pipeline task @link
64  ip.diffim.psfMatch.PsfMatchTask PSFMatchTask@endlink and @link
65  ip.diffim.psfMatch.PsfMatchConfigAL PSFMatchConfigAL@endlink uses
66  the Alard and Lupton (1998) method for matching the PSFs of the
67  template and science exposures prior to subtraction. The
68  Alard-Lupton method identifies a matching kernel, which is then
69  (typically) convolved with the template image to perform PSF
70  matching. This convolution has the effect of adding covariance
71  between neighboring pixels in the template image, which is then
72  added to the image difference by subtraction.
73 
74  The pixel covariance may be corrected by whitening the noise of
75  the image difference. This task performs such a decorrelation by
76  computing a decorrelation kernel (based upon the A&L matching
77  kernel and variances in the template and science images) and
78  convolving the image difference with it. This process is described
79  in detail in [DMTN-021](http://dmtn-021.lsst.io).
80 
81  This task has no standalone example, however it is applied as a
82  subtask of pipe.tasks.imageDifference.ImageDifferenceTask.
83  """
84  ConfigClass = DecorrelateALKernelConfig
85  _DefaultName = "ip_diffim_decorrelateALKernel"
86 
87  def __init__(self, *args, **kwargs):
88  """Create the image decorrelation Task
89 
90  Parameters
91  ----------
92  args :
93  arguments to be passed to ``lsst.pipe.base.task.Task.__init__``
94  kwargs :
95  keyword arguments to be passed to ``lsst.pipe.base.task.Task.__init__``
96  """
97  pipeBase.Task.__init__(self, *args, **kwargs)
98 
100  self.statsControl.setNumSigmaClip(3.)
101  self.statsControl.setNumIter(3)
102  self.statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes))
103 
104  def computeVarianceMean(self, exposure):
105  statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(),
106  exposure.getMaskedImage().getMask(),
107  afwMath.MEANCLIP, self.statsControl)
108  var = statObj.getValue(afwMath.MEANCLIP)
109  return var
110 
111  @pipeBase.timeMethod
112  def run(self, exposure, templateExposure, subtractedExposure, psfMatchingKernel,
113  preConvKernel=None, xcen=None, ycen=None, svar=None, tvar=None):
114  """Perform decorrelation of an image difference exposure.
115 
116  Decorrelates the diffim due to the convolution of the templateExposure with the
117  A&L PSF matching kernel. Currently can accept a spatially varying matching kernel but in
118  this case it simply uses a static kernel from the center of the exposure. The decorrelation
119  is described in [DMTN-021, Equation 1](http://dmtn-021.lsst.io/#equation-1), where
120  `exposure` is I_1; templateExposure is I_2; `subtractedExposure` is D(k);
121  `psfMatchingKernel` is kappa; and svar and tvar are their respective
122  variances (see below).
123 
124  Parameters
125  ----------
126  exposure : `lsst.afw.image.Exposure`
127  The original science exposure (before `preConvKernel` applied) used for PSF matching.
128  templateExposure : `lsst.afw.image.Exposure`
129  The original template exposure (before matched to the science exposure
130  by `psfMatchingKernel`) warped into the science exposure dimensions. Always the PSF of the
131  `templateExposure` should be matched to the PSF of `exposure`, see notes below.
132  subtractedExposure :
133  the subtracted exposure produced by
134  `ip_diffim.ImagePsfMatchTask.subtractExposures()`. The `subtractedExposure` must
135  inherit its PSF from `exposure`, see notes below.
136  psfMatchingKernel :
137  An (optionally spatially-varying) PSF matching kernel produced
138  by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
139  preConvKernel :
140  if not None, then the `exposure` was pre-convolved with this kernel
141  xcen : `float`, optional
142  X-pixel coordinate to use for computing constant matching kernel to use
143  If `None` (default), then use the center of the image.
144  ycen : `float`, optional
145  Y-pixel coordinate to use for computing constant matching kernel to use
146  If `None` (default), then use the center of the image.
147  svar : `float`, optional
148  Image variance for science image
149  If `None` (default) then compute the variance over the entire input science image.
150  tvar : `float`, optional
151  Image variance for template image
152  If `None` (default) then compute the variance over the entire input template image.
153 
154  Returns
155  -------
156  result : `lsst.pipe.base.Struct`
157  - ``correctedExposure`` : the decorrelated diffim
158 
159  Notes
160  -----
161  The `subtractedExposure` is NOT updated. The returned `correctedExposure` has an updated but
162  spatially fixed PSF. It is calculated as the center of image PSF corrected by the center of
163  image matching kernel.
164 
165  In this task, it is _always_ the `templateExposure` that was matched to the `exposure`
166  by `psfMatchingKernel`. Swap arguments accordingly if actually the science exposure was matched
167  to a co-added template. In this case, tvar > svar typically occurs.
168 
169  The `templateExposure` and `exposure` image dimensions must be the same.
170 
171  Here we currently convert a spatially-varying matching kernel into a constant kernel,
172  just by computing it at the center of the image (tickets DM-6243, DM-6244).
173 
174  We are also using a constant accross-the-image measure of sigma (sqrt(variance)) to compute
175  the decorrelation kernel.
176 
177  TODO DM-23857 As part of the spatially varying correction implementation
178  consider whether returning a Struct is still necessary.
179  """
180  spatialKernel = psfMatchingKernel
181  kimg = afwImage.ImageD(spatialKernel.getDimensions())
182  bbox = subtractedExposure.getBBox()
183  if xcen is None:
184  xcen = (bbox.getBeginX() + bbox.getEndX()) / 2.
185  if ycen is None:
186  ycen = (bbox.getBeginY() + bbox.getEndY()) / 2.
187  self.log.info("Using matching kernel computed at (%d, %d)", xcen, ycen)
188  spatialKernel.computeImage(kimg, True, xcen, ycen)
189 
190  if svar is None:
191  svar = self.computeVarianceMean(exposure)
192  if tvar is None:
193  tvar = self.computeVarianceMean(templateExposure)
194  self.log.info("Variance (science, template): (%f, %f)", svar, tvar)
195 
196  # Should not happen unless entire image has been masked, which could happen
197  # if this is a small subimage of the main exposure. In this case, just return a full NaN
198  # exposure
199  if np.isnan(svar) or np.isnan(tvar):
200  # Double check that one of the exposures is all NaNs
201  if (np.all(np.isnan(exposure.image.array))
202  or np.all(np.isnan(templateExposure.image.array))):
203  self.log.warn('Template or science image is entirely NaNs: skipping decorrelation.')
204  outExposure = subtractedExposure.clone()
205  return pipeBase.Struct(correctedExposure=outExposure, )
206 
207  # The maximal correction value converges to sqrt(tvar/svar).
208  # Correction divergence warning if the correction exceeds 4 orders of magnitude.
209  tOverSVar = tvar/svar
210  if tOverSVar > 1e8:
211  self.log.warn("Diverging correction: science image variance is much smaller than template"
212  f", tvar/svar:{tOverSVar:.2e}")
213 
214  oldVarMean = self.computeVarianceMean(subtractedExposure)
215  self.log.info("Variance (uncorrected diffim): %f", oldVarMean)
216 
217  if preConvKernel is not None:
218  self.log.info('Using a pre-convolution kernel as part of decorrelation correction.')
219  kimg2 = afwImage.ImageD(preConvKernel.getDimensions())
220  preConvKernel.computeImage(kimg2, False)
221  pckArr = kimg2.array
222 
223  kArr = kimg.array
224  diffExpArr = subtractedExposure.image.array
225  psfImg = subtractedExposure.getPsf().computeKernelImage(geom.Point2D(xcen, ycen))
226  psfDim = psfImg.getDimensions()
227  psfArr = psfImg.array
228 
229  # Determine the common shape
230  if preConvKernel is None:
231  self.computeCommonShape(kArr.shape, psfArr.shape, diffExpArr.shape)
232  corrft = self.computeCorrection(kArr, svar, tvar)
233  else:
234  self.computeCommonShape(pckArr.shape, kArr.shape,
235  psfArr.shape, diffExpArr.shape)
236  corrft = self.computeCorrection(kArr, svar, tvar, preConvArr=pckArr)
237 
238  diffExpArr = self.computeCorrectedImage(corrft, diffExpArr)
239  corrPsfArr = self.computeCorrectedDiffimPsf(corrft, psfArr)
240 
241  psfcI = afwImage.ImageD(psfDim)
242  psfcI.array = corrPsfArr
243  psfcK = afwMath.FixedKernel(psfcI)
244  psfNew = measAlg.KernelPsf(psfcK)
245 
246  correctedExposure = subtractedExposure.clone()
247  correctedExposure.image.array[...] = diffExpArr # Allow for numpy type casting
248  # The subtracted exposure variance plane is already correlated, we cannot propagate
249  # it through another convolution; instead we need to use the uncorrelated originals
250  # The whitening should scale it to svar + tvar on average
251  varImg = correctedExposure.variance.array
252  # Allow for numpy type casting
253  varImg[...] = exposure.variance.array + templateExposure.variance.array
254  correctedExposure.setPsf(psfNew)
255 
256  newVarMean = self.computeVarianceMean(correctedExposure)
257  self.log.info(f"Variance (corrected diffim): {newVarMean:.2e}")
258 
259  # TODO DM-23857 As part of the spatially varying correction implementation
260  # consider whether returning a Struct is still necessary.
261  return pipeBase.Struct(correctedExposure=correctedExposure, )
262 
263  def computeCommonShape(self, *shapes):
264  """Calculate the common shape for FFT operations. Set `self.freqSpaceShape`
265  internally.
266 
267  Parameters
268  ----------
269  shapes : one or more `tuple` of `int`
270  Shapes of the arrays. All must have the same dimensionality.
271  At least one shape must be provided.
272 
273  Returns
274  -------
275  None.
276 
277  Notes
278  -----
279  For each dimension, gets the smallest even number greater than or equal to
280  `N1+N2-1` where `N1` and `N2` are the two largest values.
281  In case of only one shape given, rounds up to even each dimension value.
282  """
283  S = np.array(shapes, dtype=int)
284  if len(shapes) > 2:
285  S.sort(axis=0)
286  S = S[-2:]
287  if len(shapes) > 1:
288  commonShape = np.sum(S, axis=0) - 1
289  else:
290  commonShape = S[0]
291  commonShape[commonShape % 2 != 0] += 1
292  self.freqSpaceShape = tuple(commonShape)
293  self.log.info(f"Common frequency space shape {self.freqSpaceShape}")
294 
295  @staticmethod
296  def padCenterOriginArray(A, newShape: tuple, useInverse=False):
297  """Zero pad an image where the origin is at the center and replace the
298  origin to the corner as required by the periodic input of FFT. Implement also
299  the inverse operation, crop the padding and re-center data.
300 
301  Parameters
302  ----------
303  A : `numpy.ndarray`
304  An array to copy from.
305  newShape : `tuple` of `int`
306  The dimensions of the resulting array. For padding, the resulting array
307  must be larger than A in each dimension. For the inverse operation this
308  must be the original, before padding size of the array.
309  useInverse : bool, optional
310  Selector of forward, add padding, operation (False)
311  or its inverse, crop padding, operation (True).
312 
313  Returns
314  -------
315  R : `numpy.ndarray`
316  The padded or unpadded array with shape of `newShape` and the same dtype as A.
317 
318  Notes
319  -----
320  For odd dimensions, the splitting is rounded to
321  put the center pixel into the new corner origin (0,0). This is to be consistent
322  e.g. for a dirac delta kernel that is originally located at the center pixel.
323  """
324 
325  # The forward and inverse operations should round odd dimension halves at the opposite
326  # sides to get the pixels back to their original positions.
327  if not useInverse:
328  # Forward operation: First and second halves with respect to the axes of A.
329  firstHalves = [x//2 for x in A.shape]
330  secondHalves = [x-y for x, y in zip(A.shape, firstHalves)]
331  else:
332  # Inverse operation: Opposite rounding
333  secondHalves = [x//2 for x in newShape]
334  firstHalves = [x-y for x, y in zip(newShape, secondHalves)]
335 
336  R = np.zeros_like(A, shape=newShape)
337  R[-firstHalves[0]:, -firstHalves[1]:] = A[:firstHalves[0], :firstHalves[1]]
338  R[:secondHalves[0], -firstHalves[1]:] = A[-secondHalves[0]:, :firstHalves[1]]
339  R[:secondHalves[0], :secondHalves[1]] = A[-secondHalves[0]:, -secondHalves[1]:]
340  R[-firstHalves[0]:, :secondHalves[1]] = A[:firstHalves[0], -secondHalves[1]:]
341  return R
342 
343  def computeCorrection(self, kappa, svar, tvar, preConvArr=None):
344  """Compute the Lupton decorrelation post-convolution kernel for decorrelating an
345  image difference, based on the PSF-matching kernel.
346 
347  Parameters
348  ----------
349  kappa : `numpy.ndarray`
350  A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching.
351  svar : `float`
352  Average variance of science image used for PSF matching.
353  tvar : `float`
354  Average variance of the template (matched) image used for PSF matching.
355  preConvArr : `numpy.ndarray`, optional
356  If not None, then pre-filtering was applied
357  to science exposure, and this is the pre-convolution kernel.
358 
359  Returns
360  -------
361  corrft : `numpy.ndarray` of `float`
362  The frequency space representation of the correction. The array is real (dtype float).
363  Shape is `self.freqSpaceShape`.
364 
365  Notes
366  -----
367  The maximum correction factor converges to `sqrt(tvar/svar)` towards high frequencies.
368  This should be a plausible value.
369  """
370  kSum = np.sum(kappa)
371  kappa = self.padCenterOriginArray(kappa, self.freqSpaceShape)
372  kft = np.fft.fft2(kappa)
373  kftAbsSq = np.real(np.conj(kft) * kft)
374  # If there is no pre-convolution kernel, use placeholder scalars
375  if preConvArr is None:
376  preSum = 1.
377  preAbsSq = 1.
378  else:
379  preSum = np.sum(preConvArr)
380  preConvArr = self.padCenterOriginArray(preConvArr, self.freqSpaceShape)
381  preK = np.fft.fft2(preConvArr)
382  preAbsSq = np.real(np.conj(preK)*preK)
383 
384  denom = svar * preAbsSq + tvar * kftAbsSq
385  # Division by zero protection, though we don't expect to hit it
386  # (rather we'll have numerical noise)
387  tiny = np.finfo(kftAbsSq.dtype).tiny * 1000.
388  flt = denom < tiny
389  sumFlt = np.sum(flt)
390  if sumFlt > 0:
391  self.log.warnf("Avoid zero division. Skip decorrelation "
392  "at {} divergent frequencies.", sumFlt)
393  denom[flt] = 1.
394  kft = np.sqrt((svar * preSum*preSum + tvar * kSum*kSum) / denom)
395  # Don't do any correction at these frequencies
396  # the difference image should be close to zero anyway, so can't be decorrelated
397  if sumFlt > 0:
398  kft[flt] = 1.
399  return kft
400 
401  def computeCorrectedDiffimPsf(self, corrft, psfOld):
402  """Compute the (decorrelated) difference image's new PSF.
403 
404  Parameters
405  ----------
406  corrft : `numpy.ndarray`
407  The frequency space representation of the correction calculated by
408  `computeCorrection`. Shape must be `self.freqSpaceShape`.
409  psfOld : `numpy.ndarray`
410  The psf of the difference image to be corrected.
411 
412  Returns
413  -------
414  psfNew : `numpy.ndarray`
415  The corrected psf, same shape as `psfOld`, sum normed to 1.
416 
417  Notes
418  ----
419  There is no algorithmic guarantee that the corrected psf can
420  meaningfully fit to the same size as the original one.
421  """
422  psfShape = psfOld.shape
423  psfNew = self.padCenterOriginArray(psfOld, self.freqSpaceShape)
424  psfNew = np.fft.fft2(psfNew)
425  psfNew *= corrft
426  psfNew = np.fft.ifft2(psfNew)
427  psfNew = psfNew.real
428  psfNew = self.padCenterOriginArray(psfNew, psfShape, useInverse=True)
429  psfNew = psfNew/psfNew.sum()
430  return psfNew
431 
432  def computeCorrectedImage(self, corrft, imgOld):
433  """Compute the decorrelated difference image.
434 
435  Parameters
436  ----------
437  corrft : `numpy.ndarray`
438  The frequency space representation of the correction calculated by
439  `computeCorrection`. Shape must be `self.freqSpaceShape`.
440  imgOld : `numpy.ndarray`
441  The difference image to be corrected.
442 
443  Returns
444  -------
445  imgNew : `numpy.ndarray`
446  The corrected image, same size as the input.
447  """
448  expShape = imgOld.shape
449  imgNew = np.copy(imgOld)
450  filtInf = np.isinf(imgNew)
451  filtNan = np.isnan(imgNew)
452  imgNew[filtInf] = np.nan
453  imgNew[filtInf | filtNan] = np.nanmean(imgNew)
454  imgNew = self.padCenterOriginArray(imgNew, self.freqSpaceShape)
455  imgNew = np.fft.fft2(imgNew)
456  imgNew *= corrft
457  imgNew = np.fft.ifft2(imgNew)
458  imgNew = imgNew.real
459  imgNew = self.padCenterOriginArray(imgNew, expShape, useInverse=True)
460  imgNew[filtNan] = np.nan
461  imgNew[filtInf] = np.inf
462  return imgNew
463 
464 
466  """Task to be used as an ImageMapper for performing
467  A&L decorrelation on subimages on a grid across a A&L difference image.
468 
469  This task subclasses DecorrelateALKernelTask in order to implement
470  all of that task's configuration parameters, as well as its `run` method.
471  """
472 
473  ConfigClass = DecorrelateALKernelConfig
474  _DefaultName = 'ip_diffim_decorrelateALKernelMapper'
475 
476  def __init__(self, *args, **kwargs):
477  DecorrelateALKernelTask.__init__(self, *args, **kwargs)
478 
479  def run(self, subExposure, expandedSubExposure, fullBBox,
480  template, science, alTaskResult=None, psfMatchingKernel=None,
481  preConvKernel=None, **kwargs):
482  """Perform decorrelation operation on `subExposure`, using
483  `expandedSubExposure` to allow for invalid edge pixels arising from
484  convolutions.
485 
486  This method performs A&L decorrelation on `subExposure` using
487  local measures for image variances and PSF. `subExposure` is a
488  sub-exposure of the non-decorrelated A&L diffim. It also
489  requires the corresponding sub-exposures of the template
490  (`template`) and science (`science`) exposures.
491 
492  Parameters
493  ----------
494  subExposure : `lsst.afw.image.Exposure`
495  the sub-exposure of the diffim
496  expandedSubExposure : `lsst.afw.image.Exposure`
497  the expanded sub-exposure upon which to operate
498  fullBBox : `lsst.geom.Box2I`
499  the bounding box of the original exposure
500  template : `lsst.afw.image.Exposure`
501  the corresponding sub-exposure of the template exposure
502  science : `lsst.afw.image.Exposure`
503  the corresponding sub-exposure of the science exposure
504  alTaskResult : `lsst.pipe.base.Struct`
505  the result of A&L image differencing on `science` and
506  `template`, importantly containing the resulting
507  `psfMatchingKernel`. Can be `None`, only if
508  `psfMatchingKernel` is not `None`.
509  psfMatchingKernel : Alternative parameter for passing the
510  A&L `psfMatchingKernel` directly.
511  preConvKernel : If not None, then pre-filtering was applied
512  to science exposure, and this is the pre-convolution
513  kernel.
514  kwargs :
515  additional keyword arguments propagated from
516  `ImageMapReduceTask.run`.
517 
518  Returns
519  -------
520  A `pipeBase.Struct` containing:
521 
522  - ``subExposure`` : the result of the `subExposure` processing.
523  - ``decorrelationKernel`` : the decorrelation kernel, currently
524  not used.
525 
526  Notes
527  -----
528  This `run` method accepts parameters identical to those of
529  `ImageMapper.run`, since it is called from the
530  `ImageMapperTask`. See that class for more information.
531  """
532  templateExposure = template # input template
533  scienceExposure = science # input science image
534  if alTaskResult is None and psfMatchingKernel is None:
535  raise RuntimeError('Both alTaskResult and psfMatchingKernel cannot be None')
536  psfMatchingKernel = alTaskResult.psfMatchingKernel if alTaskResult is not None else psfMatchingKernel
537 
538  # subExp and expandedSubExp are subimages of the (un-decorrelated) diffim!
539  # So here we compute corresponding subimages of templateExposure and scienceExposure
540  subExp2 = scienceExposure.Factory(scienceExposure, expandedSubExposure.getBBox())
541  subExp1 = templateExposure.Factory(templateExposure, expandedSubExposure.getBBox())
542 
543  # Prevent too much log INFO verbosity from DecorrelateALKernelTask.run
544  logLevel = self.log.getLevel()
545  self.log.setLevel(lsst.log.WARN)
546  res = DecorrelateALKernelTask.run(self, subExp2, subExp1, expandedSubExposure,
547  psfMatchingKernel, preConvKernel)
548  self.log.setLevel(logLevel) # reset the log level
549 
550  diffim = res.correctedExposure.Factory(res.correctedExposure, subExposure.getBBox())
551  out = pipeBase.Struct(subExposure=diffim, )
552  return out
553 
554 
556  """Configuration parameters for the ImageMapReduceTask to direct it to use
557  DecorrelateALKernelMapper as its mapper for A&L decorrelation.
558  """
559  mapper = pexConfig.ConfigurableField(
560  doc='A&L decorrelation task to run on each sub-image',
561  target=DecorrelateALKernelMapper
562  )
563 
564 
565 class DecorrelateALKernelSpatialConfig(pexConfig.Config):
566  """Configuration parameters for the DecorrelateALKernelSpatialTask.
567  """
568  decorrelateConfig = pexConfig.ConfigField(
569  dtype=DecorrelateALKernelConfig,
570  doc='DecorrelateALKernel config to use when running on complete exposure (non spatially-varying)',
571  )
572 
573  decorrelateMapReduceConfig = pexConfig.ConfigField(
574  dtype=DecorrelateALKernelMapReduceConfig,
575  doc='DecorrelateALKernelMapReduce config to use when running on each sub-image (spatially-varying)',
576  )
577 
578  ignoreMaskPlanes = pexConfig.ListField(
579  dtype=str,
580  doc="""Mask planes to ignore for sigma-clipped statistics""",
581  default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE")
582  )
583 
584  def setDefaults(self):
585  self.decorrelateMapReduceConfig.gridStepX = self.decorrelateMapReduceConfig.gridStepY = 40
586  self.decorrelateMapReduceConfig.cellSizeX = self.decorrelateMapReduceConfig.cellSizeY = 41
587  self.decorrelateMapReduceConfig.borderSizeX = self.decorrelateMapReduceConfig.borderSizeY = 8
588  self.decorrelateMapReduceConfig.reducer.reduceOperation = 'average'
589 
590 
591 class DecorrelateALKernelSpatialTask(pipeBase.Task):
592  """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference
593 
594  Notes
595  -----
596 
597  Pipe-task that removes the neighboring-pixel covariance in an
598  image difference that are added when the template image is
599  convolved with the Alard-Lupton PSF matching kernel.
600 
601  This task is a simple wrapper around @ref DecorrelateALKernelTask,
602  which takes a `spatiallyVarying` parameter in its `run` method. If
603  it is `False`, then it simply calls the `run` method of @ref
604  DecorrelateALKernelTask. If it is True, then it uses the @ref
605  ImageMapReduceTask framework to break the exposures into
606  subExposures on a grid, and performs the `run` method of @ref
607  DecorrelateALKernelTask on each subExposure. This enables it to
608  account for spatially-varying PSFs and noise in the exposures when
609  performing the decorrelation.
610 
611  This task has no standalone example, however it is applied as a
612  subtask of pipe.tasks.imageDifference.ImageDifferenceTask.
613  There is also an example of its use in `tests/testImageDecorrelation.py`.
614  """
615  ConfigClass = DecorrelateALKernelSpatialConfig
616  _DefaultName = "ip_diffim_decorrelateALKernelSpatial"
617 
618  def __init__(self, *args, **kwargs):
619  """Create the image decorrelation Task
620 
621  Parameters
622  ----------
623  args :
624  arguments to be passed to
625  `lsst.pipe.base.task.Task.__init__`
626  kwargs :
627  additional keyword arguments to be passed to
628  `lsst.pipe.base.task.Task.__init__`
629  """
630  pipeBase.Task.__init__(self, *args, **kwargs)
631 
633  self.statsControl.setNumSigmaClip(3.)
634  self.statsControl.setNumIter(3)
635  self.statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes))
636 
637  def computeVarianceMean(self, exposure):
638  """Compute the mean of the variance plane of `exposure`.
639  """
640  statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(),
641  exposure.getMaskedImage().getMask(),
642  afwMath.MEANCLIP, self.statsControl)
643  var = statObj.getValue(afwMath.MEANCLIP)
644  return var
645 
646  def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel,
647  spatiallyVarying=True, preConvKernel=None):
648  """Perform decorrelation of an image difference exposure.
649 
650  Decorrelates the diffim due to the convolution of the
651  templateExposure with the A&L psfMatchingKernel. If
652  `spatiallyVarying` is True, it utilizes the spatially varying
653  matching kernel via the `imageMapReduce` framework to perform
654  spatially-varying decorrelation on a grid of subExposures.
655 
656  Parameters
657  ----------
658  scienceExposure : `lsst.afw.image.Exposure`
659  the science Exposure used for PSF matching
660  templateExposure : `lsst.afw.image.Exposure`
661  the template Exposure used for PSF matching
662  subtractedExposure : `lsst.afw.image.Exposure`
663  the subtracted Exposure produced by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
664  psfMatchingKernel :
665  an (optionally spatially-varying) PSF matching kernel produced
666  by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
667  spatiallyVarying : `bool`
668  if True, perform the spatially-varying operation
669  preConvKernel : `lsst.meas.algorithms.Psf`
670  if not none, the scienceExposure has been pre-filtered with this kernel. (Currently
671  this option is experimental.)
672 
673  Returns
674  -------
675  results : `lsst.pipe.base.Struct`
676  a structure containing:
677 
678  - ``correctedExposure`` : the decorrelated diffim
679 
680  """
681  self.log.info('Running A&L decorrelation: spatiallyVarying=%r' % spatiallyVarying)
682 
683  svar = self.computeVarianceMean(scienceExposure)
684  tvar = self.computeVarianceMean(templateExposure)
685  if np.isnan(svar) or np.isnan(tvar): # Should not happen unless entire image has been masked.
686  # Double check that one of the exposures is all NaNs
687  if (np.all(np.isnan(scienceExposure.image.array))
688  or np.all(np.isnan(templateExposure.image.array))):
689  self.log.warn('Template or science image is entirely NaNs: skipping decorrelation.')
690  if np.isnan(svar):
691  svar = 1e-9
692  if np.isnan(tvar):
693  tvar = 1e-9
694 
695  var = self.computeVarianceMean(subtractedExposure)
696 
697  if spatiallyVarying:
698  self.log.info("Variance (science, template): (%f, %f)", svar, tvar)
699  self.log.info("Variance (uncorrected diffim): %f", var)
700  config = self.config.decorrelateMapReduceConfig
701  task = ImageMapReduceTask(config=config)
702  results = task.run(subtractedExposure, science=scienceExposure,
703  template=templateExposure, psfMatchingKernel=psfMatchingKernel,
704  preConvKernel=preConvKernel, forceEvenSized=True)
705  results.correctedExposure = results.exposure
706 
707  # Make sure masks of input image are propagated to diffim
708  def gm(exp):
709  return exp.getMaskedImage().getMask()
710  gm(results.correctedExposure)[:, :] = gm(subtractedExposure)
711 
712  var = self.computeVarianceMean(results.correctedExposure)
713  self.log.info("Variance (corrected diffim): %f", var)
714 
715  else:
716  config = self.config.decorrelateConfig
717  task = DecorrelateALKernelTask(config=config)
718  results = task.run(scienceExposure, templateExposure,
719  subtractedExposure, psfMatchingKernel, preConvKernel=preConvKernel)
720 
721  return results
lsst::afw::image
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelSpatialConfig
Definition: imageDecorrelation.py:565
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelMapReduceConfig
Definition: imageDecorrelation.py:555
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelSpatialConfig.setDefaults
def setDefaults(self)
Definition: imageDecorrelation.py:584
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelSpatialTask.computeVarianceMean
def computeVarianceMean(self, exposure)
Definition: imageDecorrelation.py:637
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelSpatialTask.__init__
def __init__(self, *args, **kwargs)
Definition: imageDecorrelation.py:618
lsst::afw::math::FixedKernel
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelMapper.run
def run(self, subExposure, expandedSubExposure, fullBBox, template, science, alTaskResult=None, psfMatchingKernel=None, preConvKernel=None, **kwargs)
Definition: imageDecorrelation.py:479
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelTask.computeCommonShape
def computeCommonShape(self, *shapes)
Definition: imageDecorrelation.py:263
lsst::afw::math::makeStatistics
Statistics makeStatistics(lsst::afw::math::MaskedVector< EntryT > const &mv, std::vector< WeightPixel > const &vweights, int const flags, StatisticsControl const &sctrl=StatisticsControl())
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelMapper
Definition: imageDecorrelation.py:465
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelTask.computeCorrection
def computeCorrection(self, kappa, svar, tvar, preConvArr=None)
Definition: imageDecorrelation.py:343
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelTask.computeCorrectedImage
def computeCorrectedImage(self, corrft, imgOld)
Definition: imageDecorrelation.py:432
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelConfig
Definition: imageDecorrelation.py:42
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelSpatialTask
Definition: imageDecorrelation.py:591
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelTask.computeVarianceMean
def computeVarianceMean(self, exposure)
Definition: imageDecorrelation.py:104
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelTask.padCenterOriginArray
def padCenterOriginArray(A, tuple newShape, useInverse=False)
Definition: imageDecorrelation.py:296
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelSpatialTask.statsControl
statsControl
Definition: imageDecorrelation.py:632
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelSpatialConfig.decorrelateMapReduceConfig
decorrelateMapReduceConfig
Definition: imageDecorrelation.py:573
lsst::pex::config
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelTask
Definition: imageDecorrelation.py:53
lsst::afw::math::StatisticsControl
lsst::geom
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelTask.freqSpaceShape
freqSpaceShape
Definition: imageDecorrelation.py:292
lsst::ip::diffim.imageMapReduce.ImageMapper
Definition: imageMapReduce.py:88
lsst::ip::diffim.imageMapReduce.ImageMapReduceConfig
Definition: imageMapReduce.py:376
lsst::afw::math
Point< double, 2 >
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelTask.statsControl
statsControl
Definition: imageDecorrelation.py:99
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelSpatialTask.run
def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel, spatiallyVarying=True, preConvKernel=None)
Definition: imageDecorrelation.py:646
lsst::ip::diffim.imageMapReduce.ImageMapReduceTask
Definition: imageMapReduce.py:487
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelMapper.__init__
def __init__(self, *args, **kwargs)
Definition: imageDecorrelation.py:476
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelTask.computeCorrectedDiffimPsf
def computeCorrectedDiffimPsf(self, corrft, psfOld)
Definition: imageDecorrelation.py:401
lsst::log
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelTask.__init__
def __init__(self, *args, **kwargs)
Definition: imageDecorrelation.py:87
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelTask.run
def run(self, exposure, templateExposure, subtractedExposure, psfMatchingKernel, preConvKernel=None, xcen=None, ycen=None, svar=None, tvar=None)
Definition: imageDecorrelation.py:112