lsst.ip.diffim  19.0.0-21-g2644856+27
imageDecorrelation.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2016 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 import numpy as np
24 
25 import lsst.afw.image as afwImage
26 import lsst.afw.math as afwMath
27 import lsst.geom as geom
28 import lsst.log
29 import lsst.meas.algorithms as measAlg
30 import lsst.pex.config as pexConfig
31 import lsst.pipe.base as pipeBase
32 
33 
34 from .imageMapReduce import (ImageMapReduceConfig, ImageMapReduceTask,
35  ImageMapper)
36 
37 __all__ = ("DecorrelateALKernelTask", "DecorrelateALKernelConfig",
38  "DecorrelateALKernelMapper", "DecorrelateALKernelMapReduceConfig",
39  "DecorrelateALKernelSpatialConfig", "DecorrelateALKernelSpatialTask")
40 
41 
42 class DecorrelateALKernelConfig(pexConfig.Config):
43  """Configuration parameters for the DecorrelateALKernelTask
44  """
45 
46  ignoreMaskPlanes = pexConfig.ListField(
47  dtype=str,
48  doc="""Mask planes to ignore for sigma-clipped statistics""",
49  default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE")
50  )
51 
52 
53 class DecorrelateALKernelTask(pipeBase.Task):
54  """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference
55 
56  Notes
57  -----
58 
59  Pipe-task that removes the neighboring-pixel covariance in an
60  image difference that are added when the template image is
61  convolved with the Alard-Lupton PSF matching kernel.
62 
63  The image differencing pipeline task @link
64  ip.diffim.psfMatch.PsfMatchTask PSFMatchTask@endlink and @link
65  ip.diffim.psfMatch.PsfMatchConfigAL PSFMatchConfigAL@endlink uses
66  the Alard and Lupton (1998) method for matching the PSFs of the
67  template and science exposures prior to subtraction. The
68  Alard-Lupton method identifies a matching kernel, which is then
69  (typically) convolved with the template image to perform PSF
70  matching. This convolution has the effect of adding covariance
71  between neighboring pixels in the template image, which is then
72  added to the image difference by subtraction.
73 
74  The pixel covariance may be corrected by whitening the noise of
75  the image difference. This task performs such a decorrelation by
76  computing a decorrelation kernel (based upon the A&L matching
77  kernel and variances in the template and science images) and
78  convolving the image difference with it. This process is described
79  in detail in [DMTN-021](http://dmtn-021.lsst.io).
80 
81  This task has no standalone example, however it is applied as a
82  subtask of pipe.tasks.imageDifference.ImageDifferenceTask.
83  """
84  ConfigClass = DecorrelateALKernelConfig
85  _DefaultName = "ip_diffim_decorrelateALKernel"
86 
87  def __init__(self, *args, **kwargs):
88  """Create the image decorrelation Task
89 
90  Parameters
91  ----------
92  args :
93  arguments to be passed to ``lsst.pipe.base.task.Task.__init__``
94  kwargs :
95  keyword arguments to be passed to ``lsst.pipe.base.task.Task.__init__``
96  """
97  pipeBase.Task.__init__(self, *args, **kwargs)
98 
100  self.statsControl.setNumSigmaClip(3.)
101  self.statsControl.setNumIter(3)
102  self.statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes))
103 
104  def computeVarianceMean(self, exposure):
105  statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(),
106  exposure.getMaskedImage().getMask(),
107  afwMath.MEANCLIP, self.statsControl)
108  var = statObj.getValue(afwMath.MEANCLIP)
109  return var
110 
111  @pipeBase.timeMethod
112  def run(self, exposure, templateExposure, subtractedExposure, psfMatchingKernel,
113  preConvKernel=None, xcen=None, ycen=None, svar=None, tvar=None):
114  """Perform decorrelation of an image difference exposure.
115 
116  Decorrelates the diffim due to the convolution of the templateExposure with the
117  A&L PSF matching kernel. Currently can accept a spatially varying matching kernel but in
118  this case it simply uses a static kernel from the center of the exposure. The decorrelation
119  is described in [DMTN-021, Equation 1](http://dmtn-021.lsst.io/#equation-1), where
120  `exposure` is I_1; templateExposure is I_2; `subtractedExposure` is D(k);
121  `psfMatchingKernel` is kappa; and svar and tvar are their respective
122  variances (see below).
123 
124  Parameters
125  ----------
126  exposure : `lsst.afw.image.Exposure`
127  The original science exposure (before `preConvKernel` applied) used for PSF matching.
128  templateExposure : `lsst.afw.image.Exposure`
129  The original template exposure (before matched to the science exposure
130  by `psfMatchingKernel`) warped into the science exposure dimensions. Always the PSF of the
131  `templateExposure` should be matched to the PSF of `exposure`, see notes below.
132  subtractedExposure :
133  the subtracted exposure produced by
134  `ip_diffim.ImagePsfMatchTask.subtractExposures()`. The `subtractedExposure` must
135  inherit its PSF from `exposure`, see notes below.
136  psfMatchingKernel :
137  An (optionally spatially-varying) PSF matching kernel produced
138  by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
139  preConvKernel :
140  if not None, then the `exposure` was pre-convolved with this kernel
141  xcen : `float`, optional
142  X-pixel coordinate to use for computing constant matching kernel to use
143  If `None` (default), then use the center of the image.
144  ycen : `float`, optional
145  Y-pixel coordinate to use for computing constant matching kernel to use
146  If `None` (default), then use the center of the image.
147  svar : `float`, optional
148  Image variance for science image
149  If `None` (default) then compute the variance over the entire input science image.
150  tvar : `float`, optional
151  Image variance for template image
152  If `None` (default) then compute the variance over the entire input template image.
153 
154  Returns
155  -------
156  result : `lsst.pipe.base.Struct`
157  - ``correctedExposure`` : the decorrelated diffim
158 
159  Notes
160  -----
161  The `subtractedExposure` is NOT updated. The returned `correctedExposure` has an updated but
162  spatially fixed PSF. It is calculated as the center of image PSF corrected by the center of
163  image matching kernel.
164 
165  In this task, it is _always_ the `templateExposure` that was matched to the `exposure`
166  by `psfMatchingKernel`. Swap arguments accordingly if actually the science exposure was matched
167  to a co-added template. In this case, tvar > svar typically occurs.
168 
169  The `templateExposure` and `exposure` image dimensions must be the same.
170 
171  Here we currently convert a spatially-varying matching kernel into a constant kernel,
172  just by computing it at the center of the image (tickets DM-6243, DM-6244).
173 
174  We are also using a constant accross-the-image measure of sigma (sqrt(variance)) to compute
175  the decorrelation kernel.
176 
177  TODO DM-23857 As part of the spatially varying correction implementation
178  consider whether returning a Struct is still necessary.
179  """
180  spatialKernel = psfMatchingKernel
181  kimg = afwImage.ImageD(spatialKernel.getDimensions())
182  bbox = subtractedExposure.getBBox()
183  if xcen is None:
184  xcen = (bbox.getBeginX() + bbox.getEndX()) / 2.
185  if ycen is None:
186  ycen = (bbox.getBeginY() + bbox.getEndY()) / 2.
187  self.log.info("Using matching kernel computed at (%d, %d)", xcen, ycen)
188  spatialKernel.computeImage(kimg, True, xcen, ycen)
189 
190  if svar is None:
191  svar = self.computeVarianceMean(exposure)
192  if tvar is None:
193  tvar = self.computeVarianceMean(templateExposure)
194  self.log.info("Variance (science, template): (%f, %f)", svar, tvar)
195 
196  # Should not happen unless entire image has been masked, which could happen
197  # if this is a small subimage of the main exposure. In this case, just return a full NaN
198  # exposure
199  if np.isnan(svar) or np.isnan(tvar):
200  # Double check that one of the exposures is all NaNs
201  if (np.all(np.isnan(exposure.image.array))
202  or np.all(np.isnan(templateExposure.image.array))):
203  self.log.warn('Template or science image is entirely NaNs: skipping decorrelation.')
204  outExposure = subtractedExposure.clone()
205  return pipeBase.Struct(correctedExposure=outExposure, )
206 
207  # The maximal correction value converges to sqrt(tvar/svar).
208  # Correction divergence warning if the correction exceeds 4 orders of magnitude.
209  tOverSVar = tvar/svar
210  if tOverSVar > 1e8:
211  self.log.warn("Diverging correction: science image variance is much smaller than template"
212  f", tvar/svar:{tOverSVar:.2e}")
213 
214  oldVarMean = self.computeVarianceMean(subtractedExposure)
215  self.log.info("Variance (uncorrected diffim): %f", oldVarMean)
216 
217  if preConvKernel is not None:
218  self.log.info('Using a pre-convolution kernel as part of decorrelation correction.')
219  kimg2 = afwImage.ImageD(preConvKernel.getDimensions())
220  preConvKernel.computeImage(kimg2, False)
221  pckArr = kimg2.array
222 
223  kArr = kimg.array
224  diffExpArr = subtractedExposure.image.array
225  psfImg = subtractedExposure.getPsf().computeKernelImage(geom.Point2D(xcen, ycen))
226  psfDim = psfImg.getDimensions()
227  psfArr = psfImg.array
228 
229  # Determine the common shape
230  if preConvKernel is None:
231  self.computeCommonShape(kArr.shape, psfArr.shape, diffExpArr.shape)
232  corrft = self.computeCorrection(kArr, svar, tvar)
233  else:
234  self.computeCommonShape(pckArr.shape, kArr.shape,
235  psfArr.shape, diffExpArr.shape)
236  corrft = self.computeCorrection(kArr, svar, tvar, preConvArr=pckArr)
237 
238  diffExpArr = self.computeCorrectedImage(corrft, diffExpArr)
239  corrPsfArr = self.computeCorrectedDiffimPsf(corrft, psfArr)
240 
241  psfcI = afwImage.ImageD(psfDim)
242  psfcI.array = corrPsfArr
243  psfcK = afwMath.FixedKernel(psfcI)
244  psfNew = measAlg.KernelPsf(psfcK)
245 
246  correctedExposure = subtractedExposure.clone()
247  correctedExposure.image.array[...] = diffExpArr # Allow for numpy type casting
248  # The subtracted exposure variance plane is already correlated, we cannot propagate
249  # it through another convolution; instead we need to use the uncorrelated originals
250  # The whitening should scale it to svar + tvar on average
251  varImg = correctedExposure.variance.array
252  # Allow for numpy type casting
253  varImg[...] = exposure.variance.array + templateExposure.variance.array
254  correctedExposure.setPsf(psfNew)
255 
256  newVarMean = self.computeVarianceMean(correctedExposure)
257  self.log.info(f"Variance (corrected diffim): {newVarMean:.2e}")
258 
259  # TODO DM-23857 As part of the spatially varying correction implementation
260  # consider whether returning a Struct is still necessary.
261  return pipeBase.Struct(correctedExposure=correctedExposure, )
262 
263  def computeCommonShape(self, *shapes):
264  """Calculate the common shape for FFT operations. Set `self.freqSpaceShape`
265  internally.
266 
267  Parameters
268  ----------
269  shapes : one or more `tuple` of `int`
270  Shapes of the arrays. All must have the same dimensionality.
271  At least one shape must be provided.
272 
273  Returns
274  -------
275  None.
276 
277  Notes
278  -----
279  For each dimension, gets the smallest even number greater than or equal to
280  `N1+N2-1` where `N1` and `N2` are the two largest values.
281  In case of only one shape given, rounds up to even each dimension value.
282  """
283  S = np.array(shapes, dtype=int)
284  if len(shapes) > 2:
285  S.sort(axis=0)
286  S = S[-2:]
287  if len(shapes) > 1:
288  commonShape = np.sum(S, axis=0) - 1
289  else:
290  commonShape = S[0]
291  commonShape[commonShape % 2 != 0] += 1
292  self.freqSpaceShape = tuple(commonShape)
293  self.log.info(f"Common frequency space shape {self.freqSpaceShape}")
294 
295  @staticmethod
296  def padCenterOriginArray(A, newShape: tuple, useInverse=False):
297  """Zero pad an image where the origin is at the center and replace the
298  origin to the corner as required by the periodic input of FFT. Implement also
299  the inverse operation, crop the padding and re-center data.
300 
301  Parameters
302  ----------
303  A : `numpy.ndarray`
304  An array to copy from.
305  newShape : `tuple` of `int`
306  The dimensions of the resulting array. For padding, the resulting array
307  must be larger than A in each dimension. For the inverse operation this
308  must be the original, before padding size of the array.
309  useInverse : bool, optional
310  Selector of forward, add padding, operation (False)
311  or its inverse, crop padding, operation (True).
312 
313  Returns
314  -------
315  R : `numpy.ndarray`
316  The padded or unpadded array with shape of `newShape` and the same dtype as A.
317 
318  Notes
319  -----
320  For odd dimensions, the splitting is rounded to
321  put the center pixel into the new corner origin (0,0). This is to be consistent
322  e.g. for a dirac delta kernel that is originally located at the center pixel.
323  """
324 
325  # The forward and inverse operations should round odd dimension halves at the opposite
326  # sides to get the pixels back to their original positions.
327  if not useInverse:
328  # Forward operation: First and second halves with respect to the axes of A.
329  firstHalves = [x//2 for x in A.shape]
330  secondHalves = [x-y for x, y in zip(A.shape, firstHalves)]
331  else:
332  # Inverse operation: Opposite rounding
333  secondHalves = [x//2 for x in newShape]
334  firstHalves = [x-y for x, y in zip(newShape, secondHalves)]
335 
336  R = np.zeros_like(A, shape=newShape)
337  R[-firstHalves[0]:, -firstHalves[1]:] = A[:firstHalves[0], :firstHalves[1]]
338  R[:secondHalves[0], -firstHalves[1]:] = A[-secondHalves[0]:, :firstHalves[1]]
339  R[:secondHalves[0], :secondHalves[1]] = A[-secondHalves[0]:, -secondHalves[1]:]
340  R[-firstHalves[0]:, :secondHalves[1]] = A[:firstHalves[0], -secondHalves[1]:]
341  return R
342 
343  def computeCorrection(self, kappa, svar, tvar, preConvArr=None):
344  """Compute the Lupton decorrelation post-convolution kernel for decorrelating an
345  image difference, based on the PSF-matching kernel.
346 
347  Parameters
348  ----------
349  kappa : `numpy.ndarray`
350  A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching.
351  svar : `float`
352  Average variance of science image used for PSF matching.
353  tvar : `float`
354  Average variance of the template (matched) image used for PSF matching.
355  preConvArr : `numpy.ndarray`, optional
356  If not None, then pre-filtering was applied
357  to science exposure, and this is the pre-convolution kernel.
358 
359  Returns
360  -------
361  corrft : `numpy.ndarray` of `float`
362  The frequency space representation of the correction. The array is real (dtype float).
363  Shape is `self.freqSpaceShape`.
364 
365  Notes
366  -----
367  The maximum correction factor converges to `sqrt(tvar/svar)` towards high frequencies.
368  This should be a plausible value.
369  """
370  kappa = self.padCenterOriginArray(kappa, self.freqSpaceShape)
371  kft = np.fft.fft2(kappa)
372  kft2 = np.real(np.conj(kft) * kft)
373  if preConvArr is None:
374  denom = svar + tvar * kft2
375  else:
376  preConvArr = self.padCenterOriginArray(preConvArr, self.freqSpaceShape)
377  mk = np.fft.fft2(preConvArr)
378  mk2 = np.real(np.conj(mk) * mk)
379  denom = svar * mk2 + tvar * kft2
380  kft = np.sqrt((svar + tvar) / denom)
381  return kft
382 
383  def computeCorrectedDiffimPsf(self, corrft, psfOld):
384  """Compute the (decorrelated) difference image's new PSF.
385 
386  Parameters
387  ----------
388  corrft : `numpy.ndarray`
389  The frequency space representation of the correction calculated by
390  `computeCorrection`. Shape must be `self.freqSpaceShape`.
391  psfOld : `numpy.ndarray`
392  The psf of the difference image to be corrected.
393 
394  Returns
395  -------
396  psfNew : `numpy.ndarray`
397  The corrected psf, same shape as `psfOld`, sum normed to 1.
398 
399  Notes
400  ----
401  There is no algorithmic guarantee that the corrected psf can
402  meaningfully fit to the same size as the original one.
403  """
404  psfShape = psfOld.shape
405  psfNew = self.padCenterOriginArray(psfOld, self.freqSpaceShape)
406  psfNew = np.fft.fft2(psfNew)
407  psfNew *= corrft
408  psfNew = np.fft.ifft2(psfNew)
409  psfNew = psfNew.real
410  psfNew = self.padCenterOriginArray(psfNew, psfShape, useInverse=True)
411  psfNew = psfNew/psfNew.sum()
412  return psfNew
413 
414  def computeCorrectedImage(self, corrft, imgOld):
415  """Compute the decorrelated difference image.
416 
417  Parameters
418  ----------
419  corrft : `numpy.ndarray`
420  The frequency space representation of the correction calculated by
421  `computeCorrection`. Shape must be `self.freqSpaceShape`.
422  imgOld : `numpy.ndarray`
423  The difference image to be corrected.
424 
425  Returns
426  -------
427  imgNew : `numpy.ndarray`
428  The corrected image, same size as the input.
429  """
430  expShape = imgOld.shape
431  imgNew = np.copy(imgOld)
432  filtInf = np.isinf(imgNew)
433  filtNan = np.isnan(imgNew)
434  imgNew[filtInf] = np.nan
435  imgNew[filtInf | filtNan] = np.nanmean(imgNew)
436  imgNew = self.padCenterOriginArray(imgNew, self.freqSpaceShape)
437  imgNew = np.fft.fft2(imgNew)
438  imgNew *= corrft
439  imgNew = np.fft.ifft2(imgNew)
440  imgNew = imgNew.real
441  imgNew = self.padCenterOriginArray(imgNew, expShape, useInverse=True)
442  imgNew[filtNan] = np.nan
443  imgNew[filtInf] = np.inf
444  return imgNew
445 
446 
448  """Task to be used as an ImageMapper for performing
449  A&L decorrelation on subimages on a grid across a A&L difference image.
450 
451  This task subclasses DecorrelateALKernelTask in order to implement
452  all of that task's configuration parameters, as well as its `run` method.
453  """
454 
455  ConfigClass = DecorrelateALKernelConfig
456  _DefaultName = 'ip_diffim_decorrelateALKernelMapper'
457 
458  def __init__(self, *args, **kwargs):
459  DecorrelateALKernelTask.__init__(self, *args, **kwargs)
460 
461  def run(self, subExposure, expandedSubExposure, fullBBox,
462  template, science, alTaskResult=None, psfMatchingKernel=None,
463  preConvKernel=None, **kwargs):
464  """Perform decorrelation operation on `subExposure`, using
465  `expandedSubExposure` to allow for invalid edge pixels arising from
466  convolutions.
467 
468  This method performs A&L decorrelation on `subExposure` using
469  local measures for image variances and PSF. `subExposure` is a
470  sub-exposure of the non-decorrelated A&L diffim. It also
471  requires the corresponding sub-exposures of the template
472  (`template`) and science (`science`) exposures.
473 
474  Parameters
475  ----------
476  subExposure : `lsst.afw.image.Exposure`
477  the sub-exposure of the diffim
478  expandedSubExposure : `lsst.afw.image.Exposure`
479  the expanded sub-exposure upon which to operate
480  fullBBox : `lsst.geom.Box2I`
481  the bounding box of the original exposure
482  template : `lsst.afw.image.Exposure`
483  the corresponding sub-exposure of the template exposure
484  science : `lsst.afw.image.Exposure`
485  the corresponding sub-exposure of the science exposure
486  alTaskResult : `lsst.pipe.base.Struct`
487  the result of A&L image differencing on `science` and
488  `template`, importantly containing the resulting
489  `psfMatchingKernel`. Can be `None`, only if
490  `psfMatchingKernel` is not `None`.
491  psfMatchingKernel : Alternative parameter for passing the
492  A&L `psfMatchingKernel` directly.
493  preConvKernel : If not None, then pre-filtering was applied
494  to science exposure, and this is the pre-convolution
495  kernel.
496  kwargs :
497  additional keyword arguments propagated from
498  `ImageMapReduceTask.run`.
499 
500  Returns
501  -------
502  A `pipeBase.Struct` containing:
503 
504  - ``subExposure`` : the result of the `subExposure` processing.
505  - ``decorrelationKernel`` : the decorrelation kernel, currently
506  not used.
507 
508  Notes
509  -----
510  This `run` method accepts parameters identical to those of
511  `ImageMapper.run`, since it is called from the
512  `ImageMapperTask`. See that class for more information.
513  """
514  templateExposure = template # input template
515  scienceExposure = science # input science image
516  if alTaskResult is None and psfMatchingKernel is None:
517  raise RuntimeError('Both alTaskResult and psfMatchingKernel cannot be None')
518  psfMatchingKernel = alTaskResult.psfMatchingKernel if alTaskResult is not None else psfMatchingKernel
519 
520  # subExp and expandedSubExp are subimages of the (un-decorrelated) diffim!
521  # So here we compute corresponding subimages of templateExposure and scienceExposure
522  subExp2 = scienceExposure.Factory(scienceExposure, expandedSubExposure.getBBox())
523  subExp1 = templateExposure.Factory(templateExposure, expandedSubExposure.getBBox())
524 
525  # Prevent too much log INFO verbosity from DecorrelateALKernelTask.run
526  logLevel = self.log.getLevel()
527  self.log.setLevel(lsst.log.WARN)
528  res = DecorrelateALKernelTask.run(self, subExp2, subExp1, expandedSubExposure,
529  psfMatchingKernel, preConvKernel)
530  self.log.setLevel(logLevel) # reset the log level
531 
532  diffim = res.correctedExposure.Factory(res.correctedExposure, subExposure.getBBox())
533  out = pipeBase.Struct(subExposure=diffim, )
534  return out
535 
536 
538  """Configuration parameters for the ImageMapReduceTask to direct it to use
539  DecorrelateALKernelMapper as its mapper for A&L decorrelation.
540  """
541  mapper = pexConfig.ConfigurableField(
542  doc='A&L decorrelation task to run on each sub-image',
543  target=DecorrelateALKernelMapper
544  )
545 
546 
547 class DecorrelateALKernelSpatialConfig(pexConfig.Config):
548  """Configuration parameters for the DecorrelateALKernelSpatialTask.
549  """
550  decorrelateConfig = pexConfig.ConfigField(
551  dtype=DecorrelateALKernelConfig,
552  doc='DecorrelateALKernel config to use when running on complete exposure (non spatially-varying)',
553  )
554 
555  decorrelateMapReduceConfig = pexConfig.ConfigField(
556  dtype=DecorrelateALKernelMapReduceConfig,
557  doc='DecorrelateALKernelMapReduce config to use when running on each sub-image (spatially-varying)',
558  )
559 
560  ignoreMaskPlanes = pexConfig.ListField(
561  dtype=str,
562  doc="""Mask planes to ignore for sigma-clipped statistics""",
563  default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE")
564  )
565 
566  def setDefaults(self):
567  self.decorrelateMapReduceConfig.gridStepX = self.decorrelateMapReduceConfig.gridStepY = 40
568  self.decorrelateMapReduceConfig.cellSizeX = self.decorrelateMapReduceConfig.cellSizeY = 41
569  self.decorrelateMapReduceConfig.borderSizeX = self.decorrelateMapReduceConfig.borderSizeY = 8
570  self.decorrelateMapReduceConfig.reducer.reduceOperation = 'average'
571 
572 
573 class DecorrelateALKernelSpatialTask(pipeBase.Task):
574  """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference
575 
576  Notes
577  -----
578 
579  Pipe-task that removes the neighboring-pixel covariance in an
580  image difference that are added when the template image is
581  convolved with the Alard-Lupton PSF matching kernel.
582 
583  This task is a simple wrapper around @ref DecorrelateALKernelTask,
584  which takes a `spatiallyVarying` parameter in its `run` method. If
585  it is `False`, then it simply calls the `run` method of @ref
586  DecorrelateALKernelTask. If it is True, then it uses the @ref
587  ImageMapReduceTask framework to break the exposures into
588  subExposures on a grid, and performs the `run` method of @ref
589  DecorrelateALKernelTask on each subExposure. This enables it to
590  account for spatially-varying PSFs and noise in the exposures when
591  performing the decorrelation.
592 
593  This task has no standalone example, however it is applied as a
594  subtask of pipe.tasks.imageDifference.ImageDifferenceTask.
595  There is also an example of its use in `tests/testImageDecorrelation.py`.
596  """
597  ConfigClass = DecorrelateALKernelSpatialConfig
598  _DefaultName = "ip_diffim_decorrelateALKernelSpatial"
599 
600  def __init__(self, *args, **kwargs):
601  """Create the image decorrelation Task
602 
603  Parameters
604  ----------
605  args :
606  arguments to be passed to
607  `lsst.pipe.base.task.Task.__init__`
608  kwargs :
609  additional keyword arguments to be passed to
610  `lsst.pipe.base.task.Task.__init__`
611  """
612  pipeBase.Task.__init__(self, *args, **kwargs)
613 
615  self.statsControl.setNumSigmaClip(3.)
616  self.statsControl.setNumIter(3)
617  self.statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes))
618 
619  def computeVarianceMean(self, exposure):
620  """Compute the mean of the variance plane of `exposure`.
621  """
622  statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(),
623  exposure.getMaskedImage().getMask(),
624  afwMath.MEANCLIP, self.statsControl)
625  var = statObj.getValue(afwMath.MEANCLIP)
626  return var
627 
628  def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel,
629  spatiallyVarying=True, preConvKernel=None):
630  """Perform decorrelation of an image difference exposure.
631 
632  Decorrelates the diffim due to the convolution of the
633  templateExposure with the A&L psfMatchingKernel. If
634  `spatiallyVarying` is True, it utilizes the spatially varying
635  matching kernel via the `imageMapReduce` framework to perform
636  spatially-varying decorrelation on a grid of subExposures.
637 
638  Parameters
639  ----------
640  scienceExposure : `lsst.afw.image.Exposure`
641  the science Exposure used for PSF matching
642  templateExposure : `lsst.afw.image.Exposure`
643  the template Exposure used for PSF matching
644  subtractedExposure : `lsst.afw.image.Exposure`
645  the subtracted Exposure produced by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
646  psfMatchingKernel :
647  an (optionally spatially-varying) PSF matching kernel produced
648  by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
649  spatiallyVarying : `bool`
650  if True, perform the spatially-varying operation
651  preConvKernel : `lsst.meas.algorithms.Psf`
652  if not none, the scienceExposure has been pre-filtered with this kernel. (Currently
653  this option is experimental.)
654 
655  Returns
656  -------
657  results : `lsst.pipe.base.Struct`
658  a structure containing:
659 
660  - ``correctedExposure`` : the decorrelated diffim
661 
662  """
663  self.log.info('Running A&L decorrelation: spatiallyVarying=%r' % spatiallyVarying)
664 
665  svar = self.computeVarianceMean(scienceExposure)
666  tvar = self.computeVarianceMean(templateExposure)
667  if np.isnan(svar) or np.isnan(tvar): # Should not happen unless entire image has been masked.
668  # Double check that one of the exposures is all NaNs
669  if (np.all(np.isnan(scienceExposure.image.array))
670  or np.all(np.isnan(templateExposure.image.array))):
671  self.log.warn('Template or science image is entirely NaNs: skipping decorrelation.')
672  if np.isnan(svar):
673  svar = 1e-9
674  if np.isnan(tvar):
675  tvar = 1e-9
676 
677  var = self.computeVarianceMean(subtractedExposure)
678 
679  if spatiallyVarying:
680  self.log.info("Variance (science, template): (%f, %f)", svar, tvar)
681  self.log.info("Variance (uncorrected diffim): %f", var)
682  config = self.config.decorrelateMapReduceConfig
683  task = ImageMapReduceTask(config=config)
684  results = task.run(subtractedExposure, science=scienceExposure,
685  template=templateExposure, psfMatchingKernel=psfMatchingKernel,
686  preConvKernel=preConvKernel, forceEvenSized=True)
687  results.correctedExposure = results.exposure
688 
689  # Make sure masks of input image are propagated to diffim
690  def gm(exp):
691  return exp.getMaskedImage().getMask()
692  gm(results.correctedExposure)[:, :] = gm(subtractedExposure)
693 
694  var = self.computeVarianceMean(results.correctedExposure)
695  self.log.info("Variance (corrected diffim): %f", var)
696 
697  else:
698  config = self.config.decorrelateConfig
699  task = DecorrelateALKernelTask(config=config)
700  results = task.run(scienceExposure, templateExposure,
701  subtractedExposure, psfMatchingKernel, preConvKernel=preConvKernel)
702 
703  return results
lsst::afw::image
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelSpatialConfig
Definition: imageDecorrelation.py:547
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelMapReduceConfig
Definition: imageDecorrelation.py:537
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelSpatialConfig.setDefaults
def setDefaults(self)
Definition: imageDecorrelation.py:566
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelSpatialTask.computeVarianceMean
def computeVarianceMean(self, exposure)
Definition: imageDecorrelation.py:619
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelSpatialTask.__init__
def __init__(self, *args, **kwargs)
Definition: imageDecorrelation.py:600
lsst::afw::math::FixedKernel
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelMapper.run
def run(self, subExposure, expandedSubExposure, fullBBox, template, science, alTaskResult=None, psfMatchingKernel=None, preConvKernel=None, **kwargs)
Definition: imageDecorrelation.py:461
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelTask.computeCommonShape
def computeCommonShape(self, *shapes)
Definition: imageDecorrelation.py:263
lsst::afw::math::makeStatistics
Statistics makeStatistics(lsst::afw::math::MaskedVector< EntryT > const &mv, std::vector< WeightPixel > const &vweights, int const flags, StatisticsControl const &sctrl=StatisticsControl())
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelMapper
Definition: imageDecorrelation.py:447
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelTask.computeCorrection
def computeCorrection(self, kappa, svar, tvar, preConvArr=None)
Definition: imageDecorrelation.py:343
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelTask.computeCorrectedImage
def computeCorrectedImage(self, corrft, imgOld)
Definition: imageDecorrelation.py:414
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelConfig
Definition: imageDecorrelation.py:42
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelSpatialTask
Definition: imageDecorrelation.py:573
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelTask.computeVarianceMean
def computeVarianceMean(self, exposure)
Definition: imageDecorrelation.py:104
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelTask.padCenterOriginArray
def padCenterOriginArray(A, tuple newShape, useInverse=False)
Definition: imageDecorrelation.py:296
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelSpatialTask.statsControl
statsControl
Definition: imageDecorrelation.py:614
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelSpatialConfig.decorrelateMapReduceConfig
decorrelateMapReduceConfig
Definition: imageDecorrelation.py:555
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelTask
Definition: imageDecorrelation.py:53
lsst::afw::math::StatisticsControl
lsst::geom
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelTask.freqSpaceShape
freqSpaceShape
Definition: imageDecorrelation.py:292
lsst::ip::diffim.imageMapReduce.ImageMapper
Definition: imageMapReduce.py:88
lsst::ip::diffim.imageMapReduce.ImageMapReduceConfig
Definition: imageMapReduce.py:376
lsst::afw::math
Point< double, 2 >
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelTask.statsControl
statsControl
Definition: imageDecorrelation.py:99
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelSpatialTask.run
def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel, spatiallyVarying=True, preConvKernel=None)
Definition: imageDecorrelation.py:628
lsst::ip::diffim.imageMapReduce.ImageMapReduceTask
Definition: imageMapReduce.py:487
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelMapper.__init__
def __init__(self, *args, **kwargs)
Definition: imageDecorrelation.py:458
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelTask.computeCorrectedDiffimPsf
def computeCorrectedDiffimPsf(self, corrft, psfOld)
Definition: imageDecorrelation.py:383
lsst::log
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelTask.__init__
def __init__(self, *args, **kwargs)
Definition: imageDecorrelation.py:87
lsst::ip::diffim.imageDecorrelation.DecorrelateALKernelTask.run
def run(self, exposure, templateExposure, subtractedExposure, psfMatchingKernel, preConvKernel=None, xcen=None, ycen=None, svar=None, tvar=None)
Definition: imageDecorrelation.py:112