lsst.ip.diffim  13.0-17-gc2cefa3
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
imageDecorrelation.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 from future import standard_library
3 standard_library.install_aliases()
4 #
5 # LSST Data Management System
6 # Copyright 2016 AURA/LSST.
7 #
8 # This product includes software developed by the
9 # LSST Project (http://www.lsst.org/).
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the LSST License Statement and
22 # the GNU General Public License along with this program. If not,
23 # see <https://www.lsstcorp.org/LegalNotices/>.
24 #
25 
26 import numpy as np
27 import scipy.fftpack
28 
29 import lsst.afw.image as afwImage
30 import lsst.afw.geom as afwGeom
31 import lsst.meas.algorithms as measAlg
32 import lsst.afw.math as afwMath
33 import lsst.pex.config as pexConfig
34 import lsst.pipe.base as pipeBase
35 import lsst.log
36 
37 from .imageMapReduce import (ImageMapReduceConfig, ImageMapperSubtask)
38 
39 __all__ = ("DecorrelateALKernelTask", "DecorrelateALKernelConfig",
40  "DecorrelateALKernelMapperSubtask", "DecorrelateALKernelMapReduceConfig")
41 
42 
43 class DecorrelateALKernelConfig(pexConfig.Config):
44  """!
45  \anchor DecorrelateALKernelConfig_
46 
47  \brief Configuration parameters for the DecorrelateALKernelTask
48  """
49 
50  ignoreMaskPlanes = pexConfig.ListField(
51  dtype=str,
52  doc="""Mask planes to ignore for sigma-clipped statistics""",
53  default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE")
54  )
55 
56 ## \addtogroup LSST_task_documentation
57 ## \{
58 ## \page DecorrelateALKernelTask
59 ## \ref DecorrelateALKernelTask_ "DecorrelateALKernelTask"
60 ## Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference
61 ## \}
62 
63 
64 class DecorrelateALKernelTask(pipeBase.Task):
65  """!
66  \anchor DecorrelateALKernelTask_
67 
68  \brief Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference
69 
70  \section pipe_tasks_multiBand_Contents Contents
71 
72  - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Purpose
73  - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Config
74  - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Run
75  - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Debug
76  - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Example
77 
78  \section ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Purpose Description
79 
80  Pipe-task that removes the neighboring-pixel covariance in an
81  image difference that are added when the template image is
82  convolved with the Alard-Lupton PSF matching kernel.
83 
84  The image differencing pipeline task \link
85  ip.diffim.psfMatch.PsfMatchTask PSFMatchTask\endlink and \link
86  ip.diffim.psfMatch.PsfMatchConfigAL PSFMatchConfigAL\endlink uses
87  the Alard and Lupton (1998) method for matching the PSFs of the
88  template and science exposures prior to subtraction. The
89  Alard-Lupton method identifies a matching kernel, which is then
90  (typically) convolved with the template image to perform PSF
91  matching. This convolution has the effect of adding covariance
92  between neighboring pixels in the template image, which is then
93  added to the image difference by subtraction.
94 
95  The pixel covariance may be corrected by whitening the noise of
96  the image difference. This task performs such a decorrelation by
97  computing a decorrelation kernel (based upon the A&L matching
98  kernel and variances in the template and science images) and
99  convolving the image difference with it. This process is described
100  in detail in [DMTN-021](http://dmtn-021.lsst.io).
101 
102  \section ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Initialize Task initialization
103 
104  \copydoc \_\_init\_\_
105 
106  \section ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Run Invoking the Task
107 
108  \copydoc run
109 
110  \section ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Config Configuration parameters
111 
112  This task currently has no relevant configuration parameters.
113  See \ref DecorrelateALKernelConfig
114 
115  \section ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Debug Debug variables
116 
117  This task has no debug variables
118 
119  \section ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Example Example of using DecorrelateALKernelTask
120 
121  This task has no standalone example, however it is applied as a
122  subtask of \link pipe.tasks.imageDifference.ImageDifferenceTask ImageDifferenceTask\endlink .
123 
124  """
125  ConfigClass = DecorrelateALKernelConfig
126  _DefaultName = "ip_diffim_decorrelateALKernel"
127 
128  def __init__(self, *args, **kwargs):
129  """! Create the image decorrelation Task
130  @param *args arguments to be passed to lsst.pipe.base.task.Task.__init__
131  @param **kwargs keyword arguments to be passed to lsst.pipe.base.task.Task.__init__
132  """
133  pipeBase.Task.__init__(self, *args, **kwargs)
134 
135  self.statsControl = afwMath.StatisticsControl()
136  self.statsControl.setNumSigmaClip(3.)
137  self.statsControl.setNumIter(3)
138  self.statsControl.setAndMask(afwImage.MaskU.getPlaneBitMask(self.config.ignoreMaskPlanes))
139 
140  def computeVarianceMean(self, exposure):
141  statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(),
142  exposure.getMaskedImage().getMask(),
143  afwMath.MEANCLIP, self.statsControl)
144  var = statObj.getValue(afwMath.MEANCLIP)
145  return var
146 
147  @pipeBase.timeMethod
148  def run(self, exposure, templateExposure, subtractedExposure, psfMatchingKernel,
149  xcen=None, ycen=None, svar=None, tvar=None):
150  """! Perform decorrelation of an image difference exposure.
151 
152  Decorrelates the diffim due to the convolution of the templateExposure with the
153  A&L PSF matching kernel. Currently can accept a spatially varying matching kernel but in
154  this case it simply uses a static kernel from the center of the exposure. The decorrelation
155  is described in [DMTN-021, Equation 1](http://dmtn-021.lsst.io/#equation-1), where
156  `exposure` is I_1; templateExposure is I_2; `subtractedExposure` is D(k);
157  `psfMatchingKernel` is kappa; and svar and tvar are their respective
158  variances (see below).
159 
160  @param[in] exposure the science afwImage.Exposure used for PSF matching
161  @param[in] templateExposure the template afwImage.Exposure used for PSF matching
162  @param[in] subtractedExposure the subtracted exposure produced by
163  `ip_diffim.ImagePsfMatchTask.subtractExposures()`
164  @param[in] psfMatchingKernel an (optionally spatially-varying) PSF matching kernel produced
165  by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
166  @param[in] xcen X-pixel coordinate to use for computing constant matching kernel to use
167  If `None` (default), then use the center of the image.
168  @param[in] ycen Y-pixel coordinate to use for computing constant matching kernel to use
169  If `None` (default), then use the center of the image.
170  @param[in] svar image variance for science image
171  If `None` (default) then compute the variance over the entire input science image.
172  @param[in] tvar image variance for template image
173  If `None` (default) then compute the variance over the entire input template image.
174 
175  @return a `pipeBase.Struct` containing:
176  * `correctedExposure`: the decorrelated diffim
177  * `correctionKernel`: the decorrelation correction kernel (which may be ignored)
178 
179  @note The `subtractedExposure` is NOT updated
180  @note The returned `correctedExposure` has an updated PSF as well.
181  @note Here we currently convert a spatially-varying matching kernel into a constant kernel,
182  just by computing it at the center of the image (tickets DM-6243, DM-6244).
183  @note We are also using a constant accross-the-image measure of sigma (sqrt(variance)) to compute
184  the decorrelation kernel.
185  @note Still TBD (ticket DM-6580): understand whether the convolution is correctly modifying
186  the variance plane of the new subtractedExposure.
187  """
188  spatialKernel = psfMatchingKernel
189  kimg = afwImage.ImageD(spatialKernel.getDimensions())
190  bbox = subtractedExposure.getBBox()
191  if xcen is None:
192  xcen = (bbox.getBeginX() + bbox.getEndX()) / 2.
193  if ycen is None:
194  ycen = (bbox.getBeginY() + bbox.getEndY()) / 2.
195  self.log.info("Using matching kernel computed at (%d, %d)", xcen, ycen)
196  spatialKernel.computeImage(kimg, True, xcen, ycen)
197 
198  if svar is None:
199  svar = self.computeVarianceMean(exposure)
200  if tvar is None:
201  tvar = self.computeVarianceMean(templateExposure)
202  self.log.info("Variance (science, template): (%f, %f)", svar, tvar)
203 
204  var = self.computeVarianceMean(subtractedExposure)
205  self.log.info("Variance (uncorrected diffim): %f", var)
206 
207  corrKernel = DecorrelateALKernelTask._computeDecorrelationKernel(kimg.getArray(), svar, tvar)
208  correctedExposure, corrKern = DecorrelateALKernelTask._doConvolve(subtractedExposure, corrKernel)
209 
210  # Compute the subtracted exposure's updated psf
211  psf = subtractedExposure.getPsf().computeKernelImage(afwGeom.Point2D(xcen, ycen)).getArray()
212  psfc = DecorrelateALKernelTask.computeCorrectedDiffimPsf(corrKernel, psf, svar=svar, tvar=tvar)
213  psfcI = afwImage.ImageD(psfc.shape[0], psfc.shape[1])
214  psfcI.getArray()[:, :] = psfc
215  psfcK = afwMath.FixedKernel(psfcI)
216  psfNew = measAlg.KernelPsf(psfcK)
217  correctedExposure.setPsf(psfNew)
218 
219  var = self.computeVarianceMean(correctedExposure)
220  self.log.info("Variance (corrected diffim): %f", var)
221 
222  return pipeBase.Struct(correctedExposure=correctedExposure, correctionKernel=corrKern)
223 
224  @staticmethod
225  def _computeDecorrelationKernel(kappa, svar=0.04, tvar=0.04):
226  """! Compute the Lupton/ZOGY post-conv. kernel for decorrelating an
227  image difference, based on the PSF-matching kernel.
228  @param kappa A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching
229  @param svar Average variance of science image used for PSF matching
230  @param tvar Average variance of template image used for PSF matching
231  @return a 2-d numpy.array containing the correction kernel
232 
233  @note As currently implemented, kappa is a static (single, non-spatially-varying) kernel.
234  """
235  kappa = DecorrelateALKernelTask._fixOddKernel(kappa)
236  kft = scipy.fftpack.fft2(kappa)
237  kft = np.sqrt((svar + tvar) / (svar + tvar * kft**2))
238  pck = scipy.fftpack.ifft2(kft)
239  pck = scipy.fftpack.ifftshift(pck.real)
240  fkernel = DecorrelateALKernelTask._fixEvenKernel(pck)
241 
242  # I think we may need to "reverse" the PSF, as in the ZOGY (and Kaiser) papers...
243  # This is the same as taking the complex conjugate in Fourier space before FFT-ing back to real space.
244  if False: # TBD: figure this out. For now, we are turning it off.
245  fkernel = fkernel[::-1, :]
246 
247  return fkernel
248 
249  @staticmethod
250  def computeCorrectedDiffimPsf(kappa, psf, svar=0.04, tvar=0.04):
251  """! Compute the (decorrelated) difference image's new PSF.
252  new_psf = psf(k) * sqrt((svar + tvar) / (svar + tvar * kappa_ft(k)**2))
253 
254  @param kappa A matching kernel array derived from Alard & Lupton PSF matching
255  @param psf The uncorrected psf array of the science image (and also of the diffim)
256  @param svar Average variance of science image used for PSF matching
257  @param tvar Average variance of template image used for PSF matching
258  @return a 2-d numpy.array containing the new PSF
259  """
260  def post_conv_psf_ft2(psf, kernel, svar, tvar):
261  # Pad psf or kernel symmetrically to make them the same size!
262  # Note this assumes they are both square (width == height)
263  if psf.shape[0] < kernel.shape[0]:
264  diff = (kernel.shape[0] - psf.shape[0]) // 2
265  psf = np.pad(psf, (diff, diff), mode='constant')
266  elif psf.shape[0] > kernel.shape[0]:
267  diff = (psf.shape[0] - kernel.shape[0]) // 2
268  kernel = np.pad(kernel, (diff, diff), mode='constant')
269  psf_ft = scipy.fftpack.fft2(psf)
270  kft = scipy.fftpack.fft2(kernel)
271  out = psf_ft * np.sqrt((svar + tvar) / (svar + tvar * kft**2))
272  return out
273 
274  def post_conv_psf(psf, kernel, svar, tvar):
275  kft = post_conv_psf_ft2(psf, kernel, svar, tvar)
276  out = scipy.fftpack.ifft2(kft)
277  return out
278 
279  pcf = post_conv_psf(psf=psf, kernel=kappa, svar=svar, tvar=tvar)
280  pcf = pcf.real / pcf.real.sum()
281  return pcf
282 
283  @staticmethod
284  def _fixOddKernel(kernel):
285  """! Take a kernel with odd dimensions and make them even for FFT
286 
287  @param kernel a numpy.array
288  @return a fixed kernel numpy.array. Returns a copy if the dimensions needed to change;
289  otherwise just return the input kernel.
290  """
291  # Note this works best for the FFT if we left-pad
292  out = kernel
293  changed = False
294  if (out.shape[0] % 2) == 1:
295  out = np.pad(out, ((1, 0), (0, 0)), mode='constant')
296  changed = True
297  if (out.shape[1] % 2) == 1:
298  out = np.pad(out, ((0, 0), (1, 0)), mode='constant')
299  changed = True
300  if changed:
301  out *= (np.mean(kernel) / np.mean(out)) # need to re-scale to same mean for FFT
302  return out
303 
304  @staticmethod
305  def _fixEvenKernel(kernel):
306  """! Take a kernel with even dimensions and make them odd, centered correctly.
307  @param kernel a numpy.array
308  @return a fixed kernel numpy.array
309  """
310  # Make sure the peak (close to a delta-function) is in the center!
311  maxloc = np.unravel_index(np.argmax(kernel), kernel.shape)
312  out = np.roll(kernel, kernel.shape[0]//2 - maxloc[0], axis=0)
313  out = np.roll(out, out.shape[1]//2 - maxloc[1], axis=1)
314  # Make sure it is odd-dimensioned by trimming it.
315  if (out.shape[0] % 2) == 0:
316  maxloc = np.unravel_index(np.argmax(out), out.shape)
317  if out.shape[0] - maxloc[0] > maxloc[0]:
318  out = out[:-1, :]
319  else:
320  out = out[1:, :]
321  if out.shape[1] - maxloc[1] > maxloc[1]:
322  out = out[:, :-1]
323  else:
324  out = out[:, 1:]
325  return out
326 
327  @staticmethod
328  def _doConvolve(exposure, kernel):
329  """! Convolve an Exposure with a decorrelation convolution kernel.
330  @param exposure Input afw.image.Exposure to be convolved.
331  @param kernel Input 2-d numpy.array to convolve the image with
332  @return a new Exposure with the convolved pixels and the (possibly
333  re-centered) kernel.
334 
335  @note We use afwMath.convolve() but keep scipy.convolve for debugging.
336  @note We re-center the kernel if necessary and return the possibly re-centered kernel
337  """
338  kernelImg = afwImage.ImageD(kernel.shape[0], kernel.shape[1])
339  kernelImg.getArray()[:, :] = kernel
340  kern = afwMath.FixedKernel(kernelImg)
341  maxloc = np.unravel_index(np.argmax(kernel), kernel.shape)
342  kern.setCtrX(maxloc[0])
343  kern.setCtrY(maxloc[1])
344  outExp = exposure.clone() # Do this to keep WCS, PSF, masks, etc.
345  convCntrl = afwMath.ConvolutionControl(False, True, 0)
346  afwMath.convolve(outExp.getMaskedImage(), exposure.getMaskedImage(), kern, convCntrl)
347 
348  return outExp, kern
349 
350 
352  """Task to be used as an ImageMapperSubtask for performing
353  A&L decorrelation on subimages on a grid across a A&L difference image.
354 
355  This task subclasses DecorrelateALKernelTask in order to implement
356  all of that task's configuration parameters, as well as its `run` method.
357  """
358  ConfigClass = DecorrelateALKernelConfig
359  _DefaultName = 'ip_diffim_decorrelateALKernelMapper'
360 
361  def __init__(self, *args, **kwargs):
362  DecorrelateALKernelTask.__init__(self, *args, **kwargs)
363 
364  def run(self, subExposure, expandedSubExposure, fullBBox,
365  template, science, alTaskResult=None, psfMatchingKernel=None,
366  preConvKernel=None, **kwargs):
367  """Perform decorrelation operation on `subExposure`, using
368  `expandedSubExposure` to allow for invalid edge pixels arising from
369  convolutions.
370 
371  This method performs A&L decorrelation on `subExposure` using
372  local measures for image variances and PSF. `subExposure` is a
373  sub-exposure of the non-decorrelated A&L diffim. It also
374  requires the corresponding sub-exposures of the template
375  (`template`) and science (`science`) exposures.
376 
377  Parameters
378  ----------
379  subExposure : afw.Exposure
380  the sub-exposure of the diffim
381  expandedSubExposure : afw.Exposure
382  the expanded sub-exposure upon which to operate
383  fullBBox : afwGeom.BoundingBox
384  the bounding box of the original exposure
385  template : afw.Exposure
386  the corresponding sub-exposure of the template exposure
387  science : afw.Exposure
388  the corresponding sub-exposure of the science exposure
389  alTaskResult : pipeBase.Struct
390  the result of A&L image differencing on `science` and
391  `template`, importantly containing the resulting
392  `psfMatchingKernel`. Can be `None`, only if
393  `psfMatchingKernel` is not `None`.
394  psfMatchingKernel : Alternative parameter for passing the
395  A&L `psfMatchingKernel` directly.
396  kwargs :
397  additional keyword arguments propagated from
398  `ImageMapReduceTask.run`.
399 
400  Returns
401  -------
402  A `pipeBase.Struct containing the result of the `subExposure`
403  processing, labelled 'subExposure'. It also returns the
404  'decorrelationKernel', although that currently is not used.
405 
406  Notes
407  -----
408  This `run` method accepts parameters identical to those of
409  `ImageMapperSubtask.run`, since it is called from the
410  `ImageMapperTask`. See that class for more information.
411  """
412  templateExposure = template # input template
413  scienceExposure = science # input science image
414  if alTaskResult is None and psfMatchingKernel is None:
415  raise RuntimeError('Both alTaskResult and psfMatchingKernel cannot be None')
416  psfMatchingKernel = alTaskResult.psfMatchingKernel if alTaskResult is not None else psfMatchingKernel
417 
418  # subExp and expandedSubExp are subimages of the (un-decorrelated) diffim!
419  # So here we compute corresponding subimages of templateExposure and scienceExposure
420  subExp2 = scienceExposure.Factory(scienceExposure, expandedSubExposure.getBBox())
421  subExp1 = templateExposure.Factory(templateExposure, expandedSubExposure.getBBox())
422 
423  # Prevent too much log INFO verbosity from DecorrelateALKernelTask.run
424  logLevel = self.log.getLevel()
425  self.log.setLevel(lsst.log.WARN)
426  res = DecorrelateALKernelTask.run(self, subExp2, subExp1, expandedSubExposure,
427  psfMatchingKernel)
428  self.log.setLevel(logLevel) # reset the log level
429 
430  diffim = res.correctedExposure.Factory(res.correctedExposure, subExposure.getBBox())
431  out = pipeBase.Struct(subExposure=diffim, decorrelationKernel=res.correctionKernel)
432  return out
433 
434 
435 class DecorrelateALKernelMapReduceConfig(ImageMapReduceConfig):
436  """Configuration parameters for the ImageMapReduceTask to direct it to use
437  DecorrelateALKernelMapperSubtask as its mapperSubtask for A&L decorrelation.
438  """
439  mapperSubtask = pexConfig.ConfigurableField(
440  doc='A&L decorrelation subtask to run on each sub-image',
441  target=DecorrelateALKernelMapperSubtask
442  )
def _fixOddKernel
Take a kernel with odd dimensions and make them even for FFT.
def run
Perform decorrelation of an image difference exposure.
def computeCorrectedDiffimPsf
Compute the (decorrelated) difference image&#39;s new PSF.
def _fixEvenKernel
Take a kernel with even dimensions and make them odd, centered correctly.
Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference.
Configuration parameters for the DecorrelateALKernelTask.
def _doConvolve
Convolve an Exposure with a decorrelation convolution kernel.
def _computeDecorrelationKernel
Compute the Lupton/ZOGY post-conv.