lsst.ip.diffim  16.0-14-g8a3b804+4
modelPsfMatch.py
Go to the documentation of this file.
1 # LSST Data Management System
2 # Copyright 2008-2016 LSST Corporation.
3 #
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 #
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
11 #
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
16 #
17 # You should have received a copy of the LSST License Statement and
18 # the GNU General Public License along with this program. If not,
19 # see <http://www.lsstcorp.org/LegalNotices/>.
20 #
21 
22 import numpy as np
23 
24 from . import diffimLib
25 import lsst.afw.geom as afwGeom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.log as log
29 import lsst.pex.config as pexConfig
30 import lsst.pipe.base as pipeBase
31 from .makeKernelBasisList import makeKernelBasisList
32 from .psfMatch import PsfMatchTask, PsfMatchConfigAL
33 from . import utils as dituils
34 import lsst.afw.display.ds9 as ds9
35 
36 __all__ = ("ModelPsfMatchTask", "ModelPsfMatchConfig")
37 
38 sigma2fwhm = 2. * np.sqrt(2. * np.log(2.))
39 
40 
42  nextInt = int(np.ceil(x))
43  return nextInt + 1 if nextInt % 2 == 0 else nextInt
44 
45 
46 class ModelPsfMatchConfig(pexConfig.Config):
47  """!Configuration for model-to-model Psf matching"""
48 
49  kernel = pexConfig.ConfigChoiceField(
50  doc="kernel type",
51  typemap=dict(
52  AL=PsfMatchConfigAL,
53  ),
54  default="AL",
55  )
56  doAutoPadPsf = pexConfig.Field(
57  dtype=bool,
58  doc=("If too small, automatically pad the science Psf? "
59  "Pad to smallest dimensions appropriate for the matching kernel dimensions, "
60  "as specified by autoPadPsfTo. If false, pad by the padPsfBy config."),
61  default=True,
62  )
63  autoPadPsfTo = pexConfig.RangeField(
64  dtype=float,
65  doc=("Minimum Science Psf dimensions as a fraction of matching kernel dimensions. "
66  "If the dimensions of the Psf to be matched are less than the "
67  "matching kernel dimensions * autoPadPsfTo, pad Science Psf to this size. "
68  "Ignored if doAutoPadPsf=False."),
69  default=1.4,
70  min=1.0,
71  max=2.0
72  )
73  padPsfBy = pexConfig.Field(
74  dtype=int,
75  doc="Pixels (even) to pad Science Psf by before matching. Ignored if doAutoPadPsf=True",
76  default=0,
77  )
78 
79  def setDefaults(self):
80  # No sigma clipping
81  self.kernel.active.singleKernelClipping = False
82  self.kernel.active.kernelSumClipping = False
83  self.kernel.active.spatialKernelClipping = False
84  self.kernel.active.checkConditionNumber = False
85 
86  # Variance is ill defined
87  self.kernel.active.constantVarianceWeighting = True
88 
89  # Do not change specified kernel size
90  self.kernel.active.scaleByFwhm = False
91 
92 
93 
99 
100 
102  """!
103 @anchor ModelPsfMatchTask_
104 
105 @brief Matching of two model Psfs, and application of the Psf-matching kernel to an input Exposure
106 
107 @section ip_diffim_modelpsfmatch_Contents Contents
108 
109  - @ref ip_diffim_modelpsfmatch_Purpose
110  - @ref ip_diffim_modelpsfmatch_Initialize
111  - @ref ip_diffim_modelpsfmatch_IO
112  - @ref ip_diffim_modelpsfmatch_Config
113  - @ref ip_diffim_modelpsfmatch_Metadata
114  - @ref ip_diffim_modelpsfmatch_Debug
115  - @ref ip_diffim_modelpsfmatch_Example
116 
117 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
118 
119 @section ip_diffim_modelpsfmatch_Purpose Description
120 
121 This Task differs from ImagePsfMatchTask in that it matches two Psf _models_, by realizing
122 them in an Exposure-sized SpatialCellSet and then inserting each Psf-image pair into KernelCandidates.
123 Because none of the pairs of sources that are to be matched should be invalid, all sigma clipping is
124 turned off in ModelPsfMatchConfig. And because there is no tracked _variance_ in the Psf images, the
125 debugging and logging QA info should be interpreted with caution.
126 
127 One item of note is that the sizes of Psf models are fixed (e.g. its defined as a 21x21 matrix). When the
128 Psf-matching kernel is being solved for, the Psf "image" is convolved with each kernel basis function,
129 leading to a loss of information around the borders. This pixel loss will be problematic for the numerical
130 stability of the kernel solution if the size of the convolution kernel (set by ModelPsfMatchConfig.kernelSize)
131 is much bigger than: psfSize//2. Thus the sizes of Psf-model matching kernels are typically smaller
132 than their image-matching counterparts. If the size of the kernel is too small, the convolved stars will
133 look "boxy"; if the kernel is too large, the kernel solution will be "noisy". This is a trade-off that
134 needs careful attention for a given dataset.
135 
136 The primary use case for this Task is in matching an Exposure to a constant-across-the-sky Psf model for the
137 purposes of image coaddition. It is important to note that in the code, the "template" Psf is the Psf
138 that the science image gets matched to. In this sense the order of template and science image are
139 reversed, compared to ImagePsfMatchTask, which operates on the template image.
140 
141 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
142 
143 @section ip_diffim_modelpsfmatch_Initialize Task initialization
144 
145 @copydoc \_\_init\_\_
146 
147 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
148 
149 @section ip_diffim_modelpsfmatch_IO Invoking the Task
150 
151 @copydoc run
152 
153 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
154 
155 @section ip_diffim_modelpsfmatch_Config Configuration parameters
156 
157 See @ref ModelPsfMatchConfig
158 
159 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
160 
161 @section ip_diffim_modelpsfmatch_Metadata Quantities set in Metadata
162 
163 See @ref ip_diffim_psfmatch_Metadata "PsfMatchTask"
164 
165 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
166 
167 @section ip_diffim_modelpsfmatch_Debug Debug variables
168 
169 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a
170 flag @c -d/--debug to import @b debug.py from your @c PYTHONPATH. The relevant contents of debug.py
171 for this Task include:
172 
173 @code{.py}
174  import sys
175  import lsstDebug
176  def DebugInfo(name):
177  di = lsstDebug.getInfo(name)
178  if name == "lsst.ip.diffim.psfMatch":
179  di.display = True # global
180  di.maskTransparency = 80 # ds9 mask transparency
181  di.displayCandidates = True # show all the candidates and residuals
182  di.displayKernelBasis = False # show kernel basis functions
183  di.displayKernelMosaic = True # show kernel realized across the image
184  di.plotKernelSpatialModel = False # show coefficients of spatial model
185  di.showBadCandidates = True # show the bad candidates (red) along with good (green)
186  elif name == "lsst.ip.diffim.modelPsfMatch":
187  di.display = True # global
188  di.maskTransparency = 30 # ds9 mask transparency
189  di.displaySpatialCells = True # show spatial cells before the fit
190  return di
191  lsstDebug.Info = DebugInfo
192  lsstDebug.frame = 1
193 @endcode
194 
195 Note that if you want addional logging info, you may add to your scripts:
196 @code{.py}
197 import lsst.log.utils as logUtils
198 logUtils.traceSetAt("ip.diffim", 4)
199 @endcode
200 
201 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
202 
203 @section ip_diffim_modelpsfmatch_Example A complete example of using ModelPsfMatchTask
204 
205 This code is modelPsfMatchTask.py in the examples directory, and can be run as @em e.g.
206 @code
207 examples/modelPsfMatchTask.py
208 examples/modelPsfMatchTask.py --debug
209 examples/modelPsfMatchTask.py --debug --template /path/to/templateExp.fits --science /path/to/scienceExp.fits
210 @endcode
211 
212 @dontinclude modelPsfMatchTask.py
213 Create a subclass of ModelPsfMatchTask that accepts two exposures. Note that the "template" exposure
214 contains the Psf that will get matched to, and the "science" exposure is the one that will be convolved:
215 @skip MyModelPsfMatchTask
216 @until return
217 
218 And allow the user the freedom to either run the script in default mode, or point to their own images on disk.
219 Note that these images must be readable as an lsst.afw.image.Exposure:
220 @skip main
221 @until parse_args
222 
223 We have enabled some minor display debugging in this script via the --debug option. However, if you
224 have an lsstDebug debug.py in your PYTHONPATH you will get additional debugging displays. The following
225 block checks for this script:
226 @skip args.debug
227 @until sys.stderr
228 
229 @dontinclude modelPsfMatchTask.py
230 Finally, we call a run method that we define below. First set up a Config and modify some of the parameters.
231 In particular we don't want to "grow" the sizes of the kernel or KernelCandidates, since we are operating with
232 fixed--size images (i.e. the size of the input Psf models).
233 @skip run(args)
234 @until False
235 
236 Make sure the images (if any) that were sent to the script exist on disk and are readable. If no images
237 are sent, make some fake data up for the sake of this example script (have a look at the code if you want
238 more details on generateFakeData):
239 @skip requested
240 @until sizeCellY
241 
242 Display the two images if --debug:
243 @skip args.debug
244 @until Science
245 
246 Create and run the Task:
247 @skip Create
248 @until result
249 
250 And finally provide optional debugging display of the Psf-matched (via the Psf models) science image:
251 @skip args.debug
252 @until result.psfMatchedExposure
253 
254 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
255 
256  """
257  ConfigClass = ModelPsfMatchConfig
258 
259  def __init__(self, *args, **kwargs):
260  """!Create a ModelPsfMatchTask
261 
262  @param *args arguments to be passed to lsst.ip.diffim.PsfMatchTask.__init__
263  @param **kwargs keyword arguments to be passed to lsst.ip.diffim.PsfMatchTask.__init__
264 
265  Upon initialization, the kernel configuration is defined by self.config.kernel.active. This Task
266  does have a run() method, which is the default way to call the Task.
267  """
268  PsfMatchTask.__init__(self, *args, **kwargs)
269  self.kConfig = self.config.kernel.active
270 
271  @pipeBase.timeMethod
272  def run(self, exposure, referencePsfModel, kernelSum=1.0):
273  """!Psf-match an exposure to a model Psf
274 
275  @param exposure: Exposure to Psf-match to the reference Psf model;
276  it must return a valid PSF model via exposure.getPsf()
277  @param referencePsfModel: The Psf model to match to (an lsst.afw.detection.Psf)
278  @param kernelSum: A multipicative factor to apply to the kernel sum (default=1.0)
279 
280  @return
281  - psfMatchedExposure: the Psf-matched Exposure. This has the same parent bbox, Wcs, Calib and
282  Filter as the input Exposure but no Psf. In theory the Psf should equal referencePsfModel but
283  the match is likely not exact.
284  - psfMatchingKernel: the spatially varying Psf-matching kernel
285  - kernelCellSet: SpatialCellSet used to solve for the Psf-matching kernel
286  - referencePsfModel: Validated and/or modified reference model used
287 
288  Raise a RuntimeError if the Exposure does not contain a Psf model
289  """
290  if not exposure.hasPsf():
291  raise RuntimeError("exposure does not contain a Psf model")
292 
293  maskedImage = exposure.getMaskedImage()
294 
295  self.log.info("compute Psf-matching kernel")
296  result = self._buildCellSet(exposure, referencePsfModel)
297  kernelCellSet = result.kernelCellSet
298  referencePsfModel = result.referencePsfModel
299  fwhmScience = exposure.getPsf().computeShape().getDeterminantRadius() * sigma2fwhm
300  fwhmModel = referencePsfModel.computeShape().getDeterminantRadius() * sigma2fwhm
301 
302  basisList = makeKernelBasisList(self.kConfig, fwhmScience, fwhmModel, metadata=self.metadata)
303  spatialSolution, psfMatchingKernel, backgroundModel = self._solve(kernelCellSet, basisList)
304 
305  if psfMatchingKernel.isSpatiallyVarying():
306  sParameters = np.array(psfMatchingKernel.getSpatialParameters())
307  sParameters[0][0] = kernelSum
308  psfMatchingKernel.setSpatialParameters(sParameters)
309  else:
310  kParameters = np.array(psfMatchingKernel.getKernelParameters())
311  kParameters[0] = kernelSum
312  psfMatchingKernel.setKernelParameters(kParameters)
313 
314  self.log.info("Psf-match science exposure to reference")
315  psfMatchedExposure = afwImage.ExposureF(exposure.getBBox(), exposure.getWcs())
316  psfMatchedExposure.setFilter(exposure.getFilter())
317  psfMatchedExposure.setCalib(exposure.getCalib())
318  psfMatchedExposure.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
319  psfMatchedExposure.setPsf(referencePsfModel)
320  psfMatchedMaskedImage = psfMatchedExposure.getMaskedImage()
321 
322  # Normalize the psf-matching kernel while convolving since its magnitude is meaningless
323  # when PSF-matching one model to another.
324  doNormalize = True
325  afwMath.convolve(psfMatchedMaskedImage, maskedImage, psfMatchingKernel, doNormalize)
326 
327  self.log.info("done")
328  return pipeBase.Struct(psfMatchedExposure=psfMatchedExposure,
329  psfMatchingKernel=psfMatchingKernel,
330  kernelCellSet=kernelCellSet,
331  metadata=self.metadata,
332  )
333 
334  def _diagnostic(self, kernelCellSet, spatialSolution, spatialKernel, spatialBg):
335  """!Print diagnostic information on spatial kernel and background fit
336 
337  The debugging diagnostics are not really useful here, since the images we are matching have
338  no variance. Thus override the _diagnostic method to generate no logging information"""
339  return
340 
341  def _buildCellSet(self, exposure, referencePsfModel):
342  """!Build a SpatialCellSet for use with the solve method
343 
344  @param exposure: The science exposure that will be convolved; must contain a Psf
345  @param referencePsfModel: Psf model to match to
346 
347  @return
348  -kernelCellSet: a SpatialCellSet to be used by self._solve
349  -referencePsfModel: Validated and/or modified reference model used to populate the SpatialCellSet
350 
351  If the reference Psf model and science Psf model have different dimensions,
352  adjust the referencePsfModel (the model to which the exposure PSF will be matched)
353  to match that of the science Psf. If the science Psf dimensions vary across the image,
354  as is common with a WarpedPsf, either pad or clip (depending on config.padPsf)
355  the dimensions to be constant.
356  """
357  sizeCellX = self.kConfig.sizeCellX
358  sizeCellY = self.kConfig.sizeCellY
359 
360  scienceBBox = exposure.getBBox()
361  # Extend for proper spatial matching kernel all the way to edge, especially for narrow strips
362  scienceBBox.grow(afwGeom.Extent2I(sizeCellX, sizeCellY))
363 
364  sciencePsfModel = exposure.getPsf()
365 
366  dimenR = referencePsfModel.getLocalKernel().getDimensions()
367  psfWidth, psfHeight = dimenR
368 
369  regionSizeX, regionSizeY = scienceBBox.getDimensions()
370  scienceX0, scienceY0 = scienceBBox.getMin()
371 
372  kernelCellSet = afwMath.SpatialCellSet(afwGeom.Box2I(scienceBBox), sizeCellX, sizeCellY)
373 
374  nCellX = regionSizeX//sizeCellX
375  nCellY = regionSizeY//sizeCellY
376 
377  if nCellX == 0 or nCellY == 0:
378  raise ValueError("Exposure dimensions=%s and sizeCell=(%s, %s). Insufficient area to match" %
379  (scienceBBox.getDimensions(), sizeCellX, sizeCellY))
380 
381  # Survey the PSF dimensions of the Spatial Cell Set
382  # to identify the minimum enclosed or maximum bounding square BBox.
383  widthList = []
384  heightList = []
385  for row in range(nCellY):
386  posY = sizeCellY*row + sizeCellY//2 + scienceY0
387  for col in range(nCellX):
388  posX = sizeCellX*col + sizeCellX//2 + scienceX0
389  widthS, heightS = sciencePsfModel.computeBBox(afwGeom.Point2D(posX, posY)).getDimensions()
390  widthList.append(widthS)
391  heightList.append(heightS)
392 
393  psfSize = max(max(heightList), max(widthList))
394 
395  if self.config.doAutoPadPsf:
396  minPsfSize = nextOddInteger(self.kConfig.kernelSize*self.config.autoPadPsfTo)
397  paddingPix = max(0, minPsfSize - psfSize)
398  else:
399  if self.config.padPsfBy % 2 != 0:
400  raise ValueError("Config padPsfBy (%i pixels) must be even number." %
401  self.config.padPsfBy)
402  paddingPix = self.config.padPsfBy
403 
404  if paddingPix > 0:
405  self.log.info("Padding Science PSF from (%s, %s) to (%s, %s) pixels" %
406  (psfSize, psfSize, paddingPix + psfSize, paddingPix + psfSize))
407  psfSize += paddingPix
408 
409  # Check that PSF is larger than the matching kernel
410  maxKernelSize = psfSize - 1
411  if maxKernelSize % 2 == 0:
412  maxKernelSize -= 1
413  if self.kConfig.kernelSize > maxKernelSize:
414  message = """
415  Kernel size (%d) too big to match Psfs of size %d.
416  Please reconfigure by setting one of the following:
417  1) kernel size to <= %d
418  2) doAutoPadPsf=True
419  3) padPsfBy to >= %s
420  """ % (self.kConfig.kernelSize, psfSize,
421  maxKernelSize, self.kConfig.kernelSize - maxKernelSize)
422  raise ValueError(message)
423 
424  dimenS = afwGeom.Extent2I(psfSize, psfSize)
425 
426  if (dimenR != dimenS):
427  try:
428  referencePsfModel = referencePsfModel.resized(psfSize, psfSize)
429  self.log.info("Adjusted dimensions of reference PSF model from %s to %s" % (dimenR, dimenS))
430  except Exception as e:
431  self.log.warn("Zero padding or clipping the reference PSF model of type %s and dimensions %s"
432  " to the science Psf dimensions %s because: %s",
433  referencePsfModel.__class__.__name__, dimenR, dimenS, e)
434  dimenR = dimenS
435 
436  policy = pexConfig.makePolicy(self.kConfig)
437  for row in range(nCellY):
438  # place at center of cell
439  posY = sizeCellY * row + sizeCellY//2 + scienceY0
440 
441  for col in range(nCellX):
442  # place at center of cell
443  posX = sizeCellX * col + sizeCellX//2 + scienceX0
444 
445  log.log("TRACE4." + self.log.getName(), log.DEBUG,
446  "Creating Psf candidate at %.1f %.1f", posX, posY)
447 
448  # reference kernel image, at location of science subimage
449  referenceMI = self._makePsfMaskedImage(referencePsfModel, posX, posY, dimensions=dimenR)
450 
451  # kernel image we are going to convolve
452  scienceMI = self._makePsfMaskedImage(sciencePsfModel, posX, posY, dimensions=dimenR)
453 
454  # The image to convolve is the science image, to the reference Psf.
455  kc = diffimLib.makeKernelCandidate(posX, posY, scienceMI, referenceMI, policy)
456  kernelCellSet.insertCandidate(kc)
457 
458  import lsstDebug
459  display = lsstDebug.Info(__name__).display
460  displaySpatialCells = lsstDebug.Info(__name__).displaySpatialCells
461  maskTransparency = lsstDebug.Info(__name__).maskTransparency
462  if not maskTransparency:
463  maskTransparency = 0
464  if display:
465  ds9.setMaskTransparency(maskTransparency)
466  if display and displaySpatialCells:
467  dituils.showKernelSpatialCells(exposure.getMaskedImage(), kernelCellSet,
468  symb="o", ctype=ds9.CYAN, ctypeUnused=ds9.YELLOW, ctypeBad=ds9.RED,
469  size=4, frame=lsstDebug.frame, title="Image to be convolved")
470  lsstDebug.frame += 1
471  return pipeBase.Struct(kernelCellSet=kernelCellSet,
472  referencePsfModel=referencePsfModel,
473  )
474 
475  def _makePsfMaskedImage(self, psfModel, posX, posY, dimensions=None):
476  """! Return a MaskedImage of the a PSF Model of specified dimensions
477  """
478  rawKernel = psfModel.computeKernelImage(afwGeom.Point2D(posX, posY)).convertF()
479  if dimensions is None:
480  dimensions = rawKernel.getDimensions()
481  if rawKernel.getDimensions() == dimensions:
482  kernelIm = rawKernel
483  else:
484  # make image of proper size
485  kernelIm = afwImage.ImageF(dimensions)
486  bboxToPlace = afwGeom.Box2I(afwGeom.Point2I((dimensions.getX() - rawKernel.getWidth())//2,
487  (dimensions.getY() - rawKernel.getHeight())//2),
488  rawKernel.getDimensions())
489  kernelIm.assign(rawKernel, bboxToPlace)
490 
491  kernelMask = afwImage.Mask(dimensions, 0x0)
492  kernelVar = afwImage.ImageF(dimensions, 1.0)
493  return afwImage.MaskedImageF(kernelIm, kernelMask, kernelVar)
Base class for Psf Matching; should not be called directly.
Definition: psfMatch.py:529
def makeKernelBasisList(config, targetFwhmPix=None, referenceFwhmPix=None, basisDegGauss=None, metadata=None)
def _buildCellSet(self, exposure, referencePsfModel)
Build a SpatialCellSet for use with the solve method.
def _solve(self, kernelCellSet, basisList, returnOnExcept=False)
Solve for the PSF matching kernel.
Definition: psfMatch.py:895
Matching of two model Psfs, and application of the Psf-matching kernel to an input Exposure...
Configuration for model-to-model Psf matching.
def run(self, exposure, referencePsfModel, kernelSum=1.0)
Psf-match an exposure to a model Psf.
def __init__(self, args, kwargs)
Create a ModelPsfMatchTask.
void convolve(OutImageT &convolvedImage, InImageT const &inImage, KernelT const &kernel, bool doNormalize, bool doCopyEdge=false)