lsst.ip.diffim  15.0-3-ge6a6747+1
psfMatch.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2016 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 from __future__ import absolute_import, division, print_function
23 import time
24 
25 import numpy as np
26 from builtins import range
27 
28 import lsst.afw.image as afwImage
29 import lsst.pex.config as pexConfig
30 import lsst.afw.math as afwMath
31 import lsst.afw.display.ds9 as ds9
32 import lsst.log as log
33 import lsst.pipe.base as pipeBase
34 from lsst.meas.algorithms import SubtractBackgroundConfig
35 from . import utils as diUtils
36 from . import diffimLib
37 
38 
39 class DetectionConfig(pexConfig.Config):
40  """!Configuration for detecting sources on images for building a PSF-matching kernel
41 
42  Configuration for turning detected lsst.afw.detection.FootPrints into an acceptable
43  (unmasked, high signal-to-noise, not too large or not too small) list of
44  lsst.ip.diffim.KernelSources that are used to build the Psf-matching kernel"""
45 
46  detThreshold = pexConfig.Field(
47  dtype=float,
48  doc="Value of footprint detection threshold",
49  default=10.0,
50  check=lambda x: x >= 3.0
51  )
52  detThresholdType = pexConfig.ChoiceField(
53  dtype=str,
54  doc="Type of detection threshold",
55  default="pixel_stdev",
56  allowed={
57  "value": "Use counts as the detection threshold type",
58  "stdev": "Use standard deviation of image plane",
59  "variance": "Use variance of image plane",
60  "pixel_stdev": "Use stdev derived from variance plane"
61  }
62  )
63  detOnTemplate = pexConfig.Field(
64  dtype=bool,
65  doc="""If true run detection on the template (image to convolve);
66  if false run detection on the science image""",
67  default=True
68  )
69  badMaskPlanes = pexConfig.ListField(
70  dtype=str,
71  doc="""Mask planes that lead to an invalid detection.
72  Options: NO_DATA EDGE SAT BAD CR INTRP""",
73  default=("NO_DATA", "EDGE", "SAT")
74  )
75  fpNpixMin = pexConfig.Field(
76  dtype=int,
77  doc="Minimum number of pixels in an acceptable Footprint",
78  default=5,
79  check=lambda x: x >= 5
80  )
81  fpNpixMax = pexConfig.Field(
82  dtype=int,
83  doc="""Maximum number of pixels in an acceptable Footprint;
84  too big and the subsequent convolutions become unwieldy""",
85  default=500,
86  check=lambda x: x <= 500
87  )
88  fpGrowKernelScaling = pexConfig.Field(
89  dtype=float,
90  doc="""If config.scaleByFwhm, grow the footprint based on
91  the final kernelSize. Each footprint will be
92  2*fpGrowKernelScaling*kernelSize x
93  2*fpGrowKernelScaling*kernelSize. With the value
94  of 1.0, the remaining pixels in each KernelCandiate
95  after convolution by the basis functions will be
96  equal to the kernel size itself.""",
97  default=1.0,
98  check=lambda x: x >= 1.0
99  )
100  fpGrowPix = pexConfig.Field(
101  dtype=int,
102  doc="""Growing radius (in pixels) for each raw detection
103  footprint. The smaller the faster; however the
104  kernel sum does not converge if the stamp is too
105  small; and the kernel is not constrained at all if
106  the stamp is the size of the kernel. The grown stamp
107  is 2 * fpGrowPix pixels larger in each dimension.
108  This is overridden by fpGrowKernelScaling if scaleByFwhm""",
109  default=30,
110  check=lambda x: x >= 10
111  )
112  scaleByFwhm = pexConfig.Field(
113  dtype=bool,
114  doc="Scale fpGrowPix by input Fwhm?",
115  default=True,
116  )
117 
118 
119 class PsfMatchConfig(pexConfig.Config):
120  """!Base configuration for Psf-matching
121 
122  The base configuration of the Psf-matching kernel, and of the warping, detection,
123  and background modeling subTasks."""
124 
125  warpingConfig = pexConfig.ConfigField("Config for warping exposures to a common alignment",
126  afwMath.warper.WarperConfig)
127  detectionConfig = pexConfig.ConfigField("Controlling the detection of sources for kernel building",
128  DetectionConfig)
129  afwBackgroundConfig = pexConfig.ConfigField("Controlling the Afw background fitting",
130  SubtractBackgroundConfig)
131 
132  useAfwBackground = pexConfig.Field(
133  dtype=bool,
134  doc="Use afw background subtraction instead of ip_diffim",
135  default=False,
136  )
137  fitForBackground = pexConfig.Field(
138  dtype=bool,
139  doc="Include terms (including kernel cross terms) for background in ip_diffim",
140  default=False,
141  )
142  kernelBasisSet = pexConfig.ChoiceField(
143  dtype=str,
144  doc="Type of basis set for PSF matching kernel.",
145  default="alard-lupton",
146  allowed={
147  "alard-lupton": """Alard-Lupton sum-of-gaussians basis set,
148  * The first term has no spatial variation
149  * The kernel sum is conserved
150  * You may want to turn off 'usePcaForSpatialKernel'""",
151  "delta-function": """Delta-function kernel basis set,
152  * You may enable the option useRegularization
153  * You should seriously consider usePcaForSpatialKernel, which will also
154  enable kernel sum conservation for the delta function kernels"""
155  }
156  )
157  kernelSize = pexConfig.Field(
158  dtype=int,
159  doc="""Number of rows/columns in the convolution kernel; should be odd-valued.
160  Modified by kernelSizeFwhmScaling if scaleByFwhm = true""",
161  default=21,
162  )
163  scaleByFwhm = pexConfig.Field(
164  dtype=bool,
165  doc="Scale kernelSize, alardGaussians by input Fwhm",
166  default=True,
167  )
168  kernelSizeFwhmScaling = pexConfig.Field(
169  dtype=float,
170  doc="""How much to scale the kernel size based on the largest AL Sigma""",
171  default=6.0,
172  check=lambda x: x >= 1.0
173  )
174  kernelSizeMin = pexConfig.Field(
175  dtype=int,
176  doc="""Minimum Kernel Size""",
177  default=21,
178  )
179  kernelSizeMax = pexConfig.Field(
180  dtype=int,
181  doc="""Maximum Kernel Size""",
182  default=35,
183  )
184  spatialModelType = pexConfig.ChoiceField(
185  dtype=str,
186  doc="Type of spatial functions for kernel and background",
187  default="chebyshev1",
188  allowed={
189  "chebyshev1": "Chebyshev polynomial of the first kind",
190  "polynomial": "Standard x,y polynomial",
191  }
192  )
193  spatialKernelOrder = pexConfig.Field(
194  dtype=int,
195  doc="Spatial order of convolution kernel variation",
196  default=2,
197  check=lambda x: x >= 0
198  )
199  spatialBgOrder = pexConfig.Field(
200  dtype=int,
201  doc="Spatial order of differential background variation",
202  default=1,
203  check=lambda x: x >= 0
204  )
205  sizeCellX = pexConfig.Field(
206  dtype=int,
207  doc="Size (rows) in pixels of each SpatialCell for spatial modeling",
208  default=128,
209  check=lambda x: x >= 32
210  )
211  sizeCellY = pexConfig.Field(
212  dtype=int,
213  doc="Size (columns) in pixels of each SpatialCell for spatial modeling",
214  default=128,
215  check=lambda x: x >= 32
216  )
217  nStarPerCell = pexConfig.Field(
218  dtype=int,
219  doc="Number of KernelCandidates in each SpatialCell to use in the spatial fitting",
220  default=3,
221  check=lambda x: x >= 1
222  )
223  maxSpatialIterations = pexConfig.Field(
224  dtype=int,
225  doc="Maximum number of iterations for rejecting bad KernelCandidates in spatial fitting",
226  default=3,
227  check=lambda x: x >= 1 and x <= 5
228  )
229  usePcaForSpatialKernel = pexConfig.Field(
230  dtype=bool,
231  doc="""Use Pca to reduce the dimensionality of the kernel basis sets.
232  This is particularly useful for delta-function kernels.
233  Functionally, after all Cells have their raw kernels determined, we run
234  a Pca on these Kernels, re-fit the Cells using the eigenKernels and then
235  fit those for spatial variation using the same technique as for Alard-Lupton kernels.
236  If this option is used, the first term will have no spatial variation and the
237  kernel sum will be conserved.""",
238  default=False,
239  )
240  subtractMeanForPca = pexConfig.Field(
241  dtype=bool,
242  doc="Subtract off the mean feature before doing the Pca",
243  default=True,
244  )
245  numPrincipalComponents = pexConfig.Field(
246  dtype=int,
247  doc="""Number of principal components to use for Pca basis, including the
248  mean kernel if requested.""",
249  default=5,
250  check=lambda x: x >= 3
251  )
252  singleKernelClipping = pexConfig.Field(
253  dtype=bool,
254  doc="Do sigma clipping on each raw kernel candidate",
255  default=True,
256  )
257  kernelSumClipping = pexConfig.Field(
258  dtype=bool,
259  doc="Do sigma clipping on the ensemble of kernel sums",
260  default=True,
261  )
262  spatialKernelClipping = pexConfig.Field(
263  dtype=bool,
264  doc="Do sigma clipping after building the spatial model",
265  default=True,
266  )
267  checkConditionNumber = pexConfig.Field(
268  dtype=bool,
269  doc="""Test for maximum condition number when inverting a kernel matrix.
270  Anything above maxConditionNumber is not used and the candidate is set as BAD.
271  Also used to truncate inverse matrix in estimateBiasedRisk. However,
272  if you are doing any deconvolution you will want to turn this off, or use
273  a large maxConditionNumber""",
274  default=False,
275  )
276  badMaskPlanes = pexConfig.ListField(
277  dtype=str,
278  doc="""Mask planes to ignore when calculating diffim statistics
279  Options: NO_DATA EDGE SAT BAD CR INTRP""",
280  default=("NO_DATA", "EDGE", "SAT")
281  )
282  candidateResidualMeanMax = pexConfig.Field(
283  dtype=float,
284  doc="""Rejects KernelCandidates yielding bad difference image quality.
285  Used by BuildSingleKernelVisitor, AssessSpatialKernelVisitor.
286  Represents average over pixels of (image/sqrt(variance)).""",
287  default=0.25,
288  check=lambda x: x >= 0.0
289  )
290  candidateResidualStdMax = pexConfig.Field(
291  dtype=float,
292  doc="""Rejects KernelCandidates yielding bad difference image quality.
293  Used by BuildSingleKernelVisitor, AssessSpatialKernelVisitor.
294  Represents stddev over pixels of (image/sqrt(variance)).""",
295  default=1.50,
296  check=lambda x: x >= 0.0
297  )
298  useCoreStats = pexConfig.Field(
299  dtype=bool,
300  doc="""Use the core of the footprint for the quality statistics, instead of the entire footprint.
301  WARNING: if there is deconvolution we probably will need to turn this off""",
302  default=False,
303  )
304  candidateCoreRadius = pexConfig.Field(
305  dtype=int,
306  doc="""Radius for calculation of stats in 'core' of KernelCandidate diffim.
307  Total number of pixels used will be (2*radius)**2.
308  This is used both for 'core' diffim quality as well as ranking of
309  KernelCandidates by their total flux in this core""",
310  default=3,
311  check=lambda x: x >= 1
312  )
313  maxKsumSigma = pexConfig.Field(
314  dtype=float,
315  doc="""Maximum allowed sigma for outliers from kernel sum distribution.
316  Used to reject variable objects from the kernel model""",
317  default=3.0,
318  check=lambda x: x >= 0.0
319  )
320  maxConditionNumber = pexConfig.Field(
321  dtype=float,
322  doc="Maximum condition number for a well conditioned matrix",
323  default=5.0e7,
324  check=lambda x: x >= 0.0
325  )
326  conditionNumberType = pexConfig.ChoiceField(
327  dtype=str,
328  doc="Use singular values (SVD) or eigen values (EIGENVALUE) to determine condition number",
329  default="EIGENVALUE",
330  allowed={
331  "SVD": "Use singular values",
332  "EIGENVALUE": "Use eigen values (faster)",
333  }
334  )
335  maxSpatialConditionNumber = pexConfig.Field(
336  dtype=float,
337  doc="Maximum condition number for a well conditioned spatial matrix",
338  default=1.0e10,
339  check=lambda x: x >= 0.0
340  )
341  iterateSingleKernel = pexConfig.Field(
342  dtype=bool,
343  doc="""Remake KernelCandidate using better variance estimate after first pass?
344  Primarily useful when convolving a single-depth image, otherwise not necessary.""",
345  default=False,
346  )
347  constantVarianceWeighting = pexConfig.Field(
348  dtype=bool,
349  doc="""Use constant variance weighting in single kernel fitting?
350  In some cases this is better for bright star residuals.""",
351  default=True,
352  )
353  calculateKernelUncertainty = pexConfig.Field(
354  dtype=bool,
355  doc="""Calculate kernel and background uncertainties for each kernel candidate?
356  This comes from the inverse of the covariance matrix.
357  Warning: regularization can cause problems for this step.""",
358  default=False,
359  )
360  useBicForKernelBasis = pexConfig.Field(
361  dtype=bool,
362  doc="""Use Bayesian Information Criterion to select the number of bases going into the kernel""",
363  default=False,
364  )
365 
366 
368  """!The parameters specific to the "Alard-Lupton" (sum-of-Gaussian) Psf-matching basis"""
369 
370  def setDefaults(self):
371  PsfMatchConfig.setDefaults(self)
372  self.kernelBasisSet = "alard-lupton"
373  self.maxConditionNumber = 5.0e7
374 
375  alardNGauss = pexConfig.Field(
376  dtype=int,
377  doc="Number of Gaussians in alard-lupton basis",
378  default=3,
379  check=lambda x: x >= 1
380  )
381  alardDegGauss = pexConfig.ListField(
382  dtype=int,
383  doc="Polynomial order of spatial modification of Gaussians. Must in number equal alardNGauss",
384  default=(4, 2, 2),
385  )
386  alardSigGauss = pexConfig.ListField(
387  dtype=float,
388  doc="""Sigma in pixels of Gaussians (FWHM = 2.35 sigma). Must in number equal alardNGauss""",
389  default=(0.7, 1.5, 3.0),
390  )
391  alardGaussBeta = pexConfig.Field(
392  dtype=float,
393  doc="""Default scale factor between Gaussian sigmas """,
394  default=2.0,
395  check=lambda x: x >= 0.0,
396  )
397  alardMinSig = pexConfig.Field(
398  dtype=float,
399  doc="""Minimum Sigma (pixels) for Gaussians""",
400  default=0.7,
401  check=lambda x: x >= 0.25
402  )
403  alardDegGaussDeconv = pexConfig.Field(
404  dtype=int,
405  doc="""Degree of spatial modification of ALL gaussians in AL basis during deconvolution""",
406  default=3,
407  check=lambda x: x >= 1
408  )
409  alardMinSigDeconv = pexConfig.Field(
410  dtype=float,
411  doc="""Minimum Sigma (pixels) for Gaussians during deconvolution;
412  make smaller than alardMinSig as this is only indirectly used""",
413  default=0.4,
414  check=lambda x: x >= 0.25
415  )
416  alardNGaussDeconv = pexConfig.Field(
417  dtype=int,
418  doc="Number of Gaussians in AL basis during deconvolution",
419  default=3,
420  check=lambda x: x >= 1
421  )
422 
423 
425  """!The parameters specific to the delta-function (one basis per-pixel) Psf-matching basis"""
426 
427  def setDefaults(self):
428  PsfMatchConfig.setDefaults(self)
429  self.kernelBasisSet = "delta-function"
430  self.maxConditionNumber = 5.0e6
432  self.subtractMeanForPca = True
433  self.useBicForKernelBasis = False
434 
435  useRegularization = pexConfig.Field(
436  dtype=bool,
437  doc="Use regularization to smooth the delta function kernels",
438  default=True,
439  )
440  regularizationType = pexConfig.ChoiceField(
441  dtype=str,
442  doc="Type of regularization.",
443  default="centralDifference",
444  allowed={
445  "centralDifference": "Penalize second derivative using 2-D stencil of central finite difference",
446  "forwardDifference": "Penalize first, second, third derivatives using forward finite differeces"
447  }
448  )
449  centralRegularizationStencil = pexConfig.ChoiceField(
450  dtype=int,
451  doc="Type of stencil to approximate central derivative (for centralDifference only)",
452  default=9,
453  allowed={
454  5: "5-point stencil including only adjacent-in-x,y elements",
455  9: "9-point stencil including diagonal elements"
456  }
457  )
458  forwardRegularizationOrders = pexConfig.ListField(
459  dtype=int,
460  doc="Array showing which order derivatives to penalize (for forwardDifference only)",
461  default=(1, 2),
462  itemCheck=lambda x: (x > 0) and (x < 4)
463  )
464  regularizationBorderPenalty = pexConfig.Field(
465  dtype=float,
466  doc="Value of the penalty for kernel border pixels",
467  default=3.0,
468  check=lambda x: x >= 0.0
469  )
470  lambdaType = pexConfig.ChoiceField(
471  dtype=str,
472  doc="How to choose the value of the regularization strength",
473  default="absolute",
474  allowed={
475  "absolute": "Use lambdaValue as the value of regularization strength",
476  "relative": "Use lambdaValue as fraction of the default regularization strength (N.R. 18.5.8)",
477  "minimizeBiasedRisk": "Minimize biased risk estimate",
478  "minimizeUnbiasedRisk": "Minimize unbiased risk estimate",
479  }
480  )
481  lambdaValue = pexConfig.Field(
482  dtype=float,
483  doc="Value used for absolute determinations of regularization strength",
484  default=0.2,
485  )
486  lambdaScaling = pexConfig.Field(
487  dtype=float,
488  doc="Fraction of the default lambda strength (N.R. 18.5.8) to use. 1e-4 or 1e-5",
489  default=1e-4,
490  )
491  lambdaStepType = pexConfig.ChoiceField(
492  dtype=str,
493  doc="""If a scan through lambda is needed (minimizeBiasedRisk, minimizeUnbiasedRisk),
494  use log or linear steps""",
495  default="log",
496  allowed={
497  "log": "Step in log intervals; e.g. lambdaMin, lambdaMax, lambdaStep = -1.0, 2.0, 0.1",
498  "linear": "Step in linear intervals; e.g. lambdaMin, lambdaMax, lambdaStep = 0.1, 100, 0.1",
499  }
500  )
501  lambdaMin = pexConfig.Field(
502  dtype=float,
503  doc="""If scan through lambda needed (minimizeBiasedRisk, minimizeUnbiasedRisk),
504  start at this value. If lambdaStepType = log:linear, suggest -1:0.1""",
505  default=-1.0,
506  )
507  lambdaMax = pexConfig.Field(
508  dtype=float,
509  doc="""If scan through lambda needed (minimizeBiasedRisk, minimizeUnbiasedRisk),
510  stop at this value. If lambdaStepType = log:linear, suggest 2:100""",
511  default=2.0,
512  )
513  lambdaStep = pexConfig.Field(
514  dtype=float,
515  doc="""If scan through lambda needed (minimizeBiasedRisk, minimizeUnbiasedRisk),
516  step in these increments. If lambdaStepType = log:linear, suggest 0.1:0.1""",
517  default=0.1,
518  )
519 
520 
521 
527 
528 class PsfMatchTask(pipeBase.Task):
529  """!
530 
531 \anchor PsfMatchTask_
532 
533 \brief Base class for Psf Matching; should not be called directly
534 
535 \section ip_diffim_psfmatch_Contents Contents
536 
537  - \ref ip_diffim_psfmatch_Purpose
538  - \ref ip_diffim_psfmatch_Initialize
539  - \ref ip_diffim_psfmatch_IO
540  - \ref ip_diffim_psfmatch_Config
541  - \ref ip_diffim_psfmatch_Metadata
542  - \ref ip_diffim_psfmatch_Debug
543  - \ref ip_diffim_psfmatch_Example
544 
545 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
546 
547 \section ip_diffim_psfmatch_Purpose Description
548 
549 PsfMatchTask is a base class that implements the core functionality for matching the
550 Psfs of two images using a spatially varying Psf-matching lsst.afw.math.LinearCombinationKernel.
551 The Task requires the user to provide an instance of an lsst.afw.math.SpatialCellSet,
552 filled with lsst.ip.diffim.KernelCandidate instances, and a list of lsst.afw.math.Kernels
553 of basis shapes that will be used for the decomposition. If requested, the Task
554 also performs background matching and returns the differential background model as an
555 lsst.afw.math.Kernel.SpatialFunction.
556 
557 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
558 
559 \section ip_diffim_psfmatch_Initialize Task initialization
560 
561 \copydoc \_\_init\_\_
562 
563 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
564 
565 \section ip_diffim_psfmatch_IO Invoking the Task
566 
567 As a base class, this Task is not directly invoked. However, run() methods that are
568 implemented on derived classes will make use of the core _solve() functionality,
569 which defines a sequence of lsst.afw.math.CandidateVisitor classes that iterate
570 through the KernelCandidates, first building up a per-candidate solution and then
571 building up a spatial model from the ensemble of candidates. Sigma clipping is
572 performed using the mean and standard deviation of all kernel sums (to reject
573 variable objects), on the per-candidate substamp diffim residuals
574 (to indicate a bad choice of kernel basis shapes for that particular object),
575 and on the substamp diffim residuals using the spatial kernel fit (to indicate a bad
576 choice of spatial kernel order, or poor constraints on the spatial model). The
577 _diagnostic() method logs information on the quality of the spatial fit, and also
578 modifies the Task metadata.
579 
580 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
581 
582 \section ip_diffim_psfmatch_Config Configuration parameters
583 
584 See \ref PsfMatchConfig, \ref PsfMatchConfigAL, \ref PsfMatchConfigDF, and \ref DetectionConfig.
585 
586 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
587 
588 \section ip_diffim_psfmatch_Metadata Quantities set in Metadata
589 
590 
591 <DL>
592 <DT> spatialConditionNum <DD> Condition number of the spatial kernel fit;
593  via \link lsst.ip.diffim.PsfMatchTask._diagnostic PsfMatchTask._diagnostic \endlink </DD> </DT>
594 <DT> spatialKernelSum <DD> Kernel sum (10^{-0.4 * &Delta; zeropoint}) of the spatial Psf-matching kernel;
595  via \link lsst.ip.diffim.PsfMatchTask._diagnostic PsfMatchTask._diagnostic \endlink </DD> </DT>
596 
597 <DT> ALBasisNGauss <DD> If using sum-of-Gaussian basis, the number of gaussians used;
598  via \link lsst.ip.diffim.makeKernelBasisList.generateAlardLuptonBasisList
599  generateAlardLuptonBasisList\endlink </DD> </DT>
600 <DT> ALBasisDegGauss <DD> If using sum-of-Gaussian basis, the degree of spatial variation of the Gaussians;
601  via \link lsst.ip.diffim.makeKernelBasisList.generateAlardLuptonBasisList
602  generateAlardLuptonBasisList\endlink </DD> </DT>
603 <DT> ALBasisSigGauss <DD> If using sum-of-Gaussian basis, the widths (sigma) of the Gaussians;
604  via \link lsst.ip.diffim.makeKernelBasisList.generateAlardLuptonBasisList
605  generateAlardLuptonBasisList\endlink </DD> </DT>
606 <DT> ALKernelSize <DD> If using sum-of-Gaussian basis, the kernel size;
607  via \link lsst.ip.diffim.makeKernelBasisList.generateAlardLuptonBasisList
608  generateAlardLuptonBasisList\endlink </DD> </DT>
609 
610 <DT> NFalsePositivesTotal <DD> Total number of diaSources;
611  via \link lsst.ip.diffim.KernelCandidateQa.aggregate KernelCandidateQa.aggregate\endlink </DD> </DT>
612 <DT> NFalsePositivesRefAssociated <DD> Number of diaSources that associate with the reference catalog;
613  via \link lsst.ip.diffim.KernelCandidateQa.aggregate KernelCandidateQa.aggregate\endlink </DD> </DT>
614 <DT> NFalsePositivesRefAssociated <DD> Number of diaSources that associate with the source catalog;
615  via \link lsst.ip.diffim.KernelCandidateQa.aggregate KernelCandidateQa.aggregate\endlink </DD> </DT>
616 <DT> NFalsePositivesUnassociated <DD> Number of diaSources that are orphans;
617  via \link lsst.ip.diffim.KernelCandidateQa.aggregate KernelCandidateQa.aggregate\endlink </DD> </DT>
618 <DT> metric_MEAN <DD> Mean value of substamp diffim quality metrics across all KernelCandidates,
619  for both the per-candidate (LOCAL) and SPATIAL residuals;
620  via \link lsst.ip.diffim.KernelCandidateQa.aggregate KernelCandidateQa.aggregate\endlink </DD> </DT>
621 <DT> metric_MEDIAN <DD> Median value of substamp diffim quality metrics across all KernelCandidates,
622  for both the per-candidate (LOCAL) and SPATIAL residuals;
623  via \link lsst.ip.diffim.KernelCandidateQa.aggregate KernelCandidateQa.aggregate\endlink </DD> </DT>
624 <DT> metric_STDEV <DD> Standard deviation of substamp diffim quality metrics across all KernelCandidates,
625  for both the per-candidate (LOCAL) and SPATIAL residuals;
626  via \link lsst.ip.diffim.KernelCandidateQa.aggregate KernelCandidateQa.aggregate\endlink </DD> </DT>
627 </DL>
628 
629 
630 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
631 
632 \section ip_diffim_psfmatch_Debug Debug variables
633 
634 
635 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
636 flag \c -d/--debug to import \b debug.py from your \c PYTHONPATH. The relevant contents of debug.py
637 for this Task include:
638 
639 \code{.py}
640  import sys
641  import lsstDebug
642  def DebugInfo(name):
643  di = lsstDebug.getInfo(name)
644  if name == "lsst.ip.diffim.psfMatch":
645  di.display = True # enable debug output
646  di.maskTransparency = 80 # ds9 mask transparency
647  di.displayCandidates = True # show all the candidates and residuals
648  di.displayKernelBasis = False # show kernel basis functions
649  di.displayKernelMosaic = True # show kernel realized across the image
650  di.plotKernelSpatialModel = False # show coefficients of spatial model
651  di.showBadCandidates = True # show the bad candidates (red) along with good (green)
652  return di
653  lsstDebug.Info = DebugInfo
654  lsstDebug.frame = 1
655 
656 \endcode
657 
658 Note that if you want addional logging info, you may add to your scripts:
659 \code{.py}
660 import lsst.log.utils as logUtils
661 logUtils.traceSetAt("ip.diffim", 4)
662 \endcode
663 
664 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
665 
666 \section ip_diffim_psfmatch_Example Example code
667 
668 As a base class, there is no example code for PsfMatchTask.
669 However, see \link lsst.ip.diffim.imagePsfMatch.ImagePsfMatchTask ImagePsfMatchTask\endlink,
670 \link lsst.ip.diffim.snapPsfMatch.SnapPsfMatchTask SnapPsfMatchTask\endlink, and
671 \link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask\endlink.
672 
673 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
674 
675  """
676  ConfigClass = PsfMatchConfig
677  _DefaultName = "psfMatch"
678 
679  def __init__(self, *args, **kwargs):
680  """!Create the psf-matching Task
681 
682  \param *args arguments to be passed to lsst.pipe.base.task.Task.__init__
683  \param **kwargs keyword arguments to be passed to lsst.pipe.base.task.Task.__init__
684 
685  The initialization sets the Psf-matching kernel configuration using the value of
686  self.config.kernel.active. If the kernel is requested with regularization to moderate
687  the bias/variance tradeoff, currently only used when a delta function kernel basis
688  is provided, it creates a regularization matrix stored as member variable
689  self.hMat.
690  """
691  pipeBase.Task.__init__(self, *args, **kwargs)
692  self.kConfig = self.config.kernel.active
693 
694  #
695  if 'useRegularization' in self.kConfig:
696  self.useRegularization = self.kConfig.useRegularization
697  else:
698  self.useRegularization = False
699 
700  if self.useRegularization:
701  self.hMat = diffimLib.makeRegularizationMatrix(pexConfig.makePolicy(self.kConfig))
702 
703  def _diagnostic(self, kernelCellSet, spatialSolution, spatialKernel, spatialBg):
704  """!Provide logging diagnostics on quality of spatial kernel fit
705 
706  @param kernelCellSet: Cellset that contains the KernelCandidates used in the fitting
707  @param spatialSolution: KernelSolution of best-fit
708  @param spatialKernel: Best-fit spatial Kernel model
709  @param spatialBg: Best-fit spatial background model
710 
711  """
712  # What is the final kernel sum
713  kImage = afwImage.ImageD(spatialKernel.getDimensions())
714  kSum = spatialKernel.computeImage(kImage, False)
715  self.log.info("Final spatial kernel sum %.3f" % (kSum))
716 
717  # Look at how well conditioned the matrix is
718  conditionNum = spatialSolution.getConditionNumber(
719  getattr(diffimLib.KernelSolution, self.kConfig.conditionNumberType))
720  self.log.info("Spatial model condition number %.3e" % (conditionNum))
721 
722  if conditionNum < 0.0:
723  self.log.warn("Condition number is negative (%.3e)" % (conditionNum))
724  if conditionNum > self.kConfig.maxSpatialConditionNumber:
725  self.log.warn("Spatial solution exceeds max condition number (%.3e > %.3e)" % (
726  conditionNum, self.kConfig.maxSpatialConditionNumber))
727 
728  self.metadata.set("spatialConditionNum", conditionNum)
729  self.metadata.set("spatialKernelSum", kSum)
730 
731  # Look at how well the solution is constrained
732  nBasisKernels = spatialKernel.getNBasisKernels()
733  nKernelTerms = spatialKernel.getNSpatialParameters()
734  if nKernelTerms == 0: # order 0
735  nKernelTerms = 1
736 
737  # Not fit for
738  nBgTerms = spatialBg.getNParameters()
739  if nBgTerms == 1:
740  if spatialBg.getParameters()[0] == 0.0:
741  nBgTerms = 0
742 
743  nGood = 0
744  nBad = 0
745  nTot = 0
746  for cell in kernelCellSet.getCellList():
747  for cand in cell.begin(False): # False = include bad candidates
748  nTot += 1
749  if cand.getStatus() == afwMath.SpatialCellCandidate.GOOD:
750  nGood += 1
751  if cand.getStatus() == afwMath.SpatialCellCandidate.BAD:
752  nBad += 1
753 
754  self.log.info("Doing stats of kernel candidates used in the spatial fit.")
755 
756  # Counting statistics
757  if nBad > 2*nGood:
758  self.log.warn("Many more candidates rejected than accepted; %d total, %d rejected, %d used" % (
759  nTot, nBad, nGood))
760  else:
761  self.log.info("%d candidates total, %d rejected, %d used" % (nTot, nBad, nGood))
762 
763  # Some judgements on the quality of the spatial models
764  if nGood < nKernelTerms:
765  self.log.warn("Spatial kernel model underconstrained; %d candidates, %d terms, %d bases" % (
766  nGood, nKernelTerms, nBasisKernels))
767  self.log.warn("Consider lowering the spatial order")
768  elif nGood <= 2*nKernelTerms:
769  self.log.warn("Spatial kernel model poorly constrained; %d candidates, %d terms, %d bases" % (
770  nGood, nKernelTerms, nBasisKernels))
771  self.log.warn("Consider lowering the spatial order")
772  else:
773  self.log.info("Spatial kernel model well constrained; %d candidates, %d terms, %d bases" % (
774  nGood, nKernelTerms, nBasisKernels))
775 
776  if nGood < nBgTerms:
777  self.log.warn("Spatial background model underconstrained; %d candidates, %d terms" % (
778  nGood, nBgTerms))
779  self.log.warn("Consider lowering the spatial order")
780  elif nGood <= 2*nBgTerms:
781  self.log.warn("Spatial background model poorly constrained; %d candidates, %d terms" % (
782  nGood, nBgTerms))
783  self.log.warn("Consider lowering the spatial order")
784  else:
785  self.log.info("Spatial background model appears well constrained; %d candidates, %d terms" % (
786  nGood, nBgTerms))
787 
788  def _displayDebug(self, kernelCellSet, spatialKernel, spatialBackground):
789  """!Provide visualization of the inputs and ouputs to the Psf-matching code
790 
791  @param kernelCellSet: the SpatialCellSet used in determining the matching kernel and background
792  @param spatialKernel: spatially varying Psf-matching kernel
793  @param spatialBackground: spatially varying background-matching function
794 
795  """
796  import lsstDebug
797  displayCandidates = lsstDebug.Info(__name__).displayCandidates
798  displayKernelBasis = lsstDebug.Info(__name__).displayKernelBasis
799  displayKernelMosaic = lsstDebug.Info(__name__).displayKernelMosaic
800  plotKernelSpatialModel = lsstDebug.Info(__name__).plotKernelSpatialModel
801  showBadCandidates = lsstDebug.Info(__name__).showBadCandidates
802  maskTransparency = lsstDebug.Info(__name__).maskTransparency
803  if not maskTransparency:
804  maskTransparency = 0
805  ds9.setMaskTransparency(maskTransparency)
806 
807  if displayCandidates:
808  diUtils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground,
809  frame=lsstDebug.frame,
810  showBadCandidates=showBadCandidates)
811  lsstDebug.frame += 1
812  diUtils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground,
813  frame=lsstDebug.frame,
814  showBadCandidates=showBadCandidates,
815  kernels=True)
816  lsstDebug.frame += 1
817  diUtils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground,
818  frame=lsstDebug.frame,
819  showBadCandidates=showBadCandidates,
820  resids=True)
821  lsstDebug.frame += 1
822 
823  if displayKernelBasis:
824  diUtils.showKernelBasis(spatialKernel, frame=lsstDebug.frame)
825  lsstDebug.frame += 1
826 
827  if displayKernelMosaic:
828  diUtils.showKernelMosaic(kernelCellSet.getBBox(), spatialKernel, frame=lsstDebug.frame)
829  lsstDebug.frame += 1
830 
831  if plotKernelSpatialModel:
832  diUtils.plotKernelSpatialModel(spatialKernel, kernelCellSet, showBadCandidates=showBadCandidates)
833 
834  def _createPcaBasis(self, kernelCellSet, nStarPerCell, policy):
835  """!Create Principal Component basis
836 
837  If a principal component analysis is requested, typically when using a delta function basis,
838  perform the PCA here and return a new basis list containing the new principal components.
839 
840  @param kernelCellSet: a SpatialCellSet containing KernelCandidates, from which components are derived
841  @param nStarPerCell: the number of stars per cell to visit when doing the PCA
842  @param policy: input policy controlling the single kernel visitor
843 
844  @return
845  - nRejectedPca: number of KernelCandidates rejected during PCA loop
846  - spatialBasisList: basis list containing the principal shapes as Kernels
847 
848  """
849  nComponents = self.kConfig.numPrincipalComponents
850  imagePca = diffimLib.KernelPcaD()
851  importStarVisitor = diffimLib.KernelPcaVisitorF(imagePca)
852  kernelCellSet.visitCandidates(importStarVisitor, nStarPerCell)
853  if self.kConfig.subtractMeanForPca:
854  importStarVisitor.subtractMean()
855  imagePca.analyze()
856 
857  eigenValues = imagePca.getEigenValues()
858  pcaBasisList = importStarVisitor.getEigenKernels()
859 
860  eSum = np.sum(eigenValues)
861  if eSum == 0.0:
862  raise RuntimeError("Eigenvalues sum to zero")
863  for j in range(len(eigenValues)):
864  log.log("TRACE5." + self.log.getName() + "._solve", log.DEBUG,
865  "Eigenvalue %d : %f (%f)", j, eigenValues[j], eigenValues[j]/eSum)
866 
867  nToUse = min(nComponents, len(eigenValues))
868  trimBasisList = []
869  for j in range(nToUse):
870  # Check for NaNs?
871  kimage = afwImage.ImageD(pcaBasisList[j].getDimensions())
872  pcaBasisList[j].computeImage(kimage, False)
873  if not (True in np.isnan(kimage.getArray())):
874  trimBasisList.append(pcaBasisList[j])
875 
876  # Put all the power in the first kernel, which will not vary spatially
877  spatialBasisList = diffimLib.renormalizeKernelList(trimBasisList)
878 
879  # New Kernel visitor for this new basis list (no regularization explicitly)
880  singlekvPca = diffimLib.BuildSingleKernelVisitorF(spatialBasisList, policy)
881  singlekvPca.setSkipBuilt(False)
882  kernelCellSet.visitCandidates(singlekvPca, nStarPerCell)
883  singlekvPca.setSkipBuilt(True)
884  nRejectedPca = singlekvPca.getNRejected()
885 
886  return nRejectedPca, spatialBasisList
887 
888  def _buildCellSet(self, *args):
889  """!Fill a SpatialCellSet with KernelCandidates for the Psf-matching process;
890  override in derived classes"""
891  return
892 
893  @pipeBase.timeMethod
894  def _solve(self, kernelCellSet, basisList, returnOnExcept=False):
895  """!Solve for the PSF matching kernel
896 
897  @param kernelCellSet: a SpatialCellSet to use in determining the matching kernel
898  (typically as provided by _buildCellSet)
899  @param basisList: list of Kernels to be used in the decomposition of the spatially varying kernel
900  (typically as provided by makeKernelBasisList)
901  @param returnOnExcept: if True then return (None, None) if an error occurs, else raise the exception
902 
903  @return
904  - psfMatchingKernel: PSF matching kernel
905  - backgroundModel: differential background model
906 
907  Raise Exception if unable to determine PSF matching kernel and returnOnExcept False
908  """
909 
910  import lsstDebug
911  display = lsstDebug.Info(__name__).display
912 
913  maxSpatialIterations = self.kConfig.maxSpatialIterations
914  nStarPerCell = self.kConfig.nStarPerCell
915  usePcaForSpatialKernel = self.kConfig.usePcaForSpatialKernel
916 
917  # Visitor for the single kernel fit
918  policy = pexConfig.makePolicy(self.kConfig)
919  if self.useRegularization:
920  singlekv = diffimLib.BuildSingleKernelVisitorF(basisList, policy, self.hMat)
921  else:
922  singlekv = diffimLib.BuildSingleKernelVisitorF(basisList, policy)
923 
924  # Visitor for the kernel sum rejection
925  ksv = diffimLib.KernelSumVisitorF(policy)
926 
927  # Main loop
928  t0 = time.time()
929  try:
930  totalIterations = 0
931  thisIteration = 0
932  while (thisIteration < maxSpatialIterations):
933 
934  # Make sure there are no uninitialized candidates as active occupants of Cell
935  nRejectedSkf = -1
936  while (nRejectedSkf != 0):
937  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG,
938  "Building single kernels...")
939  kernelCellSet.visitCandidates(singlekv, nStarPerCell)
940  nRejectedSkf = singlekv.getNRejected()
941  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG,
942  "Iteration %d, rejected %d candidates due to initial kernel fit",
943  thisIteration, nRejectedSkf)
944 
945  # Reject outliers in kernel sum
946  ksv.resetKernelSum()
947  ksv.setMode(diffimLib.KernelSumVisitorF.AGGREGATE)
948  kernelCellSet.visitCandidates(ksv, nStarPerCell)
949  ksv.processKsumDistribution()
950  ksv.setMode(diffimLib.KernelSumVisitorF.REJECT)
951  kernelCellSet.visitCandidates(ksv, nStarPerCell)
952 
953  nRejectedKsum = ksv.getNRejected()
954  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG,
955  "Iteration %d, rejected %d candidates due to kernel sum",
956  thisIteration, nRejectedKsum)
957 
958  # Do we jump back to the top without incrementing thisIteration?
959  if nRejectedKsum > 0:
960  totalIterations += 1
961  continue
962 
963  # At this stage we can either apply the spatial fit to
964  # the kernels, or we run a PCA, use these as a *new*
965  # basis set with lower dimensionality, and then apply
966  # the spatial fit to these kernels
967 
968  if (usePcaForSpatialKernel):
969  log.log("TRACE0." + self.log.getName() + "._solve", log.DEBUG,
970  "Building Pca basis")
971 
972  nRejectedPca, spatialBasisList = self._createPcaBasis(kernelCellSet, nStarPerCell, policy)
973  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG,
974  "Iteration %d, rejected %d candidates due to Pca kernel fit",
975  thisIteration, nRejectedPca)
976 
977  # We don't want to continue on (yet) with the
978  # spatial modeling, because we have bad objects
979  # contributing to the Pca basis. We basically
980  # need to restart from the beginning of this loop,
981  # since the cell-mates of those objects that were
982  # rejected need their original Kernels built by
983  # singleKernelFitter.
984 
985  # Don't count against thisIteration
986  if (nRejectedPca > 0):
987  totalIterations += 1
988  continue
989  else:
990  spatialBasisList = basisList
991 
992  # We have gotten on to the spatial modeling part
993  regionBBox = kernelCellSet.getBBox()
994  spatialkv = diffimLib.BuildSpatialKernelVisitorF(spatialBasisList, regionBBox, policy)
995  kernelCellSet.visitCandidates(spatialkv, nStarPerCell)
996  spatialkv.solveLinearEquation()
997  log.log("TRACE2." + self.log.getName() + "._solve", log.DEBUG,
998  "Spatial kernel built with %d candidates", spatialkv.getNCandidates())
999  spatialKernel, spatialBackground = spatialkv.getSolutionPair()
1000 
1001  # Check the quality of the spatial fit (look at residuals)
1002  assesskv = diffimLib.AssessSpatialKernelVisitorF(spatialKernel, spatialBackground, policy)
1003  kernelCellSet.visitCandidates(assesskv, nStarPerCell)
1004  nRejectedSpatial = assesskv.getNRejected()
1005  nGoodSpatial = assesskv.getNGood()
1006  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG,
1007  "Iteration %d, rejected %d candidates due to spatial kernel fit",
1008  thisIteration, nRejectedSpatial)
1009  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG,
1010  "%d candidates used in fit", nGoodSpatial)
1011 
1012  # If only nGoodSpatial == 0, might be other candidates in the cells
1013  if nGoodSpatial == 0 and nRejectedSpatial == 0:
1014  raise RuntimeError("No kernel candidates for spatial fit")
1015 
1016  if nRejectedSpatial == 0:
1017  # Nothing rejected, finished with spatial fit
1018  break
1019 
1020  # Otherwise, iterate on...
1021  thisIteration += 1
1022 
1023  # Final fit if above did not converge
1024  if (nRejectedSpatial > 0) and (thisIteration == maxSpatialIterations):
1025  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG, "Final spatial fit")
1026  if (usePcaForSpatialKernel):
1027  nRejectedPca, spatialBasisList = self._createPcaBasis(kernelCellSet, nStarPerCell, policy)
1028  regionBBox = kernelCellSet.getBBox()
1029  spatialkv = diffimLib.BuildSpatialKernelVisitorF(spatialBasisList, regionBBox, policy)
1030  kernelCellSet.visitCandidates(spatialkv, nStarPerCell)
1031  spatialkv.solveLinearEquation()
1032  log.log("TRACE2." + self.log.getName() + "._solve", log.DEBUG,
1033  "Spatial kernel built with %d candidates", spatialkv.getNCandidates())
1034  spatialKernel, spatialBackground = spatialkv.getSolutionPair()
1035 
1036  spatialSolution = spatialkv.getKernelSolution()
1037 
1038  except Exception as e:
1039  self.log.error("ERROR: Unable to calculate psf matching kernel")
1040 
1041  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG, str(e))
1042  raise e
1043 
1044  t1 = time.time()
1045  log.log("TRACE0." + self.log.getName() + "._solve", log.DEBUG,
1046  "Total time to compute the spatial kernel : %.2f s", (t1 - t0))
1047 
1048  if display:
1049  self._displayDebug(kernelCellSet, spatialKernel, spatialBackground)
1050 
1051  self._diagnostic(kernelCellSet, spatialSolution, spatialKernel, spatialBackground)
1052 
1053  return spatialSolution, spatialKernel, spatialBackground
1054 
1055 PsfMatch = PsfMatchTask
Configuration for detecting sources on images for building a PSF-matching kernel. ...
Definition: psfMatch.py:39
The parameters specific to the "Alard-Lupton" (sum-of-Gaussian) Psf-matching basis.
Definition: psfMatch.py:367
Base configuration for Psf-matching.
Definition: psfMatch.py:119
The parameters specific to the delta-function (one basis per-pixel) Psf-matching basis.
Definition: psfMatch.py:424
Base class for Psf Matching; should not be called directly.
Definition: psfMatch.py:528
def _diagnostic(self, kernelCellSet, spatialSolution, spatialKernel, spatialBg)
Provide logging diagnostics on quality of spatial kernel fit.
Definition: psfMatch.py:703
def __init__(self, args, kwargs)
Create the psf-matching Task.
Definition: psfMatch.py:679
def _createPcaBasis(self, kernelCellSet, nStarPerCell, policy)
Create Principal Component basis.
Definition: psfMatch.py:834
def _displayDebug(self, kernelCellSet, spatialKernel, spatialBackground)
Provide visualization of the inputs and ouputs to the Psf-matching code.
Definition: psfMatch.py:788