lsst.ip.diffim  16.0-12-g1dc09ba+6
psfMatch.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2016 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 __all__ = ["DetectionConfig", "PsfMatchConfig", "PsfMatchConfigAL", "PsfMatchConfigDF", "PsfMatchTask"]
24 
25 import time
26 
27 import numpy as np
28 
29 import lsst.afw.image as afwImage
30 import lsst.pex.config as pexConfig
31 import lsst.afw.math as afwMath
32 import lsst.afw.display.ds9 as ds9
33 import lsst.log as log
34 import lsst.pipe.base as pipeBase
35 from lsst.meas.algorithms import SubtractBackgroundConfig
36 from . import utils as diutils
37 from . import diffimLib
38 
39 
40 class DetectionConfig(pexConfig.Config):
41  """!Configuration for detecting sources on images for building a PSF-matching kernel
42 
43  Configuration for turning detected lsst.afw.detection.FootPrints into an acceptable
44  (unmasked, high signal-to-noise, not too large or not too small) list of
45  lsst.ip.diffim.KernelSources that are used to build the Psf-matching kernel"""
46 
47  detThreshold = pexConfig.Field(
48  dtype=float,
49  doc="Value of footprint detection threshold",
50  default=10.0,
51  check=lambda x: x >= 3.0
52  )
53  detThresholdType = pexConfig.ChoiceField(
54  dtype=str,
55  doc="Type of detection threshold",
56  default="pixel_stdev",
57  allowed={
58  "value": "Use counts as the detection threshold type",
59  "stdev": "Use standard deviation of image plane",
60  "variance": "Use variance of image plane",
61  "pixel_stdev": "Use stdev derived from variance plane"
62  }
63  )
64  detOnTemplate = pexConfig.Field(
65  dtype=bool,
66  doc="""If true run detection on the template (image to convolve);
67  if false run detection on the science image""",
68  default=True
69  )
70  badMaskPlanes = pexConfig.ListField(
71  dtype=str,
72  doc="""Mask planes that lead to an invalid detection.
73  Options: NO_DATA EDGE SAT BAD CR INTRP""",
74  default=("NO_DATA", "EDGE", "SAT")
75  )
76  fpNpixMin = pexConfig.Field(
77  dtype=int,
78  doc="Minimum number of pixels in an acceptable Footprint",
79  default=5,
80  check=lambda x: x >= 5
81  )
82  fpNpixMax = pexConfig.Field(
83  dtype=int,
84  doc="""Maximum number of pixels in an acceptable Footprint;
85  too big and the subsequent convolutions become unwieldy""",
86  default=500,
87  check=lambda x: x <= 500
88  )
89  fpGrowKernelScaling = pexConfig.Field(
90  dtype=float,
91  doc="""If config.scaleByFwhm, grow the footprint based on
92  the final kernelSize. Each footprint will be
93  2*fpGrowKernelScaling*kernelSize x
94  2*fpGrowKernelScaling*kernelSize. With the value
95  of 1.0, the remaining pixels in each KernelCandiate
96  after convolution by the basis functions will be
97  equal to the kernel size itself.""",
98  default=1.0,
99  check=lambda x: x >= 1.0
100  )
101  fpGrowPix = pexConfig.Field(
102  dtype=int,
103  doc="""Growing radius (in pixels) for each raw detection
104  footprint. The smaller the faster; however the
105  kernel sum does not converge if the stamp is too
106  small; and the kernel is not constrained at all if
107  the stamp is the size of the kernel. The grown stamp
108  is 2 * fpGrowPix pixels larger in each dimension.
109  This is overridden by fpGrowKernelScaling if scaleByFwhm""",
110  default=30,
111  check=lambda x: x >= 10
112  )
113  scaleByFwhm = pexConfig.Field(
114  dtype=bool,
115  doc="Scale fpGrowPix by input Fwhm?",
116  default=True,
117  )
118 
119 
120 class PsfMatchConfig(pexConfig.Config):
121  """!Base configuration for Psf-matching
122 
123  The base configuration of the Psf-matching kernel, and of the warping, detection,
124  and background modeling subTasks."""
125 
126  warpingConfig = pexConfig.ConfigField("Config for warping exposures to a common alignment",
127  afwMath.warper.WarperConfig)
128  detectionConfig = pexConfig.ConfigField("Controlling the detection of sources for kernel building",
129  DetectionConfig)
130  afwBackgroundConfig = pexConfig.ConfigField("Controlling the Afw background fitting",
131  SubtractBackgroundConfig)
132 
133  useAfwBackground = pexConfig.Field(
134  dtype=bool,
135  doc="Use afw background subtraction instead of ip_diffim",
136  default=False,
137  )
138  fitForBackground = pexConfig.Field(
139  dtype=bool,
140  doc="Include terms (including kernel cross terms) for background in ip_diffim",
141  default=False,
142  )
143  kernelBasisSet = pexConfig.ChoiceField(
144  dtype=str,
145  doc="Type of basis set for PSF matching kernel.",
146  default="alard-lupton",
147  allowed={
148  "alard-lupton": """Alard-Lupton sum-of-gaussians basis set,
149  * The first term has no spatial variation
150  * The kernel sum is conserved
151  * You may want to turn off 'usePcaForSpatialKernel'""",
152  "delta-function": """Delta-function kernel basis set,
153  * You may enable the option useRegularization
154  * You should seriously consider usePcaForSpatialKernel, which will also
155  enable kernel sum conservation for the delta function kernels"""
156  }
157  )
158  kernelSize = pexConfig.Field(
159  dtype=int,
160  doc="""Number of rows/columns in the convolution kernel; should be odd-valued.
161  Modified by kernelSizeFwhmScaling if scaleByFwhm = true""",
162  default=21,
163  )
164  scaleByFwhm = pexConfig.Field(
165  dtype=bool,
166  doc="Scale kernelSize, alardGaussians by input Fwhm",
167  default=True,
168  )
169  kernelSizeFwhmScaling = pexConfig.Field(
170  dtype=float,
171  doc="""How much to scale the kernel size based on the largest AL Sigma""",
172  default=6.0,
173  check=lambda x: x >= 1.0
174  )
175  kernelSizeMin = pexConfig.Field(
176  dtype=int,
177  doc="""Minimum Kernel Size""",
178  default=21,
179  )
180  kernelSizeMax = pexConfig.Field(
181  dtype=int,
182  doc="""Maximum Kernel Size""",
183  default=35,
184  )
185  spatialModelType = pexConfig.ChoiceField(
186  dtype=str,
187  doc="Type of spatial functions for kernel and background",
188  default="chebyshev1",
189  allowed={
190  "chebyshev1": "Chebyshev polynomial of the first kind",
191  "polynomial": "Standard x,y polynomial",
192  }
193  )
194  spatialKernelOrder = pexConfig.Field(
195  dtype=int,
196  doc="Spatial order of convolution kernel variation",
197  default=2,
198  check=lambda x: x >= 0
199  )
200  spatialBgOrder = pexConfig.Field(
201  dtype=int,
202  doc="Spatial order of differential background variation",
203  default=1,
204  check=lambda x: x >= 0
205  )
206  sizeCellX = pexConfig.Field(
207  dtype=int,
208  doc="Size (rows) in pixels of each SpatialCell for spatial modeling",
209  default=128,
210  check=lambda x: x >= 32
211  )
212  sizeCellY = pexConfig.Field(
213  dtype=int,
214  doc="Size (columns) in pixels of each SpatialCell for spatial modeling",
215  default=128,
216  check=lambda x: x >= 32
217  )
218  nStarPerCell = pexConfig.Field(
219  dtype=int,
220  doc="Number of KernelCandidates in each SpatialCell to use in the spatial fitting",
221  default=3,
222  check=lambda x: x >= 1
223  )
224  maxSpatialIterations = pexConfig.Field(
225  dtype=int,
226  doc="Maximum number of iterations for rejecting bad KernelCandidates in spatial fitting",
227  default=3,
228  check=lambda x: x >= 1 and x <= 5
229  )
230  usePcaForSpatialKernel = pexConfig.Field(
231  dtype=bool,
232  doc="""Use Pca to reduce the dimensionality of the kernel basis sets.
233  This is particularly useful for delta-function kernels.
234  Functionally, after all Cells have their raw kernels determined, we run
235  a Pca on these Kernels, re-fit the Cells using the eigenKernels and then
236  fit those for spatial variation using the same technique as for Alard-Lupton kernels.
237  If this option is used, the first term will have no spatial variation and the
238  kernel sum will be conserved.""",
239  default=False,
240  )
241  subtractMeanForPca = pexConfig.Field(
242  dtype=bool,
243  doc="Subtract off the mean feature before doing the Pca",
244  default=True,
245  )
246  numPrincipalComponents = pexConfig.Field(
247  dtype=int,
248  doc="""Number of principal components to use for Pca basis, including the
249  mean kernel if requested.""",
250  default=5,
251  check=lambda x: x >= 3
252  )
253  singleKernelClipping = pexConfig.Field(
254  dtype=bool,
255  doc="Do sigma clipping on each raw kernel candidate",
256  default=True,
257  )
258  kernelSumClipping = pexConfig.Field(
259  dtype=bool,
260  doc="Do sigma clipping on the ensemble of kernel sums",
261  default=True,
262  )
263  spatialKernelClipping = pexConfig.Field(
264  dtype=bool,
265  doc="Do sigma clipping after building the spatial model",
266  default=True,
267  )
268  checkConditionNumber = pexConfig.Field(
269  dtype=bool,
270  doc="""Test for maximum condition number when inverting a kernel matrix.
271  Anything above maxConditionNumber is not used and the candidate is set as BAD.
272  Also used to truncate inverse matrix in estimateBiasedRisk. However,
273  if you are doing any deconvolution you will want to turn this off, or use
274  a large maxConditionNumber""",
275  default=False,
276  )
277  badMaskPlanes = pexConfig.ListField(
278  dtype=str,
279  doc="""Mask planes to ignore when calculating diffim statistics
280  Options: NO_DATA EDGE SAT BAD CR INTRP""",
281  default=("NO_DATA", "EDGE", "SAT")
282  )
283  candidateResidualMeanMax = pexConfig.Field(
284  dtype=float,
285  doc="""Rejects KernelCandidates yielding bad difference image quality.
286  Used by BuildSingleKernelVisitor, AssessSpatialKernelVisitor.
287  Represents average over pixels of (image/sqrt(variance)).""",
288  default=0.25,
289  check=lambda x: x >= 0.0
290  )
291  candidateResidualStdMax = pexConfig.Field(
292  dtype=float,
293  doc="""Rejects KernelCandidates yielding bad difference image quality.
294  Used by BuildSingleKernelVisitor, AssessSpatialKernelVisitor.
295  Represents stddev over pixels of (image/sqrt(variance)).""",
296  default=1.50,
297  check=lambda x: x >= 0.0
298  )
299  useCoreStats = pexConfig.Field(
300  dtype=bool,
301  doc="""Use the core of the footprint for the quality statistics, instead of the entire footprint.
302  WARNING: if there is deconvolution we probably will need to turn this off""",
303  default=False,
304  )
305  candidateCoreRadius = pexConfig.Field(
306  dtype=int,
307  doc="""Radius for calculation of stats in 'core' of KernelCandidate diffim.
308  Total number of pixels used will be (2*radius)**2.
309  This is used both for 'core' diffim quality as well as ranking of
310  KernelCandidates by their total flux in this core""",
311  default=3,
312  check=lambda x: x >= 1
313  )
314  maxKsumSigma = pexConfig.Field(
315  dtype=float,
316  doc="""Maximum allowed sigma for outliers from kernel sum distribution.
317  Used to reject variable objects from the kernel model""",
318  default=3.0,
319  check=lambda x: x >= 0.0
320  )
321  maxConditionNumber = pexConfig.Field(
322  dtype=float,
323  doc="Maximum condition number for a well conditioned matrix",
324  default=5.0e7,
325  check=lambda x: x >= 0.0
326  )
327  conditionNumberType = pexConfig.ChoiceField(
328  dtype=str,
329  doc="Use singular values (SVD) or eigen values (EIGENVALUE) to determine condition number",
330  default="EIGENVALUE",
331  allowed={
332  "SVD": "Use singular values",
333  "EIGENVALUE": "Use eigen values (faster)",
334  }
335  )
336  maxSpatialConditionNumber = pexConfig.Field(
337  dtype=float,
338  doc="Maximum condition number for a well conditioned spatial matrix",
339  default=1.0e10,
340  check=lambda x: x >= 0.0
341  )
342  iterateSingleKernel = pexConfig.Field(
343  dtype=bool,
344  doc="""Remake KernelCandidate using better variance estimate after first pass?
345  Primarily useful when convolving a single-depth image, otherwise not necessary.""",
346  default=False,
347  )
348  constantVarianceWeighting = pexConfig.Field(
349  dtype=bool,
350  doc="""Use constant variance weighting in single kernel fitting?
351  In some cases this is better for bright star residuals.""",
352  default=True,
353  )
354  calculateKernelUncertainty = pexConfig.Field(
355  dtype=bool,
356  doc="""Calculate kernel and background uncertainties for each kernel candidate?
357  This comes from the inverse of the covariance matrix.
358  Warning: regularization can cause problems for this step.""",
359  default=False,
360  )
361  useBicForKernelBasis = pexConfig.Field(
362  dtype=bool,
363  doc="""Use Bayesian Information Criterion to select the number of bases going into the kernel""",
364  default=False,
365  )
366 
367 
369  """!The parameters specific to the "Alard-Lupton" (sum-of-Gaussian) Psf-matching basis"""
370 
371  def setDefaults(self):
372  PsfMatchConfig.setDefaults(self)
373  self.kernelBasisSet = "alard-lupton"
374  self.maxConditionNumber = 5.0e7
375 
376  alardNGauss = pexConfig.Field(
377  dtype=int,
378  doc="Number of Gaussians in alard-lupton basis",
379  default=3,
380  check=lambda x: x >= 1
381  )
382  alardDegGauss = pexConfig.ListField(
383  dtype=int,
384  doc="Polynomial order of spatial modification of Gaussians. Must in number equal alardNGauss",
385  default=(4, 2, 2),
386  )
387  alardSigGauss = pexConfig.ListField(
388  dtype=float,
389  doc="""Sigma in pixels of Gaussians (FWHM = 2.35 sigma). Must in number equal alardNGauss""",
390  default=(0.7, 1.5, 3.0),
391  )
392  alardGaussBeta = pexConfig.Field(
393  dtype=float,
394  doc="""Default scale factor between Gaussian sigmas """,
395  default=2.0,
396  check=lambda x: x >= 0.0,
397  )
398  alardMinSig = pexConfig.Field(
399  dtype=float,
400  doc="""Minimum Sigma (pixels) for Gaussians""",
401  default=0.7,
402  check=lambda x: x >= 0.25
403  )
404  alardDegGaussDeconv = pexConfig.Field(
405  dtype=int,
406  doc="""Degree of spatial modification of ALL gaussians in AL basis during deconvolution""",
407  default=3,
408  check=lambda x: x >= 1
409  )
410  alardMinSigDeconv = pexConfig.Field(
411  dtype=float,
412  doc="""Minimum Sigma (pixels) for Gaussians during deconvolution;
413  make smaller than alardMinSig as this is only indirectly used""",
414  default=0.4,
415  check=lambda x: x >= 0.25
416  )
417  alardNGaussDeconv = pexConfig.Field(
418  dtype=int,
419  doc="Number of Gaussians in AL basis during deconvolution",
420  default=3,
421  check=lambda x: x >= 1
422  )
423 
424 
426  """!The parameters specific to the delta-function (one basis per-pixel) Psf-matching basis"""
427 
428  def setDefaults(self):
429  PsfMatchConfig.setDefaults(self)
430  self.kernelBasisSet = "delta-function"
431  self.maxConditionNumber = 5.0e6
433  self.subtractMeanForPca = True
434  self.useBicForKernelBasis = False
435 
436  useRegularization = pexConfig.Field(
437  dtype=bool,
438  doc="Use regularization to smooth the delta function kernels",
439  default=True,
440  )
441  regularizationType = pexConfig.ChoiceField(
442  dtype=str,
443  doc="Type of regularization.",
444  default="centralDifference",
445  allowed={
446  "centralDifference": "Penalize second derivative using 2-D stencil of central finite difference",
447  "forwardDifference": "Penalize first, second, third derivatives using forward finite differeces"
448  }
449  )
450  centralRegularizationStencil = pexConfig.ChoiceField(
451  dtype=int,
452  doc="Type of stencil to approximate central derivative (for centralDifference only)",
453  default=9,
454  allowed={
455  5: "5-point stencil including only adjacent-in-x,y elements",
456  9: "9-point stencil including diagonal elements"
457  }
458  )
459  forwardRegularizationOrders = pexConfig.ListField(
460  dtype=int,
461  doc="Array showing which order derivatives to penalize (for forwardDifference only)",
462  default=(1, 2),
463  itemCheck=lambda x: (x > 0) and (x < 4)
464  )
465  regularizationBorderPenalty = pexConfig.Field(
466  dtype=float,
467  doc="Value of the penalty for kernel border pixels",
468  default=3.0,
469  check=lambda x: x >= 0.0
470  )
471  lambdaType = pexConfig.ChoiceField(
472  dtype=str,
473  doc="How to choose the value of the regularization strength",
474  default="absolute",
475  allowed={
476  "absolute": "Use lambdaValue as the value of regularization strength",
477  "relative": "Use lambdaValue as fraction of the default regularization strength (N.R. 18.5.8)",
478  "minimizeBiasedRisk": "Minimize biased risk estimate",
479  "minimizeUnbiasedRisk": "Minimize unbiased risk estimate",
480  }
481  )
482  lambdaValue = pexConfig.Field(
483  dtype=float,
484  doc="Value used for absolute determinations of regularization strength",
485  default=0.2,
486  )
487  lambdaScaling = pexConfig.Field(
488  dtype=float,
489  doc="Fraction of the default lambda strength (N.R. 18.5.8) to use. 1e-4 or 1e-5",
490  default=1e-4,
491  )
492  lambdaStepType = pexConfig.ChoiceField(
493  dtype=str,
494  doc="""If a scan through lambda is needed (minimizeBiasedRisk, minimizeUnbiasedRisk),
495  use log or linear steps""",
496  default="log",
497  allowed={
498  "log": "Step in log intervals; e.g. lambdaMin, lambdaMax, lambdaStep = -1.0, 2.0, 0.1",
499  "linear": "Step in linear intervals; e.g. lambdaMin, lambdaMax, lambdaStep = 0.1, 100, 0.1",
500  }
501  )
502  lambdaMin = pexConfig.Field(
503  dtype=float,
504  doc="""If scan through lambda needed (minimizeBiasedRisk, minimizeUnbiasedRisk),
505  start at this value. If lambdaStepType = log:linear, suggest -1:0.1""",
506  default=-1.0,
507  )
508  lambdaMax = pexConfig.Field(
509  dtype=float,
510  doc="""If scan through lambda needed (minimizeBiasedRisk, minimizeUnbiasedRisk),
511  stop at this value. If lambdaStepType = log:linear, suggest 2:100""",
512  default=2.0,
513  )
514  lambdaStep = pexConfig.Field(
515  dtype=float,
516  doc="""If scan through lambda needed (minimizeBiasedRisk, minimizeUnbiasedRisk),
517  step in these increments. If lambdaStepType = log:linear, suggest 0.1:0.1""",
518  default=0.1,
519  )
520 
521 
522 
528 
529 class PsfMatchTask(pipeBase.Task):
530  """!
531 
532 @anchor PsfMatchTask_
533 
534 @brief Base class for Psf Matching; should not be called directly
535 
536 @section ip_diffim_psfmatch_Contents Contents
537 
538  - @ref ip_diffim_psfmatch_Purpose
539  - @ref ip_diffim_psfmatch_Initialize
540  - @ref ip_diffim_psfmatch_IO
541  - @ref ip_diffim_psfmatch_Config
542  - @ref ip_diffim_psfmatch_Metadata
543  - @ref ip_diffim_psfmatch_Debug
544  - @ref ip_diffim_psfmatch_Example
545 
546 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
547 
548 @section ip_diffim_psfmatch_Purpose Description
549 
550 PsfMatchTask is a base class that implements the core functionality for matching the
551 Psfs of two images using a spatially varying Psf-matching lsst.afw.math.LinearCombinationKernel.
552 The Task requires the user to provide an instance of an lsst.afw.math.SpatialCellSet,
553 filled with lsst.ip.diffim.KernelCandidate instances, and a list of lsst.afw.math.Kernels
554 of basis shapes that will be used for the decomposition. If requested, the Task
555 also performs background matching and returns the differential background model as an
556 lsst.afw.math.Kernel.SpatialFunction.
557 
558 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
559 
560 @ection ip_diffim_psfmatch_Initialize Task initialization
561 
562 @copydoc \_\_init\_\_
563 
564 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
565 
566 @section ip_diffim_psfmatch_IO Invoking the Task
567 
568 As a base class, this Task is not directly invoked. However, run() methods that are
569 implemented on derived classes will make use of the core _solve() functionality,
570 which defines a sequence of lsst.afw.math.CandidateVisitor classes that iterate
571 through the KernelCandidates, first building up a per-candidate solution and then
572 building up a spatial model from the ensemble of candidates. Sigma clipping is
573 performed using the mean and standard deviation of all kernel sums (to reject
574 variable objects), on the per-candidate substamp diffim residuals
575 (to indicate a bad choice of kernel basis shapes for that particular object),
576 and on the substamp diffim residuals using the spatial kernel fit (to indicate a bad
577 choice of spatial kernel order, or poor constraints on the spatial model). The
578 _diagnostic() method logs information on the quality of the spatial fit, and also
579 modifies the Task metadata.
580 
581 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
582 
583 @section ip_diffim_psfmatch_Config Configuration parameters
584 
585 See @ref PsfMatchConfig, @ref PsfMatchConfigAL, @ref PsfMatchConfigDF, and @ref DetectionConfig.
586 
587 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
588 
589 @section ip_diffim_psfmatch_Metadata Quantities set in Metadata
590 
591 
592 <DL>
593 <DT> spatialConditionNum <DD> Condition number of the spatial kernel fit;
594  via @link lsst.ip.diffim.PsfMatchTask._diagnostic PsfMatchTask._diagnostic @endlink </DD> </DT>
595 <DT> spatialKernelSum <DD> Kernel sum (10^{-0.4 * &Delta; zeropoint}) of the spatial Psf-matching kernel;
596  via @link lsst.ip.diffim.PsfMatchTask._diagnostic PsfMatchTask._diagnostic @endlink </DD> </DT>
597 
598 <DT> ALBasisNGauss <DD> If using sum-of-Gaussian basis, the number of gaussians used;
599  via @link lsst.ip.diffim.makeKernelBasisList.generateAlardLuptonBasisList
600  generateAlardLuptonBasisList@endlink </DD> </DT>
601 <DT> ALBasisDegGauss <DD> If using sum-of-Gaussian basis, the degree of spatial variation of the Gaussians;
602  via @link lsst.ip.diffim.makeKernelBasisList.generateAlardLuptonBasisList
603  generateAlardLuptonBasisList@endlink </DD> </DT>
604 <DT> ALBasisSigGauss <DD> If using sum-of-Gaussian basis, the widths (sigma) of the Gaussians;
605  via @link lsst.ip.diffim.makeKernelBasisList.generateAlardLuptonBasisList
606  generateAlardLuptonBasisList@endlink </DD> </DT>
607 <DT> ALKernelSize <DD> If using sum-of-Gaussian basis, the kernel size;
608  via @link lsst.ip.diffim.makeKernelBasisList.generateAlardLuptonBasisList
609  generateAlardLuptonBasisList@endlink </DD> </DT>
610 
611 <DT> NFalsePositivesTotal <DD> Total number of diaSources;
612  via @link lsst.ip.diffim.KernelCandidateQa.aggregate KernelCandidateQa.aggregate@endlink </DD> </DT>
613 <DT> NFalsePositivesRefAssociated <DD> Number of diaSources that associate with the reference catalog;
614  via @link lsst.ip.diffim.KernelCandidateQa.aggregate KernelCandidateQa.aggregate@endlink </DD> </DT>
615 <DT> NFalsePositivesRefAssociated <DD> Number of diaSources that associate with the source catalog;
616  via @link lsst.ip.diffim.KernelCandidateQa.aggregate KernelCandidateQa.aggregate@endlink </DD> </DT>
617 <DT> NFalsePositivesUnassociated <DD> Number of diaSources that are orphans;
618  via @link lsst.ip.diffim.KernelCandidateQa.aggregate KernelCandidateQa.aggregate@endlink </DD> </DT>
619 <DT> metric_MEAN <DD> Mean value of substamp diffim quality metrics across all KernelCandidates,
620  for both the per-candidate (LOCAL) and SPATIAL residuals;
621  via @link lsst.ip.diffim.KernelCandidateQa.aggregate KernelCandidateQa.aggregate@endlink </DD> </DT>
622 <DT> metric_MEDIAN <DD> Median value of substamp diffim quality metrics across all KernelCandidates,
623  for both the per-candidate (LOCAL) and SPATIAL residuals;
624  via @link lsst.ip.diffim.KernelCandidateQa.aggregate KernelCandidateQa.aggregate@endlink </DD> </DT>
625 <DT> metric_STDEV <DD> Standard deviation of substamp diffim quality metrics across all KernelCandidates,
626  for both the per-candidate (LOCAL) and SPATIAL residuals;
627  via @link lsst.ip.diffim.KernelCandidateQa.aggregate KernelCandidateQa.aggregate@endlink </DD> </DT>
628 </DL>
629 
630 
631 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
632 
633 @section ip_diffim_psfmatch_Debug Debug variables
634 
635 
636 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a
637 flag @c -d/--debug to import @b debug.py from your @c PYTHONPATH. The relevant contents of debug.py
638 for this Task include:
639 
640 @code{.py}
641  import sys
642  import lsstDebug
643  def DebugInfo(name):
644  di = lsstDebug.getInfo(name)
645  if name == "lsst.ip.diffim.psfMatch":
646  di.display = True # enable debug output
647  di.maskTransparency = 80 # ds9 mask transparency
648  di.displayCandidates = True # show all the candidates and residuals
649  di.displayKernelBasis = False # show kernel basis functions
650  di.displayKernelMosaic = True # show kernel realized across the image
651  di.plotKernelSpatialModel = False # show coefficients of spatial model
652  di.showBadCandidates = True # show the bad candidates (red) along with good (green)
653  return di
654  lsstDebug.Info = DebugInfo
655  lsstDebug.frame = 1
656 
657 @endcode
658 
659 Note that if you want addional logging info, you may add to your scripts:
660 @code{.py}
661 import lsst.log.utils as logUtils
662 logUtils.traceSetAt("ip.diffim", 4)
663 @endcode
664 
665 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
666 
667 @section ip_diffim_psfmatch_Example Example code
668 
669 As a base class, there is no example code for PsfMatchTask.
670 However, see @link lsst.ip.diffim.imagePsfMatch.ImagePsfMatchTask ImagePsfMatchTask@endlink,
671 @link lsst.ip.diffim.snapPsfMatch.SnapPsfMatchTask SnapPsfMatchTask@endlink, and
672 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask@endlink.
673 
674 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
675 
676  """
677  ConfigClass = PsfMatchConfig
678  _DefaultName = "psfMatch"
679 
680  def __init__(self, *args, **kwargs):
681  """!Create the psf-matching Task
682 
683  @param *args arguments to be passed to lsst.pipe.base.task.Task.__init__
684  @param **kwargs keyword arguments to be passed to lsst.pipe.base.task.Task.__init__
685 
686  The initialization sets the Psf-matching kernel configuration using the value of
687  self.config.kernel.active. If the kernel is requested with regularization to moderate
688  the bias/variance tradeoff, currently only used when a delta function kernel basis
689  is provided, it creates a regularization matrix stored as member variable
690  self.hMat.
691  """
692  pipeBase.Task.__init__(self, *args, **kwargs)
693  self.kConfig = self.config.kernel.active
694 
695  #
696  if 'useRegularization' in self.kConfig:
697  self.useRegularization = self.kConfig.useRegularization
698  else:
699  self.useRegularization = False
700 
701  if self.useRegularization:
702  self.hMat = diffimLib.makeRegularizationMatrix(pexConfig.makePolicy(self.kConfig))
703 
704  def _diagnostic(self, kernelCellSet, spatialSolution, spatialKernel, spatialBg):
705  """!Provide logging diagnostics on quality of spatial kernel fit
706 
707  @param kernelCellSet: Cellset that contains the KernelCandidates used in the fitting
708  @param spatialSolution: KernelSolution of best-fit
709  @param spatialKernel: Best-fit spatial Kernel model
710  @param spatialBg: Best-fit spatial background model
711 
712  """
713  # What is the final kernel sum
714  kImage = afwImage.ImageD(spatialKernel.getDimensions())
715  kSum = spatialKernel.computeImage(kImage, False)
716  self.log.info("Final spatial kernel sum %.3f" % (kSum))
717 
718  # Look at how well conditioned the matrix is
719  conditionNum = spatialSolution.getConditionNumber(
720  getattr(diffimLib.KernelSolution, self.kConfig.conditionNumberType))
721  self.log.info("Spatial model condition number %.3e" % (conditionNum))
722 
723  if conditionNum < 0.0:
724  self.log.warn("Condition number is negative (%.3e)" % (conditionNum))
725  if conditionNum > self.kConfig.maxSpatialConditionNumber:
726  self.log.warn("Spatial solution exceeds max condition number (%.3e > %.3e)" % (
727  conditionNum, self.kConfig.maxSpatialConditionNumber))
728 
729  self.metadata.set("spatialConditionNum", conditionNum)
730  self.metadata.set("spatialKernelSum", kSum)
731 
732  # Look at how well the solution is constrained
733  nBasisKernels = spatialKernel.getNBasisKernels()
734  nKernelTerms = spatialKernel.getNSpatialParameters()
735  if nKernelTerms == 0: # order 0
736  nKernelTerms = 1
737 
738  # Not fit for
739  nBgTerms = spatialBg.getNParameters()
740  if nBgTerms == 1:
741  if spatialBg.getParameters()[0] == 0.0:
742  nBgTerms = 0
743 
744  nGood = 0
745  nBad = 0
746  nTot = 0
747  for cell in kernelCellSet.getCellList():
748  for cand in cell.begin(False): # False = include bad candidates
749  nTot += 1
750  if cand.getStatus() == afwMath.SpatialCellCandidate.GOOD:
751  nGood += 1
752  if cand.getStatus() == afwMath.SpatialCellCandidate.BAD:
753  nBad += 1
754 
755  self.log.info("Doing stats of kernel candidates used in the spatial fit.")
756 
757  # Counting statistics
758  if nBad > 2*nGood:
759  self.log.warn("Many more candidates rejected than accepted; %d total, %d rejected, %d used" % (
760  nTot, nBad, nGood))
761  else:
762  self.log.info("%d candidates total, %d rejected, %d used" % (nTot, nBad, nGood))
763 
764  # Some judgements on the quality of the spatial models
765  if nGood < nKernelTerms:
766  self.log.warn("Spatial kernel model underconstrained; %d candidates, %d terms, %d bases" % (
767  nGood, nKernelTerms, nBasisKernels))
768  self.log.warn("Consider lowering the spatial order")
769  elif nGood <= 2*nKernelTerms:
770  self.log.warn("Spatial kernel model poorly constrained; %d candidates, %d terms, %d bases" % (
771  nGood, nKernelTerms, nBasisKernels))
772  self.log.warn("Consider lowering the spatial order")
773  else:
774  self.log.info("Spatial kernel model well constrained; %d candidates, %d terms, %d bases" % (
775  nGood, nKernelTerms, nBasisKernels))
776 
777  if nGood < nBgTerms:
778  self.log.warn("Spatial background model underconstrained; %d candidates, %d terms" % (
779  nGood, nBgTerms))
780  self.log.warn("Consider lowering the spatial order")
781  elif nGood <= 2*nBgTerms:
782  self.log.warn("Spatial background model poorly constrained; %d candidates, %d terms" % (
783  nGood, nBgTerms))
784  self.log.warn("Consider lowering the spatial order")
785  else:
786  self.log.info("Spatial background model appears well constrained; %d candidates, %d terms" % (
787  nGood, nBgTerms))
788 
789  def _displayDebug(self, kernelCellSet, spatialKernel, spatialBackground):
790  """!Provide visualization of the inputs and ouputs to the Psf-matching code
791 
792  @param kernelCellSet: the SpatialCellSet used in determining the matching kernel and background
793  @param spatialKernel: spatially varying Psf-matching kernel
794  @param spatialBackground: spatially varying background-matching function
795 
796  """
797  import lsstDebug
798  displayCandidates = lsstDebug.Info(__name__).displayCandidates
799  displayKernelBasis = lsstDebug.Info(__name__).displayKernelBasis
800  displayKernelMosaic = lsstDebug.Info(__name__).displayKernelMosaic
801  plotKernelSpatialModel = lsstDebug.Info(__name__).plotKernelSpatialModel
802  showBadCandidates = lsstDebug.Info(__name__).showBadCandidates
803  maskTransparency = lsstDebug.Info(__name__).maskTransparency
804  if not maskTransparency:
805  maskTransparency = 0
806  ds9.setMaskTransparency(maskTransparency)
807 
808  if displayCandidates:
809  diutils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground,
810  frame=lsstDebug.frame,
811  showBadCandidates=showBadCandidates)
812  lsstDebug.frame += 1
813  diutils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground,
814  frame=lsstDebug.frame,
815  showBadCandidates=showBadCandidates,
816  kernels=True)
817  lsstDebug.frame += 1
818  diutils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground,
819  frame=lsstDebug.frame,
820  showBadCandidates=showBadCandidates,
821  resids=True)
822  lsstDebug.frame += 1
823 
824  if displayKernelBasis:
825  diutils.showKernelBasis(spatialKernel, frame=lsstDebug.frame)
826  lsstDebug.frame += 1
827 
828  if displayKernelMosaic:
829  diutils.showKernelMosaic(kernelCellSet.getBBox(), spatialKernel, frame=lsstDebug.frame)
830  lsstDebug.frame += 1
831 
832  if plotKernelSpatialModel:
833  diutils.plotKernelSpatialModel(spatialKernel, kernelCellSet, showBadCandidates=showBadCandidates)
834 
835  def _createPcaBasis(self, kernelCellSet, nStarPerCell, policy):
836  """!Create Principal Component basis
837 
838  If a principal component analysis is requested, typically when using a delta function basis,
839  perform the PCA here and return a new basis list containing the new principal components.
840 
841  @param kernelCellSet: a SpatialCellSet containing KernelCandidates, from which components are derived
842  @param nStarPerCell: the number of stars per cell to visit when doing the PCA
843  @param policy: input policy controlling the single kernel visitor
844 
845  @return
846  - nRejectedPca: number of KernelCandidates rejected during PCA loop
847  - spatialBasisList: basis list containing the principal shapes as Kernels
848 
849  """
850  nComponents = self.kConfig.numPrincipalComponents
851  imagePca = diffimLib.KernelPcaD()
852  importStarVisitor = diffimLib.KernelPcaVisitorF(imagePca)
853  kernelCellSet.visitCandidates(importStarVisitor, nStarPerCell)
854  if self.kConfig.subtractMeanForPca:
855  importStarVisitor.subtractMean()
856  imagePca.analyze()
857 
858  eigenValues = imagePca.getEigenValues()
859  pcaBasisList = importStarVisitor.getEigenKernels()
860 
861  eSum = np.sum(eigenValues)
862  if eSum == 0.0:
863  raise RuntimeError("Eigenvalues sum to zero")
864  for j in range(len(eigenValues)):
865  log.log("TRACE5." + self.log.getName() + "._solve", log.DEBUG,
866  "Eigenvalue %d : %f (%f)", j, eigenValues[j], eigenValues[j]/eSum)
867 
868  nToUse = min(nComponents, len(eigenValues))
869  trimBasisList = []
870  for j in range(nToUse):
871  # Check for NaNs?
872  kimage = afwImage.ImageD(pcaBasisList[j].getDimensions())
873  pcaBasisList[j].computeImage(kimage, False)
874  if not (True in np.isnan(kimage.getArray())):
875  trimBasisList.append(pcaBasisList[j])
876 
877  # Put all the power in the first kernel, which will not vary spatially
878  spatialBasisList = diffimLib.renormalizeKernelList(trimBasisList)
879 
880  # New Kernel visitor for this new basis list (no regularization explicitly)
881  singlekvPca = diffimLib.BuildSingleKernelVisitorF(spatialBasisList, policy)
882  singlekvPca.setSkipBuilt(False)
883  kernelCellSet.visitCandidates(singlekvPca, nStarPerCell)
884  singlekvPca.setSkipBuilt(True)
885  nRejectedPca = singlekvPca.getNRejected()
886 
887  return nRejectedPca, spatialBasisList
888 
889  def _buildCellSet(self, *args):
890  """!Fill a SpatialCellSet with KernelCandidates for the Psf-matching process;
891  override in derived classes"""
892  return
893 
894  @pipeBase.timeMethod
895  def _solve(self, kernelCellSet, basisList, returnOnExcept=False):
896  """!Solve for the PSF matching kernel
897 
898  @param kernelCellSet: a SpatialCellSet to use in determining the matching kernel
899  (typically as provided by _buildCellSet)
900  @param basisList: list of Kernels to be used in the decomposition of the spatially varying kernel
901  (typically as provided by makeKernelBasisList)
902  @param returnOnExcept: if True then return (None, None) if an error occurs, else raise the exception
903 
904  @return
905  - psfMatchingKernel: PSF matching kernel
906  - backgroundModel: differential background model
907 
908  Raise Exception if unable to determine PSF matching kernel and returnOnExcept False
909  """
910 
911  import lsstDebug
912  display = lsstDebug.Info(__name__).display
913 
914  maxSpatialIterations = self.kConfig.maxSpatialIterations
915  nStarPerCell = self.kConfig.nStarPerCell
916  usePcaForSpatialKernel = self.kConfig.usePcaForSpatialKernel
917 
918  # Visitor for the single kernel fit
919  policy = pexConfig.makePolicy(self.kConfig)
920  if self.useRegularization:
921  singlekv = diffimLib.BuildSingleKernelVisitorF(basisList, policy, self.hMat)
922  else:
923  singlekv = diffimLib.BuildSingleKernelVisitorF(basisList, policy)
924 
925  # Visitor for the kernel sum rejection
926  ksv = diffimLib.KernelSumVisitorF(policy)
927 
928  # Main loop
929  t0 = time.time()
930  try:
931  totalIterations = 0
932  thisIteration = 0
933  while (thisIteration < maxSpatialIterations):
934 
935  # Make sure there are no uninitialized candidates as active occupants of Cell
936  nRejectedSkf = -1
937  while (nRejectedSkf != 0):
938  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG,
939  "Building single kernels...")
940  kernelCellSet.visitCandidates(singlekv, nStarPerCell)
941  nRejectedSkf = singlekv.getNRejected()
942  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG,
943  "Iteration %d, rejected %d candidates due to initial kernel fit",
944  thisIteration, nRejectedSkf)
945 
946  # Reject outliers in kernel sum
947  ksv.resetKernelSum()
948  ksv.setMode(diffimLib.KernelSumVisitorF.AGGREGATE)
949  kernelCellSet.visitCandidates(ksv, nStarPerCell)
950  ksv.processKsumDistribution()
951  ksv.setMode(diffimLib.KernelSumVisitorF.REJECT)
952  kernelCellSet.visitCandidates(ksv, nStarPerCell)
953 
954  nRejectedKsum = ksv.getNRejected()
955  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG,
956  "Iteration %d, rejected %d candidates due to kernel sum",
957  thisIteration, nRejectedKsum)
958 
959  # Do we jump back to the top without incrementing thisIteration?
960  if nRejectedKsum > 0:
961  totalIterations += 1
962  continue
963 
964  # At this stage we can either apply the spatial fit to
965  # the kernels, or we run a PCA, use these as a *new*
966  # basis set with lower dimensionality, and then apply
967  # the spatial fit to these kernels
968 
969  if (usePcaForSpatialKernel):
970  log.log("TRACE0." + self.log.getName() + "._solve", log.DEBUG,
971  "Building Pca basis")
972 
973  nRejectedPca, spatialBasisList = self._createPcaBasis(kernelCellSet, nStarPerCell, policy)
974  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG,
975  "Iteration %d, rejected %d candidates due to Pca kernel fit",
976  thisIteration, nRejectedPca)
977 
978  # We don't want to continue on (yet) with the
979  # spatial modeling, because we have bad objects
980  # contributing to the Pca basis. We basically
981  # need to restart from the beginning of this loop,
982  # since the cell-mates of those objects that were
983  # rejected need their original Kernels built by
984  # singleKernelFitter.
985 
986  # Don't count against thisIteration
987  if (nRejectedPca > 0):
988  totalIterations += 1
989  continue
990  else:
991  spatialBasisList = basisList
992 
993  # We have gotten on to the spatial modeling part
994  regionBBox = kernelCellSet.getBBox()
995  spatialkv = diffimLib.BuildSpatialKernelVisitorF(spatialBasisList, regionBBox, policy)
996  kernelCellSet.visitCandidates(spatialkv, nStarPerCell)
997  spatialkv.solveLinearEquation()
998  log.log("TRACE2." + self.log.getName() + "._solve", log.DEBUG,
999  "Spatial kernel built with %d candidates", spatialkv.getNCandidates())
1000  spatialKernel, spatialBackground = spatialkv.getSolutionPair()
1001 
1002  # Check the quality of the spatial fit (look at residuals)
1003  assesskv = diffimLib.AssessSpatialKernelVisitorF(spatialKernel, spatialBackground, policy)
1004  kernelCellSet.visitCandidates(assesskv, nStarPerCell)
1005  nRejectedSpatial = assesskv.getNRejected()
1006  nGoodSpatial = assesskv.getNGood()
1007  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG,
1008  "Iteration %d, rejected %d candidates due to spatial kernel fit",
1009  thisIteration, nRejectedSpatial)
1010  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG,
1011  "%d candidates used in fit", nGoodSpatial)
1012 
1013  # If only nGoodSpatial == 0, might be other candidates in the cells
1014  if nGoodSpatial == 0 and nRejectedSpatial == 0:
1015  raise RuntimeError("No kernel candidates for spatial fit")
1016 
1017  if nRejectedSpatial == 0:
1018  # Nothing rejected, finished with spatial fit
1019  break
1020 
1021  # Otherwise, iterate on...
1022  thisIteration += 1
1023 
1024  # Final fit if above did not converge
1025  if (nRejectedSpatial > 0) and (thisIteration == maxSpatialIterations):
1026  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG, "Final spatial fit")
1027  if (usePcaForSpatialKernel):
1028  nRejectedPca, spatialBasisList = self._createPcaBasis(kernelCellSet, nStarPerCell, policy)
1029  regionBBox = kernelCellSet.getBBox()
1030  spatialkv = diffimLib.BuildSpatialKernelVisitorF(spatialBasisList, regionBBox, policy)
1031  kernelCellSet.visitCandidates(spatialkv, nStarPerCell)
1032  spatialkv.solveLinearEquation()
1033  log.log("TRACE2." + self.log.getName() + "._solve", log.DEBUG,
1034  "Spatial kernel built with %d candidates", spatialkv.getNCandidates())
1035  spatialKernel, spatialBackground = spatialkv.getSolutionPair()
1036 
1037  spatialSolution = spatialkv.getKernelSolution()
1038 
1039  except Exception as e:
1040  self.log.error("ERROR: Unable to calculate psf matching kernel")
1041 
1042  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG, str(e))
1043  raise e
1044 
1045  t1 = time.time()
1046  log.log("TRACE0." + self.log.getName() + "._solve", log.DEBUG,
1047  "Total time to compute the spatial kernel : %.2f s", (t1 - t0))
1048 
1049  if display:
1050  self._displayDebug(kernelCellSet, spatialKernel, spatialBackground)
1051 
1052  self._diagnostic(kernelCellSet, spatialSolution, spatialKernel, spatialBackground)
1053 
1054  return spatialSolution, spatialKernel, spatialBackground
1055 
1056 
1057 PsfMatch = PsfMatchTask
Configuration for detecting sources on images for building a PSF-matching kernel. ...
Definition: psfMatch.py:40
The parameters specific to the "Alard-Lupton" (sum-of-Gaussian) Psf-matching basis.
Definition: psfMatch.py:368
Base configuration for Psf-matching.
Definition: psfMatch.py:120
The parameters specific to the delta-function (one basis per-pixel) Psf-matching basis.
Definition: psfMatch.py:425
Base class for Psf Matching; should not be called directly.
Definition: psfMatch.py:529
def _diagnostic(self, kernelCellSet, spatialSolution, spatialKernel, spatialBg)
Provide logging diagnostics on quality of spatial kernel fit.
Definition: psfMatch.py:704
def __init__(self, args, kwargs)
Create the psf-matching Task.
Definition: psfMatch.py:680
def _createPcaBasis(self, kernelCellSet, nStarPerCell, policy)
Create Principal Component basis.
Definition: psfMatch.py:835
def _displayDebug(self, kernelCellSet, spatialKernel, spatialBackground)
Provide visualization of the inputs and ouputs to the Psf-matching code.
Definition: psfMatch.py:789