lsst.ip.diffim  15.0-4-gd76abed
psfMatch.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2016 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 from __future__ import absolute_import, division, print_function
23 
24 __all__ = ["DetectionConfig", "PsfMatchConfig", "PsfMatchConfigAL", "PsfMatchConfigDF", "PsfMatchTask"]
25 
26 import time
27 
28 import numpy as np
29 from builtins import range
30 
31 import lsst.afw.image as afwImage
32 import lsst.pex.config as pexConfig
33 import lsst.afw.math as afwMath
34 import lsst.afw.display.ds9 as ds9
35 import lsst.log as log
36 import lsst.pipe.base as pipeBase
37 from lsst.meas.algorithms import SubtractBackgroundConfig
38 from . import utils as diUtils
39 from . import diffimLib
40 
41 
42 class DetectionConfig(pexConfig.Config):
43  """!Configuration for detecting sources on images for building a PSF-matching kernel
44 
45  Configuration for turning detected lsst.afw.detection.FootPrints into an acceptable
46  (unmasked, high signal-to-noise, not too large or not too small) list of
47  lsst.ip.diffim.KernelSources that are used to build the Psf-matching kernel"""
48 
49  detThreshold = pexConfig.Field(
50  dtype=float,
51  doc="Value of footprint detection threshold",
52  default=10.0,
53  check=lambda x: x >= 3.0
54  )
55  detThresholdType = pexConfig.ChoiceField(
56  dtype=str,
57  doc="Type of detection threshold",
58  default="pixel_stdev",
59  allowed={
60  "value": "Use counts as the detection threshold type",
61  "stdev": "Use standard deviation of image plane",
62  "variance": "Use variance of image plane",
63  "pixel_stdev": "Use stdev derived from variance plane"
64  }
65  )
66  detOnTemplate = pexConfig.Field(
67  dtype=bool,
68  doc="""If true run detection on the template (image to convolve);
69  if false run detection on the science image""",
70  default=True
71  )
72  badMaskPlanes = pexConfig.ListField(
73  dtype=str,
74  doc="""Mask planes that lead to an invalid detection.
75  Options: NO_DATA EDGE SAT BAD CR INTRP""",
76  default=("NO_DATA", "EDGE", "SAT")
77  )
78  fpNpixMin = pexConfig.Field(
79  dtype=int,
80  doc="Minimum number of pixels in an acceptable Footprint",
81  default=5,
82  check=lambda x: x >= 5
83  )
84  fpNpixMax = pexConfig.Field(
85  dtype=int,
86  doc="""Maximum number of pixels in an acceptable Footprint;
87  too big and the subsequent convolutions become unwieldy""",
88  default=500,
89  check=lambda x: x <= 500
90  )
91  fpGrowKernelScaling = pexConfig.Field(
92  dtype=float,
93  doc="""If config.scaleByFwhm, grow the footprint based on
94  the final kernelSize. Each footprint will be
95  2*fpGrowKernelScaling*kernelSize x
96  2*fpGrowKernelScaling*kernelSize. With the value
97  of 1.0, the remaining pixels in each KernelCandiate
98  after convolution by the basis functions will be
99  equal to the kernel size itself.""",
100  default=1.0,
101  check=lambda x: x >= 1.0
102  )
103  fpGrowPix = pexConfig.Field(
104  dtype=int,
105  doc="""Growing radius (in pixels) for each raw detection
106  footprint. The smaller the faster; however the
107  kernel sum does not converge if the stamp is too
108  small; and the kernel is not constrained at all if
109  the stamp is the size of the kernel. The grown stamp
110  is 2 * fpGrowPix pixels larger in each dimension.
111  This is overridden by fpGrowKernelScaling if scaleByFwhm""",
112  default=30,
113  check=lambda x: x >= 10
114  )
115  scaleByFwhm = pexConfig.Field(
116  dtype=bool,
117  doc="Scale fpGrowPix by input Fwhm?",
118  default=True,
119  )
120 
121 
122 class PsfMatchConfig(pexConfig.Config):
123  """!Base configuration for Psf-matching
124 
125  The base configuration of the Psf-matching kernel, and of the warping, detection,
126  and background modeling subTasks."""
127 
128  warpingConfig = pexConfig.ConfigField("Config for warping exposures to a common alignment",
129  afwMath.warper.WarperConfig)
130  detectionConfig = pexConfig.ConfigField("Controlling the detection of sources for kernel building",
131  DetectionConfig)
132  afwBackgroundConfig = pexConfig.ConfigField("Controlling the Afw background fitting",
133  SubtractBackgroundConfig)
134 
135  useAfwBackground = pexConfig.Field(
136  dtype=bool,
137  doc="Use afw background subtraction instead of ip_diffim",
138  default=False,
139  )
140  fitForBackground = pexConfig.Field(
141  dtype=bool,
142  doc="Include terms (including kernel cross terms) for background in ip_diffim",
143  default=False,
144  )
145  kernelBasisSet = pexConfig.ChoiceField(
146  dtype=str,
147  doc="Type of basis set for PSF matching kernel.",
148  default="alard-lupton",
149  allowed={
150  "alard-lupton": """Alard-Lupton sum-of-gaussians basis set,
151  * The first term has no spatial variation
152  * The kernel sum is conserved
153  * You may want to turn off 'usePcaForSpatialKernel'""",
154  "delta-function": """Delta-function kernel basis set,
155  * You may enable the option useRegularization
156  * You should seriously consider usePcaForSpatialKernel, which will also
157  enable kernel sum conservation for the delta function kernels"""
158  }
159  )
160  kernelSize = pexConfig.Field(
161  dtype=int,
162  doc="""Number of rows/columns in the convolution kernel; should be odd-valued.
163  Modified by kernelSizeFwhmScaling if scaleByFwhm = true""",
164  default=21,
165  )
166  scaleByFwhm = pexConfig.Field(
167  dtype=bool,
168  doc="Scale kernelSize, alardGaussians by input Fwhm",
169  default=True,
170  )
171  kernelSizeFwhmScaling = pexConfig.Field(
172  dtype=float,
173  doc="""How much to scale the kernel size based on the largest AL Sigma""",
174  default=6.0,
175  check=lambda x: x >= 1.0
176  )
177  kernelSizeMin = pexConfig.Field(
178  dtype=int,
179  doc="""Minimum Kernel Size""",
180  default=21,
181  )
182  kernelSizeMax = pexConfig.Field(
183  dtype=int,
184  doc="""Maximum Kernel Size""",
185  default=35,
186  )
187  spatialModelType = pexConfig.ChoiceField(
188  dtype=str,
189  doc="Type of spatial functions for kernel and background",
190  default="chebyshev1",
191  allowed={
192  "chebyshev1": "Chebyshev polynomial of the first kind",
193  "polynomial": "Standard x,y polynomial",
194  }
195  )
196  spatialKernelOrder = pexConfig.Field(
197  dtype=int,
198  doc="Spatial order of convolution kernel variation",
199  default=2,
200  check=lambda x: x >= 0
201  )
202  spatialBgOrder = pexConfig.Field(
203  dtype=int,
204  doc="Spatial order of differential background variation",
205  default=1,
206  check=lambda x: x >= 0
207  )
208  sizeCellX = pexConfig.Field(
209  dtype=int,
210  doc="Size (rows) in pixels of each SpatialCell for spatial modeling",
211  default=128,
212  check=lambda x: x >= 32
213  )
214  sizeCellY = pexConfig.Field(
215  dtype=int,
216  doc="Size (columns) in pixels of each SpatialCell for spatial modeling",
217  default=128,
218  check=lambda x: x >= 32
219  )
220  nStarPerCell = pexConfig.Field(
221  dtype=int,
222  doc="Number of KernelCandidates in each SpatialCell to use in the spatial fitting",
223  default=3,
224  check=lambda x: x >= 1
225  )
226  maxSpatialIterations = pexConfig.Field(
227  dtype=int,
228  doc="Maximum number of iterations for rejecting bad KernelCandidates in spatial fitting",
229  default=3,
230  check=lambda x: x >= 1 and x <= 5
231  )
232  usePcaForSpatialKernel = pexConfig.Field(
233  dtype=bool,
234  doc="""Use Pca to reduce the dimensionality of the kernel basis sets.
235  This is particularly useful for delta-function kernels.
236  Functionally, after all Cells have their raw kernels determined, we run
237  a Pca on these Kernels, re-fit the Cells using the eigenKernels and then
238  fit those for spatial variation using the same technique as for Alard-Lupton kernels.
239  If this option is used, the first term will have no spatial variation and the
240  kernel sum will be conserved.""",
241  default=False,
242  )
243  subtractMeanForPca = pexConfig.Field(
244  dtype=bool,
245  doc="Subtract off the mean feature before doing the Pca",
246  default=True,
247  )
248  numPrincipalComponents = pexConfig.Field(
249  dtype=int,
250  doc="""Number of principal components to use for Pca basis, including the
251  mean kernel if requested.""",
252  default=5,
253  check=lambda x: x >= 3
254  )
255  singleKernelClipping = pexConfig.Field(
256  dtype=bool,
257  doc="Do sigma clipping on each raw kernel candidate",
258  default=True,
259  )
260  kernelSumClipping = pexConfig.Field(
261  dtype=bool,
262  doc="Do sigma clipping on the ensemble of kernel sums",
263  default=True,
264  )
265  spatialKernelClipping = pexConfig.Field(
266  dtype=bool,
267  doc="Do sigma clipping after building the spatial model",
268  default=True,
269  )
270  checkConditionNumber = pexConfig.Field(
271  dtype=bool,
272  doc="""Test for maximum condition number when inverting a kernel matrix.
273  Anything above maxConditionNumber is not used and the candidate is set as BAD.
274  Also used to truncate inverse matrix in estimateBiasedRisk. However,
275  if you are doing any deconvolution you will want to turn this off, or use
276  a large maxConditionNumber""",
277  default=False,
278  )
279  badMaskPlanes = pexConfig.ListField(
280  dtype=str,
281  doc="""Mask planes to ignore when calculating diffim statistics
282  Options: NO_DATA EDGE SAT BAD CR INTRP""",
283  default=("NO_DATA", "EDGE", "SAT")
284  )
285  candidateResidualMeanMax = pexConfig.Field(
286  dtype=float,
287  doc="""Rejects KernelCandidates yielding bad difference image quality.
288  Used by BuildSingleKernelVisitor, AssessSpatialKernelVisitor.
289  Represents average over pixels of (image/sqrt(variance)).""",
290  default=0.25,
291  check=lambda x: x >= 0.0
292  )
293  candidateResidualStdMax = pexConfig.Field(
294  dtype=float,
295  doc="""Rejects KernelCandidates yielding bad difference image quality.
296  Used by BuildSingleKernelVisitor, AssessSpatialKernelVisitor.
297  Represents stddev over pixels of (image/sqrt(variance)).""",
298  default=1.50,
299  check=lambda x: x >= 0.0
300  )
301  useCoreStats = pexConfig.Field(
302  dtype=bool,
303  doc="""Use the core of the footprint for the quality statistics, instead of the entire footprint.
304  WARNING: if there is deconvolution we probably will need to turn this off""",
305  default=False,
306  )
307  candidateCoreRadius = pexConfig.Field(
308  dtype=int,
309  doc="""Radius for calculation of stats in 'core' of KernelCandidate diffim.
310  Total number of pixels used will be (2*radius)**2.
311  This is used both for 'core' diffim quality as well as ranking of
312  KernelCandidates by their total flux in this core""",
313  default=3,
314  check=lambda x: x >= 1
315  )
316  maxKsumSigma = pexConfig.Field(
317  dtype=float,
318  doc="""Maximum allowed sigma for outliers from kernel sum distribution.
319  Used to reject variable objects from the kernel model""",
320  default=3.0,
321  check=lambda x: x >= 0.0
322  )
323  maxConditionNumber = pexConfig.Field(
324  dtype=float,
325  doc="Maximum condition number for a well conditioned matrix",
326  default=5.0e7,
327  check=lambda x: x >= 0.0
328  )
329  conditionNumberType = pexConfig.ChoiceField(
330  dtype=str,
331  doc="Use singular values (SVD) or eigen values (EIGENVALUE) to determine condition number",
332  default="EIGENVALUE",
333  allowed={
334  "SVD": "Use singular values",
335  "EIGENVALUE": "Use eigen values (faster)",
336  }
337  )
338  maxSpatialConditionNumber = pexConfig.Field(
339  dtype=float,
340  doc="Maximum condition number for a well conditioned spatial matrix",
341  default=1.0e10,
342  check=lambda x: x >= 0.0
343  )
344  iterateSingleKernel = pexConfig.Field(
345  dtype=bool,
346  doc="""Remake KernelCandidate using better variance estimate after first pass?
347  Primarily useful when convolving a single-depth image, otherwise not necessary.""",
348  default=False,
349  )
350  constantVarianceWeighting = pexConfig.Field(
351  dtype=bool,
352  doc="""Use constant variance weighting in single kernel fitting?
353  In some cases this is better for bright star residuals.""",
354  default=True,
355  )
356  calculateKernelUncertainty = pexConfig.Field(
357  dtype=bool,
358  doc="""Calculate kernel and background uncertainties for each kernel candidate?
359  This comes from the inverse of the covariance matrix.
360  Warning: regularization can cause problems for this step.""",
361  default=False,
362  )
363  useBicForKernelBasis = pexConfig.Field(
364  dtype=bool,
365  doc="""Use Bayesian Information Criterion to select the number of bases going into the kernel""",
366  default=False,
367  )
368 
369 
371  """!The parameters specific to the "Alard-Lupton" (sum-of-Gaussian) Psf-matching basis"""
372 
373  def setDefaults(self):
374  PsfMatchConfig.setDefaults(self)
375  self.kernelBasisSet = "alard-lupton"
376  self.maxConditionNumber = 5.0e7
377 
378  alardNGauss = pexConfig.Field(
379  dtype=int,
380  doc="Number of Gaussians in alard-lupton basis",
381  default=3,
382  check=lambda x: x >= 1
383  )
384  alardDegGauss = pexConfig.ListField(
385  dtype=int,
386  doc="Polynomial order of spatial modification of Gaussians. Must in number equal alardNGauss",
387  default=(4, 2, 2),
388  )
389  alardSigGauss = pexConfig.ListField(
390  dtype=float,
391  doc="""Sigma in pixels of Gaussians (FWHM = 2.35 sigma). Must in number equal alardNGauss""",
392  default=(0.7, 1.5, 3.0),
393  )
394  alardGaussBeta = pexConfig.Field(
395  dtype=float,
396  doc="""Default scale factor between Gaussian sigmas """,
397  default=2.0,
398  check=lambda x: x >= 0.0,
399  )
400  alardMinSig = pexConfig.Field(
401  dtype=float,
402  doc="""Minimum Sigma (pixels) for Gaussians""",
403  default=0.7,
404  check=lambda x: x >= 0.25
405  )
406  alardDegGaussDeconv = pexConfig.Field(
407  dtype=int,
408  doc="""Degree of spatial modification of ALL gaussians in AL basis during deconvolution""",
409  default=3,
410  check=lambda x: x >= 1
411  )
412  alardMinSigDeconv = pexConfig.Field(
413  dtype=float,
414  doc="""Minimum Sigma (pixels) for Gaussians during deconvolution;
415  make smaller than alardMinSig as this is only indirectly used""",
416  default=0.4,
417  check=lambda x: x >= 0.25
418  )
419  alardNGaussDeconv = pexConfig.Field(
420  dtype=int,
421  doc="Number of Gaussians in AL basis during deconvolution",
422  default=3,
423  check=lambda x: x >= 1
424  )
425 
426 
428  """!The parameters specific to the delta-function (one basis per-pixel) Psf-matching basis"""
429 
430  def setDefaults(self):
431  PsfMatchConfig.setDefaults(self)
432  self.kernelBasisSet = "delta-function"
433  self.maxConditionNumber = 5.0e6
435  self.subtractMeanForPca = True
436  self.useBicForKernelBasis = False
437 
438  useRegularization = pexConfig.Field(
439  dtype=bool,
440  doc="Use regularization to smooth the delta function kernels",
441  default=True,
442  )
443  regularizationType = pexConfig.ChoiceField(
444  dtype=str,
445  doc="Type of regularization.",
446  default="centralDifference",
447  allowed={
448  "centralDifference": "Penalize second derivative using 2-D stencil of central finite difference",
449  "forwardDifference": "Penalize first, second, third derivatives using forward finite differeces"
450  }
451  )
452  centralRegularizationStencil = pexConfig.ChoiceField(
453  dtype=int,
454  doc="Type of stencil to approximate central derivative (for centralDifference only)",
455  default=9,
456  allowed={
457  5: "5-point stencil including only adjacent-in-x,y elements",
458  9: "9-point stencil including diagonal elements"
459  }
460  )
461  forwardRegularizationOrders = pexConfig.ListField(
462  dtype=int,
463  doc="Array showing which order derivatives to penalize (for forwardDifference only)",
464  default=(1, 2),
465  itemCheck=lambda x: (x > 0) and (x < 4)
466  )
467  regularizationBorderPenalty = pexConfig.Field(
468  dtype=float,
469  doc="Value of the penalty for kernel border pixels",
470  default=3.0,
471  check=lambda x: x >= 0.0
472  )
473  lambdaType = pexConfig.ChoiceField(
474  dtype=str,
475  doc="How to choose the value of the regularization strength",
476  default="absolute",
477  allowed={
478  "absolute": "Use lambdaValue as the value of regularization strength",
479  "relative": "Use lambdaValue as fraction of the default regularization strength (N.R. 18.5.8)",
480  "minimizeBiasedRisk": "Minimize biased risk estimate",
481  "minimizeUnbiasedRisk": "Minimize unbiased risk estimate",
482  }
483  )
484  lambdaValue = pexConfig.Field(
485  dtype=float,
486  doc="Value used for absolute determinations of regularization strength",
487  default=0.2,
488  )
489  lambdaScaling = pexConfig.Field(
490  dtype=float,
491  doc="Fraction of the default lambda strength (N.R. 18.5.8) to use. 1e-4 or 1e-5",
492  default=1e-4,
493  )
494  lambdaStepType = pexConfig.ChoiceField(
495  dtype=str,
496  doc="""If a scan through lambda is needed (minimizeBiasedRisk, minimizeUnbiasedRisk),
497  use log or linear steps""",
498  default="log",
499  allowed={
500  "log": "Step in log intervals; e.g. lambdaMin, lambdaMax, lambdaStep = -1.0, 2.0, 0.1",
501  "linear": "Step in linear intervals; e.g. lambdaMin, lambdaMax, lambdaStep = 0.1, 100, 0.1",
502  }
503  )
504  lambdaMin = pexConfig.Field(
505  dtype=float,
506  doc="""If scan through lambda needed (minimizeBiasedRisk, minimizeUnbiasedRisk),
507  start at this value. If lambdaStepType = log:linear, suggest -1:0.1""",
508  default=-1.0,
509  )
510  lambdaMax = pexConfig.Field(
511  dtype=float,
512  doc="""If scan through lambda needed (minimizeBiasedRisk, minimizeUnbiasedRisk),
513  stop at this value. If lambdaStepType = log:linear, suggest 2:100""",
514  default=2.0,
515  )
516  lambdaStep = pexConfig.Field(
517  dtype=float,
518  doc="""If scan through lambda needed (minimizeBiasedRisk, minimizeUnbiasedRisk),
519  step in these increments. If lambdaStepType = log:linear, suggest 0.1:0.1""",
520  default=0.1,
521  )
522 
523 
524 
530 
531 class PsfMatchTask(pipeBase.Task):
532  """!
533 
534 \anchor PsfMatchTask_
535 
536 \brief Base class for Psf Matching; should not be called directly
537 
538 \section ip_diffim_psfmatch_Contents Contents
539 
540  - \ref ip_diffim_psfmatch_Purpose
541  - \ref ip_diffim_psfmatch_Initialize
542  - \ref ip_diffim_psfmatch_IO
543  - \ref ip_diffim_psfmatch_Config
544  - \ref ip_diffim_psfmatch_Metadata
545  - \ref ip_diffim_psfmatch_Debug
546  - \ref ip_diffim_psfmatch_Example
547 
548 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
549 
550 \section ip_diffim_psfmatch_Purpose Description
551 
552 PsfMatchTask is a base class that implements the core functionality for matching the
553 Psfs of two images using a spatially varying Psf-matching lsst.afw.math.LinearCombinationKernel.
554 The Task requires the user to provide an instance of an lsst.afw.math.SpatialCellSet,
555 filled with lsst.ip.diffim.KernelCandidate instances, and a list of lsst.afw.math.Kernels
556 of basis shapes that will be used for the decomposition. If requested, the Task
557 also performs background matching and returns the differential background model as an
558 lsst.afw.math.Kernel.SpatialFunction.
559 
560 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
561 
562 \section ip_diffim_psfmatch_Initialize Task initialization
563 
564 \copydoc \_\_init\_\_
565 
566 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
567 
568 \section ip_diffim_psfmatch_IO Invoking the Task
569 
570 As a base class, this Task is not directly invoked. However, run() methods that are
571 implemented on derived classes will make use of the core _solve() functionality,
572 which defines a sequence of lsst.afw.math.CandidateVisitor classes that iterate
573 through the KernelCandidates, first building up a per-candidate solution and then
574 building up a spatial model from the ensemble of candidates. Sigma clipping is
575 performed using the mean and standard deviation of all kernel sums (to reject
576 variable objects), on the per-candidate substamp diffim residuals
577 (to indicate a bad choice of kernel basis shapes for that particular object),
578 and on the substamp diffim residuals using the spatial kernel fit (to indicate a bad
579 choice of spatial kernel order, or poor constraints on the spatial model). The
580 _diagnostic() method logs information on the quality of the spatial fit, and also
581 modifies the Task metadata.
582 
583 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
584 
585 \section ip_diffim_psfmatch_Config Configuration parameters
586 
587 See \ref PsfMatchConfig, \ref PsfMatchConfigAL, \ref PsfMatchConfigDF, and \ref DetectionConfig.
588 
589 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
590 
591 \section ip_diffim_psfmatch_Metadata Quantities set in Metadata
592 
593 
594 <DL>
595 <DT> spatialConditionNum <DD> Condition number of the spatial kernel fit;
596  via \link lsst.ip.diffim.PsfMatchTask._diagnostic PsfMatchTask._diagnostic \endlink </DD> </DT>
597 <DT> spatialKernelSum <DD> Kernel sum (10^{-0.4 * &Delta; zeropoint}) of the spatial Psf-matching kernel;
598  via \link lsst.ip.diffim.PsfMatchTask._diagnostic PsfMatchTask._diagnostic \endlink </DD> </DT>
599 
600 <DT> ALBasisNGauss <DD> If using sum-of-Gaussian basis, the number of gaussians used;
601  via \link lsst.ip.diffim.makeKernelBasisList.generateAlardLuptonBasisList
602  generateAlardLuptonBasisList\endlink </DD> </DT>
603 <DT> ALBasisDegGauss <DD> If using sum-of-Gaussian basis, the degree of spatial variation of the Gaussians;
604  via \link lsst.ip.diffim.makeKernelBasisList.generateAlardLuptonBasisList
605  generateAlardLuptonBasisList\endlink </DD> </DT>
606 <DT> ALBasisSigGauss <DD> If using sum-of-Gaussian basis, the widths (sigma) of the Gaussians;
607  via \link lsst.ip.diffim.makeKernelBasisList.generateAlardLuptonBasisList
608  generateAlardLuptonBasisList\endlink </DD> </DT>
609 <DT> ALKernelSize <DD> If using sum-of-Gaussian basis, the kernel size;
610  via \link lsst.ip.diffim.makeKernelBasisList.generateAlardLuptonBasisList
611  generateAlardLuptonBasisList\endlink </DD> </DT>
612 
613 <DT> NFalsePositivesTotal <DD> Total number of diaSources;
614  via \link lsst.ip.diffim.KernelCandidateQa.aggregate KernelCandidateQa.aggregate\endlink </DD> </DT>
615 <DT> NFalsePositivesRefAssociated <DD> Number of diaSources that associate with the reference catalog;
616  via \link lsst.ip.diffim.KernelCandidateQa.aggregate KernelCandidateQa.aggregate\endlink </DD> </DT>
617 <DT> NFalsePositivesRefAssociated <DD> Number of diaSources that associate with the source catalog;
618  via \link lsst.ip.diffim.KernelCandidateQa.aggregate KernelCandidateQa.aggregate\endlink </DD> </DT>
619 <DT> NFalsePositivesUnassociated <DD> Number of diaSources that are orphans;
620  via \link lsst.ip.diffim.KernelCandidateQa.aggregate KernelCandidateQa.aggregate\endlink </DD> </DT>
621 <DT> metric_MEAN <DD> Mean value of substamp diffim quality metrics across all KernelCandidates,
622  for both the per-candidate (LOCAL) and SPATIAL residuals;
623  via \link lsst.ip.diffim.KernelCandidateQa.aggregate KernelCandidateQa.aggregate\endlink </DD> </DT>
624 <DT> metric_MEDIAN <DD> Median value of substamp diffim quality metrics across all KernelCandidates,
625  for both the per-candidate (LOCAL) and SPATIAL residuals;
626  via \link lsst.ip.diffim.KernelCandidateQa.aggregate KernelCandidateQa.aggregate\endlink </DD> </DT>
627 <DT> metric_STDEV <DD> Standard deviation of substamp diffim quality metrics across all KernelCandidates,
628  for both the per-candidate (LOCAL) and SPATIAL residuals;
629  via \link lsst.ip.diffim.KernelCandidateQa.aggregate KernelCandidateQa.aggregate\endlink </DD> </DT>
630 </DL>
631 
632 
633 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
634 
635 \section ip_diffim_psfmatch_Debug Debug variables
636 
637 
638 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
639 flag \c -d/--debug to import \b debug.py from your \c PYTHONPATH. The relevant contents of debug.py
640 for this Task include:
641 
642 \code{.py}
643  import sys
644  import lsstDebug
645  def DebugInfo(name):
646  di = lsstDebug.getInfo(name)
647  if name == "lsst.ip.diffim.psfMatch":
648  di.display = True # enable debug output
649  di.maskTransparency = 80 # ds9 mask transparency
650  di.displayCandidates = True # show all the candidates and residuals
651  di.displayKernelBasis = False # show kernel basis functions
652  di.displayKernelMosaic = True # show kernel realized across the image
653  di.plotKernelSpatialModel = False # show coefficients of spatial model
654  di.showBadCandidates = True # show the bad candidates (red) along with good (green)
655  return di
656  lsstDebug.Info = DebugInfo
657  lsstDebug.frame = 1
658 
659 \endcode
660 
661 Note that if you want addional logging info, you may add to your scripts:
662 \code{.py}
663 import lsst.log.utils as logUtils
664 logUtils.traceSetAt("ip.diffim", 4)
665 \endcode
666 
667 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
668 
669 \section ip_diffim_psfmatch_Example Example code
670 
671 As a base class, there is no example code for PsfMatchTask.
672 However, see \link lsst.ip.diffim.imagePsfMatch.ImagePsfMatchTask ImagePsfMatchTask\endlink,
673 \link lsst.ip.diffim.snapPsfMatch.SnapPsfMatchTask SnapPsfMatchTask\endlink, and
674 \link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask\endlink.
675 
676 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
677 
678  """
679  ConfigClass = PsfMatchConfig
680  _DefaultName = "psfMatch"
681 
682  def __init__(self, *args, **kwargs):
683  """!Create the psf-matching Task
684 
685  \param *args arguments to be passed to lsst.pipe.base.task.Task.__init__
686  \param **kwargs keyword arguments to be passed to lsst.pipe.base.task.Task.__init__
687 
688  The initialization sets the Psf-matching kernel configuration using the value of
689  self.config.kernel.active. If the kernel is requested with regularization to moderate
690  the bias/variance tradeoff, currently only used when a delta function kernel basis
691  is provided, it creates a regularization matrix stored as member variable
692  self.hMat.
693  """
694  pipeBase.Task.__init__(self, *args, **kwargs)
695  self.kConfig = self.config.kernel.active
696 
697  #
698  if 'useRegularization' in self.kConfig:
699  self.useRegularization = self.kConfig.useRegularization
700  else:
701  self.useRegularization = False
702 
703  if self.useRegularization:
704  self.hMat = diffimLib.makeRegularizationMatrix(pexConfig.makePolicy(self.kConfig))
705 
706  def _diagnostic(self, kernelCellSet, spatialSolution, spatialKernel, spatialBg):
707  """!Provide logging diagnostics on quality of spatial kernel fit
708 
709  @param kernelCellSet: Cellset that contains the KernelCandidates used in the fitting
710  @param spatialSolution: KernelSolution of best-fit
711  @param spatialKernel: Best-fit spatial Kernel model
712  @param spatialBg: Best-fit spatial background model
713 
714  """
715  # What is the final kernel sum
716  kImage = afwImage.ImageD(spatialKernel.getDimensions())
717  kSum = spatialKernel.computeImage(kImage, False)
718  self.log.info("Final spatial kernel sum %.3f" % (kSum))
719 
720  # Look at how well conditioned the matrix is
721  conditionNum = spatialSolution.getConditionNumber(
722  getattr(diffimLib.KernelSolution, self.kConfig.conditionNumberType))
723  self.log.info("Spatial model condition number %.3e" % (conditionNum))
724 
725  if conditionNum < 0.0:
726  self.log.warn("Condition number is negative (%.3e)" % (conditionNum))
727  if conditionNum > self.kConfig.maxSpatialConditionNumber:
728  self.log.warn("Spatial solution exceeds max condition number (%.3e > %.3e)" % (
729  conditionNum, self.kConfig.maxSpatialConditionNumber))
730 
731  self.metadata.set("spatialConditionNum", conditionNum)
732  self.metadata.set("spatialKernelSum", kSum)
733 
734  # Look at how well the solution is constrained
735  nBasisKernels = spatialKernel.getNBasisKernels()
736  nKernelTerms = spatialKernel.getNSpatialParameters()
737  if nKernelTerms == 0: # order 0
738  nKernelTerms = 1
739 
740  # Not fit for
741  nBgTerms = spatialBg.getNParameters()
742  if nBgTerms == 1:
743  if spatialBg.getParameters()[0] == 0.0:
744  nBgTerms = 0
745 
746  nGood = 0
747  nBad = 0
748  nTot = 0
749  for cell in kernelCellSet.getCellList():
750  for cand in cell.begin(False): # False = include bad candidates
751  nTot += 1
752  if cand.getStatus() == afwMath.SpatialCellCandidate.GOOD:
753  nGood += 1
754  if cand.getStatus() == afwMath.SpatialCellCandidate.BAD:
755  nBad += 1
756 
757  self.log.info("Doing stats of kernel candidates used in the spatial fit.")
758 
759  # Counting statistics
760  if nBad > 2*nGood:
761  self.log.warn("Many more candidates rejected than accepted; %d total, %d rejected, %d used" % (
762  nTot, nBad, nGood))
763  else:
764  self.log.info("%d candidates total, %d rejected, %d used" % (nTot, nBad, nGood))
765 
766  # Some judgements on the quality of the spatial models
767  if nGood < nKernelTerms:
768  self.log.warn("Spatial kernel model underconstrained; %d candidates, %d terms, %d bases" % (
769  nGood, nKernelTerms, nBasisKernels))
770  self.log.warn("Consider lowering the spatial order")
771  elif nGood <= 2*nKernelTerms:
772  self.log.warn("Spatial kernel model poorly constrained; %d candidates, %d terms, %d bases" % (
773  nGood, nKernelTerms, nBasisKernels))
774  self.log.warn("Consider lowering the spatial order")
775  else:
776  self.log.info("Spatial kernel model well constrained; %d candidates, %d terms, %d bases" % (
777  nGood, nKernelTerms, nBasisKernels))
778 
779  if nGood < nBgTerms:
780  self.log.warn("Spatial background model underconstrained; %d candidates, %d terms" % (
781  nGood, nBgTerms))
782  self.log.warn("Consider lowering the spatial order")
783  elif nGood <= 2*nBgTerms:
784  self.log.warn("Spatial background model poorly constrained; %d candidates, %d terms" % (
785  nGood, nBgTerms))
786  self.log.warn("Consider lowering the spatial order")
787  else:
788  self.log.info("Spatial background model appears well constrained; %d candidates, %d terms" % (
789  nGood, nBgTerms))
790 
791  def _displayDebug(self, kernelCellSet, spatialKernel, spatialBackground):
792  """!Provide visualization of the inputs and ouputs to the Psf-matching code
793 
794  @param kernelCellSet: the SpatialCellSet used in determining the matching kernel and background
795  @param spatialKernel: spatially varying Psf-matching kernel
796  @param spatialBackground: spatially varying background-matching function
797 
798  """
799  import lsstDebug
800  displayCandidates = lsstDebug.Info(__name__).displayCandidates
801  displayKernelBasis = lsstDebug.Info(__name__).displayKernelBasis
802  displayKernelMosaic = lsstDebug.Info(__name__).displayKernelMosaic
803  plotKernelSpatialModel = lsstDebug.Info(__name__).plotKernelSpatialModel
804  showBadCandidates = lsstDebug.Info(__name__).showBadCandidates
805  maskTransparency = lsstDebug.Info(__name__).maskTransparency
806  if not maskTransparency:
807  maskTransparency = 0
808  ds9.setMaskTransparency(maskTransparency)
809 
810  if displayCandidates:
811  diUtils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground,
812  frame=lsstDebug.frame,
813  showBadCandidates=showBadCandidates)
814  lsstDebug.frame += 1
815  diUtils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground,
816  frame=lsstDebug.frame,
817  showBadCandidates=showBadCandidates,
818  kernels=True)
819  lsstDebug.frame += 1
820  diUtils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground,
821  frame=lsstDebug.frame,
822  showBadCandidates=showBadCandidates,
823  resids=True)
824  lsstDebug.frame += 1
825 
826  if displayKernelBasis:
827  diUtils.showKernelBasis(spatialKernel, frame=lsstDebug.frame)
828  lsstDebug.frame += 1
829 
830  if displayKernelMosaic:
831  diUtils.showKernelMosaic(kernelCellSet.getBBox(), spatialKernel, frame=lsstDebug.frame)
832  lsstDebug.frame += 1
833 
834  if plotKernelSpatialModel:
835  diUtils.plotKernelSpatialModel(spatialKernel, kernelCellSet, showBadCandidates=showBadCandidates)
836 
837  def _createPcaBasis(self, kernelCellSet, nStarPerCell, policy):
838  """!Create Principal Component basis
839 
840  If a principal component analysis is requested, typically when using a delta function basis,
841  perform the PCA here and return a new basis list containing the new principal components.
842 
843  @param kernelCellSet: a SpatialCellSet containing KernelCandidates, from which components are derived
844  @param nStarPerCell: the number of stars per cell to visit when doing the PCA
845  @param policy: input policy controlling the single kernel visitor
846 
847  @return
848  - nRejectedPca: number of KernelCandidates rejected during PCA loop
849  - spatialBasisList: basis list containing the principal shapes as Kernels
850 
851  """
852  nComponents = self.kConfig.numPrincipalComponents
853  imagePca = diffimLib.KernelPcaD()
854  importStarVisitor = diffimLib.KernelPcaVisitorF(imagePca)
855  kernelCellSet.visitCandidates(importStarVisitor, nStarPerCell)
856  if self.kConfig.subtractMeanForPca:
857  importStarVisitor.subtractMean()
858  imagePca.analyze()
859 
860  eigenValues = imagePca.getEigenValues()
861  pcaBasisList = importStarVisitor.getEigenKernels()
862 
863  eSum = np.sum(eigenValues)
864  if eSum == 0.0:
865  raise RuntimeError("Eigenvalues sum to zero")
866  for j in range(len(eigenValues)):
867  log.log("TRACE5." + self.log.getName() + "._solve", log.DEBUG,
868  "Eigenvalue %d : %f (%f)", j, eigenValues[j], eigenValues[j]/eSum)
869 
870  nToUse = min(nComponents, len(eigenValues))
871  trimBasisList = []
872  for j in range(nToUse):
873  # Check for NaNs?
874  kimage = afwImage.ImageD(pcaBasisList[j].getDimensions())
875  pcaBasisList[j].computeImage(kimage, False)
876  if not (True in np.isnan(kimage.getArray())):
877  trimBasisList.append(pcaBasisList[j])
878 
879  # Put all the power in the first kernel, which will not vary spatially
880  spatialBasisList = diffimLib.renormalizeKernelList(trimBasisList)
881 
882  # New Kernel visitor for this new basis list (no regularization explicitly)
883  singlekvPca = diffimLib.BuildSingleKernelVisitorF(spatialBasisList, policy)
884  singlekvPca.setSkipBuilt(False)
885  kernelCellSet.visitCandidates(singlekvPca, nStarPerCell)
886  singlekvPca.setSkipBuilt(True)
887  nRejectedPca = singlekvPca.getNRejected()
888 
889  return nRejectedPca, spatialBasisList
890 
891  def _buildCellSet(self, *args):
892  """!Fill a SpatialCellSet with KernelCandidates for the Psf-matching process;
893  override in derived classes"""
894  return
895 
896  @pipeBase.timeMethod
897  def _solve(self, kernelCellSet, basisList, returnOnExcept=False):
898  """!Solve for the PSF matching kernel
899 
900  @param kernelCellSet: a SpatialCellSet to use in determining the matching kernel
901  (typically as provided by _buildCellSet)
902  @param basisList: list of Kernels to be used in the decomposition of the spatially varying kernel
903  (typically as provided by makeKernelBasisList)
904  @param returnOnExcept: if True then return (None, None) if an error occurs, else raise the exception
905 
906  @return
907  - psfMatchingKernel: PSF matching kernel
908  - backgroundModel: differential background model
909 
910  Raise Exception if unable to determine PSF matching kernel and returnOnExcept False
911  """
912 
913  import lsstDebug
914  display = lsstDebug.Info(__name__).display
915 
916  maxSpatialIterations = self.kConfig.maxSpatialIterations
917  nStarPerCell = self.kConfig.nStarPerCell
918  usePcaForSpatialKernel = self.kConfig.usePcaForSpatialKernel
919 
920  # Visitor for the single kernel fit
921  policy = pexConfig.makePolicy(self.kConfig)
922  if self.useRegularization:
923  singlekv = diffimLib.BuildSingleKernelVisitorF(basisList, policy, self.hMat)
924  else:
925  singlekv = diffimLib.BuildSingleKernelVisitorF(basisList, policy)
926 
927  # Visitor for the kernel sum rejection
928  ksv = diffimLib.KernelSumVisitorF(policy)
929 
930  # Main loop
931  t0 = time.time()
932  try:
933  totalIterations = 0
934  thisIteration = 0
935  while (thisIteration < maxSpatialIterations):
936 
937  # Make sure there are no uninitialized candidates as active occupants of Cell
938  nRejectedSkf = -1
939  while (nRejectedSkf != 0):
940  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG,
941  "Building single kernels...")
942  kernelCellSet.visitCandidates(singlekv, nStarPerCell)
943  nRejectedSkf = singlekv.getNRejected()
944  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG,
945  "Iteration %d, rejected %d candidates due to initial kernel fit",
946  thisIteration, nRejectedSkf)
947 
948  # Reject outliers in kernel sum
949  ksv.resetKernelSum()
950  ksv.setMode(diffimLib.KernelSumVisitorF.AGGREGATE)
951  kernelCellSet.visitCandidates(ksv, nStarPerCell)
952  ksv.processKsumDistribution()
953  ksv.setMode(diffimLib.KernelSumVisitorF.REJECT)
954  kernelCellSet.visitCandidates(ksv, nStarPerCell)
955 
956  nRejectedKsum = ksv.getNRejected()
957  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG,
958  "Iteration %d, rejected %d candidates due to kernel sum",
959  thisIteration, nRejectedKsum)
960 
961  # Do we jump back to the top without incrementing thisIteration?
962  if nRejectedKsum > 0:
963  totalIterations += 1
964  continue
965 
966  # At this stage we can either apply the spatial fit to
967  # the kernels, or we run a PCA, use these as a *new*
968  # basis set with lower dimensionality, and then apply
969  # the spatial fit to these kernels
970 
971  if (usePcaForSpatialKernel):
972  log.log("TRACE0." + self.log.getName() + "._solve", log.DEBUG,
973  "Building Pca basis")
974 
975  nRejectedPca, spatialBasisList = self._createPcaBasis(kernelCellSet, nStarPerCell, policy)
976  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG,
977  "Iteration %d, rejected %d candidates due to Pca kernel fit",
978  thisIteration, nRejectedPca)
979 
980  # We don't want to continue on (yet) with the
981  # spatial modeling, because we have bad objects
982  # contributing to the Pca basis. We basically
983  # need to restart from the beginning of this loop,
984  # since the cell-mates of those objects that were
985  # rejected need their original Kernels built by
986  # singleKernelFitter.
987 
988  # Don't count against thisIteration
989  if (nRejectedPca > 0):
990  totalIterations += 1
991  continue
992  else:
993  spatialBasisList = basisList
994 
995  # We have gotten on to the spatial modeling part
996  regionBBox = kernelCellSet.getBBox()
997  spatialkv = diffimLib.BuildSpatialKernelVisitorF(spatialBasisList, regionBBox, policy)
998  kernelCellSet.visitCandidates(spatialkv, nStarPerCell)
999  spatialkv.solveLinearEquation()
1000  log.log("TRACE2." + self.log.getName() + "._solve", log.DEBUG,
1001  "Spatial kernel built with %d candidates", spatialkv.getNCandidates())
1002  spatialKernel, spatialBackground = spatialkv.getSolutionPair()
1003 
1004  # Check the quality of the spatial fit (look at residuals)
1005  assesskv = diffimLib.AssessSpatialKernelVisitorF(spatialKernel, spatialBackground, policy)
1006  kernelCellSet.visitCandidates(assesskv, nStarPerCell)
1007  nRejectedSpatial = assesskv.getNRejected()
1008  nGoodSpatial = assesskv.getNGood()
1009  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG,
1010  "Iteration %d, rejected %d candidates due to spatial kernel fit",
1011  thisIteration, nRejectedSpatial)
1012  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG,
1013  "%d candidates used in fit", nGoodSpatial)
1014 
1015  # If only nGoodSpatial == 0, might be other candidates in the cells
1016  if nGoodSpatial == 0 and nRejectedSpatial == 0:
1017  raise RuntimeError("No kernel candidates for spatial fit")
1018 
1019  if nRejectedSpatial == 0:
1020  # Nothing rejected, finished with spatial fit
1021  break
1022 
1023  # Otherwise, iterate on...
1024  thisIteration += 1
1025 
1026  # Final fit if above did not converge
1027  if (nRejectedSpatial > 0) and (thisIteration == maxSpatialIterations):
1028  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG, "Final spatial fit")
1029  if (usePcaForSpatialKernel):
1030  nRejectedPca, spatialBasisList = self._createPcaBasis(kernelCellSet, nStarPerCell, policy)
1031  regionBBox = kernelCellSet.getBBox()
1032  spatialkv = diffimLib.BuildSpatialKernelVisitorF(spatialBasisList, regionBBox, policy)
1033  kernelCellSet.visitCandidates(spatialkv, nStarPerCell)
1034  spatialkv.solveLinearEquation()
1035  log.log("TRACE2." + self.log.getName() + "._solve", log.DEBUG,
1036  "Spatial kernel built with %d candidates", spatialkv.getNCandidates())
1037  spatialKernel, spatialBackground = spatialkv.getSolutionPair()
1038 
1039  spatialSolution = spatialkv.getKernelSolution()
1040 
1041  except Exception as e:
1042  self.log.error("ERROR: Unable to calculate psf matching kernel")
1043 
1044  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG, str(e))
1045  raise e
1046 
1047  t1 = time.time()
1048  log.log("TRACE0." + self.log.getName() + "._solve", log.DEBUG,
1049  "Total time to compute the spatial kernel : %.2f s", (t1 - t0))
1050 
1051  if display:
1052  self._displayDebug(kernelCellSet, spatialKernel, spatialBackground)
1053 
1054  self._diagnostic(kernelCellSet, spatialSolution, spatialKernel, spatialBackground)
1055 
1056  return spatialSolution, spatialKernel, spatialBackground
1057 
1058 PsfMatch = PsfMatchTask
Configuration for detecting sources on images for building a PSF-matching kernel. ...
Definition: psfMatch.py:42
The parameters specific to the "Alard-Lupton" (sum-of-Gaussian) Psf-matching basis.
Definition: psfMatch.py:370
Base configuration for Psf-matching.
Definition: psfMatch.py:122
The parameters specific to the delta-function (one basis per-pixel) Psf-matching basis.
Definition: psfMatch.py:427
Base class for Psf Matching; should not be called directly.
Definition: psfMatch.py:531
def _diagnostic(self, kernelCellSet, spatialSolution, spatialKernel, spatialBg)
Provide logging diagnostics on quality of spatial kernel fit.
Definition: psfMatch.py:706
def __init__(self, args, kwargs)
Create the psf-matching Task.
Definition: psfMatch.py:682
def _createPcaBasis(self, kernelCellSet, nStarPerCell, policy)
Create Principal Component basis.
Definition: psfMatch.py:837
def _displayDebug(self, kernelCellSet, spatialKernel, spatialBackground)
Provide visualization of the inputs and ouputs to the Psf-matching code.
Definition: psfMatch.py:791