lsst.ip.diffim  17.0.1-4-g088434c
psfMatch.py
Go to the documentation of this file.
1 # This file is part of ip_diffim.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 __all__ = ["DetectionConfig", "PsfMatchConfig", "PsfMatchConfigAL", "PsfMatchConfigDF", "PsfMatchTask"]
23 
24 import time
25 
26 import numpy as np
27 
28 import lsst.afw.image as afwImage
29 import lsst.pex.config as pexConfig
30 import lsst.afw.math as afwMath
31 import lsst.afw.display as afwDisplay
32 import lsst.log as log
33 import lsst.pipe.base as pipeBase
34 from lsst.meas.algorithms import SubtractBackgroundConfig
35 from . import utils as diutils
36 from . import diffimLib
37 
38 
39 class DetectionConfig(pexConfig.Config):
40  """Configuration for detecting sources on images for building a
41  PSF-matching kernel
42 
43  Configuration for turning detected lsst.afw.detection.FootPrints into an
44  acceptable (unmasked, high signal-to-noise, not too large or not too small)
45  list of `lsst.ip.diffim.KernelSources` that are used to build the
46  Psf-matching kernel"""
47 
48  detThreshold = pexConfig.Field(
49  dtype=float,
50  doc="Value of footprint detection threshold",
51  default=10.0,
52  check=lambda x: x >= 3.0
53  )
54  detThresholdType = pexConfig.ChoiceField(
55  dtype=str,
56  doc="Type of detection threshold",
57  default="pixel_stdev",
58  allowed={
59  "value": "Use counts as the detection threshold type",
60  "stdev": "Use standard deviation of image plane",
61  "variance": "Use variance of image plane",
62  "pixel_stdev": "Use stdev derived from variance plane"
63  }
64  )
65  detOnTemplate = pexConfig.Field(
66  dtype=bool,
67  doc="""If true run detection on the template (image to convolve);
68  if false run detection on the science image""",
69  default=True
70  )
71  badMaskPlanes = pexConfig.ListField(
72  dtype=str,
73  doc="""Mask planes that lead to an invalid detection.
74  Options: NO_DATA EDGE SAT BAD CR INTRP""",
75  default=("NO_DATA", "EDGE", "SAT")
76  )
77  fpNpixMin = pexConfig.Field(
78  dtype=int,
79  doc="Minimum number of pixels in an acceptable Footprint",
80  default=5,
81  check=lambda x: x >= 5
82  )
83  fpNpixMax = pexConfig.Field(
84  dtype=int,
85  doc="""Maximum number of pixels in an acceptable Footprint;
86  too big and the subsequent convolutions become unwieldy""",
87  default=500,
88  check=lambda x: x <= 500
89  )
90  fpGrowKernelScaling = pexConfig.Field(
91  dtype=float,
92  doc="""If config.scaleByFwhm, grow the footprint based on
93  the final kernelSize. Each footprint will be
94  2*fpGrowKernelScaling*kernelSize x
95  2*fpGrowKernelScaling*kernelSize. With the value
96  of 1.0, the remaining pixels in each KernelCandiate
97  after convolution by the basis functions will be
98  equal to the kernel size itself.""",
99  default=1.0,
100  check=lambda x: x >= 1.0
101  )
102  fpGrowPix = pexConfig.Field(
103  dtype=int,
104  doc="""Growing radius (in pixels) for each raw detection
105  footprint. The smaller the faster; however the
106  kernel sum does not converge if the stamp is too
107  small; and the kernel is not constrained at all if
108  the stamp is the size of the kernel. The grown stamp
109  is 2 * fpGrowPix pixels larger in each dimension.
110  This is overridden by fpGrowKernelScaling if scaleByFwhm""",
111  default=30,
112  check=lambda x: x >= 10
113  )
114  scaleByFwhm = pexConfig.Field(
115  dtype=bool,
116  doc="Scale fpGrowPix by input Fwhm?",
117  default=True,
118  )
119 
120 
121 class PsfMatchConfig(pexConfig.Config):
122  """Base configuration for Psf-matching
123 
124  The base configuration of the Psf-matching kernel, and of the warping, detection,
125  and background modeling subTasks."""
126 
127  warpingConfig = pexConfig.ConfigField("Config for warping exposures to a common alignment",
128  afwMath.warper.WarperConfig)
129  detectionConfig = pexConfig.ConfigField("Controlling the detection of sources for kernel building",
130  DetectionConfig)
131  afwBackgroundConfig = pexConfig.ConfigField("Controlling the Afw background fitting",
132  SubtractBackgroundConfig)
133 
134  useAfwBackground = pexConfig.Field(
135  dtype=bool,
136  doc="Use afw background subtraction instead of ip_diffim",
137  default=False,
138  )
139  fitForBackground = pexConfig.Field(
140  dtype=bool,
141  doc="Include terms (including kernel cross terms) for background in ip_diffim",
142  default=False,
143  )
144  kernelBasisSet = pexConfig.ChoiceField(
145  dtype=str,
146  doc="Type of basis set for PSF matching kernel.",
147  default="alard-lupton",
148  allowed={
149  "alard-lupton": """Alard-Lupton sum-of-gaussians basis set,
150  * The first term has no spatial variation
151  * The kernel sum is conserved
152  * You may want to turn off 'usePcaForSpatialKernel'""",
153  "delta-function": """Delta-function kernel basis set,
154  * You may enable the option useRegularization
155  * You should seriously consider usePcaForSpatialKernel, which will also
156  enable kernel sum conservation for the delta function kernels"""
157  }
158  )
159  kernelSize = pexConfig.Field(
160  dtype=int,
161  doc="""Number of rows/columns in the convolution kernel; should be odd-valued.
162  Modified by kernelSizeFwhmScaling if scaleByFwhm = true""",
163  default=21,
164  )
165  scaleByFwhm = pexConfig.Field(
166  dtype=bool,
167  doc="Scale kernelSize, alardGaussians by input Fwhm",
168  default=True,
169  )
170  kernelSizeFwhmScaling = pexConfig.Field(
171  dtype=float,
172  doc="""How much to scale the kernel size based on the largest AL Sigma""",
173  default=6.0,
174  check=lambda x: x >= 1.0
175  )
176  kernelSizeMin = pexConfig.Field(
177  dtype=int,
178  doc="""Minimum Kernel Size""",
179  default=21,
180  )
181  kernelSizeMax = pexConfig.Field(
182  dtype=int,
183  doc="""Maximum Kernel Size""",
184  default=35,
185  )
186  spatialModelType = pexConfig.ChoiceField(
187  dtype=str,
188  doc="Type of spatial functions for kernel and background",
189  default="chebyshev1",
190  allowed={
191  "chebyshev1": "Chebyshev polynomial of the first kind",
192  "polynomial": "Standard x,y polynomial",
193  }
194  )
195  spatialKernelOrder = pexConfig.Field(
196  dtype=int,
197  doc="Spatial order of convolution kernel variation",
198  default=2,
199  check=lambda x: x >= 0
200  )
201  spatialBgOrder = pexConfig.Field(
202  dtype=int,
203  doc="Spatial order of differential background variation",
204  default=1,
205  check=lambda x: x >= 0
206  )
207  sizeCellX = pexConfig.Field(
208  dtype=int,
209  doc="Size (rows) in pixels of each SpatialCell for spatial modeling",
210  default=128,
211  check=lambda x: x >= 32
212  )
213  sizeCellY = pexConfig.Field(
214  dtype=int,
215  doc="Size (columns) in pixels of each SpatialCell for spatial modeling",
216  default=128,
217  check=lambda x: x >= 32
218  )
219  nStarPerCell = pexConfig.Field(
220  dtype=int,
221  doc="Number of KernelCandidates in each SpatialCell to use in the spatial fitting",
222  default=3,
223  check=lambda x: x >= 1
224  )
225  maxSpatialIterations = pexConfig.Field(
226  dtype=int,
227  doc="Maximum number of iterations for rejecting bad KernelCandidates in spatial fitting",
228  default=3,
229  check=lambda x: x >= 1 and x <= 5
230  )
231  usePcaForSpatialKernel = pexConfig.Field(
232  dtype=bool,
233  doc="""Use Pca to reduce the dimensionality of the kernel basis sets.
234  This is particularly useful for delta-function kernels.
235  Functionally, after all Cells have their raw kernels determined, we run
236  a Pca on these Kernels, re-fit the Cells using the eigenKernels and then
237  fit those for spatial variation using the same technique as for Alard-Lupton kernels.
238  If this option is used, the first term will have no spatial variation and the
239  kernel sum will be conserved.""",
240  default=False,
241  )
242  subtractMeanForPca = pexConfig.Field(
243  dtype=bool,
244  doc="Subtract off the mean feature before doing the Pca",
245  default=True,
246  )
247  numPrincipalComponents = pexConfig.Field(
248  dtype=int,
249  doc="""Number of principal components to use for Pca basis, including the
250  mean kernel if requested.""",
251  default=5,
252  check=lambda x: x >= 3
253  )
254  singleKernelClipping = pexConfig.Field(
255  dtype=bool,
256  doc="Do sigma clipping on each raw kernel candidate",
257  default=True,
258  )
259  kernelSumClipping = pexConfig.Field(
260  dtype=bool,
261  doc="Do sigma clipping on the ensemble of kernel sums",
262  default=True,
263  )
264  spatialKernelClipping = pexConfig.Field(
265  dtype=bool,
266  doc="Do sigma clipping after building the spatial model",
267  default=True,
268  )
269  checkConditionNumber = pexConfig.Field(
270  dtype=bool,
271  doc="""Test for maximum condition number when inverting a kernel matrix.
272  Anything above maxConditionNumber is not used and the candidate is set as BAD.
273  Also used to truncate inverse matrix in estimateBiasedRisk. However,
274  if you are doing any deconvolution you will want to turn this off, or use
275  a large maxConditionNumber""",
276  default=False,
277  )
278  badMaskPlanes = pexConfig.ListField(
279  dtype=str,
280  doc="""Mask planes to ignore when calculating diffim statistics
281  Options: NO_DATA EDGE SAT BAD CR INTRP""",
282  default=("NO_DATA", "EDGE", "SAT")
283  )
284  candidateResidualMeanMax = pexConfig.Field(
285  dtype=float,
286  doc="""Rejects KernelCandidates yielding bad difference image quality.
287  Used by BuildSingleKernelVisitor, AssessSpatialKernelVisitor.
288  Represents average over pixels of (image/sqrt(variance)).""",
289  default=0.25,
290  check=lambda x: x >= 0.0
291  )
292  candidateResidualStdMax = pexConfig.Field(
293  dtype=float,
294  doc="""Rejects KernelCandidates yielding bad difference image quality.
295  Used by BuildSingleKernelVisitor, AssessSpatialKernelVisitor.
296  Represents stddev over pixels of (image/sqrt(variance)).""",
297  default=1.50,
298  check=lambda x: x >= 0.0
299  )
300  useCoreStats = pexConfig.Field(
301  dtype=bool,
302  doc="""Use the core of the footprint for the quality statistics, instead of the entire footprint.
303  WARNING: if there is deconvolution we probably will need to turn this off""",
304  default=False,
305  )
306  candidateCoreRadius = pexConfig.Field(
307  dtype=int,
308  doc="""Radius for calculation of stats in 'core' of KernelCandidate diffim.
309  Total number of pixels used will be (2*radius)**2.
310  This is used both for 'core' diffim quality as well as ranking of
311  KernelCandidates by their total flux in this core""",
312  default=3,
313  check=lambda x: x >= 1
314  )
315  maxKsumSigma = pexConfig.Field(
316  dtype=float,
317  doc="""Maximum allowed sigma for outliers from kernel sum distribution.
318  Used to reject variable objects from the kernel model""",
319  default=3.0,
320  check=lambda x: x >= 0.0
321  )
322  maxConditionNumber = pexConfig.Field(
323  dtype=float,
324  doc="Maximum condition number for a well conditioned matrix",
325  default=5.0e7,
326  check=lambda x: x >= 0.0
327  )
328  conditionNumberType = pexConfig.ChoiceField(
329  dtype=str,
330  doc="Use singular values (SVD) or eigen values (EIGENVALUE) to determine condition number",
331  default="EIGENVALUE",
332  allowed={
333  "SVD": "Use singular values",
334  "EIGENVALUE": "Use eigen values (faster)",
335  }
336  )
337  maxSpatialConditionNumber = pexConfig.Field(
338  dtype=float,
339  doc="Maximum condition number for a well conditioned spatial matrix",
340  default=1.0e10,
341  check=lambda x: x >= 0.0
342  )
343  iterateSingleKernel = pexConfig.Field(
344  dtype=bool,
345  doc="""Remake KernelCandidate using better variance estimate after first pass?
346  Primarily useful when convolving a single-depth image, otherwise not necessary.""",
347  default=False,
348  )
349  constantVarianceWeighting = pexConfig.Field(
350  dtype=bool,
351  doc="""Use constant variance weighting in single kernel fitting?
352  In some cases this is better for bright star residuals.""",
353  default=True,
354  )
355  calculateKernelUncertainty = pexConfig.Field(
356  dtype=bool,
357  doc="""Calculate kernel and background uncertainties for each kernel candidate?
358  This comes from the inverse of the covariance matrix.
359  Warning: regularization can cause problems for this step.""",
360  default=False,
361  )
362  useBicForKernelBasis = pexConfig.Field(
363  dtype=bool,
364  doc="""Use Bayesian Information Criterion to select the number of bases going into the kernel""",
365  default=False,
366  )
367 
368 
370  """The parameters specific to the "Alard-Lupton" (sum-of-Gaussian) Psf-matching basis"""
371 
372  def setDefaults(self):
373  PsfMatchConfig.setDefaults(self)
374  self.kernelBasisSet = "alard-lupton"
375  self.maxConditionNumber = 5.0e7
376 
377  alardNGauss = pexConfig.Field(
378  dtype=int,
379  doc="Number of Gaussians in alard-lupton basis",
380  default=3,
381  check=lambda x: x >= 1
382  )
383  alardDegGauss = pexConfig.ListField(
384  dtype=int,
385  doc="Polynomial order of spatial modification of Gaussians. Must in number equal alardNGauss",
386  default=(4, 2, 2),
387  )
388  alardSigGauss = pexConfig.ListField(
389  dtype=float,
390  doc="""Sigma in pixels of Gaussians (FWHM = 2.35 sigma). Must in number equal alardNGauss""",
391  default=(0.7, 1.5, 3.0),
392  )
393  alardGaussBeta = pexConfig.Field(
394  dtype=float,
395  doc="""Default scale factor between Gaussian sigmas """,
396  default=2.0,
397  check=lambda x: x >= 0.0,
398  )
399  alardMinSig = pexConfig.Field(
400  dtype=float,
401  doc="""Minimum Sigma (pixels) for Gaussians""",
402  default=0.7,
403  check=lambda x: x >= 0.25
404  )
405  alardDegGaussDeconv = pexConfig.Field(
406  dtype=int,
407  doc="""Degree of spatial modification of ALL gaussians in AL basis during deconvolution""",
408  default=3,
409  check=lambda x: x >= 1
410  )
411  alardMinSigDeconv = pexConfig.Field(
412  dtype=float,
413  doc="""Minimum Sigma (pixels) for Gaussians during deconvolution;
414  make smaller than alardMinSig as this is only indirectly used""",
415  default=0.4,
416  check=lambda x: x >= 0.25
417  )
418  alardNGaussDeconv = pexConfig.Field(
419  dtype=int,
420  doc="Number of Gaussians in AL basis during deconvolution",
421  default=3,
422  check=lambda x: x >= 1
423  )
424 
425 
427  """The parameters specific to the delta-function (one basis per-pixel) Psf-matching basis"""
428 
429  def setDefaults(self):
430  PsfMatchConfig.setDefaults(self)
431  self.kernelBasisSet = "delta-function"
432  self.maxConditionNumber = 5.0e6
434  self.subtractMeanForPca = True
435  self.useBicForKernelBasis = False
436 
437  useRegularization = pexConfig.Field(
438  dtype=bool,
439  doc="Use regularization to smooth the delta function kernels",
440  default=True,
441  )
442  regularizationType = pexConfig.ChoiceField(
443  dtype=str,
444  doc="Type of regularization.",
445  default="centralDifference",
446  allowed={
447  "centralDifference": "Penalize second derivative using 2-D stencil of central finite difference",
448  "forwardDifference": "Penalize first, second, third derivatives using forward finite differeces"
449  }
450  )
451  centralRegularizationStencil = pexConfig.ChoiceField(
452  dtype=int,
453  doc="Type of stencil to approximate central derivative (for centralDifference only)",
454  default=9,
455  allowed={
456  5: "5-point stencil including only adjacent-in-x,y elements",
457  9: "9-point stencil including diagonal elements"
458  }
459  )
460  forwardRegularizationOrders = pexConfig.ListField(
461  dtype=int,
462  doc="Array showing which order derivatives to penalize (for forwardDifference only)",
463  default=(1, 2),
464  itemCheck=lambda x: (x > 0) and (x < 4)
465  )
466  regularizationBorderPenalty = pexConfig.Field(
467  dtype=float,
468  doc="Value of the penalty for kernel border pixels",
469  default=3.0,
470  check=lambda x: x >= 0.0
471  )
472  lambdaType = pexConfig.ChoiceField(
473  dtype=str,
474  doc="How to choose the value of the regularization strength",
475  default="absolute",
476  allowed={
477  "absolute": "Use lambdaValue as the value of regularization strength",
478  "relative": "Use lambdaValue as fraction of the default regularization strength (N.R. 18.5.8)",
479  "minimizeBiasedRisk": "Minimize biased risk estimate",
480  "minimizeUnbiasedRisk": "Minimize unbiased risk estimate",
481  }
482  )
483  lambdaValue = pexConfig.Field(
484  dtype=float,
485  doc="Value used for absolute determinations of regularization strength",
486  default=0.2,
487  )
488  lambdaScaling = pexConfig.Field(
489  dtype=float,
490  doc="Fraction of the default lambda strength (N.R. 18.5.8) to use. 1e-4 or 1e-5",
491  default=1e-4,
492  )
493  lambdaStepType = pexConfig.ChoiceField(
494  dtype=str,
495  doc="""If a scan through lambda is needed (minimizeBiasedRisk, minimizeUnbiasedRisk),
496  use log or linear steps""",
497  default="log",
498  allowed={
499  "log": "Step in log intervals; e.g. lambdaMin, lambdaMax, lambdaStep = -1.0, 2.0, 0.1",
500  "linear": "Step in linear intervals; e.g. lambdaMin, lambdaMax, lambdaStep = 0.1, 100, 0.1",
501  }
502  )
503  lambdaMin = pexConfig.Field(
504  dtype=float,
505  doc="""If scan through lambda needed (minimizeBiasedRisk, minimizeUnbiasedRisk),
506  start at this value. If lambdaStepType = log:linear, suggest -1:0.1""",
507  default=-1.0,
508  )
509  lambdaMax = pexConfig.Field(
510  dtype=float,
511  doc="""If scan through lambda needed (minimizeBiasedRisk, minimizeUnbiasedRisk),
512  stop at this value. If lambdaStepType = log:linear, suggest 2:100""",
513  default=2.0,
514  )
515  lambdaStep = pexConfig.Field(
516  dtype=float,
517  doc="""If scan through lambda needed (minimizeBiasedRisk, minimizeUnbiasedRisk),
518  step in these increments. If lambdaStepType = log:linear, suggest 0.1:0.1""",
519  default=0.1,
520  )
521 
522 
523 class PsfMatchTask(pipeBase.Task):
524  """Base class for Psf Matching; should not be called directly
525 
526  Notes
527  -----
528  PsfMatchTask is a base class that implements the core functionality for matching the
529  Psfs of two images using a spatially varying Psf-matching lsst.afw.math.LinearCombinationKernel.
530  The Task requires the user to provide an instance of an lsst.afw.math.SpatialCellSet,
531  filled with lsst.ip.diffim.KernelCandidate instances, and a list of lsst.afw.math.Kernels
532  of basis shapes that will be used for the decomposition. If requested, the Task
533  also performs background matching and returns the differential background model as an
534  lsst.afw.math.Kernel.SpatialFunction.
535 
536  Invoking the Task
537 
538  As a base class, this Task is not directly invoked. However, run() methods that are
539  implemented on derived classes will make use of the core _solve() functionality,
540  which defines a sequence of lsst.afw.math.CandidateVisitor classes that iterate
541  through the KernelCandidates, first building up a per-candidate solution and then
542  building up a spatial model from the ensemble of candidates. Sigma clipping is
543  performed using the mean and standard deviation of all kernel sums (to reject
544  variable objects), on the per-candidate substamp diffim residuals
545  (to indicate a bad choice of kernel basis shapes for that particular object),
546  and on the substamp diffim residuals using the spatial kernel fit (to indicate a bad
547  choice of spatial kernel order, or poor constraints on the spatial model). The
548  _diagnostic() method logs information on the quality of the spatial fit, and also
549  modifies the Task metadata.
550 
551  .. list-table:: Quantities set in Metadata
552  :header-rows: 1
553 
554  * - Parameter
555  - Description
556  * - `spatialConditionNum`
557  - Condition number of the spatial kernel fit
558  * - `spatialKernelSum`
559  - Kernel sum (10^{-0.4 * ``Delta``; zeropoint}) of the spatial Psf-matching kernel
560  * - `ALBasisNGauss`
561  - If using sum-of-Gaussian basis, the number of gaussians used
562  * - `ALBasisDegGauss`
563  - If using sum-of-Gaussian basis, the deg of spatial variation of the Gaussians
564  * - `ALBasisSigGauss`
565  - If using sum-of-Gaussian basis, the widths (sigma) of the Gaussians
566  * - `ALKernelSize`
567  - If using sum-of-Gaussian basis, the kernel size
568  * - `NFalsePositivesTotal`
569  - Total number of diaSources
570  * - `NFalsePositivesRefAssociated`
571  - Number of diaSources that associate with the reference catalog
572  * - `NFalsePositivesRefAssociated`
573  - Number of diaSources that associate with the source catalog
574  * - `NFalsePositivesUnassociated`
575  - Number of diaSources that are orphans
576  * - `metric_MEAN`
577  - Mean value of substamp diffim quality metrics across all KernelCandidates,
578  for both the per-candidate (LOCAL) and SPATIAL residuals
579  * - `metric_MEDIAN`
580  - Median value of substamp diffim quality metrics across all KernelCandidates,
581  for both the per-candidate (LOCAL) and SPATIAL residuals
582  * - `metric_STDEV`
583  - Standard deviation of substamp diffim quality metrics across all KernelCandidates,
584  for both the per-candidate (LOCAL) and SPATIAL residuals
585 
586  Debug variables
587 
588  The lsst.pipe.base.cmdLineTask.CmdLineTask command line task interface supports a
589  flag -d/--debug to import @b debug.py from your PYTHONPATH. The relevant contents of debug.py
590  for this Task include:
591 
592  .. code-block:: py
593 
594  import sys
595  import lsstDebug
596  def DebugInfo(name):
597  di = lsstDebug.getInfo(name)
598  if name == "lsst.ip.diffim.psfMatch":
599  # enable debug output
600  di.display = True
601  # display mask transparency
602  di.maskTransparency = 80
603  # show all the candidates and residuals
604  di.displayCandidates = True
605  # show kernel basis functions
606  di.displayKernelBasis = False
607  # show kernel realized across the image
608  di.displayKernelMosaic = True
609  # show coefficients of spatial model
610  di.plotKernelSpatialModel = False
611  # show the bad candidates (red) along with good (green)
612  di.showBadCandidates = True
613  return di
614  lsstDebug.Info = DebugInfo
615  lsstDebug.frame = 1
616 
617  Note that if you want addional logging info, you may add to your scripts:
618 
619  .. code-block:: py
620 
621  import lsst.log.utils as logUtils
622  logUtils.traceSetAt("ip.diffim", 4)
623  """
624  ConfigClass = PsfMatchConfig
625  _DefaultName = "psfMatch"
626 
627  def __init__(self, *args, **kwargs):
628  """Create the psf-matching Task
629 
630  Parameters
631  ----------
632  *args
633  Arguments to be passed to ``lsst.pipe.base.task.Task.__init__``
634  **kwargs
635  Keyword arguments to be passed to ``lsst.pipe.base.task.Task.__init__``
636 
637  Notes
638  -----
639  The initialization sets the Psf-matching kernel configuration using the value of
640  self.config.kernel.active. If the kernel is requested with regularization to moderate
641  the bias/variance tradeoff, currently only used when a delta function kernel basis
642  is provided, it creates a regularization matrix stored as member variable
643  self.hMat.
644  """
645  pipeBase.Task.__init__(self, *args, **kwargs)
646  self.kConfig = self.config.kernel.active
647 
648  if 'useRegularization' in self.kConfig:
649  self.useRegularization = self.kConfig.useRegularization
650  else:
651  self.useRegularization = False
652 
653  if self.useRegularization:
654  self.hMat = diffimLib.makeRegularizationMatrix(pexConfig.makePolicy(self.kConfig))
655 
656  def _diagnostic(self, kernelCellSet, spatialSolution, spatialKernel, spatialBg):
657  """Provide logging diagnostics on quality of spatial kernel fit
658 
659  Parameters
660  ----------
661  kernelCellSet : TYPE
662  Cellset that contains the KernelCandidates used in the fitting
663  spatialSolution : TYPE
664  KernelSolution of best-fit
665  spatialKernel : TYPE
666  Best-fit spatial Kernel model
667  spatialBg : TYPE
668  Best-fit spatial background model
669  """
670  # What is the final kernel sum
671  kImage = afwImage.ImageD(spatialKernel.getDimensions())
672  kSum = spatialKernel.computeImage(kImage, False)
673  self.log.info("Final spatial kernel sum %.3f" % (kSum))
674 
675  # Look at how well conditioned the matrix is
676  conditionNum = spatialSolution.getConditionNumber(
677  getattr(diffimLib.KernelSolution, self.kConfig.conditionNumberType))
678  self.log.info("Spatial model condition number %.3e" % (conditionNum))
679 
680  if conditionNum < 0.0:
681  self.log.warn("Condition number is negative (%.3e)" % (conditionNum))
682  if conditionNum > self.kConfig.maxSpatialConditionNumber:
683  self.log.warn("Spatial solution exceeds max condition number (%.3e > %.3e)" % (
684  conditionNum, self.kConfig.maxSpatialConditionNumber))
685 
686  self.metadata.set("spatialConditionNum", conditionNum)
687  self.metadata.set("spatialKernelSum", kSum)
688 
689  # Look at how well the solution is constrained
690  nBasisKernels = spatialKernel.getNBasisKernels()
691  nKernelTerms = spatialKernel.getNSpatialParameters()
692  if nKernelTerms == 0: # order 0
693  nKernelTerms = 1
694 
695  # Not fit for
696  nBgTerms = spatialBg.getNParameters()
697  if nBgTerms == 1:
698  if spatialBg.getParameters()[0] == 0.0:
699  nBgTerms = 0
700 
701  nGood = 0
702  nBad = 0
703  nTot = 0
704  for cell in kernelCellSet.getCellList():
705  for cand in cell.begin(False): # False = include bad candidates
706  nTot += 1
707  if cand.getStatus() == afwMath.SpatialCellCandidate.GOOD:
708  nGood += 1
709  if cand.getStatus() == afwMath.SpatialCellCandidate.BAD:
710  nBad += 1
711 
712  self.log.info("Doing stats of kernel candidates used in the spatial fit.")
713 
714  # Counting statistics
715  if nBad > 2*nGood:
716  self.log.warn("Many more candidates rejected than accepted; %d total, %d rejected, %d used" % (
717  nTot, nBad, nGood))
718  else:
719  self.log.info("%d candidates total, %d rejected, %d used" % (nTot, nBad, nGood))
720 
721  # Some judgements on the quality of the spatial models
722  if nGood < nKernelTerms:
723  self.log.warn("Spatial kernel model underconstrained; %d candidates, %d terms, %d bases" % (
724  nGood, nKernelTerms, nBasisKernels))
725  self.log.warn("Consider lowering the spatial order")
726  elif nGood <= 2*nKernelTerms:
727  self.log.warn("Spatial kernel model poorly constrained; %d candidates, %d terms, %d bases" % (
728  nGood, nKernelTerms, nBasisKernels))
729  self.log.warn("Consider lowering the spatial order")
730  else:
731  self.log.info("Spatial kernel model well constrained; %d candidates, %d terms, %d bases" % (
732  nGood, nKernelTerms, nBasisKernels))
733 
734  if nGood < nBgTerms:
735  self.log.warn("Spatial background model underconstrained; %d candidates, %d terms" % (
736  nGood, nBgTerms))
737  self.log.warn("Consider lowering the spatial order")
738  elif nGood <= 2*nBgTerms:
739  self.log.warn("Spatial background model poorly constrained; %d candidates, %d terms" % (
740  nGood, nBgTerms))
741  self.log.warn("Consider lowering the spatial order")
742  else:
743  self.log.info("Spatial background model appears well constrained; %d candidates, %d terms" % (
744  nGood, nBgTerms))
745 
746  def _displayDebug(self, kernelCellSet, spatialKernel, spatialBackground):
747  """Provide visualization of the inputs and ouputs to the Psf-matching code
748 
749  Parameters
750  ----------
751  kernelCellSet : TYPE
752  The SpatialCellSet used in determining the matching kernel and background
753  spatialKernel : TYPE
754  Spatially varying Psf-matching kernel
755  spatialBackground : TYPE
756  Spatially varying background-matching function
757  """
758  import lsstDebug
759  displayCandidates = lsstDebug.Info(__name__).displayCandidates
760  displayKernelBasis = lsstDebug.Info(__name__).displayKernelBasis
761  displayKernelMosaic = lsstDebug.Info(__name__).displayKernelMosaic
762  plotKernelSpatialModel = lsstDebug.Info(__name__).plotKernelSpatialModel
763  showBadCandidates = lsstDebug.Info(__name__).showBadCandidates
764  maskTransparency = lsstDebug.Info(__name__).maskTransparency
765  if not maskTransparency:
766  maskTransparency = 0
767  afwDisplay.setDefaultMaskTransparency(maskTransparency)
768 
769  if displayCandidates:
770  diutils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground,
771  frame=lsstDebug.frame,
772  showBadCandidates=showBadCandidates)
773  lsstDebug.frame += 1
774  diutils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground,
775  frame=lsstDebug.frame,
776  showBadCandidates=showBadCandidates,
777  kernels=True)
778  lsstDebug.frame += 1
779  diutils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground,
780  frame=lsstDebug.frame,
781  showBadCandidates=showBadCandidates,
782  resids=True)
783  lsstDebug.frame += 1
784 
785  if displayKernelBasis:
786  diutils.showKernelBasis(spatialKernel, frame=lsstDebug.frame)
787  lsstDebug.frame += 1
788 
789  if displayKernelMosaic:
790  diutils.showKernelMosaic(kernelCellSet.getBBox(), spatialKernel, frame=lsstDebug.frame)
791  lsstDebug.frame += 1
792 
793  if plotKernelSpatialModel:
794  diutils.plotKernelSpatialModel(spatialKernel, kernelCellSet, showBadCandidates=showBadCandidates)
795 
796  def _createPcaBasis(self, kernelCellSet, nStarPerCell, policy):
797  """Create Principal Component basis
798 
799  If a principal component analysis is requested, typically when using a delta function basis,
800  perform the PCA here and return a new basis list containing the new principal components.
801 
802  Parameters
803  ----------
804  kernelCellSet : TYPE
805  a SpatialCellSet containing KernelCandidates, from which components are derived
806  nStarPerCell : TYPE
807  the number of stars per cell to visit when doing the PCA
808  policy : TYPE
809  input policy controlling the single kernel visitor
810 
811  Returns
812  -------
813  nRejectedPca : TYPE
814  number of KernelCandidates rejected during PCA loop
815  spatialBasisList : TYPE
816  basis list containing the principal shapes as Kernels
817 
818  Raises
819  ------
820  RuntimeError
821  If the Eigenvalues sum to zero.
822  """
823  nComponents = self.kConfig.numPrincipalComponents
824  imagePca = diffimLib.KernelPcaD()
825  importStarVisitor = diffimLib.KernelPcaVisitorF(imagePca)
826  kernelCellSet.visitCandidates(importStarVisitor, nStarPerCell)
827  if self.kConfig.subtractMeanForPca:
828  importStarVisitor.subtractMean()
829  imagePca.analyze()
830 
831  eigenValues = imagePca.getEigenValues()
832  pcaBasisList = importStarVisitor.getEigenKernels()
833 
834  eSum = np.sum(eigenValues)
835  if eSum == 0.0:
836  raise RuntimeError("Eigenvalues sum to zero")
837  for j in range(len(eigenValues)):
838  log.log("TRACE5." + self.log.getName() + "._solve", log.DEBUG,
839  "Eigenvalue %d : %f (%f)", j, eigenValues[j], eigenValues[j]/eSum)
840 
841  nToUse = min(nComponents, len(eigenValues))
842  trimBasisList = []
843  for j in range(nToUse):
844  # Check for NaNs?
845  kimage = afwImage.ImageD(pcaBasisList[j].getDimensions())
846  pcaBasisList[j].computeImage(kimage, False)
847  if not (True in np.isnan(kimage.getArray())):
848  trimBasisList.append(pcaBasisList[j])
849 
850  # Put all the power in the first kernel, which will not vary spatially
851  spatialBasisList = diffimLib.renormalizeKernelList(trimBasisList)
852 
853  # New Kernel visitor for this new basis list (no regularization explicitly)
854  singlekvPca = diffimLib.BuildSingleKernelVisitorF(spatialBasisList, policy)
855  singlekvPca.setSkipBuilt(False)
856  kernelCellSet.visitCandidates(singlekvPca, nStarPerCell)
857  singlekvPca.setSkipBuilt(True)
858  nRejectedPca = singlekvPca.getNRejected()
859 
860  return nRejectedPca, spatialBasisList
861 
862  def _buildCellSet(self, *args):
863  """Fill a SpatialCellSet with KernelCandidates for the Psf-matching process;
864  override in derived classes"""
865  return
866 
867  @pipeBase.timeMethod
868  def _solve(self, kernelCellSet, basisList, returnOnExcept=False):
869  """Solve for the PSF matching kernel
870 
871  Parameters
872  ----------
873  kernelCellSet : TYPE
874  a SpatialCellSet to use in determining the matching kernel
875  (typically as provided by _buildCellSet)
876  basisList : TYPE
877  list of Kernels to be used in the decomposition of the spatially varying kernel
878  (typically as provided by makeKernelBasisList)
879  returnOnExcept : `bool`, optional
880  if True then return (None, None) if an error occurs, else raise the exception
881 
882  Returns
883  -------
884  psfMatchingKernel : TYPE
885  PSF matching kernel
886  backgroundModel : TYPE
887  differential background model
888 
889  Raises
890  ------
891  Exception
892  if unable to determine PSF matching kernel and returnOnExcept False
893  """
894 
895  import lsstDebug
896  display = lsstDebug.Info(__name__).display
897 
898  maxSpatialIterations = self.kConfig.maxSpatialIterations
899  nStarPerCell = self.kConfig.nStarPerCell
900  usePcaForSpatialKernel = self.kConfig.usePcaForSpatialKernel
901 
902  # Visitor for the single kernel fit
903  policy = pexConfig.makePolicy(self.kConfig)
904  if self.useRegularization:
905  singlekv = diffimLib.BuildSingleKernelVisitorF(basisList, policy, self.hMat)
906  else:
907  singlekv = diffimLib.BuildSingleKernelVisitorF(basisList, policy)
908 
909  # Visitor for the kernel sum rejection
910  ksv = diffimLib.KernelSumVisitorF(policy)
911 
912  # Main loop
913  t0 = time.time()
914  try:
915  totalIterations = 0
916  thisIteration = 0
917  while (thisIteration < maxSpatialIterations):
918 
919  # Make sure there are no uninitialized candidates as active occupants of Cell
920  nRejectedSkf = -1
921  while (nRejectedSkf != 0):
922  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG,
923  "Building single kernels...")
924  kernelCellSet.visitCandidates(singlekv, nStarPerCell)
925  nRejectedSkf = singlekv.getNRejected()
926  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG,
927  "Iteration %d, rejected %d candidates due to initial kernel fit",
928  thisIteration, nRejectedSkf)
929 
930  # Reject outliers in kernel sum
931  ksv.resetKernelSum()
932  ksv.setMode(diffimLib.KernelSumVisitorF.AGGREGATE)
933  kernelCellSet.visitCandidates(ksv, nStarPerCell)
934  ksv.processKsumDistribution()
935  ksv.setMode(diffimLib.KernelSumVisitorF.REJECT)
936  kernelCellSet.visitCandidates(ksv, nStarPerCell)
937 
938  nRejectedKsum = ksv.getNRejected()
939  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG,
940  "Iteration %d, rejected %d candidates due to kernel sum",
941  thisIteration, nRejectedKsum)
942 
943  # Do we jump back to the top without incrementing thisIteration?
944  if nRejectedKsum > 0:
945  totalIterations += 1
946  continue
947 
948  # At this stage we can either apply the spatial fit to
949  # the kernels, or we run a PCA, use these as a *new*
950  # basis set with lower dimensionality, and then apply
951  # the spatial fit to these kernels
952 
953  if (usePcaForSpatialKernel):
954  log.log("TRACE0." + self.log.getName() + "._solve", log.DEBUG,
955  "Building Pca basis")
956 
957  nRejectedPca, spatialBasisList = self._createPcaBasis(kernelCellSet, nStarPerCell, policy)
958  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG,
959  "Iteration %d, rejected %d candidates due to Pca kernel fit",
960  thisIteration, nRejectedPca)
961 
962  # We don't want to continue on (yet) with the
963  # spatial modeling, because we have bad objects
964  # contributing to the Pca basis. We basically
965  # need to restart from the beginning of this loop,
966  # since the cell-mates of those objects that were
967  # rejected need their original Kernels built by
968  # singleKernelFitter.
969 
970  # Don't count against thisIteration
971  if (nRejectedPca > 0):
972  totalIterations += 1
973  continue
974  else:
975  spatialBasisList = basisList
976 
977  # We have gotten on to the spatial modeling part
978  regionBBox = kernelCellSet.getBBox()
979  spatialkv = diffimLib.BuildSpatialKernelVisitorF(spatialBasisList, regionBBox, policy)
980  kernelCellSet.visitCandidates(spatialkv, nStarPerCell)
981  spatialkv.solveLinearEquation()
982  log.log("TRACE2." + self.log.getName() + "._solve", log.DEBUG,
983  "Spatial kernel built with %d candidates", spatialkv.getNCandidates())
984  spatialKernel, spatialBackground = spatialkv.getSolutionPair()
985 
986  # Check the quality of the spatial fit (look at residuals)
987  assesskv = diffimLib.AssessSpatialKernelVisitorF(spatialKernel, spatialBackground, policy)
988  kernelCellSet.visitCandidates(assesskv, nStarPerCell)
989  nRejectedSpatial = assesskv.getNRejected()
990  nGoodSpatial = assesskv.getNGood()
991  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG,
992  "Iteration %d, rejected %d candidates due to spatial kernel fit",
993  thisIteration, nRejectedSpatial)
994  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG,
995  "%d candidates used in fit", nGoodSpatial)
996 
997  # If only nGoodSpatial == 0, might be other candidates in the cells
998  if nGoodSpatial == 0 and nRejectedSpatial == 0:
999  raise RuntimeError("No kernel candidates for spatial fit")
1000 
1001  if nRejectedSpatial == 0:
1002  # Nothing rejected, finished with spatial fit
1003  break
1004 
1005  # Otherwise, iterate on...
1006  thisIteration += 1
1007 
1008  # Final fit if above did not converge
1009  if (nRejectedSpatial > 0) and (thisIteration == maxSpatialIterations):
1010  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG, "Final spatial fit")
1011  if (usePcaForSpatialKernel):
1012  nRejectedPca, spatialBasisList = self._createPcaBasis(kernelCellSet, nStarPerCell, policy)
1013  regionBBox = kernelCellSet.getBBox()
1014  spatialkv = diffimLib.BuildSpatialKernelVisitorF(spatialBasisList, regionBBox, policy)
1015  kernelCellSet.visitCandidates(spatialkv, nStarPerCell)
1016  spatialkv.solveLinearEquation()
1017  log.log("TRACE2." + self.log.getName() + "._solve", log.DEBUG,
1018  "Spatial kernel built with %d candidates", spatialkv.getNCandidates())
1019  spatialKernel, spatialBackground = spatialkv.getSolutionPair()
1020 
1021  spatialSolution = spatialkv.getKernelSolution()
1022 
1023  except Exception as e:
1024  self.log.error("ERROR: Unable to calculate psf matching kernel")
1025 
1026  log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG, str(e))
1027  raise e
1028 
1029  t1 = time.time()
1030  log.log("TRACE0." + self.log.getName() + "._solve", log.DEBUG,
1031  "Total time to compute the spatial kernel : %.2f s", (t1 - t0))
1032 
1033  if display:
1034  self._displayDebug(kernelCellSet, spatialKernel, spatialBackground)
1035 
1036  self._diagnostic(kernelCellSet, spatialSolution, spatialKernel, spatialBackground)
1037 
1038  return spatialSolution, spatialKernel, spatialBackground
1039 
1040 
1041 PsfMatch = PsfMatchTask
def _diagnostic(self, kernelCellSet, spatialSolution, spatialKernel, spatialBg)
Definition: psfMatch.py:656
def __init__(self, args, kwargs)
Definition: psfMatch.py:627
def _createPcaBasis(self, kernelCellSet, nStarPerCell, policy)
Definition: psfMatch.py:796
def _displayDebug(self, kernelCellSet, spatialKernel, spatialBackground)
Definition: psfMatch.py:746