Coverage for python/lsst/ip/diffim/psfMatch.py: 29%
294 statements
« prev ^ index » next coverage.py v7.1.0, created at 2023-02-05 18:32 -0800
« prev ^ index » next coverage.py v7.1.0, created at 2023-02-05 18:32 -0800
1# This file is part of ip_diffim.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ["DetectionConfig", "PsfMatchConfig", "PsfMatchConfigAL", "PsfMatchConfigDF", "PsfMatchTask"]
24import time
26import numpy as np
28import lsst.afw.image as afwImage
29import lsst.pex.config as pexConfig
30import lsst.afw.math as afwMath
31import lsst.afw.display as afwDisplay
32import lsst.log as log
33import lsst.pipe.base as pipeBase
34from lsst.meas.algorithms import SubtractBackgroundConfig
35from . import utils as diutils
36from . import diffimLib
39class DetectionConfig(pexConfig.Config):
40 """Configuration for detecting sources on images for building a
41 PSF-matching kernel
43 Configuration for turning detected lsst.afw.detection.FootPrints into an
44 acceptable (unmasked, high signal-to-noise, not too large or not too small)
45 list of `lsst.ip.diffim.KernelSources` that are used to build the
46 Psf-matching kernel"""
48 detThreshold = pexConfig.Field( 48 ↛ exitline 48 didn't jump to the function exit
49 dtype=float,
50 doc="Value of footprint detection threshold",
51 default=10.0,
52 check=lambda x: x >= 3.0
53 )
54 detThresholdType = pexConfig.ChoiceField(
55 dtype=str,
56 doc="Type of detection threshold",
57 default="pixel_stdev",
58 allowed={
59 "value": "Use counts as the detection threshold type",
60 "stdev": "Use standard deviation of image plane",
61 "variance": "Use variance of image plane",
62 "pixel_stdev": "Use stdev derived from variance plane"
63 }
64 )
65 detOnTemplate = pexConfig.Field(
66 dtype=bool,
67 doc="""If true run detection on the template (image to convolve);
68 if false run detection on the science image""",
69 default=True
70 )
71 badMaskPlanes = pexConfig.ListField(
72 dtype=str,
73 doc="""Mask planes that lead to an invalid detection.
74 Options: NO_DATA EDGE SAT BAD CR INTRP""",
75 default=("NO_DATA", "EDGE", "SAT")
76 )
77 fpNpixMin = pexConfig.Field( 77 ↛ exitline 77 didn't jump to the function exit
78 dtype=int,
79 doc="Minimum number of pixels in an acceptable Footprint",
80 default=5,
81 check=lambda x: x >= 5
82 )
83 fpNpixMax = pexConfig.Field( 83 ↛ exitline 83 didn't jump to the function exit
84 dtype=int,
85 doc="""Maximum number of pixels in an acceptable Footprint;
86 too big and the subsequent convolutions become unwieldy""",
87 default=500,
88 check=lambda x: x <= 500
89 )
90 fpGrowKernelScaling = pexConfig.Field( 90 ↛ exitline 90 didn't jump to the function exit
91 dtype=float,
92 doc="""If config.scaleByFwhm, grow the footprint based on
93 the final kernelSize. Each footprint will be
94 2*fpGrowKernelScaling*kernelSize x
95 2*fpGrowKernelScaling*kernelSize. With the value
96 of 1.0, the remaining pixels in each KernelCandiate
97 after convolution by the basis functions will be
98 equal to the kernel size itself.""",
99 default=1.0,
100 check=lambda x: x >= 1.0
101 )
102 fpGrowPix = pexConfig.Field( 102 ↛ exitline 102 didn't jump to the function exit
103 dtype=int,
104 doc="""Growing radius (in pixels) for each raw detection
105 footprint. The smaller the faster; however the
106 kernel sum does not converge if the stamp is too
107 small; and the kernel is not constrained at all if
108 the stamp is the size of the kernel. The grown stamp
109 is 2 * fpGrowPix pixels larger in each dimension.
110 This is overridden by fpGrowKernelScaling if scaleByFwhm""",
111 default=30,
112 check=lambda x: x >= 10
113 )
114 scaleByFwhm = pexConfig.Field(
115 dtype=bool,
116 doc="Scale fpGrowPix by input Fwhm?",
117 default=True,
118 )
121class PsfMatchConfig(pexConfig.Config):
122 """Base configuration for Psf-matching
124 The base configuration of the Psf-matching kernel, and of the warping, detection,
125 and background modeling subTasks."""
127 warpingConfig = pexConfig.ConfigField("Config for warping exposures to a common alignment",
128 afwMath.WarperConfig)
129 detectionConfig = pexConfig.ConfigField("Controlling the detection of sources for kernel building",
130 DetectionConfig)
131 afwBackgroundConfig = pexConfig.ConfigField("Controlling the Afw background fitting",
132 SubtractBackgroundConfig)
134 useAfwBackground = pexConfig.Field(
135 dtype=bool,
136 doc="Use afw background subtraction instead of ip_diffim",
137 default=False,
138 )
139 fitForBackground = pexConfig.Field(
140 dtype=bool,
141 doc="Include terms (including kernel cross terms) for background in ip_diffim",
142 default=False,
143 )
144 kernelBasisSet = pexConfig.ChoiceField(
145 dtype=str,
146 doc="Type of basis set for PSF matching kernel.",
147 default="alard-lupton",
148 allowed={
149 "alard-lupton": """Alard-Lupton sum-of-gaussians basis set,
150 * The first term has no spatial variation
151 * The kernel sum is conserved
152 * You may want to turn off 'usePcaForSpatialKernel'""",
153 "delta-function": """Delta-function kernel basis set,
154 * You may enable the option useRegularization
155 * You should seriously consider usePcaForSpatialKernel, which will also
156 enable kernel sum conservation for the delta function kernels"""
157 }
158 )
159 kernelSize = pexConfig.Field(
160 dtype=int,
161 doc="""Number of rows/columns in the convolution kernel; should be odd-valued.
162 Modified by kernelSizeFwhmScaling if scaleByFwhm = true""",
163 default=21,
164 )
165 scaleByFwhm = pexConfig.Field(
166 dtype=bool,
167 doc="Scale kernelSize, alardGaussians by input Fwhm",
168 default=True,
169 )
170 kernelSizeFwhmScaling = pexConfig.Field( 170 ↛ exitline 170 didn't jump to the function exit
171 dtype=float,
172 doc="Multiplier of the largest AL Gaussian basis sigma to get the kernel bbox (pixel) size.",
173 default=6.0,
174 check=lambda x: x >= 1.0
175 )
176 kernelSizeMin = pexConfig.Field(
177 dtype=int,
178 doc="Minimum kernel bbox (pixel) size.",
179 default=21,
180 )
181 kernelSizeMax = pexConfig.Field(
182 dtype=int,
183 doc="Maximum kernel bbox (pixel) size.",
184 default=35,
185 )
186 spatialModelType = pexConfig.ChoiceField(
187 dtype=str,
188 doc="Type of spatial functions for kernel and background",
189 default="chebyshev1",
190 allowed={
191 "chebyshev1": "Chebyshev polynomial of the first kind",
192 "polynomial": "Standard x,y polynomial",
193 }
194 )
195 spatialKernelOrder = pexConfig.Field( 195 ↛ exitline 195 didn't jump to the function exit
196 dtype=int,
197 doc="Spatial order of convolution kernel variation",
198 default=2,
199 check=lambda x: x >= 0
200 )
201 spatialBgOrder = pexConfig.Field( 201 ↛ exitline 201 didn't jump to the function exit
202 dtype=int,
203 doc="Spatial order of differential background variation",
204 default=1,
205 check=lambda x: x >= 0
206 )
207 sizeCellX = pexConfig.Field( 207 ↛ exitline 207 didn't jump to the function exit
208 dtype=int,
209 doc="Size (rows) in pixels of each SpatialCell for spatial modeling",
210 default=128,
211 check=lambda x: x >= 32
212 )
213 sizeCellY = pexConfig.Field( 213 ↛ exitline 213 didn't jump to the function exit
214 dtype=int,
215 doc="Size (columns) in pixels of each SpatialCell for spatial modeling",
216 default=128,
217 check=lambda x: x >= 32
218 )
219 nStarPerCell = pexConfig.Field( 219 ↛ exitline 219 didn't jump to the function exit
220 dtype=int,
221 doc="Number of KernelCandidates in each SpatialCell to use in the spatial fitting",
222 default=3,
223 check=lambda x: x >= 1
224 )
225 maxSpatialIterations = pexConfig.Field( 225 ↛ exitline 225 didn't jump to the function exit
226 dtype=int,
227 doc="Maximum number of iterations for rejecting bad KernelCandidates in spatial fitting",
228 default=3,
229 check=lambda x: x >= 1 and x <= 5
230 )
231 usePcaForSpatialKernel = pexConfig.Field(
232 dtype=bool,
233 doc="""Use Pca to reduce the dimensionality of the kernel basis sets.
234 This is particularly useful for delta-function kernels.
235 Functionally, after all Cells have their raw kernels determined, we run
236 a Pca on these Kernels, re-fit the Cells using the eigenKernels and then
237 fit those for spatial variation using the same technique as for Alard-Lupton kernels.
238 If this option is used, the first term will have no spatial variation and the
239 kernel sum will be conserved.""",
240 default=False,
241 )
242 subtractMeanForPca = pexConfig.Field(
243 dtype=bool,
244 doc="Subtract off the mean feature before doing the Pca",
245 default=True,
246 )
247 numPrincipalComponents = pexConfig.Field( 247 ↛ exitline 247 didn't jump to the function exit
248 dtype=int,
249 doc="""Number of principal components to use for Pca basis, including the
250 mean kernel if requested.""",
251 default=5,
252 check=lambda x: x >= 3
253 )
254 singleKernelClipping = pexConfig.Field(
255 dtype=bool,
256 doc="Do sigma clipping on each raw kernel candidate",
257 default=True,
258 )
259 kernelSumClipping = pexConfig.Field(
260 dtype=bool,
261 doc="Do sigma clipping on the ensemble of kernel sums",
262 default=True,
263 )
264 spatialKernelClipping = pexConfig.Field(
265 dtype=bool,
266 doc="Do sigma clipping after building the spatial model",
267 default=True,
268 )
269 checkConditionNumber = pexConfig.Field(
270 dtype=bool,
271 doc="""Test for maximum condition number when inverting a kernel matrix.
272 Anything above maxConditionNumber is not used and the candidate is set as BAD.
273 Also used to truncate inverse matrix in estimateBiasedRisk. However,
274 if you are doing any deconvolution you will want to turn this off, or use
275 a large maxConditionNumber""",
276 default=False,
277 )
278 badMaskPlanes = pexConfig.ListField(
279 dtype=str,
280 doc="""Mask planes to ignore when calculating diffim statistics
281 Options: NO_DATA EDGE SAT BAD CR INTRP""",
282 default=("NO_DATA", "EDGE", "SAT")
283 )
284 candidateResidualMeanMax = pexConfig.Field( 284 ↛ exitline 284 didn't jump to the function exit
285 dtype=float,
286 doc="""Rejects KernelCandidates yielding bad difference image quality.
287 Used by BuildSingleKernelVisitor, AssessSpatialKernelVisitor.
288 Represents average over pixels of (image/sqrt(variance)).""",
289 default=0.25,
290 check=lambda x: x >= 0.0
291 )
292 candidateResidualStdMax = pexConfig.Field( 292 ↛ exitline 292 didn't jump to the function exit
293 dtype=float,
294 doc="""Rejects KernelCandidates yielding bad difference image quality.
295 Used by BuildSingleKernelVisitor, AssessSpatialKernelVisitor.
296 Represents stddev over pixels of (image/sqrt(variance)).""",
297 default=1.50,
298 check=lambda x: x >= 0.0
299 )
300 useCoreStats = pexConfig.Field(
301 dtype=bool,
302 doc="""Use the core of the footprint for the quality statistics, instead of the entire footprint.
303 WARNING: if there is deconvolution we probably will need to turn this off""",
304 default=False,
305 )
306 candidateCoreRadius = pexConfig.Field( 306 ↛ exitline 306 didn't jump to the function exit
307 dtype=int,
308 doc="""Radius for calculation of stats in 'core' of KernelCandidate diffim.
309 Total number of pixels used will be (2*radius)**2.
310 This is used both for 'core' diffim quality as well as ranking of
311 KernelCandidates by their total flux in this core""",
312 default=3,
313 check=lambda x: x >= 1
314 )
315 maxKsumSigma = pexConfig.Field( 315 ↛ exitline 315 didn't jump to the function exit
316 dtype=float,
317 doc="""Maximum allowed sigma for outliers from kernel sum distribution.
318 Used to reject variable objects from the kernel model""",
319 default=3.0,
320 check=lambda x: x >= 0.0
321 )
322 maxConditionNumber = pexConfig.Field( 322 ↛ exitline 322 didn't jump to the function exit
323 dtype=float,
324 doc="Maximum condition number for a well conditioned matrix",
325 default=5.0e7,
326 check=lambda x: x >= 0.0
327 )
328 conditionNumberType = pexConfig.ChoiceField(
329 dtype=str,
330 doc="Use singular values (SVD) or eigen values (EIGENVALUE) to determine condition number",
331 default="EIGENVALUE",
332 allowed={
333 "SVD": "Use singular values",
334 "EIGENVALUE": "Use eigen values (faster)",
335 }
336 )
337 maxSpatialConditionNumber = pexConfig.Field( 337 ↛ exitline 337 didn't jump to the function exit
338 dtype=float,
339 doc="Maximum condition number for a well conditioned spatial matrix",
340 default=1.0e10,
341 check=lambda x: x >= 0.0
342 )
343 iterateSingleKernel = pexConfig.Field(
344 dtype=bool,
345 doc="""Remake KernelCandidate using better variance estimate after first pass?
346 Primarily useful when convolving a single-depth image, otherwise not necessary.""",
347 default=False,
348 )
349 constantVarianceWeighting = pexConfig.Field(
350 dtype=bool,
351 doc="""Use constant variance weighting in single kernel fitting?
352 In some cases this is better for bright star residuals.""",
353 default=True,
354 )
355 calculateKernelUncertainty = pexConfig.Field(
356 dtype=bool,
357 doc="""Calculate kernel and background uncertainties for each kernel candidate?
358 This comes from the inverse of the covariance matrix.
359 Warning: regularization can cause problems for this step.""",
360 default=False,
361 )
362 useBicForKernelBasis = pexConfig.Field(
363 dtype=bool,
364 doc="""Use Bayesian Information Criterion to select the number of bases going into the kernel""",
365 default=False,
366 )
369class PsfMatchConfigAL(PsfMatchConfig):
370 """The parameters specific to the "Alard-Lupton" (sum-of-Gaussian) Psf-matching basis"""
372 def setDefaults(self):
373 PsfMatchConfig.setDefaults(self)
374 self.kernelBasisSet = "alard-lupton"
375 self.maxConditionNumber = 5.0e7
377 alardNGauss = pexConfig.Field( 377 ↛ exitline 377 didn't jump to the function exit
378 dtype=int,
379 doc="Number of base Gaussians in alard-lupton kernel basis function generation.",
380 default=3,
381 check=lambda x: x >= 1
382 )
383 alardDegGauss = pexConfig.ListField(
384 dtype=int,
385 doc="Polynomial order of spatial modification of base Gaussians. "
386 "List length must be `alardNGauss`.",
387 default=(4, 2, 2),
388 )
389 alardSigGauss = pexConfig.ListField(
390 dtype=float,
391 doc="Default sigma values in pixels of base Gaussians. "
392 "List length must be `alardNGauss`.",
393 default=(0.7, 1.5, 3.0),
394 )
395 alardGaussBeta = pexConfig.Field( 395 ↛ exitline 395 didn't jump to the function exit
396 dtype=float,
397 doc="Used if `scaleByFwhm==True`, scaling multiplier of base "
398 "Gaussian sigmas for automated sigma determination",
399 default=2.0,
400 check=lambda x: x >= 0.0,
401 )
402 alardMinSig = pexConfig.Field( 402 ↛ exitline 402 didn't jump to the function exit
403 dtype=float,
404 doc="Used if `scaleByFwhm==True`, minimum sigma (pixels) for base Gaussians",
405 default=0.7,
406 check=lambda x: x >= 0.25
407 )
408 alardDegGaussDeconv = pexConfig.Field( 408 ↛ exitline 408 didn't jump to the function exit
409 dtype=int,
410 doc="Used if `scaleByFwhm==True`, degree of spatial modification of ALL base Gaussians "
411 "in AL basis during deconvolution",
412 default=3,
413 check=lambda x: x >= 1
414 )
415 alardMinSigDeconv = pexConfig.Field( 415 ↛ exitline 415 didn't jump to the function exit
416 dtype=float,
417 doc="Used if `scaleByFwhm==True`, minimum sigma (pixels) for base Gaussians during deconvolution; "
418 "make smaller than `alardMinSig` as this is only indirectly used",
419 default=0.4,
420 check=lambda x: x >= 0.25
421 )
422 alardNGaussDeconv = pexConfig.Field( 422 ↛ exitline 422 didn't jump to the function exit
423 dtype=int,
424 doc="Used if `scaleByFwhm==True`, number of base Gaussians in AL basis during deconvolution",
425 default=3,
426 check=lambda x: x >= 1
427 )
430class PsfMatchConfigDF(PsfMatchConfig):
431 """The parameters specific to the delta-function (one basis per-pixel) Psf-matching basis"""
433 def setDefaults(self):
434 PsfMatchConfig.setDefaults(self)
435 self.kernelBasisSet = "delta-function"
436 self.maxConditionNumber = 5.0e6
437 self.usePcaForSpatialKernel = True
438 self.subtractMeanForPca = True
439 self.useBicForKernelBasis = False
441 useRegularization = pexConfig.Field(
442 dtype=bool,
443 doc="Use regularization to smooth the delta function kernels",
444 default=True,
445 )
446 regularizationType = pexConfig.ChoiceField(
447 dtype=str,
448 doc="Type of regularization.",
449 default="centralDifference",
450 allowed={
451 "centralDifference": "Penalize second derivative using 2-D stencil of central finite difference",
452 "forwardDifference": "Penalize first, second, third derivatives using forward finite differeces"
453 }
454 )
455 centralRegularizationStencil = pexConfig.ChoiceField(
456 dtype=int,
457 doc="Type of stencil to approximate central derivative (for centralDifference only)",
458 default=9,
459 allowed={
460 5: "5-point stencil including only adjacent-in-x,y elements",
461 9: "9-point stencil including diagonal elements"
462 }
463 )
464 forwardRegularizationOrders = pexConfig.ListField( 464 ↛ exitline 464 didn't jump to the function exit
465 dtype=int,
466 doc="Array showing which order derivatives to penalize (for forwardDifference only)",
467 default=(1, 2),
468 itemCheck=lambda x: (x > 0) and (x < 4)
469 )
470 regularizationBorderPenalty = pexConfig.Field( 470 ↛ exitline 470 didn't jump to the function exit
471 dtype=float,
472 doc="Value of the penalty for kernel border pixels",
473 default=3.0,
474 check=lambda x: x >= 0.0
475 )
476 lambdaType = pexConfig.ChoiceField(
477 dtype=str,
478 doc="How to choose the value of the regularization strength",
479 default="absolute",
480 allowed={
481 "absolute": "Use lambdaValue as the value of regularization strength",
482 "relative": "Use lambdaValue as fraction of the default regularization strength (N.R. 18.5.8)",
483 "minimizeBiasedRisk": "Minimize biased risk estimate",
484 "minimizeUnbiasedRisk": "Minimize unbiased risk estimate",
485 }
486 )
487 lambdaValue = pexConfig.Field(
488 dtype=float,
489 doc="Value used for absolute determinations of regularization strength",
490 default=0.2,
491 )
492 lambdaScaling = pexConfig.Field(
493 dtype=float,
494 doc="Fraction of the default lambda strength (N.R. 18.5.8) to use. 1e-4 or 1e-5",
495 default=1e-4,
496 )
497 lambdaStepType = pexConfig.ChoiceField(
498 dtype=str,
499 doc="""If a scan through lambda is needed (minimizeBiasedRisk, minimizeUnbiasedRisk),
500 use log or linear steps""",
501 default="log",
502 allowed={
503 "log": "Step in log intervals; e.g. lambdaMin, lambdaMax, lambdaStep = -1.0, 2.0, 0.1",
504 "linear": "Step in linear intervals; e.g. lambdaMin, lambdaMax, lambdaStep = 0.1, 100, 0.1",
505 }
506 )
507 lambdaMin = pexConfig.Field(
508 dtype=float,
509 doc="""If scan through lambda needed (minimizeBiasedRisk, minimizeUnbiasedRisk),
510 start at this value. If lambdaStepType = log:linear, suggest -1:0.1""",
511 default=-1.0,
512 )
513 lambdaMax = pexConfig.Field(
514 dtype=float,
515 doc="""If scan through lambda needed (minimizeBiasedRisk, minimizeUnbiasedRisk),
516 stop at this value. If lambdaStepType = log:linear, suggest 2:100""",
517 default=2.0,
518 )
519 lambdaStep = pexConfig.Field(
520 dtype=float,
521 doc="""If scan through lambda needed (minimizeBiasedRisk, minimizeUnbiasedRisk),
522 step in these increments. If lambdaStepType = log:linear, suggest 0.1:0.1""",
523 default=0.1,
524 )
527class PsfMatchTask(pipeBase.Task):
528 """Base class for Psf Matching; should not be called directly
530 Notes
531 -----
532 PsfMatchTask is a base class that implements the core functionality for matching the
533 Psfs of two images using a spatially varying Psf-matching `lsst.afw.math.LinearCombinationKernel`.
534 The Task requires the user to provide an instance of an `lsst.afw.math.SpatialCellSet`,
535 filled with `lsst.ip.diffim.KernelCandidate` instances, and a list of `lsst.afw.math.Kernels`
536 of basis shapes that will be used for the decomposition. If requested, the Task
537 also performs background matching and returns the differential background model as an
538 `lsst.afw.math.Kernel.SpatialFunction`.
540 **Invoking the Task**
542 As a base class, this Task is not directly invoked. However, ``run()`` methods that are
543 implemented on derived classes will make use of the core ``_solve()`` functionality,
544 which defines a sequence of `lsst.afw.math.CandidateVisitor` classes that iterate
545 through the KernelCandidates, first building up a per-candidate solution and then
546 building up a spatial model from the ensemble of candidates. Sigma clipping is
547 performed using the mean and standard deviation of all kernel sums (to reject
548 variable objects), on the per-candidate substamp diffim residuals
549 (to indicate a bad choice of kernel basis shapes for that particular object),
550 and on the substamp diffim residuals using the spatial kernel fit (to indicate a bad
551 choice of spatial kernel order, or poor constraints on the spatial model). The
552 ``_diagnostic()`` method logs information on the quality of the spatial fit, and also
553 modifies the Task metadata.
555 .. list-table:: Quantities set in Metadata
556 :header-rows: 1
558 * - Parameter
559 - Description
560 * - ``spatialConditionNum``
561 - Condition number of the spatial kernel fit
562 * - ``spatialKernelSum``
563 - Kernel sum (10^{-0.4 * ``Delta``; zeropoint}) of the spatial Psf-matching kernel
564 * - ``ALBasisNGauss``
565 - If using sum-of-Gaussian basis, the number of gaussians used
566 * - ``ALBasisDegGauss``
567 - If using sum-of-Gaussian basis, the deg of spatial variation of the Gaussians
568 * - ``ALBasisSigGauss``
569 - If using sum-of-Gaussian basis, the widths (sigma) of the Gaussians
570 * - ``ALKernelSize``
571 - If using sum-of-Gaussian basis, the kernel size
572 * - ``NFalsePositivesTotal``
573 - Total number of diaSources
574 * - ``NFalsePositivesRefAssociated``
575 - Number of diaSources that associate with the reference catalog
576 * - ``NFalsePositivesRefAssociated``
577 - Number of diaSources that associate with the source catalog
578 * - ``NFalsePositivesUnassociated``
579 - Number of diaSources that are orphans
580 * - ``metric_MEAN``
581 - Mean value of substamp diffim quality metrics across all KernelCandidates,
582 for both the per-candidate (LOCAL) and SPATIAL residuals
583 * - ``metric_MEDIAN``
584 - Median value of substamp diffim quality metrics across all KernelCandidates,
585 for both the per-candidate (LOCAL) and SPATIAL residuals
586 * - ``metric_STDEV``
587 - Standard deviation of substamp diffim quality metrics across all KernelCandidates,
588 for both the per-candidate (LOCAL) and SPATIAL residuals
590 **Debug variables**
592 The `lsst.pipe.base.cmdLineTask.CmdLineTask` command line task interface supports a
593 flag -d/--debug to import @b debug.py from your PYTHONPATH. The relevant contents of debug.py
594 for this Task include:
596 .. code-block:: py
598 import sys
599 import lsstDebug
600 def DebugInfo(name):
601 di = lsstDebug.getInfo(name)
602 if name == "lsst.ip.diffim.psfMatch":
603 # enable debug output
604 di.display = True
605 # display mask transparency
606 di.maskTransparency = 80
607 # show all the candidates and residuals
608 di.displayCandidates = True
609 # show kernel basis functions
610 di.displayKernelBasis = False
611 # show kernel realized across the image
612 di.displayKernelMosaic = True
613 # show coefficients of spatial model
614 di.plotKernelSpatialModel = False
615 # show fixed and spatial coefficients and coefficient histograms
616 di.plotKernelCoefficients = True
617 # show the bad candidates (red) along with good (green)
618 di.showBadCandidates = True
619 return di
620 lsstDebug.Info = DebugInfo
621 lsstDebug.frame = 1
623 Note that if you want additional logging info, you may add to your scripts:
625 .. code-block:: py
627 import lsst.log.utils as logUtils
628 logUtils.traceSetAt("ip.diffim", 4)
629 """
630 ConfigClass = PsfMatchConfig
631 _DefaultName = "psfMatch"
633 def __init__(self, *args, **kwargs):
634 """Create the psf-matching Task
636 Parameters
637 ----------
638 *args
639 Arguments to be passed to ``lsst.pipe.base.task.Task.__init__``
640 **kwargs
641 Keyword arguments to be passed to ``lsst.pipe.base.task.Task.__init__``
643 Notes
644 -----
645 The initialization sets the Psf-matching kernel configuration using the value of
646 self.config.kernel.active. If the kernel is requested with regularization to moderate
647 the bias/variance tradeoff, currently only used when a delta function kernel basis
648 is provided, it creates a regularization matrix stored as member variable
649 self.hMat.
650 """
651 pipeBase.Task.__init__(self, *args, **kwargs)
652 self.kConfig = self.config.kernel.active
654 if 'useRegularization' in self.kConfig:
655 self.useRegularization = self.kConfig.useRegularization
656 else:
657 self.useRegularization = False
659 if self.useRegularization:
660 self.hMat = diffimLib.makeRegularizationMatrix(pexConfig.makePropertySet(self.kConfig))
662 def _diagnostic(self, kernelCellSet, spatialSolution, spatialKernel, spatialBg):
663 """Provide logging diagnostics on quality of spatial kernel fit
665 Parameters
666 ----------
667 kernelCellSet : `lsst.afw.math.SpatialCellSet`
668 Cellset that contains the KernelCandidates used in the fitting
669 spatialSolution : `lsst.ip.diffim.SpatialKernelSolution`
670 KernelSolution of best-fit
671 spatialKernel : `lsst.afw.math.LinearCombinationKernel`
672 Best-fit spatial Kernel model
673 spatialBg : `lsst.afw.math.Function2D`
674 Best-fit spatial background model
675 """
676 # What is the final kernel sum
677 kImage = afwImage.ImageD(spatialKernel.getDimensions())
678 kSum = spatialKernel.computeImage(kImage, False)
679 self.log.info("Final spatial kernel sum %.3f", kSum)
681 # Look at how well conditioned the matrix is
682 conditionNum = spatialSolution.getConditionNumber(
683 getattr(diffimLib.KernelSolution, self.kConfig.conditionNumberType))
684 self.log.info("Spatial model condition number %.3e", conditionNum)
686 if conditionNum < 0.0:
687 self.log.warning("Condition number is negative (%.3e)", conditionNum)
688 if conditionNum > self.kConfig.maxSpatialConditionNumber:
689 self.log.warning("Spatial solution exceeds max condition number (%.3e > %.3e)",
690 conditionNum, self.kConfig.maxSpatialConditionNumber)
692 self.metadata.set("spatialConditionNum", conditionNum)
693 self.metadata.set("spatialKernelSum", kSum)
695 # Look at how well the solution is constrained
696 nBasisKernels = spatialKernel.getNBasisKernels()
697 nKernelTerms = spatialKernel.getNSpatialParameters()
698 if nKernelTerms == 0: # order 0
699 nKernelTerms = 1
701 # Not fit for
702 nBgTerms = spatialBg.getNParameters()
703 if nBgTerms == 1:
704 if spatialBg.getParameters()[0] == 0.0:
705 nBgTerms = 0
707 nGood = 0
708 nBad = 0
709 nTot = 0
710 for cell in kernelCellSet.getCellList():
711 for cand in cell.begin(False): # False = include bad candidates
712 nTot += 1
713 if cand.getStatus() == afwMath.SpatialCellCandidate.GOOD:
714 nGood += 1
715 if cand.getStatus() == afwMath.SpatialCellCandidate.BAD:
716 nBad += 1
718 self.log.info("Doing stats of kernel candidates used in the spatial fit.")
720 # Counting statistics
721 if nBad > 2*nGood:
722 self.log.warning("Many more candidates rejected than accepted; %d total, %d rejected, %d used",
723 nTot, nBad, nGood)
724 else:
725 self.log.info("%d candidates total, %d rejected, %d used", nTot, nBad, nGood)
727 # Some judgements on the quality of the spatial models
728 if nGood < nKernelTerms:
729 self.log.warning("Spatial kernel model underconstrained; %d candidates, %d terms, %d bases",
730 nGood, nKernelTerms, nBasisKernels)
731 self.log.warning("Consider lowering the spatial order")
732 elif nGood <= 2*nKernelTerms:
733 self.log.warning("Spatial kernel model poorly constrained; %d candidates, %d terms, %d bases",
734 nGood, nKernelTerms, nBasisKernels)
735 self.log.warning("Consider lowering the spatial order")
736 else:
737 self.log.info("Spatial kernel model well constrained; %d candidates, %d terms, %d bases",
738 nGood, nKernelTerms, nBasisKernels)
740 if nGood < nBgTerms:
741 self.log.warning("Spatial background model underconstrained; %d candidates, %d terms",
742 nGood, nBgTerms)
743 self.log.warning("Consider lowering the spatial order")
744 elif nGood <= 2*nBgTerms:
745 self.log.warning("Spatial background model poorly constrained; %d candidates, %d terms",
746 nGood, nBgTerms)
747 self.log.warning("Consider lowering the spatial order")
748 else:
749 self.log.info("Spatial background model appears well constrained; %d candidates, %d terms",
750 nGood, nBgTerms)
752 def _displayDebug(self, kernelCellSet, spatialKernel, spatialBackground):
753 """Provide visualization of the inputs and ouputs to the Psf-matching code
755 Parameters
756 ----------
757 kernelCellSet : `lsst.afw.math.SpatialCellSet`
758 The SpatialCellSet used in determining the matching kernel and background
759 spatialKernel : `lsst.afw.math.LinearCombinationKernel`
760 Spatially varying Psf-matching kernel
761 spatialBackground : `lsst.afw.math.Function2D`
762 Spatially varying background-matching function
763 """
764 import lsstDebug
765 displayCandidates = lsstDebug.Info(__name__).displayCandidates
766 displayKernelBasis = lsstDebug.Info(__name__).displayKernelBasis
767 displayKernelMosaic = lsstDebug.Info(__name__).displayKernelMosaic
768 plotKernelSpatialModel = lsstDebug.Info(__name__).plotKernelSpatialModel
769 plotKernelCoefficients = lsstDebug.Info(__name__).plotKernelCoefficients
770 showBadCandidates = lsstDebug.Info(__name__).showBadCandidates
771 maskTransparency = lsstDebug.Info(__name__).maskTransparency
772 if not maskTransparency:
773 maskTransparency = 0
774 afwDisplay.setDefaultMaskTransparency(maskTransparency)
776 if displayCandidates:
777 diutils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground,
778 frame=lsstDebug.frame,
779 showBadCandidates=showBadCandidates)
780 lsstDebug.frame += 1
781 diutils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground,
782 frame=lsstDebug.frame,
783 showBadCandidates=showBadCandidates,
784 kernels=True)
785 lsstDebug.frame += 1
786 diutils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground,
787 frame=lsstDebug.frame,
788 showBadCandidates=showBadCandidates,
789 resids=True)
790 lsstDebug.frame += 1
792 if displayKernelBasis:
793 diutils.showKernelBasis(spatialKernel, frame=lsstDebug.frame)
794 lsstDebug.frame += 1
796 if displayKernelMosaic:
797 diutils.showKernelMosaic(kernelCellSet.getBBox(), spatialKernel, frame=lsstDebug.frame)
798 lsstDebug.frame += 1
800 if plotKernelSpatialModel:
801 diutils.plotKernelSpatialModel(spatialKernel, kernelCellSet, showBadCandidates=showBadCandidates)
803 if plotKernelCoefficients:
804 diutils.plotKernelCoefficients(spatialKernel, kernelCellSet)
806 def _createPcaBasis(self, kernelCellSet, nStarPerCell, ps):
807 """Create Principal Component basis
809 If a principal component analysis is requested, typically when using a delta function basis,
810 perform the PCA here and return a new basis list containing the new principal components.
812 Parameters
813 ----------
814 kernelCellSet : `lsst.afw.math.SpatialCellSet`
815 a SpatialCellSet containing KernelCandidates, from which components are derived
816 nStarPerCell : `int`
817 the number of stars per cell to visit when doing the PCA
818 ps : `lsst.daf.base.PropertySet`
819 input property set controlling the single kernel visitor
821 Returns
822 -------
823 nRejectedPca : `int`
824 number of KernelCandidates rejected during PCA loop
825 spatialBasisList : `list` of `lsst.afw.math.kernel.FixedKernel`
826 basis list containing the principal shapes as Kernels
828 Raises
829 ------
830 RuntimeError
831 If the Eigenvalues sum to zero.
832 """
833 nComponents = self.kConfig.numPrincipalComponents
834 imagePca = diffimLib.KernelPcaD()
835 importStarVisitor = diffimLib.KernelPcaVisitorF(imagePca)
836 kernelCellSet.visitCandidates(importStarVisitor, nStarPerCell)
837 if self.kConfig.subtractMeanForPca:
838 importStarVisitor.subtractMean()
839 imagePca.analyze()
841 eigenValues = imagePca.getEigenValues()
842 pcaBasisList = importStarVisitor.getEigenKernels()
844 eSum = np.sum(eigenValues)
845 if eSum == 0.0:
846 raise RuntimeError("Eigenvalues sum to zero")
847 for j in range(len(eigenValues)):
848 log.log("TRACE5." + self.log.name + "._solve", log.DEBUG,
849 "Eigenvalue %d : %f (%f)", j, eigenValues[j], eigenValues[j]/eSum)
851 nToUse = min(nComponents, len(eigenValues))
852 trimBasisList = []
853 for j in range(nToUse):
854 # Check for NaNs?
855 kimage = afwImage.ImageD(pcaBasisList[j].getDimensions())
856 pcaBasisList[j].computeImage(kimage, False)
857 if not (True in np.isnan(kimage.getArray())):
858 trimBasisList.append(pcaBasisList[j])
860 # Put all the power in the first kernel, which will not vary spatially
861 spatialBasisList = diffimLib.renormalizeKernelList(trimBasisList)
863 # New Kernel visitor for this new basis list (no regularization explicitly)
864 singlekvPca = diffimLib.BuildSingleKernelVisitorF(spatialBasisList, ps)
865 singlekvPca.setSkipBuilt(False)
866 kernelCellSet.visitCandidates(singlekvPca, nStarPerCell)
867 singlekvPca.setSkipBuilt(True)
868 nRejectedPca = singlekvPca.getNRejected()
870 return nRejectedPca, spatialBasisList
872 def _buildCellSet(self, *args):
873 """Fill a SpatialCellSet with KernelCandidates for the Psf-matching process;
874 override in derived classes"""
875 return
877 @pipeBase.timeMethod
878 def _solve(self, kernelCellSet, basisList, returnOnExcept=False):
879 """Solve for the PSF matching kernel
881 Parameters
882 ----------
883 kernelCellSet : `lsst.afw.math.SpatialCellSet`
884 a SpatialCellSet to use in determining the matching kernel
885 (typically as provided by _buildCellSet)
886 basisList : `list` of `lsst.afw.math.kernel.FixedKernel`
887 list of Kernels to be used in the decomposition of the spatially varying kernel
888 (typically as provided by makeKernelBasisList)
889 returnOnExcept : `bool`, optional
890 if True then return (None, None) if an error occurs, else raise the exception
892 Returns
893 -------
894 psfMatchingKernel : `lsst.afw.math.LinearCombinationKernel`
895 Spatially varying Psf-matching kernel
896 backgroundModel : `lsst.afw.math.Function2D`
897 Spatially varying background-matching function
899 Raises
900 ------
901 RuntimeError :
902 If unable to determine PSF matching kernel and ``returnOnExcept==False``.
903 """
905 import lsstDebug
906 display = lsstDebug.Info(__name__).display
908 maxSpatialIterations = self.kConfig.maxSpatialIterations
909 nStarPerCell = self.kConfig.nStarPerCell
910 usePcaForSpatialKernel = self.kConfig.usePcaForSpatialKernel
912 # Visitor for the single kernel fit
913 ps = pexConfig.makePropertySet(self.kConfig)
914 if self.useRegularization:
915 singlekv = diffimLib.BuildSingleKernelVisitorF(basisList, ps, self.hMat)
916 else:
917 singlekv = diffimLib.BuildSingleKernelVisitorF(basisList, ps)
919 # Visitor for the kernel sum rejection
920 ksv = diffimLib.KernelSumVisitorF(ps)
922 # Main loop
923 t0 = time.time()
924 try:
925 totalIterations = 0
926 thisIteration = 0
927 while (thisIteration < maxSpatialIterations):
929 # Make sure there are no uninitialized candidates as active occupants of Cell
930 nRejectedSkf = -1
931 while (nRejectedSkf != 0):
932 log.log("TRACE1." + self.log.name + "._solve", log.DEBUG,
933 "Building single kernels...")
934 kernelCellSet.visitCandidates(singlekv, nStarPerCell)
935 nRejectedSkf = singlekv.getNRejected()
936 log.log("TRACE1." + self.log.name + "._solve", log.DEBUG,
937 "Iteration %d, rejected %d candidates due to initial kernel fit",
938 thisIteration, nRejectedSkf)
940 # Reject outliers in kernel sum
941 ksv.resetKernelSum()
942 ksv.setMode(diffimLib.KernelSumVisitorF.AGGREGATE)
943 kernelCellSet.visitCandidates(ksv, nStarPerCell)
944 ksv.processKsumDistribution()
945 ksv.setMode(diffimLib.KernelSumVisitorF.REJECT)
946 kernelCellSet.visitCandidates(ksv, nStarPerCell)
948 nRejectedKsum = ksv.getNRejected()
949 log.log("TRACE1." + self.log.name + "._solve", log.DEBUG,
950 "Iteration %d, rejected %d candidates due to kernel sum",
951 thisIteration, nRejectedKsum)
953 # Do we jump back to the top without incrementing thisIteration?
954 if nRejectedKsum > 0:
955 totalIterations += 1
956 continue
958 # At this stage we can either apply the spatial fit to
959 # the kernels, or we run a PCA, use these as a *new*
960 # basis set with lower dimensionality, and then apply
961 # the spatial fit to these kernels
963 if (usePcaForSpatialKernel):
964 log.log("TRACE0." + self.log.name + "._solve", log.DEBUG,
965 "Building Pca basis")
967 nRejectedPca, spatialBasisList = self._createPcaBasis(kernelCellSet, nStarPerCell, ps)
968 log.log("TRACE1." + self.log.name + "._solve", log.DEBUG,
969 "Iteration %d, rejected %d candidates due to Pca kernel fit",
970 thisIteration, nRejectedPca)
972 # We don't want to continue on (yet) with the
973 # spatial modeling, because we have bad objects
974 # contributing to the Pca basis. We basically
975 # need to restart from the beginning of this loop,
976 # since the cell-mates of those objects that were
977 # rejected need their original Kernels built by
978 # singleKernelFitter.
980 # Don't count against thisIteration
981 if (nRejectedPca > 0):
982 totalIterations += 1
983 continue
984 else:
985 spatialBasisList = basisList
987 # We have gotten on to the spatial modeling part
988 regionBBox = kernelCellSet.getBBox()
989 spatialkv = diffimLib.BuildSpatialKernelVisitorF(spatialBasisList, regionBBox, ps)
990 kernelCellSet.visitCandidates(spatialkv, nStarPerCell)
991 spatialkv.solveLinearEquation()
992 log.log("TRACE2." + self.log.name + "._solve", log.DEBUG,
993 "Spatial kernel built with %d candidates", spatialkv.getNCandidates())
994 spatialKernel, spatialBackground = spatialkv.getSolutionPair()
996 # Check the quality of the spatial fit (look at residuals)
997 assesskv = diffimLib.AssessSpatialKernelVisitorF(spatialKernel, spatialBackground, ps)
998 kernelCellSet.visitCandidates(assesskv, nStarPerCell)
999 nRejectedSpatial = assesskv.getNRejected()
1000 nGoodSpatial = assesskv.getNGood()
1001 log.log("TRACE1." + self.log.name + "._solve", log.DEBUG,
1002 "Iteration %d, rejected %d candidates due to spatial kernel fit",
1003 thisIteration, nRejectedSpatial)
1004 log.log("TRACE1." + self.log.name + "._solve", log.DEBUG,
1005 "%d candidates used in fit", nGoodSpatial)
1007 # If only nGoodSpatial == 0, might be other candidates in the cells
1008 if nGoodSpatial == 0 and nRejectedSpatial == 0:
1009 raise RuntimeError("No kernel candidates for spatial fit")
1011 if nRejectedSpatial == 0:
1012 # Nothing rejected, finished with spatial fit
1013 break
1015 # Otherwise, iterate on...
1016 thisIteration += 1
1018 # Final fit if above did not converge
1019 if (nRejectedSpatial > 0) and (thisIteration == maxSpatialIterations):
1020 log.log("TRACE1." + self.log.name + "._solve", log.DEBUG, "Final spatial fit")
1021 if (usePcaForSpatialKernel):
1022 nRejectedPca, spatialBasisList = self._createPcaBasis(kernelCellSet, nStarPerCell, ps)
1023 regionBBox = kernelCellSet.getBBox()
1024 spatialkv = diffimLib.BuildSpatialKernelVisitorF(spatialBasisList, regionBBox, ps)
1025 kernelCellSet.visitCandidates(spatialkv, nStarPerCell)
1026 spatialkv.solveLinearEquation()
1027 log.log("TRACE2." + self.log.name + "._solve", log.DEBUG,
1028 "Spatial kernel built with %d candidates", spatialkv.getNCandidates())
1029 spatialKernel, spatialBackground = spatialkv.getSolutionPair()
1031 spatialSolution = spatialkv.getKernelSolution()
1033 except Exception as e:
1034 self.log.error("ERROR: Unable to calculate psf matching kernel")
1036 log.log("TRACE1." + self.log.name + "._solve", log.DEBUG, "%s", e)
1037 raise e
1039 t1 = time.time()
1040 log.log("TRACE0." + self.log.name + "._solve", log.DEBUG,
1041 "Total time to compute the spatial kernel : %.2f s", (t1 - t0))
1043 if display:
1044 self._displayDebug(kernelCellSet, spatialKernel, spatialBackground)
1046 self._diagnostic(kernelCellSet, spatialSolution, spatialKernel, spatialBackground)
1048 return spatialSolution, spatialKernel, spatialBackground
1051PsfMatch = PsfMatchTask