Coverage for python/lsst/ip/diffim/psfMatch.py: 29%

299 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-12-04 02:16 -0800

1# This file is part of ip_diffim. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["DetectionConfig", "PsfMatchConfig", "PsfMatchConfigAL", "PsfMatchConfigDF", "PsfMatchTask"] 

23 

24import abc 

25import time 

26 

27import numpy as np 

28 

29import lsst.afw.image as afwImage 

30import lsst.pex.config as pexConfig 

31import lsst.afw.math as afwMath 

32import lsst.afw.display as afwDisplay 

33import lsst.pipe.base as pipeBase 

34from lsst.meas.algorithms import SubtractBackgroundConfig 

35from lsst.utils.logging import getTraceLogger 

36from lsst.utils.timer import timeMethod 

37from . import utils as diutils 

38from . import diffimLib 

39 

40 

41class DetectionConfig(pexConfig.Config): 

42 """Configuration for detecting sources on images for building a 

43 PSF-matching kernel 

44 

45 Configuration for turning detected lsst.afw.detection.FootPrints into an 

46 acceptable (unmasked, high signal-to-noise, not too large or not too small) 

47 list of `lsst.ip.diffim.KernelSources` that are used to build the 

48 Psf-matching kernel""" 

49 

50 detThreshold = pexConfig.Field( 50 ↛ exitline 50 didn't jump to the function exit

51 dtype=float, 

52 doc="Value of footprint detection threshold", 

53 default=10.0, 

54 check=lambda x: x >= 3.0 

55 ) 

56 detThresholdType = pexConfig.ChoiceField( 

57 dtype=str, 

58 doc="Type of detection threshold", 

59 default="pixel_stdev", 

60 allowed={ 

61 "value": "Use counts as the detection threshold type", 

62 "stdev": "Use standard deviation of image plane", 

63 "variance": "Use variance of image plane", 

64 "pixel_stdev": "Use stdev derived from variance plane" 

65 } 

66 ) 

67 detOnTemplate = pexConfig.Field( 

68 dtype=bool, 

69 doc="""If true run detection on the template (image to convolve); 

70 if false run detection on the science image""", 

71 default=True 

72 ) 

73 badMaskPlanes = pexConfig.ListField( 

74 dtype=str, 

75 doc="""Mask planes that lead to an invalid detection. 

76 Options: NO_DATA EDGE SAT BAD CR INTRP""", 

77 default=("NO_DATA", "EDGE", "SAT") 

78 ) 

79 fpNpixMin = pexConfig.Field( 79 ↛ exitline 79 didn't jump to the function exit

80 dtype=int, 

81 doc="Minimum number of pixels in an acceptable Footprint", 

82 default=5, 

83 check=lambda x: x >= 5 

84 ) 

85 fpNpixMax = pexConfig.Field( 85 ↛ exitline 85 didn't jump to the function exit

86 dtype=int, 

87 doc="""Maximum number of pixels in an acceptable Footprint; 

88 too big and the subsequent convolutions become unwieldy""", 

89 default=500, 

90 check=lambda x: x <= 500 

91 ) 

92 fpGrowKernelScaling = pexConfig.Field( 92 ↛ exitline 92 didn't jump to the function exit

93 dtype=float, 

94 doc="""If config.scaleByFwhm, grow the footprint based on 

95 the final kernelSize. Each footprint will be 

96 2*fpGrowKernelScaling*kernelSize x 

97 2*fpGrowKernelScaling*kernelSize. With the value 

98 of 1.0, the remaining pixels in each KernelCandiate 

99 after convolution by the basis functions will be 

100 equal to the kernel size itself.""", 

101 default=1.0, 

102 check=lambda x: x >= 1.0 

103 ) 

104 fpGrowPix = pexConfig.Field( 104 ↛ exitline 104 didn't jump to the function exit

105 dtype=int, 

106 doc="""Growing radius (in pixels) for each raw detection 

107 footprint. The smaller the faster; however the 

108 kernel sum does not converge if the stamp is too 

109 small; and the kernel is not constrained at all if 

110 the stamp is the size of the kernel. The grown stamp 

111 is 2 * fpGrowPix pixels larger in each dimension. 

112 This is overridden by fpGrowKernelScaling if scaleByFwhm""", 

113 default=30, 

114 check=lambda x: x >= 10 

115 ) 

116 scaleByFwhm = pexConfig.Field( 

117 dtype=bool, 

118 doc="Scale fpGrowPix by input Fwhm?", 

119 default=True, 

120 ) 

121 

122 

123class PsfMatchConfig(pexConfig.Config): 

124 """Base configuration for Psf-matching 

125 

126 The base configuration of the Psf-matching kernel, and of the warping, detection, 

127 and background modeling subTasks.""" 

128 

129 warpingConfig = pexConfig.ConfigField("Config for warping exposures to a common alignment", 

130 afwMath.WarperConfig) 

131 detectionConfig = pexConfig.ConfigField("Controlling the detection of sources for kernel building", 

132 DetectionConfig) 

133 afwBackgroundConfig = pexConfig.ConfigField("Controlling the Afw background fitting", 

134 SubtractBackgroundConfig) 

135 

136 useAfwBackground = pexConfig.Field( 

137 dtype=bool, 

138 doc="Use afw background subtraction instead of ip_diffim", 

139 default=False, 

140 ) 

141 fitForBackground = pexConfig.Field( 

142 dtype=bool, 

143 doc="Include terms (including kernel cross terms) for background in ip_diffim", 

144 default=False, 

145 ) 

146 kernelBasisSet = pexConfig.ChoiceField( 

147 dtype=str, 

148 doc="Type of basis set for PSF matching kernel.", 

149 default="alard-lupton", 

150 allowed={ 

151 "alard-lupton": """Alard-Lupton sum-of-gaussians basis set, 

152 * The first term has no spatial variation 

153 * The kernel sum is conserved 

154 * You may want to turn off 'usePcaForSpatialKernel'""", 

155 "delta-function": """Delta-function kernel basis set, 

156 * You may enable the option useRegularization 

157 * You should seriously consider usePcaForSpatialKernel, which will also 

158 enable kernel sum conservation for the delta function kernels""" 

159 } 

160 ) 

161 kernelSize = pexConfig.Field( 

162 dtype=int, 

163 doc="""Number of rows/columns in the convolution kernel; should be odd-valued. 

164 Modified by kernelSizeFwhmScaling if scaleByFwhm = true""", 

165 default=21, 

166 ) 

167 scaleByFwhm = pexConfig.Field( 

168 dtype=bool, 

169 doc="Scale kernelSize, alardGaussians by input Fwhm", 

170 default=True, 

171 ) 

172 kernelSizeFwhmScaling = pexConfig.Field( 172 ↛ exitline 172 didn't jump to the function exit

173 dtype=float, 

174 doc="Multiplier of the largest AL Gaussian basis sigma to get the kernel bbox (pixel) size.", 

175 default=6.0, 

176 check=lambda x: x >= 1.0 

177 ) 

178 kernelSizeMin = pexConfig.Field( 

179 dtype=int, 

180 doc="Minimum kernel bbox (pixel) size.", 

181 default=21, 

182 ) 

183 kernelSizeMax = pexConfig.Field( 

184 dtype=int, 

185 doc="Maximum kernel bbox (pixel) size.", 

186 default=35, 

187 ) 

188 spatialModelType = pexConfig.ChoiceField( 

189 dtype=str, 

190 doc="Type of spatial functions for kernel and background", 

191 default="chebyshev1", 

192 allowed={ 

193 "chebyshev1": "Chebyshev polynomial of the first kind", 

194 "polynomial": "Standard x,y polynomial", 

195 } 

196 ) 

197 spatialKernelOrder = pexConfig.Field( 197 ↛ exitline 197 didn't jump to the function exit

198 dtype=int, 

199 doc="Spatial order of convolution kernel variation", 

200 default=2, 

201 check=lambda x: x >= 0 

202 ) 

203 spatialBgOrder = pexConfig.Field( 203 ↛ exitline 203 didn't jump to the function exit

204 dtype=int, 

205 doc="Spatial order of differential background variation", 

206 default=1, 

207 check=lambda x: x >= 0 

208 ) 

209 sizeCellX = pexConfig.Field( 209 ↛ exitline 209 didn't jump to the function exit

210 dtype=int, 

211 doc="Size (rows) in pixels of each SpatialCell for spatial modeling", 

212 default=128, 

213 check=lambda x: x >= 32 

214 ) 

215 sizeCellY = pexConfig.Field( 215 ↛ exitline 215 didn't jump to the function exit

216 dtype=int, 

217 doc="Size (columns) in pixels of each SpatialCell for spatial modeling", 

218 default=128, 

219 check=lambda x: x >= 32 

220 ) 

221 nStarPerCell = pexConfig.Field( 221 ↛ exitline 221 didn't jump to the function exit

222 dtype=int, 

223 doc="Number of KernelCandidates in each SpatialCell to use in the spatial fitting", 

224 default=3, 

225 check=lambda x: x >= 1 

226 ) 

227 maxSpatialIterations = pexConfig.Field( 227 ↛ exitline 227 didn't jump to the function exit

228 dtype=int, 

229 doc="Maximum number of iterations for rejecting bad KernelCandidates in spatial fitting", 

230 default=3, 

231 check=lambda x: x >= 1 and x <= 5 

232 ) 

233 usePcaForSpatialKernel = pexConfig.Field( 

234 dtype=bool, 

235 doc="""Use Pca to reduce the dimensionality of the kernel basis sets. 

236 This is particularly useful for delta-function kernels. 

237 Functionally, after all Cells have their raw kernels determined, we run 

238 a Pca on these Kernels, re-fit the Cells using the eigenKernels and then 

239 fit those for spatial variation using the same technique as for Alard-Lupton kernels. 

240 If this option is used, the first term will have no spatial variation and the 

241 kernel sum will be conserved.""", 

242 default=False, 

243 ) 

244 subtractMeanForPca = pexConfig.Field( 

245 dtype=bool, 

246 doc="Subtract off the mean feature before doing the Pca", 

247 default=True, 

248 ) 

249 numPrincipalComponents = pexConfig.Field( 249 ↛ exitline 249 didn't jump to the function exit

250 dtype=int, 

251 doc="""Number of principal components to use for Pca basis, including the 

252 mean kernel if requested.""", 

253 default=5, 

254 check=lambda x: x >= 3 

255 ) 

256 singleKernelClipping = pexConfig.Field( 

257 dtype=bool, 

258 doc="Do sigma clipping on each raw kernel candidate", 

259 default=True, 

260 ) 

261 kernelSumClipping = pexConfig.Field( 

262 dtype=bool, 

263 doc="Do sigma clipping on the ensemble of kernel sums", 

264 default=True, 

265 ) 

266 spatialKernelClipping = pexConfig.Field( 

267 dtype=bool, 

268 doc="Do sigma clipping after building the spatial model", 

269 default=True, 

270 ) 

271 checkConditionNumber = pexConfig.Field( 

272 dtype=bool, 

273 doc="""Test for maximum condition number when inverting a kernel matrix. 

274 Anything above maxConditionNumber is not used and the candidate is set as BAD. 

275 Also used to truncate inverse matrix in estimateBiasedRisk. However, 

276 if you are doing any deconvolution you will want to turn this off, or use 

277 a large maxConditionNumber""", 

278 default=False, 

279 ) 

280 badMaskPlanes = pexConfig.ListField( 

281 dtype=str, 

282 doc="""Mask planes to ignore when calculating diffim statistics 

283 Options: NO_DATA EDGE SAT BAD CR INTRP""", 

284 default=("NO_DATA", "EDGE", "SAT") 

285 ) 

286 candidateResidualMeanMax = pexConfig.Field( 286 ↛ exitline 286 didn't jump to the function exit

287 dtype=float, 

288 doc="""Rejects KernelCandidates yielding bad difference image quality. 

289 Used by BuildSingleKernelVisitor, AssessSpatialKernelVisitor. 

290 Represents average over pixels of (image/sqrt(variance)).""", 

291 default=0.25, 

292 check=lambda x: x >= 0.0 

293 ) 

294 candidateResidualStdMax = pexConfig.Field( 294 ↛ exitline 294 didn't jump to the function exit

295 dtype=float, 

296 doc="""Rejects KernelCandidates yielding bad difference image quality. 

297 Used by BuildSingleKernelVisitor, AssessSpatialKernelVisitor. 

298 Represents stddev over pixels of (image/sqrt(variance)).""", 

299 default=1.50, 

300 check=lambda x: x >= 0.0 

301 ) 

302 useCoreStats = pexConfig.Field( 

303 dtype=bool, 

304 doc="""Use the core of the footprint for the quality statistics, instead of the entire footprint. 

305 WARNING: if there is deconvolution we probably will need to turn this off""", 

306 default=False, 

307 ) 

308 candidateCoreRadius = pexConfig.Field( 308 ↛ exitline 308 didn't jump to the function exit

309 dtype=int, 

310 doc="""Radius for calculation of stats in 'core' of KernelCandidate diffim. 

311 Total number of pixels used will be (2*radius)**2. 

312 This is used both for 'core' diffim quality as well as ranking of 

313 KernelCandidates by their total flux in this core""", 

314 default=3, 

315 check=lambda x: x >= 1 

316 ) 

317 maxKsumSigma = pexConfig.Field( 317 ↛ exitline 317 didn't jump to the function exit

318 dtype=float, 

319 doc="""Maximum allowed sigma for outliers from kernel sum distribution. 

320 Used to reject variable objects from the kernel model""", 

321 default=3.0, 

322 check=lambda x: x >= 0.0 

323 ) 

324 maxConditionNumber = pexConfig.Field( 324 ↛ exitline 324 didn't jump to the function exit

325 dtype=float, 

326 doc="Maximum condition number for a well conditioned matrix", 

327 default=5.0e7, 

328 check=lambda x: x >= 0.0 

329 ) 

330 conditionNumberType = pexConfig.ChoiceField( 

331 dtype=str, 

332 doc="Use singular values (SVD) or eigen values (EIGENVALUE) to determine condition number", 

333 default="EIGENVALUE", 

334 allowed={ 

335 "SVD": "Use singular values", 

336 "EIGENVALUE": "Use eigen values (faster)", 

337 } 

338 ) 

339 maxSpatialConditionNumber = pexConfig.Field( 339 ↛ exitline 339 didn't jump to the function exit

340 dtype=float, 

341 doc="Maximum condition number for a well conditioned spatial matrix", 

342 default=1.0e10, 

343 check=lambda x: x >= 0.0 

344 ) 

345 iterateSingleKernel = pexConfig.Field( 

346 dtype=bool, 

347 doc="""Remake KernelCandidate using better variance estimate after first pass? 

348 Primarily useful when convolving a single-depth image, otherwise not necessary.""", 

349 default=False, 

350 ) 

351 constantVarianceWeighting = pexConfig.Field( 

352 dtype=bool, 

353 doc="""Use constant variance weighting in single kernel fitting? 

354 In some cases this is better for bright star residuals.""", 

355 default=True, 

356 ) 

357 calculateKernelUncertainty = pexConfig.Field( 

358 dtype=bool, 

359 doc="""Calculate kernel and background uncertainties for each kernel candidate? 

360 This comes from the inverse of the covariance matrix. 

361 Warning: regularization can cause problems for this step.""", 

362 default=False, 

363 ) 

364 useBicForKernelBasis = pexConfig.Field( 

365 dtype=bool, 

366 doc="""Use Bayesian Information Criterion to select the number of bases going into the kernel""", 

367 default=False, 

368 ) 

369 

370 

371class PsfMatchConfigAL(PsfMatchConfig): 

372 """The parameters specific to the "Alard-Lupton" (sum-of-Gaussian) Psf-matching basis""" 

373 

374 def setDefaults(self): 

375 PsfMatchConfig.setDefaults(self) 

376 self.kernelBasisSet = "alard-lupton" 

377 self.maxConditionNumber = 5.0e7 

378 

379 alardNGauss = pexConfig.Field( 379 ↛ exitline 379 didn't jump to the function exit

380 dtype=int, 

381 doc="Number of base Gaussians in alard-lupton kernel basis function generation.", 

382 default=3, 

383 check=lambda x: x >= 1 

384 ) 

385 alardDegGauss = pexConfig.ListField( 

386 dtype=int, 

387 doc="Polynomial order of spatial modification of base Gaussians. " 

388 "List length must be `alardNGauss`.", 

389 default=(4, 2, 2), 

390 ) 

391 alardSigGauss = pexConfig.ListField( 

392 dtype=float, 

393 doc="Default sigma values in pixels of base Gaussians. " 

394 "List length must be `alardNGauss`.", 

395 default=(0.7, 1.5, 3.0), 

396 ) 

397 alardGaussBeta = pexConfig.Field( 397 ↛ exitline 397 didn't jump to the function exit

398 dtype=float, 

399 doc="Used if `scaleByFwhm==True`, scaling multiplier of base " 

400 "Gaussian sigmas for automated sigma determination", 

401 default=2.0, 

402 check=lambda x: x >= 0.0, 

403 ) 

404 alardMinSig = pexConfig.Field( 404 ↛ exitline 404 didn't jump to the function exit

405 dtype=float, 

406 doc="Used if `scaleByFwhm==True`, minimum sigma (pixels) for base Gaussians", 

407 default=0.7, 

408 check=lambda x: x >= 0.25 

409 ) 

410 alardDegGaussDeconv = pexConfig.Field( 410 ↛ exitline 410 didn't jump to the function exit

411 dtype=int, 

412 doc="Used if `scaleByFwhm==True`, degree of spatial modification of ALL base Gaussians " 

413 "in AL basis during deconvolution", 

414 default=3, 

415 check=lambda x: x >= 1 

416 ) 

417 alardMinSigDeconv = pexConfig.Field( 417 ↛ exitline 417 didn't jump to the function exit

418 dtype=float, 

419 doc="Used if `scaleByFwhm==True`, minimum sigma (pixels) for base Gaussians during deconvolution; " 

420 "make smaller than `alardMinSig` as this is only indirectly used", 

421 default=0.4, 

422 check=lambda x: x >= 0.25 

423 ) 

424 alardNGaussDeconv = pexConfig.Field( 424 ↛ exitline 424 didn't jump to the function exit

425 dtype=int, 

426 doc="Used if `scaleByFwhm==True`, number of base Gaussians in AL basis during deconvolution", 

427 default=3, 

428 check=lambda x: x >= 1 

429 ) 

430 

431 

432class PsfMatchConfigDF(PsfMatchConfig): 

433 """The parameters specific to the delta-function (one basis per-pixel) Psf-matching basis""" 

434 

435 def setDefaults(self): 

436 PsfMatchConfig.setDefaults(self) 

437 self.kernelBasisSet = "delta-function" 

438 self.maxConditionNumber = 5.0e6 

439 self.usePcaForSpatialKernel = True 

440 self.subtractMeanForPca = True 

441 self.useBicForKernelBasis = False 

442 

443 useRegularization = pexConfig.Field( 

444 dtype=bool, 

445 doc="Use regularization to smooth the delta function kernels", 

446 default=True, 

447 ) 

448 regularizationType = pexConfig.ChoiceField( 

449 dtype=str, 

450 doc="Type of regularization.", 

451 default="centralDifference", 

452 allowed={ 

453 "centralDifference": "Penalize second derivative using 2-D stencil of central finite difference", 

454 "forwardDifference": "Penalize first, second, third derivatives using forward finite differeces" 

455 } 

456 ) 

457 centralRegularizationStencil = pexConfig.ChoiceField( 

458 dtype=int, 

459 doc="Type of stencil to approximate central derivative (for centralDifference only)", 

460 default=9, 

461 allowed={ 

462 5: "5-point stencil including only adjacent-in-x,y elements", 

463 9: "9-point stencil including diagonal elements" 

464 } 

465 ) 

466 forwardRegularizationOrders = pexConfig.ListField( 466 ↛ exitline 466 didn't jump to the function exit

467 dtype=int, 

468 doc="Array showing which order derivatives to penalize (for forwardDifference only)", 

469 default=(1, 2), 

470 itemCheck=lambda x: (x > 0) and (x < 4) 

471 ) 

472 regularizationBorderPenalty = pexConfig.Field( 472 ↛ exitline 472 didn't jump to the function exit

473 dtype=float, 

474 doc="Value of the penalty for kernel border pixels", 

475 default=3.0, 

476 check=lambda x: x >= 0.0 

477 ) 

478 lambdaType = pexConfig.ChoiceField( 

479 dtype=str, 

480 doc="How to choose the value of the regularization strength", 

481 default="absolute", 

482 allowed={ 

483 "absolute": "Use lambdaValue as the value of regularization strength", 

484 "relative": "Use lambdaValue as fraction of the default regularization strength (N.R. 18.5.8)", 

485 "minimizeBiasedRisk": "Minimize biased risk estimate", 

486 "minimizeUnbiasedRisk": "Minimize unbiased risk estimate", 

487 } 

488 ) 

489 lambdaValue = pexConfig.Field( 

490 dtype=float, 

491 doc="Value used for absolute determinations of regularization strength", 

492 default=0.2, 

493 ) 

494 lambdaScaling = pexConfig.Field( 

495 dtype=float, 

496 doc="Fraction of the default lambda strength (N.R. 18.5.8) to use. 1e-4 or 1e-5", 

497 default=1e-4, 

498 ) 

499 lambdaStepType = pexConfig.ChoiceField( 

500 dtype=str, 

501 doc="""If a scan through lambda is needed (minimizeBiasedRisk, minimizeUnbiasedRisk), 

502 use log or linear steps""", 

503 default="log", 

504 allowed={ 

505 "log": "Step in log intervals; e.g. lambdaMin, lambdaMax, lambdaStep = -1.0, 2.0, 0.1", 

506 "linear": "Step in linear intervals; e.g. lambdaMin, lambdaMax, lambdaStep = 0.1, 100, 0.1", 

507 } 

508 ) 

509 lambdaMin = pexConfig.Field( 

510 dtype=float, 

511 doc="""If scan through lambda needed (minimizeBiasedRisk, minimizeUnbiasedRisk), 

512 start at this value. If lambdaStepType = log:linear, suggest -1:0.1""", 

513 default=-1.0, 

514 ) 

515 lambdaMax = pexConfig.Field( 

516 dtype=float, 

517 doc="""If scan through lambda needed (minimizeBiasedRisk, minimizeUnbiasedRisk), 

518 stop at this value. If lambdaStepType = log:linear, suggest 2:100""", 

519 default=2.0, 

520 ) 

521 lambdaStep = pexConfig.Field( 

522 dtype=float, 

523 doc="""If scan through lambda needed (minimizeBiasedRisk, minimizeUnbiasedRisk), 

524 step in these increments. If lambdaStepType = log:linear, suggest 0.1:0.1""", 

525 default=0.1, 

526 ) 

527 

528 

529class PsfMatchTask(pipeBase.Task, abc.ABC): 

530 """Base class for Psf Matching; should not be called directly 

531 

532 Notes 

533 ----- 

534 PsfMatchTask is a base class that implements the core functionality for matching the 

535 Psfs of two images using a spatially varying Psf-matching `lsst.afw.math.LinearCombinationKernel`. 

536 The Task requires the user to provide an instance of an `lsst.afw.math.SpatialCellSet`, 

537 filled with `lsst.ip.diffim.KernelCandidate` instances, and a list of `lsst.afw.math.Kernels` 

538 of basis shapes that will be used for the decomposition. If requested, the Task 

539 also performs background matching and returns the differential background model as an 

540 `lsst.afw.math.Kernel.SpatialFunction`. 

541 

542 **Invoking the Task** 

543 

544 As a base class, this Task is not directly invoked. However, ``run()`` methods that are 

545 implemented on derived classes will make use of the core ``_solve()`` functionality, 

546 which defines a sequence of `lsst.afw.math.CandidateVisitor` classes that iterate 

547 through the KernelCandidates, first building up a per-candidate solution and then 

548 building up a spatial model from the ensemble of candidates. Sigma clipping is 

549 performed using the mean and standard deviation of all kernel sums (to reject 

550 variable objects), on the per-candidate substamp diffim residuals 

551 (to indicate a bad choice of kernel basis shapes for that particular object), 

552 and on the substamp diffim residuals using the spatial kernel fit (to indicate a bad 

553 choice of spatial kernel order, or poor constraints on the spatial model). The 

554 ``_diagnostic()`` method logs information on the quality of the spatial fit, and also 

555 modifies the Task metadata. 

556 

557 .. list-table:: Quantities set in Metadata 

558 :header-rows: 1 

559 

560 * - Parameter 

561 - Description 

562 * - ``spatialConditionNum`` 

563 - Condition number of the spatial kernel fit 

564 * - ``spatialKernelSum`` 

565 - Kernel sum (10^{-0.4 * ``Delta``; zeropoint}) of the spatial Psf-matching kernel 

566 * - ``ALBasisNGauss`` 

567 - If using sum-of-Gaussian basis, the number of gaussians used 

568 * - ``ALBasisDegGauss`` 

569 - If using sum-of-Gaussian basis, the deg of spatial variation of the Gaussians 

570 * - ``ALBasisSigGauss`` 

571 - If using sum-of-Gaussian basis, the widths (sigma) of the Gaussians 

572 * - ``ALKernelSize`` 

573 - If using sum-of-Gaussian basis, the kernel size 

574 * - ``NFalsePositivesTotal`` 

575 - Total number of diaSources 

576 * - ``NFalsePositivesRefAssociated`` 

577 - Number of diaSources that associate with the reference catalog 

578 * - ``NFalsePositivesRefAssociated`` 

579 - Number of diaSources that associate with the source catalog 

580 * - ``NFalsePositivesUnassociated`` 

581 - Number of diaSources that are orphans 

582 * - ``metric_MEAN`` 

583 - Mean value of substamp diffim quality metrics across all KernelCandidates, 

584 for both the per-candidate (LOCAL) and SPATIAL residuals 

585 * - ``metric_MEDIAN`` 

586 - Median value of substamp diffim quality metrics across all KernelCandidates, 

587 for both the per-candidate (LOCAL) and SPATIAL residuals 

588 * - ``metric_STDEV`` 

589 - Standard deviation of substamp diffim quality metrics across all KernelCandidates, 

590 for both the per-candidate (LOCAL) and SPATIAL residuals 

591 

592 **Debug variables** 

593 

594 The ``pipetask`` command line interface supports a 

595 flag --debug to import @b debug.py from your PYTHONPATH. The relevant contents of debug.py 

596 for this Task include: 

597 

598 .. code-block:: py 

599 

600 import sys 

601 import lsstDebug 

602 def DebugInfo(name): 

603 di = lsstDebug.getInfo(name) 

604 if name == "lsst.ip.diffim.psfMatch": 

605 # enable debug output 

606 di.display = True 

607 # display mask transparency 

608 di.maskTransparency = 80 

609 # show all the candidates and residuals 

610 di.displayCandidates = True 

611 # show kernel basis functions 

612 di.displayKernelBasis = False 

613 # show kernel realized across the image 

614 di.displayKernelMosaic = True 

615 # show coefficients of spatial model 

616 di.plotKernelSpatialModel = False 

617 # show fixed and spatial coefficients and coefficient histograms 

618 di.plotKernelCoefficients = True 

619 # show the bad candidates (red) along with good (green) 

620 di.showBadCandidates = True 

621 return di 

622 lsstDebug.Info = DebugInfo 

623 lsstDebug.frame = 1 

624 

625 Note that if you want additional logging info, you may add to your scripts: 

626 

627 .. code-block:: py 

628 

629 import lsst.utils.logging as logUtils 

630 logUtils.trace_set_at("lsst.ip.diffim", 4) 

631 """ 

632 ConfigClass = PsfMatchConfig 

633 _DefaultName = "psfMatch" 

634 

635 def __init__(self, *args, **kwargs): 

636 """Create the psf-matching Task 

637 

638 Parameters 

639 ---------- 

640 *args 

641 Arguments to be passed to ``lsst.pipe.base.task.Task.__init__`` 

642 **kwargs 

643 Keyword arguments to be passed to ``lsst.pipe.base.task.Task.__init__`` 

644 

645 Notes 

646 ----- 

647 The initialization sets the Psf-matching kernel configuration using the value of 

648 self.config.kernel.active. If the kernel is requested with regularization to moderate 

649 the bias/variance tradeoff, currently only used when a delta function kernel basis 

650 is provided, it creates a regularization matrix stored as member variable 

651 self.hMat. 

652 """ 

653 pipeBase.Task.__init__(self, *args, **kwargs) 

654 self.kConfig = self.config.kernel.active 

655 

656 if 'useRegularization' in self.kConfig: 

657 self.useRegularization = self.kConfig.useRegularization 

658 else: 

659 self.useRegularization = False 

660 

661 if self.useRegularization: 

662 self.hMat = diffimLib.makeRegularizationMatrix(pexConfig.makePropertySet(self.kConfig)) 

663 

664 def _diagnostic(self, kernelCellSet, spatialSolution, spatialKernel, spatialBg): 

665 """Provide logging diagnostics on quality of spatial kernel fit 

666 

667 Parameters 

668 ---------- 

669 kernelCellSet : `lsst.afw.math.SpatialCellSet` 

670 Cellset that contains the KernelCandidates used in the fitting 

671 spatialSolution : `lsst.ip.diffim.SpatialKernelSolution` 

672 KernelSolution of best-fit 

673 spatialKernel : `lsst.afw.math.LinearCombinationKernel` 

674 Best-fit spatial Kernel model 

675 spatialBg : `lsst.afw.math.Function2D` 

676 Best-fit spatial background model 

677 """ 

678 # What is the final kernel sum 

679 kImage = afwImage.ImageD(spatialKernel.getDimensions()) 

680 kSum = spatialKernel.computeImage(kImage, False) 

681 self.log.info("Final spatial kernel sum %.3f", kSum) 

682 

683 # Look at how well conditioned the matrix is 

684 conditionNum = spatialSolution.getConditionNumber( 

685 getattr(diffimLib.KernelSolution, self.kConfig.conditionNumberType)) 

686 self.log.info("Spatial model condition number %.3e", conditionNum) 

687 

688 if conditionNum < 0.0: 

689 self.log.warning("Condition number is negative (%.3e)", conditionNum) 

690 if conditionNum > self.kConfig.maxSpatialConditionNumber: 

691 self.log.warning("Spatial solution exceeds max condition number (%.3e > %.3e)", 

692 conditionNum, self.kConfig.maxSpatialConditionNumber) 

693 

694 self.metadata["spatialConditionNum"] = conditionNum 

695 self.metadata["spatialKernelSum"] = kSum 

696 

697 # Look at how well the solution is constrained 

698 nBasisKernels = spatialKernel.getNBasisKernels() 

699 nKernelTerms = spatialKernel.getNSpatialParameters() 

700 if nKernelTerms == 0: # order 0 

701 nKernelTerms = 1 

702 

703 # Not fit for 

704 nBgTerms = spatialBg.getNParameters() 

705 if nBgTerms == 1: 

706 if spatialBg.getParameters()[0] == 0.0: 

707 nBgTerms = 0 

708 

709 nGood = 0 

710 nBad = 0 

711 nTot = 0 

712 for cell in kernelCellSet.getCellList(): 

713 for cand in cell.begin(False): # False = include bad candidates 

714 nTot += 1 

715 if cand.getStatus() == afwMath.SpatialCellCandidate.GOOD: 

716 nGood += 1 

717 if cand.getStatus() == afwMath.SpatialCellCandidate.BAD: 

718 nBad += 1 

719 

720 self.log.info("Doing stats of kernel candidates used in the spatial fit.") 

721 

722 # Counting statistics 

723 if nBad > 2*nGood: 

724 self.log.warning("Many more candidates rejected than accepted; %d total, %d rejected, %d used", 

725 nTot, nBad, nGood) 

726 else: 

727 self.log.info("%d candidates total, %d rejected, %d used", nTot, nBad, nGood) 

728 

729 # Some judgements on the quality of the spatial models 

730 if nGood < nKernelTerms: 

731 self.log.warning("Spatial kernel model underconstrained; %d candidates, %d terms, %d bases", 

732 nGood, nKernelTerms, nBasisKernels) 

733 self.log.warning("Consider lowering the spatial order") 

734 elif nGood <= 2*nKernelTerms: 

735 self.log.warning("Spatial kernel model poorly constrained; %d candidates, %d terms, %d bases", 

736 nGood, nKernelTerms, nBasisKernels) 

737 self.log.warning("Consider lowering the spatial order") 

738 else: 

739 self.log.info("Spatial kernel model well constrained; %d candidates, %d terms, %d bases", 

740 nGood, nKernelTerms, nBasisKernels) 

741 

742 if nGood < nBgTerms: 

743 self.log.warning("Spatial background model underconstrained; %d candidates, %d terms", 

744 nGood, nBgTerms) 

745 self.log.warning("Consider lowering the spatial order") 

746 elif nGood <= 2*nBgTerms: 

747 self.log.warning("Spatial background model poorly constrained; %d candidates, %d terms", 

748 nGood, nBgTerms) 

749 self.log.warning("Consider lowering the spatial order") 

750 else: 

751 self.log.info("Spatial background model appears well constrained; %d candidates, %d terms", 

752 nGood, nBgTerms) 

753 

754 def _displayDebug(self, kernelCellSet, spatialKernel, spatialBackground): 

755 """Provide visualization of the inputs and ouputs to the Psf-matching code 

756 

757 Parameters 

758 ---------- 

759 kernelCellSet : `lsst.afw.math.SpatialCellSet` 

760 The SpatialCellSet used in determining the matching kernel and background 

761 spatialKernel : `lsst.afw.math.LinearCombinationKernel` 

762 Spatially varying Psf-matching kernel 

763 spatialBackground : `lsst.afw.math.Function2D` 

764 Spatially varying background-matching function 

765 """ 

766 import lsstDebug 

767 displayCandidates = lsstDebug.Info(__name__).displayCandidates 

768 displayKernelBasis = lsstDebug.Info(__name__).displayKernelBasis 

769 displayKernelMosaic = lsstDebug.Info(__name__).displayKernelMosaic 

770 plotKernelSpatialModel = lsstDebug.Info(__name__).plotKernelSpatialModel 

771 plotKernelCoefficients = lsstDebug.Info(__name__).plotKernelCoefficients 

772 showBadCandidates = lsstDebug.Info(__name__).showBadCandidates 

773 maskTransparency = lsstDebug.Info(__name__).maskTransparency 

774 if not maskTransparency: 

775 maskTransparency = 0 

776 afwDisplay.setDefaultMaskTransparency(maskTransparency) 

777 

778 if displayCandidates: 

779 diutils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground, 

780 frame=lsstDebug.frame, 

781 showBadCandidates=showBadCandidates) 

782 lsstDebug.frame += 1 

783 diutils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground, 

784 frame=lsstDebug.frame, 

785 showBadCandidates=showBadCandidates, 

786 kernels=True) 

787 lsstDebug.frame += 1 

788 diutils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground, 

789 frame=lsstDebug.frame, 

790 showBadCandidates=showBadCandidates, 

791 resids=True) 

792 lsstDebug.frame += 1 

793 

794 if displayKernelBasis: 

795 diutils.showKernelBasis(spatialKernel, frame=lsstDebug.frame) 

796 lsstDebug.frame += 1 

797 

798 if displayKernelMosaic: 

799 diutils.showKernelMosaic(kernelCellSet.getBBox(), spatialKernel, frame=lsstDebug.frame) 

800 lsstDebug.frame += 1 

801 

802 if plotKernelSpatialModel: 

803 diutils.plotKernelSpatialModel(spatialKernel, kernelCellSet, showBadCandidates=showBadCandidates) 

804 

805 if plotKernelCoefficients: 

806 diutils.plotKernelCoefficients(spatialKernel, kernelCellSet) 

807 

808 def _createPcaBasis(self, kernelCellSet, nStarPerCell, ps): 

809 """Create Principal Component basis 

810 

811 If a principal component analysis is requested, typically when using a delta function basis, 

812 perform the PCA here and return a new basis list containing the new principal components. 

813 

814 Parameters 

815 ---------- 

816 kernelCellSet : `lsst.afw.math.SpatialCellSet` 

817 a SpatialCellSet containing KernelCandidates, from which components are derived 

818 nStarPerCell : `int` 

819 the number of stars per cell to visit when doing the PCA 

820 ps : `lsst.daf.base.PropertySet` 

821 input property set controlling the single kernel visitor 

822 

823 Returns 

824 ------- 

825 nRejectedPca : `int` 

826 number of KernelCandidates rejected during PCA loop 

827 spatialBasisList : `list` of `lsst.afw.math.kernel.FixedKernel` 

828 basis list containing the principal shapes as Kernels 

829 

830 Raises 

831 ------ 

832 RuntimeError 

833 If the Eigenvalues sum to zero. 

834 """ 

835 nComponents = self.kConfig.numPrincipalComponents 

836 imagePca = diffimLib.KernelPcaD() 

837 importStarVisitor = diffimLib.KernelPcaVisitorF(imagePca) 

838 kernelCellSet.visitCandidates(importStarVisitor, nStarPerCell) 

839 if self.kConfig.subtractMeanForPca: 

840 importStarVisitor.subtractMean() 

841 imagePca.analyze() 

842 

843 eigenValues = imagePca.getEigenValues() 

844 pcaBasisList = importStarVisitor.getEigenKernels() 

845 

846 eSum = np.sum(eigenValues) 

847 if eSum == 0.0: 

848 raise RuntimeError("Eigenvalues sum to zero") 

849 trace_logger = getTraceLogger(self.log.getChild("_solve"), 5) 

850 for j in range(len(eigenValues)): 

851 trace_logger.debug("Eigenvalue %d : %f (%f)", j, eigenValues[j], eigenValues[j]/eSum) 

852 

853 nToUse = min(nComponents, len(eigenValues)) 

854 trimBasisList = [] 

855 for j in range(nToUse): 

856 # Check for NaNs? 

857 kimage = afwImage.ImageD(pcaBasisList[j].getDimensions()) 

858 pcaBasisList[j].computeImage(kimage, False) 

859 if not (True in np.isnan(kimage.getArray())): 

860 trimBasisList.append(pcaBasisList[j]) 

861 

862 # Put all the power in the first kernel, which will not vary spatially 

863 spatialBasisList = diffimLib.renormalizeKernelList(trimBasisList) 

864 

865 # New Kernel visitor for this new basis list (no regularization explicitly) 

866 singlekvPca = diffimLib.BuildSingleKernelVisitorF(spatialBasisList, ps) 

867 singlekvPca.setSkipBuilt(False) 

868 kernelCellSet.visitCandidates(singlekvPca, nStarPerCell) 

869 singlekvPca.setSkipBuilt(True) 

870 nRejectedPca = singlekvPca.getNRejected() 

871 

872 return nRejectedPca, spatialBasisList 

873 

874 @abc.abstractmethod 

875 def _buildCellSet(self, *args): 

876 """Fill a SpatialCellSet with KernelCandidates for the Psf-matching process; 

877 override in derived classes""" 

878 return 

879 

880 @timeMethod 

881 def _solve(self, kernelCellSet, basisList, returnOnExcept=False): 

882 """Solve for the PSF matching kernel 

883 

884 Parameters 

885 ---------- 

886 kernelCellSet : `lsst.afw.math.SpatialCellSet` 

887 a SpatialCellSet to use in determining the matching kernel 

888 (typically as provided by _buildCellSet) 

889 basisList : `list` of `lsst.afw.math.kernel.FixedKernel` 

890 list of Kernels to be used in the decomposition of the spatially varying kernel 

891 (typically as provided by makeKernelBasisList) 

892 returnOnExcept : `bool`, optional 

893 if True then return (None, None) if an error occurs, else raise the exception 

894 

895 Returns 

896 ------- 

897 psfMatchingKernel : `lsst.afw.math.LinearCombinationKernel` 

898 Spatially varying Psf-matching kernel 

899 backgroundModel : `lsst.afw.math.Function2D` 

900 Spatially varying background-matching function 

901 

902 Raises 

903 ------ 

904 RuntimeError : 

905 If unable to determine PSF matching kernel and ``returnOnExcept==False``. 

906 """ 

907 

908 import lsstDebug 

909 display = lsstDebug.Info(__name__).display 

910 

911 maxSpatialIterations = self.kConfig.maxSpatialIterations 

912 nStarPerCell = self.kConfig.nStarPerCell 

913 usePcaForSpatialKernel = self.kConfig.usePcaForSpatialKernel 

914 

915 # Visitor for the single kernel fit 

916 ps = pexConfig.makePropertySet(self.kConfig) 

917 if self.useRegularization: 

918 singlekv = diffimLib.BuildSingleKernelVisitorF(basisList, ps, self.hMat) 

919 else: 

920 singlekv = diffimLib.BuildSingleKernelVisitorF(basisList, ps) 

921 

922 # Visitor for the kernel sum rejection 

923 ksv = diffimLib.KernelSumVisitorF(ps) 

924 

925 # Main loop 

926 trace_loggers = [getTraceLogger(self.log.getChild("_solve"), i) for i in range(5)] 

927 t0 = time.time() 

928 try: 

929 totalIterations = 0 

930 thisIteration = 0 

931 while (thisIteration < maxSpatialIterations): 

932 

933 # Make sure there are no uninitialized candidates as active occupants of Cell 

934 nRejectedSkf = -1 

935 while (nRejectedSkf != 0): 

936 trace_loggers[1].debug("Building single kernels...") 

937 kernelCellSet.visitCandidates(singlekv, nStarPerCell) 

938 nRejectedSkf = singlekv.getNRejected() 

939 trace_loggers[1].debug( 

940 "Iteration %d, rejected %d candidates due to initial kernel fit", 

941 thisIteration, nRejectedSkf 

942 ) 

943 

944 # Reject outliers in kernel sum 

945 ksv.resetKernelSum() 

946 ksv.setMode(diffimLib.KernelSumVisitorF.AGGREGATE) 

947 kernelCellSet.visitCandidates(ksv, nStarPerCell) 

948 ksv.processKsumDistribution() 

949 ksv.setMode(diffimLib.KernelSumVisitorF.REJECT) 

950 kernelCellSet.visitCandidates(ksv, nStarPerCell) 

951 

952 nRejectedKsum = ksv.getNRejected() 

953 trace_loggers[1].debug( 

954 "Iteration %d, rejected %d candidates due to kernel sum", 

955 thisIteration, nRejectedKsum 

956 ) 

957 

958 # Do we jump back to the top without incrementing thisIteration? 

959 if nRejectedKsum > 0: 

960 totalIterations += 1 

961 continue 

962 

963 # At this stage we can either apply the spatial fit to 

964 # the kernels, or we run a PCA, use these as a *new* 

965 # basis set with lower dimensionality, and then apply 

966 # the spatial fit to these kernels 

967 

968 if (usePcaForSpatialKernel): 

969 trace_loggers[0].debug("Building Pca basis") 

970 

971 nRejectedPca, spatialBasisList = self._createPcaBasis(kernelCellSet, nStarPerCell, ps) 

972 trace_loggers[1].debug( 

973 "Iteration %d, rejected %d candidates due to Pca kernel fit", 

974 thisIteration, nRejectedPca 

975 ) 

976 

977 # We don't want to continue on (yet) with the 

978 # spatial modeling, because we have bad objects 

979 # contributing to the Pca basis. We basically 

980 # need to restart from the beginning of this loop, 

981 # since the cell-mates of those objects that were 

982 # rejected need their original Kernels built by 

983 # singleKernelFitter. 

984 

985 # Don't count against thisIteration 

986 if (nRejectedPca > 0): 

987 totalIterations += 1 

988 continue 

989 else: 

990 spatialBasisList = basisList 

991 

992 # We have gotten on to the spatial modeling part 

993 regionBBox = kernelCellSet.getBBox() 

994 spatialkv = diffimLib.BuildSpatialKernelVisitorF(spatialBasisList, regionBBox, ps) 

995 kernelCellSet.visitCandidates(spatialkv, nStarPerCell) 

996 spatialkv.solveLinearEquation() 

997 trace_loggers[2].debug("Spatial kernel built with %d candidates", spatialkv.getNCandidates()) 

998 spatialKernel, spatialBackground = spatialkv.getSolutionPair() 

999 

1000 # Check the quality of the spatial fit (look at residuals) 

1001 assesskv = diffimLib.AssessSpatialKernelVisitorF(spatialKernel, spatialBackground, ps) 

1002 kernelCellSet.visitCandidates(assesskv, nStarPerCell) 

1003 nRejectedSpatial = assesskv.getNRejected() 

1004 nGoodSpatial = assesskv.getNGood() 

1005 trace_loggers[1].debug( 

1006 "Iteration %d, rejected %d candidates due to spatial kernel fit", 

1007 thisIteration, nRejectedSpatial 

1008 ) 

1009 trace_loggers[1].debug("%d candidates used in fit", nGoodSpatial) 

1010 

1011 # If only nGoodSpatial == 0, might be other candidates in the cells 

1012 if nGoodSpatial == 0 and nRejectedSpatial == 0: 

1013 raise RuntimeError("No kernel candidates for spatial fit") 

1014 

1015 if nRejectedSpatial == 0: 

1016 # Nothing rejected, finished with spatial fit 

1017 break 

1018 

1019 # Otherwise, iterate on... 

1020 thisIteration += 1 

1021 

1022 # Final fit if above did not converge 

1023 if (nRejectedSpatial > 0) and (thisIteration == maxSpatialIterations): 

1024 trace_loggers[1].debug("Final spatial fit") 

1025 if (usePcaForSpatialKernel): 

1026 nRejectedPca, spatialBasisList = self._createPcaBasis(kernelCellSet, nStarPerCell, ps) 

1027 regionBBox = kernelCellSet.getBBox() 

1028 spatialkv = diffimLib.BuildSpatialKernelVisitorF(spatialBasisList, regionBBox, ps) 

1029 kernelCellSet.visitCandidates(spatialkv, nStarPerCell) 

1030 spatialkv.solveLinearEquation() 

1031 trace_loggers[2].debug("Spatial kernel built with %d candidates", spatialkv.getNCandidates()) 

1032 spatialKernel, spatialBackground = spatialkv.getSolutionPair() 

1033 

1034 spatialSolution = spatialkv.getKernelSolution() 

1035 

1036 except Exception as e: 

1037 self.log.error("ERROR: Unable to calculate psf matching kernel") 

1038 

1039 trace_loggers[1].debug("%s", e) 

1040 raise e 

1041 

1042 t1 = time.time() 

1043 trace_loggers[0].debug("Total time to compute the spatial kernel : %.2f s", (t1 - t0)) 

1044 

1045 if display: 

1046 self._displayDebug(kernelCellSet, spatialKernel, spatialBackground) 

1047 

1048 self._diagnostic(kernelCellSet, spatialSolution, spatialKernel, spatialBackground) 

1049 

1050 return spatialSolution, spatialKernel, spatialBackground 

1051 

1052 

1053PsfMatch = PsfMatchTask