Coverage for python/lsst/ip/diffim/psfMatch.py: 30%

294 statements  

« prev     ^ index     » next       coverage.py v7.4.3, created at 2024-03-05 13:22 +0000

1# This file is part of ip_diffim. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["DetectionConfig", "PsfMatchConfig", "PsfMatchConfigAL", "PsfMatchConfigDF", "PsfMatchTask"] 

23 

24import abc 

25import time 

26 

27import numpy as np 

28 

29import lsst.afw.image as afwImage 

30import lsst.pex.config as pexConfig 

31import lsst.afw.math as afwMath 

32import lsst.afw.display as afwDisplay 

33import lsst.pipe.base as pipeBase 

34from lsst.meas.algorithms import SubtractBackgroundConfig 

35from lsst.utils.logging import getTraceLogger 

36from lsst.utils.timer import timeMethod 

37from . import utils as diutils 

38from . import diffimLib 

39 

40 

41# Remove this class on DM-42980. 

42# Not deprecated-decorated to prevent excessive warnings when using PsfMatchTask. 

43class DetectionConfig(pexConfig.Config): 

44 """Configuration for detecting sources on images for building a 

45 PSF-matching kernel 

46 

47 Configuration for turning detected lsst.afw.detection.FootPrints into an 

48 acceptable (unmasked, high signal-to-noise, not too large or not too small) 

49 list of `lsst.ip.diffim.KernelSources` that are used to build the 

50 Psf-matching kernel""" 

51 

52 detThreshold = pexConfig.Field( 52 ↛ exitline 52 didn't jump to the function exit

53 deprecated="This field is no longer used and will be removed after v27.", 

54 dtype=float, 

55 doc="Value of footprint detection threshold", 

56 default=10.0, 

57 check=lambda x: x >= 3.0 

58 ) 

59 detThresholdType = pexConfig.ChoiceField( 

60 deprecated="This field is no longer used and will be removed after v27.", 

61 dtype=str, 

62 doc="Type of detection threshold", 

63 default="pixel_stdev", 

64 allowed={ 

65 "value": "Use counts as the detection threshold type", 

66 "stdev": "Use standard deviation of image plane", 

67 "variance": "Use variance of image plane", 

68 "pixel_stdev": "Use stdev derived from variance plane" 

69 } 

70 ) 

71 detOnTemplate = pexConfig.Field( 

72 dtype=bool, 

73 doc="""If true run detection on the template (image to convolve); 

74 if false run detection on the science image""", 

75 deprecated="This field is no longer used and will be removed after v27.", 

76 default=True 

77 ) 

78 badMaskPlanes = pexConfig.ListField( 

79 dtype=str, 

80 doc="""Mask planes that lead to an invalid detection. 

81 Options: NO_DATA EDGE SAT BAD CR INTRP""", 

82 default=("NO_DATA", "EDGE", "SAT") 

83 ) 

84 fpNpixMin = pexConfig.Field( 84 ↛ exitline 84 didn't jump to the function exit

85 dtype=int, 

86 doc="Minimum number of pixels in an acceptable Footprint", 

87 default=5, 

88 deprecated="This field is no longer used and will be removed after v27.", 

89 check=lambda x: x >= 5 

90 ) 

91 fpNpixMax = pexConfig.Field( 91 ↛ exitline 91 didn't jump to the function exit

92 dtype=int, 

93 doc="""Maximum number of pixels in an acceptable Footprint; 

94 too big and the subsequent convolutions become unwieldy""", 

95 default=500, 

96 deprecated="This field is no longer used and will be removed after v27.", 

97 check=lambda x: x <= 500 

98 ) 

99 fpGrowKernelScaling = pexConfig.Field( 99 ↛ exitline 99 didn't jump to the function exit

100 dtype=float, 

101 doc="""If config.scaleByFwhm, grow the footprint based on 

102 the final kernelSize. Each footprint will be 

103 2*fpGrowKernelScaling*kernelSize x 

104 2*fpGrowKernelScaling*kernelSize. With the value 

105 of 1.0, the remaining pixels in each KernelCandiate 

106 after convolution by the basis functions will be 

107 equal to the kernel size itself.""", 

108 default=1.0, 

109 deprecated="This field is no longer used and will be removed after v27.", 

110 check=lambda x: x >= 1.0 

111 ) 

112 fpGrowPix = pexConfig.Field( 112 ↛ exitline 112 didn't jump to the function exit

113 dtype=int, 

114 doc="""Growing radius (in pixels) for each raw detection 

115 footprint. The smaller the faster; however the 

116 kernel sum does not converge if the stamp is too 

117 small; and the kernel is not constrained at all if 

118 the stamp is the size of the kernel. The grown stamp 

119 is 2 * fpGrowPix pixels larger in each dimension. 

120 This is overridden by fpGrowKernelScaling if scaleByFwhm""", 

121 default=30, 

122 deprecated="This field is no longer used and will be removed after v27.", 

123 check=lambda x: x >= 10 

124 ) 

125 scaleByFwhm = pexConfig.Field( 

126 dtype=bool, 

127 doc="Scale fpGrowPix by input Fwhm?", 

128 deprecated="This field is no longer used and will be removed after v27.", 

129 default=True, 

130 ) 

131 

132 

133class PsfMatchConfig(pexConfig.Config): 

134 """Base configuration for Psf-matching 

135 

136 The base configuration of the Psf-matching kernel, and of the warping, detection, 

137 and background modeling subTasks.""" 

138 

139 warpingConfig = pexConfig.ConfigField("Config for warping exposures to a common alignment", 

140 afwMath.WarperConfig) 

141 # Remove this field on DM-42980. 

142 detectionConfig = pexConfig.ConfigField( 

143 "Controlling the detection of sources for kernel building", 

144 DetectionConfig, 

145 deprecated="This field is no longer used and will be removed after v27.") 

146 afwBackgroundConfig = pexConfig.ConfigField("Controlling the Afw background fitting", 

147 SubtractBackgroundConfig) 

148 

149 useAfwBackground = pexConfig.Field( 

150 dtype=bool, 

151 doc="Use afw background subtraction instead of ip_diffim", 

152 default=False, 

153 ) 

154 fitForBackground = pexConfig.Field( 

155 dtype=bool, 

156 doc="Include terms (including kernel cross terms) for background in ip_diffim", 

157 default=False, 

158 ) 

159 kernelBasisSet = pexConfig.ChoiceField( 

160 dtype=str, 

161 doc="Type of basis set for PSF matching kernel.", 

162 default="alard-lupton", 

163 allowed={ 

164 "alard-lupton": """Alard-Lupton sum-of-gaussians basis set, 

165 * The first term has no spatial variation 

166 * The kernel sum is conserved 

167 * You may want to turn off 'usePcaForSpatialKernel'""", 

168 "delta-function": """Delta-function kernel basis set, 

169 * You may enable the option useRegularization 

170 * You should seriously consider usePcaForSpatialKernel, which will also 

171 enable kernel sum conservation for the delta function kernels""" 

172 } 

173 ) 

174 kernelSize = pexConfig.Field( 

175 dtype=int, 

176 doc="""Number of rows/columns in the convolution kernel; should be odd-valued. 

177 Modified by kernelSizeFwhmScaling if scaleByFwhm = true""", 

178 default=21, 

179 ) 

180 scaleByFwhm = pexConfig.Field( 

181 dtype=bool, 

182 doc="Scale kernelSize, alardGaussians by input Fwhm", 

183 default=True, 

184 ) 

185 kernelSizeFwhmScaling = pexConfig.Field( 185 ↛ exitline 185 didn't jump to the function exit

186 dtype=float, 

187 doc="Multiplier of the largest AL Gaussian basis sigma to get the kernel bbox (pixel) size.", 

188 default=6.0, 

189 check=lambda x: x >= 1.0 

190 ) 

191 kernelSizeMin = pexConfig.Field( 

192 dtype=int, 

193 doc="Minimum kernel bbox (pixel) size.", 

194 default=21, 

195 ) 

196 kernelSizeMax = pexConfig.Field( 

197 dtype=int, 

198 doc="Maximum kernel bbox (pixel) size.", 

199 default=35, 

200 ) 

201 spatialModelType = pexConfig.ChoiceField( 

202 dtype=str, 

203 doc="Type of spatial functions for kernel and background", 

204 default="chebyshev1", 

205 allowed={ 

206 "chebyshev1": "Chebyshev polynomial of the first kind", 

207 "polynomial": "Standard x,y polynomial", 

208 } 

209 ) 

210 spatialKernelOrder = pexConfig.Field( 210 ↛ exitline 210 didn't jump to the function exit

211 dtype=int, 

212 doc="Spatial order of convolution kernel variation", 

213 default=2, 

214 check=lambda x: x >= 0 

215 ) 

216 spatialBgOrder = pexConfig.Field( 216 ↛ exitline 216 didn't jump to the function exit

217 dtype=int, 

218 doc="Spatial order of differential background variation", 

219 default=1, 

220 check=lambda x: x >= 0 

221 ) 

222 sizeCellX = pexConfig.Field( 222 ↛ exitline 222 didn't jump to the function exit

223 dtype=int, 

224 doc="Size (rows) in pixels of each SpatialCell for spatial modeling", 

225 default=128, 

226 check=lambda x: x >= 32 

227 ) 

228 sizeCellY = pexConfig.Field( 228 ↛ exitline 228 didn't jump to the function exit

229 dtype=int, 

230 doc="Size (columns) in pixels of each SpatialCell for spatial modeling", 

231 default=128, 

232 check=lambda x: x >= 32 

233 ) 

234 nStarPerCell = pexConfig.Field( 

235 dtype=int, 

236 doc="Maximum number of KernelCandidates in each SpatialCell to use in the spatial fitting. " 

237 "Set to -1 to use all candidates in each cell.", 

238 default=5, 

239 ) 

240 maxSpatialIterations = pexConfig.Field( 240 ↛ exitline 240 didn't jump to the function exit

241 dtype=int, 

242 doc="Maximum number of iterations for rejecting bad KernelCandidates in spatial fitting", 

243 default=3, 

244 check=lambda x: x >= 1 and x <= 5 

245 ) 

246 usePcaForSpatialKernel = pexConfig.Field( 

247 dtype=bool, 

248 doc="""Use Pca to reduce the dimensionality of the kernel basis sets. 

249 This is particularly useful for delta-function kernels. 

250 Functionally, after all Cells have their raw kernels determined, we run 

251 a Pca on these Kernels, re-fit the Cells using the eigenKernels and then 

252 fit those for spatial variation using the same technique as for Alard-Lupton kernels. 

253 If this option is used, the first term will have no spatial variation and the 

254 kernel sum will be conserved.""", 

255 default=False, 

256 ) 

257 subtractMeanForPca = pexConfig.Field( 

258 dtype=bool, 

259 doc="Subtract off the mean feature before doing the Pca", 

260 default=True, 

261 ) 

262 numPrincipalComponents = pexConfig.Field( 262 ↛ exitline 262 didn't jump to the function exit

263 dtype=int, 

264 doc="""Number of principal components to use for Pca basis, including the 

265 mean kernel if requested.""", 

266 default=5, 

267 check=lambda x: x >= 3 

268 ) 

269 singleKernelClipping = pexConfig.Field( 

270 dtype=bool, 

271 doc="Do sigma clipping on each raw kernel candidate", 

272 default=True, 

273 ) 

274 kernelSumClipping = pexConfig.Field( 

275 dtype=bool, 

276 doc="Do sigma clipping on the ensemble of kernel sums", 

277 default=True, 

278 ) 

279 spatialKernelClipping = pexConfig.Field( 

280 dtype=bool, 

281 doc="Do sigma clipping after building the spatial model", 

282 default=True, 

283 ) 

284 checkConditionNumber = pexConfig.Field( 

285 dtype=bool, 

286 doc="""Test for maximum condition number when inverting a kernel matrix. 

287 Anything above maxConditionNumber is not used and the candidate is set as BAD. 

288 Also used to truncate inverse matrix in estimateBiasedRisk. However, 

289 if you are doing any deconvolution you will want to turn this off, or use 

290 a large maxConditionNumber""", 

291 default=False, 

292 ) 

293 badMaskPlanes = pexConfig.ListField( 

294 dtype=str, 

295 doc="""Mask planes to ignore when calculating diffim statistics 

296 Options: NO_DATA EDGE SAT BAD CR INTRP""", 

297 default=("NO_DATA", "EDGE", "SAT") 

298 ) 

299 candidateResidualMeanMax = pexConfig.Field( 299 ↛ exitline 299 didn't jump to the function exit

300 dtype=float, 

301 doc="""Rejects KernelCandidates yielding bad difference image quality. 

302 Used by BuildSingleKernelVisitor, AssessSpatialKernelVisitor. 

303 Represents average over pixels of (image/sqrt(variance)).""", 

304 default=0.25, 

305 check=lambda x: x >= 0.0 

306 ) 

307 candidateResidualStdMax = pexConfig.Field( 307 ↛ exitline 307 didn't jump to the function exit

308 dtype=float, 

309 doc="""Rejects KernelCandidates yielding bad difference image quality. 

310 Used by BuildSingleKernelVisitor, AssessSpatialKernelVisitor. 

311 Represents stddev over pixels of (image/sqrt(variance)).""", 

312 default=1.50, 

313 check=lambda x: x >= 0.0 

314 ) 

315 useCoreStats = pexConfig.Field( 

316 dtype=bool, 

317 doc="""Use the core of the footprint for the quality statistics, instead of the entire footprint. 

318 WARNING: if there is deconvolution we probably will need to turn this off""", 

319 default=False, 

320 ) 

321 candidateCoreRadius = pexConfig.Field( 321 ↛ exitline 321 didn't jump to the function exit

322 dtype=int, 

323 doc="""Radius for calculation of stats in 'core' of KernelCandidate diffim. 

324 Total number of pixels used will be (2*radius)**2. 

325 This is used both for 'core' diffim quality as well as ranking of 

326 KernelCandidates by their total flux in this core""", 

327 default=3, 

328 check=lambda x: x >= 1 

329 ) 

330 maxKsumSigma = pexConfig.Field( 330 ↛ exitline 330 didn't jump to the function exit

331 dtype=float, 

332 doc="""Maximum allowed sigma for outliers from kernel sum distribution. 

333 Used to reject variable objects from the kernel model""", 

334 default=3.0, 

335 check=lambda x: x >= 0.0 

336 ) 

337 maxConditionNumber = pexConfig.Field( 337 ↛ exitline 337 didn't jump to the function exit

338 dtype=float, 

339 doc="Maximum condition number for a well conditioned matrix", 

340 default=5.0e7, 

341 check=lambda x: x >= 0.0 

342 ) 

343 conditionNumberType = pexConfig.ChoiceField( 

344 dtype=str, 

345 doc="Use singular values (SVD) or eigen values (EIGENVALUE) to determine condition number", 

346 default="EIGENVALUE", 

347 allowed={ 

348 "SVD": "Use singular values", 

349 "EIGENVALUE": "Use eigen values (faster)", 

350 } 

351 ) 

352 maxSpatialConditionNumber = pexConfig.Field( 352 ↛ exitline 352 didn't jump to the function exit

353 dtype=float, 

354 doc="Maximum condition number for a well conditioned spatial matrix", 

355 default=1.0e10, 

356 check=lambda x: x >= 0.0 

357 ) 

358 iterateSingleKernel = pexConfig.Field( 

359 dtype=bool, 

360 doc="""Remake KernelCandidate using better variance estimate after first pass? 

361 Primarily useful when convolving a single-depth image, otherwise not necessary.""", 

362 default=False, 

363 ) 

364 constantVarianceWeighting = pexConfig.Field( 

365 dtype=bool, 

366 doc="""Use constant variance weighting in single kernel fitting? 

367 In some cases this is better for bright star residuals.""", 

368 default=True, 

369 ) 

370 calculateKernelUncertainty = pexConfig.Field( 

371 dtype=bool, 

372 doc="""Calculate kernel and background uncertainties for each kernel candidate? 

373 This comes from the inverse of the covariance matrix. 

374 Warning: regularization can cause problems for this step.""", 

375 default=False, 

376 ) 

377 useBicForKernelBasis = pexConfig.Field( 

378 dtype=bool, 

379 doc="""Use Bayesian Information Criterion to select the number of bases going into the kernel""", 

380 default=False, 

381 ) 

382 

383 

384class PsfMatchConfigAL(PsfMatchConfig): 

385 """The parameters specific to the "Alard-Lupton" (sum-of-Gaussian) Psf-matching basis""" 

386 

387 def setDefaults(self): 

388 PsfMatchConfig.setDefaults(self) 

389 self.kernelBasisSet = "alard-lupton" 

390 self.maxConditionNumber = 5.0e7 

391 

392 alardNGauss = pexConfig.Field( 392 ↛ exitline 392 didn't jump to the function exit

393 dtype=int, 

394 doc="Number of base Gaussians in alard-lupton kernel basis function generation.", 

395 default=3, 

396 check=lambda x: x >= 1 

397 ) 

398 alardDegGauss = pexConfig.ListField( 

399 dtype=int, 

400 doc="Polynomial order of spatial modification of base Gaussians. " 

401 "List length must be `alardNGauss`.", 

402 default=(4, 2, 2), 

403 ) 

404 alardSigGauss = pexConfig.ListField( 

405 dtype=float, 

406 doc="Default sigma values in pixels of base Gaussians. " 

407 "List length must be `alardNGauss`.", 

408 default=(0.7, 1.5, 3.0), 

409 ) 

410 alardGaussBeta = pexConfig.Field( 410 ↛ exitline 410 didn't jump to the function exit

411 dtype=float, 

412 doc="Used if `scaleByFwhm==True`, scaling multiplier of base " 

413 "Gaussian sigmas for automated sigma determination", 

414 default=2.0, 

415 check=lambda x: x >= 0.0, 

416 ) 

417 alardMinSig = pexConfig.Field( 417 ↛ exitline 417 didn't jump to the function exit

418 dtype=float, 

419 doc="Used if `scaleByFwhm==True`, minimum sigma (pixels) for base Gaussians", 

420 default=0.7, 

421 check=lambda x: x >= 0.25 

422 ) 

423 alardDegGaussDeconv = pexConfig.Field( 423 ↛ exitline 423 didn't jump to the function exit

424 dtype=int, 

425 doc="Used if `scaleByFwhm==True`, degree of spatial modification of ALL base Gaussians " 

426 "in AL basis during deconvolution", 

427 default=3, 

428 check=lambda x: x >= 1 

429 ) 

430 alardMinSigDeconv = pexConfig.Field( 430 ↛ exitline 430 didn't jump to the function exit

431 dtype=float, 

432 doc="Used if `scaleByFwhm==True`, minimum sigma (pixels) for base Gaussians during deconvolution; " 

433 "make smaller than `alardMinSig` as this is only indirectly used", 

434 default=0.4, 

435 check=lambda x: x >= 0.25 

436 ) 

437 alardNGaussDeconv = pexConfig.Field( 437 ↛ exitline 437 didn't jump to the function exit

438 dtype=int, 

439 doc="Used if `scaleByFwhm==True`, number of base Gaussians in AL basis during deconvolution", 

440 default=3, 

441 check=lambda x: x >= 1 

442 ) 

443 

444 

445class PsfMatchConfigDF(PsfMatchConfig): 

446 """The parameters specific to the delta-function (one basis per-pixel) Psf-matching basis""" 

447 

448 def setDefaults(self): 

449 PsfMatchConfig.setDefaults(self) 

450 self.kernelBasisSet = "delta-function" 

451 self.maxConditionNumber = 5.0e6 

452 self.usePcaForSpatialKernel = True 

453 self.subtractMeanForPca = True 

454 self.useBicForKernelBasis = False 

455 

456 useRegularization = pexConfig.Field( 

457 dtype=bool, 

458 doc="Use regularization to smooth the delta function kernels", 

459 default=True, 

460 ) 

461 regularizationType = pexConfig.ChoiceField( 

462 dtype=str, 

463 doc="Type of regularization.", 

464 default="centralDifference", 

465 allowed={ 

466 "centralDifference": "Penalize second derivative using 2-D stencil of central finite difference", 

467 "forwardDifference": "Penalize first, second, third derivatives using forward finite differeces" 

468 } 

469 ) 

470 centralRegularizationStencil = pexConfig.ChoiceField( 

471 dtype=int, 

472 doc="Type of stencil to approximate central derivative (for centralDifference only)", 

473 default=9, 

474 allowed={ 

475 5: "5-point stencil including only adjacent-in-x,y elements", 

476 9: "9-point stencil including diagonal elements" 

477 } 

478 ) 

479 forwardRegularizationOrders = pexConfig.ListField( 479 ↛ exitline 479 didn't jump to the function exit

480 dtype=int, 

481 doc="Array showing which order derivatives to penalize (for forwardDifference only)", 

482 default=(1, 2), 

483 itemCheck=lambda x: (x > 0) and (x < 4) 

484 ) 

485 regularizationBorderPenalty = pexConfig.Field( 485 ↛ exitline 485 didn't jump to the function exit

486 dtype=float, 

487 doc="Value of the penalty for kernel border pixels", 

488 default=3.0, 

489 check=lambda x: x >= 0.0 

490 ) 

491 lambdaType = pexConfig.ChoiceField( 

492 dtype=str, 

493 doc="How to choose the value of the regularization strength", 

494 default="absolute", 

495 allowed={ 

496 "absolute": "Use lambdaValue as the value of regularization strength", 

497 "relative": "Use lambdaValue as fraction of the default regularization strength (N.R. 18.5.8)", 

498 "minimizeBiasedRisk": "Minimize biased risk estimate", 

499 "minimizeUnbiasedRisk": "Minimize unbiased risk estimate", 

500 } 

501 ) 

502 lambdaValue = pexConfig.Field( 

503 dtype=float, 

504 doc="Value used for absolute determinations of regularization strength", 

505 default=0.2, 

506 ) 

507 lambdaScaling = pexConfig.Field( 

508 dtype=float, 

509 doc="Fraction of the default lambda strength (N.R. 18.5.8) to use. 1e-4 or 1e-5", 

510 default=1e-4, 

511 ) 

512 lambdaStepType = pexConfig.ChoiceField( 

513 dtype=str, 

514 doc="""If a scan through lambda is needed (minimizeBiasedRisk, minimizeUnbiasedRisk), 

515 use log or linear steps""", 

516 default="log", 

517 allowed={ 

518 "log": "Step in log intervals; e.g. lambdaMin, lambdaMax, lambdaStep = -1.0, 2.0, 0.1", 

519 "linear": "Step in linear intervals; e.g. lambdaMin, lambdaMax, lambdaStep = 0.1, 100, 0.1", 

520 } 

521 ) 

522 lambdaMin = pexConfig.Field( 

523 dtype=float, 

524 doc="""If scan through lambda needed (minimizeBiasedRisk, minimizeUnbiasedRisk), 

525 start at this value. If lambdaStepType = log:linear, suggest -1:0.1""", 

526 default=-1.0, 

527 ) 

528 lambdaMax = pexConfig.Field( 

529 dtype=float, 

530 doc="""If scan through lambda needed (minimizeBiasedRisk, minimizeUnbiasedRisk), 

531 stop at this value. If lambdaStepType = log:linear, suggest 2:100""", 

532 default=2.0, 

533 ) 

534 lambdaStep = pexConfig.Field( 

535 dtype=float, 

536 doc="""If scan through lambda needed (minimizeBiasedRisk, minimizeUnbiasedRisk), 

537 step in these increments. If lambdaStepType = log:linear, suggest 0.1:0.1""", 

538 default=0.1, 

539 ) 

540 

541 

542class PsfMatchTask(pipeBase.Task, abc.ABC): 

543 """Base class for Psf Matching; should not be called directly 

544 

545 Notes 

546 ----- 

547 PsfMatchTask is a base class that implements the core functionality for matching the 

548 Psfs of two images using a spatially varying Psf-matching `lsst.afw.math.LinearCombinationKernel`. 

549 The Task requires the user to provide an instance of an `lsst.afw.math.SpatialCellSet`, 

550 filled with `lsst.ip.diffim.KernelCandidate` instances, and a list of `lsst.afw.math.Kernels` 

551 of basis shapes that will be used for the decomposition. If requested, the Task 

552 also performs background matching and returns the differential background model as an 

553 `lsst.afw.math.Kernel.SpatialFunction`. 

554 

555 **Invoking the Task** 

556 

557 As a base class, this Task is not directly invoked. However, ``run()`` methods that are 

558 implemented on derived classes will make use of the core ``_solve()`` functionality, 

559 which defines a sequence of `lsst.afw.math.CandidateVisitor` classes that iterate 

560 through the KernelCandidates, first building up a per-candidate solution and then 

561 building up a spatial model from the ensemble of candidates. Sigma clipping is 

562 performed using the mean and standard deviation of all kernel sums (to reject 

563 variable objects), on the per-candidate substamp diffim residuals 

564 (to indicate a bad choice of kernel basis shapes for that particular object), 

565 and on the substamp diffim residuals using the spatial kernel fit (to indicate a bad 

566 choice of spatial kernel order, or poor constraints on the spatial model). The 

567 ``_diagnostic()`` method logs information on the quality of the spatial fit, and also 

568 modifies the Task metadata. 

569 

570 .. list-table:: Quantities set in Metadata 

571 :header-rows: 1 

572 

573 * - Parameter 

574 - Description 

575 * - ``spatialConditionNum`` 

576 - Condition number of the spatial kernel fit 

577 * - ``spatialKernelSum`` 

578 - Kernel sum (10^{-0.4 * ``Delta``; zeropoint}) of the spatial Psf-matching kernel 

579 * - ``ALBasisNGauss`` 

580 - If using sum-of-Gaussian basis, the number of gaussians used 

581 * - ``ALBasisDegGauss`` 

582 - If using sum-of-Gaussian basis, the deg of spatial variation of the Gaussians 

583 * - ``ALBasisSigGauss`` 

584 - If using sum-of-Gaussian basis, the widths (sigma) of the Gaussians 

585 * - ``ALKernelSize`` 

586 - If using sum-of-Gaussian basis, the kernel size 

587 * - ``NFalsePositivesTotal`` 

588 - Total number of diaSources 

589 * - ``NFalsePositivesRefAssociated`` 

590 - Number of diaSources that associate with the reference catalog 

591 * - ``NFalsePositivesRefAssociated`` 

592 - Number of diaSources that associate with the source catalog 

593 * - ``NFalsePositivesUnassociated`` 

594 - Number of diaSources that are orphans 

595 * - ``metric_MEAN`` 

596 - Mean value of substamp diffim quality metrics across all KernelCandidates, 

597 for both the per-candidate (LOCAL) and SPATIAL residuals 

598 * - ``metric_MEDIAN`` 

599 - Median value of substamp diffim quality metrics across all KernelCandidates, 

600 for both the per-candidate (LOCAL) and SPATIAL residuals 

601 * - ``metric_STDEV`` 

602 - Standard deviation of substamp diffim quality metrics across all KernelCandidates, 

603 for both the per-candidate (LOCAL) and SPATIAL residuals 

604 

605 **Debug variables** 

606 

607 The ``pipetask`` command line interface supports a 

608 flag --debug to import @b debug.py from your PYTHONPATH. The relevant contents of debug.py 

609 for this Task include: 

610 

611 .. code-block:: py 

612 

613 import sys 

614 import lsstDebug 

615 def DebugInfo(name): 

616 di = lsstDebug.getInfo(name) 

617 if name == "lsst.ip.diffim.psfMatch": 

618 # enable debug output 

619 di.display = True 

620 # display mask transparency 

621 di.maskTransparency = 80 

622 # show all the candidates and residuals 

623 di.displayCandidates = True 

624 # show kernel basis functions 

625 di.displayKernelBasis = False 

626 # show kernel realized across the image 

627 di.displayKernelMosaic = True 

628 # show coefficients of spatial model 

629 di.plotKernelSpatialModel = False 

630 # show fixed and spatial coefficients and coefficient histograms 

631 di.plotKernelCoefficients = True 

632 # show the bad candidates (red) along with good (green) 

633 di.showBadCandidates = True 

634 return di 

635 lsstDebug.Info = DebugInfo 

636 lsstDebug.frame = 1 

637 

638 Note that if you want additional logging info, you may add to your scripts: 

639 

640 .. code-block:: py 

641 

642 import lsst.utils.logging as logUtils 

643 logUtils.trace_set_at("lsst.ip.diffim", 4) 

644 """ 

645 ConfigClass = PsfMatchConfig 

646 _DefaultName = "psfMatch" 

647 

648 def __init__(self, *args, **kwargs): 

649 """Create the psf-matching Task 

650 

651 Parameters 

652 ---------- 

653 *args 

654 Arguments to be passed to ``lsst.pipe.base.task.Task.__init__`` 

655 **kwargs 

656 Keyword arguments to be passed to ``lsst.pipe.base.task.Task.__init__`` 

657 

658 Notes 

659 ----- 

660 The initialization sets the Psf-matching kernel configuration using the value of 

661 self.config.kernel.active. If the kernel is requested with regularization to moderate 

662 the bias/variance tradeoff, currently only used when a delta function kernel basis 

663 is provided, it creates a regularization matrix stored as member variable 

664 self.hMat. 

665 """ 

666 pipeBase.Task.__init__(self, *args, **kwargs) 

667 self.kConfig = self.config.kernel.active 

668 

669 if 'useRegularization' in self.kConfig: 

670 self.useRegularization = self.kConfig.useRegularization 

671 else: 

672 self.useRegularization = False 

673 

674 if self.useRegularization: 

675 self.hMat = diffimLib.makeRegularizationMatrix(pexConfig.makePropertySet(self.kConfig)) 

676 

677 def _diagnostic(self, kernelCellSet, spatialSolution, spatialKernel, spatialBg): 

678 """Provide logging diagnostics on quality of spatial kernel fit 

679 

680 Parameters 

681 ---------- 

682 kernelCellSet : `lsst.afw.math.SpatialCellSet` 

683 Cellset that contains the KernelCandidates used in the fitting 

684 spatialSolution : `lsst.ip.diffim.SpatialKernelSolution` 

685 KernelSolution of best-fit 

686 spatialKernel : `lsst.afw.math.LinearCombinationKernel` 

687 Best-fit spatial Kernel model 

688 spatialBg : `lsst.afw.math.Function2D` 

689 Best-fit spatial background model 

690 """ 

691 # What is the final kernel sum 

692 kImage = afwImage.ImageD(spatialKernel.getDimensions()) 

693 kSum = spatialKernel.computeImage(kImage, False) 

694 self.log.info("Final spatial kernel sum %.3f", kSum) 

695 

696 # Look at how well conditioned the matrix is 

697 conditionNum = spatialSolution.getConditionNumber( 

698 getattr(diffimLib.KernelSolution, self.kConfig.conditionNumberType)) 

699 self.log.info("Spatial model condition number %.3e", conditionNum) 

700 

701 if conditionNum < 0.0: 

702 self.log.warning("Condition number is negative (%.3e)", conditionNum) 

703 if conditionNum > self.kConfig.maxSpatialConditionNumber: 

704 self.log.warning("Spatial solution exceeds max condition number (%.3e > %.3e)", 

705 conditionNum, self.kConfig.maxSpatialConditionNumber) 

706 

707 self.metadata["spatialConditionNum"] = conditionNum 

708 self.metadata["spatialKernelSum"] = kSum 

709 

710 # Look at how well the solution is constrained 

711 nBasisKernels = spatialKernel.getNBasisKernels() 

712 nKernelTerms = spatialKernel.getNSpatialParameters() 

713 if nKernelTerms == 0: # order 0 

714 nKernelTerms = 1 

715 

716 # Not fit for 

717 nBgTerms = spatialBg.getNParameters() 

718 if nBgTerms == 1: 

719 if spatialBg.getParameters()[0] == 0.0: 

720 nBgTerms = 0 

721 

722 nGood = 0 

723 nBad = 0 

724 nTot = 0 

725 for cell in kernelCellSet.getCellList(): 

726 for cand in cell.begin(False): # False = include bad candidates 

727 nTot += 1 

728 if cand.getStatus() == afwMath.SpatialCellCandidate.GOOD: 

729 nGood += 1 

730 if cand.getStatus() == afwMath.SpatialCellCandidate.BAD: 

731 nBad += 1 

732 

733 self.log.info("Doing stats of kernel candidates used in the spatial fit.") 

734 

735 # Counting statistics 

736 if nBad > 2*nGood: 

737 self.log.warning("Many more candidates rejected than accepted; %d total, %d rejected, %d used", 

738 nTot, nBad, nGood) 

739 else: 

740 self.log.info("%d candidates total, %d rejected, %d used", nTot, nBad, nGood) 

741 

742 # Some judgements on the quality of the spatial models 

743 if nGood < nKernelTerms: 

744 self.log.warning("Spatial kernel model underconstrained; %d candidates, %d terms, %d bases", 

745 nGood, nKernelTerms, nBasisKernels) 

746 self.log.warning("Consider lowering the spatial order") 

747 elif nGood <= 2*nKernelTerms: 

748 self.log.warning("Spatial kernel model poorly constrained; %d candidates, %d terms, %d bases", 

749 nGood, nKernelTerms, nBasisKernels) 

750 self.log.warning("Consider lowering the spatial order") 

751 else: 

752 self.log.info("Spatial kernel model well constrained; %d candidates, %d terms, %d bases", 

753 nGood, nKernelTerms, nBasisKernels) 

754 

755 if nGood < nBgTerms: 

756 self.log.warning("Spatial background model underconstrained; %d candidates, %d terms", 

757 nGood, nBgTerms) 

758 self.log.warning("Consider lowering the spatial order") 

759 elif nGood <= 2*nBgTerms: 

760 self.log.warning("Spatial background model poorly constrained; %d candidates, %d terms", 

761 nGood, nBgTerms) 

762 self.log.warning("Consider lowering the spatial order") 

763 else: 

764 self.log.info("Spatial background model appears well constrained; %d candidates, %d terms", 

765 nGood, nBgTerms) 

766 

767 def _displayDebug(self, kernelCellSet, spatialKernel, spatialBackground): 

768 """Provide visualization of the inputs and ouputs to the Psf-matching code 

769 

770 Parameters 

771 ---------- 

772 kernelCellSet : `lsst.afw.math.SpatialCellSet` 

773 The SpatialCellSet used in determining the matching kernel and background 

774 spatialKernel : `lsst.afw.math.LinearCombinationKernel` 

775 Spatially varying Psf-matching kernel 

776 spatialBackground : `lsst.afw.math.Function2D` 

777 Spatially varying background-matching function 

778 """ 

779 import lsstDebug 

780 displayCandidates = lsstDebug.Info(__name__).displayCandidates 

781 displayKernelBasis = lsstDebug.Info(__name__).displayKernelBasis 

782 displayKernelMosaic = lsstDebug.Info(__name__).displayKernelMosaic 

783 plotKernelSpatialModel = lsstDebug.Info(__name__).plotKernelSpatialModel 

784 plotKernelCoefficients = lsstDebug.Info(__name__).plotKernelCoefficients 

785 showBadCandidates = lsstDebug.Info(__name__).showBadCandidates 

786 maskTransparency = lsstDebug.Info(__name__).maskTransparency 

787 if not maskTransparency: 

788 maskTransparency = 0 

789 afwDisplay.setDefaultMaskTransparency(maskTransparency) 

790 

791 if displayCandidates: 

792 diutils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground, 

793 frame=lsstDebug.frame, 

794 showBadCandidates=showBadCandidates) 

795 lsstDebug.frame += 1 

796 diutils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground, 

797 frame=lsstDebug.frame, 

798 showBadCandidates=showBadCandidates, 

799 kernels=True) 

800 lsstDebug.frame += 1 

801 diutils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground, 

802 frame=lsstDebug.frame, 

803 showBadCandidates=showBadCandidates, 

804 resids=True) 

805 lsstDebug.frame += 1 

806 

807 if displayKernelBasis: 

808 diutils.showKernelBasis(spatialKernel, frame=lsstDebug.frame) 

809 lsstDebug.frame += 1 

810 

811 if displayKernelMosaic: 

812 diutils.showKernelMosaic(kernelCellSet.getBBox(), spatialKernel, frame=lsstDebug.frame) 

813 lsstDebug.frame += 1 

814 

815 if plotKernelSpatialModel: 

816 diutils.plotKernelSpatialModel(spatialKernel, kernelCellSet, showBadCandidates=showBadCandidates) 

817 

818 if plotKernelCoefficients: 

819 diutils.plotKernelCoefficients(spatialKernel, kernelCellSet) 

820 

821 def _createPcaBasis(self, kernelCellSet, nStarPerCell, ps): 

822 """Create Principal Component basis 

823 

824 If a principal component analysis is requested, typically when using a delta function basis, 

825 perform the PCA here and return a new basis list containing the new principal components. 

826 

827 Parameters 

828 ---------- 

829 kernelCellSet : `lsst.afw.math.SpatialCellSet` 

830 a SpatialCellSet containing KernelCandidates, from which components are derived 

831 nStarPerCell : `int` 

832 the number of stars per cell to visit when doing the PCA 

833 ps : `lsst.daf.base.PropertySet` 

834 input property set controlling the single kernel visitor 

835 

836 Returns 

837 ------- 

838 nRejectedPca : `int` 

839 number of KernelCandidates rejected during PCA loop 

840 spatialBasisList : `list` of `lsst.afw.math.kernel.FixedKernel` 

841 basis list containing the principal shapes as Kernels 

842 

843 Raises 

844 ------ 

845 RuntimeError 

846 If the Eigenvalues sum to zero. 

847 """ 

848 nComponents = self.kConfig.numPrincipalComponents 

849 imagePca = diffimLib.KernelPcaD() 

850 importStarVisitor = diffimLib.KernelPcaVisitorF(imagePca) 

851 kernelCellSet.visitCandidates(importStarVisitor, nStarPerCell) 

852 if self.kConfig.subtractMeanForPca: 

853 importStarVisitor.subtractMean() 

854 imagePca.analyze() 

855 

856 eigenValues = imagePca.getEigenValues() 

857 pcaBasisList = importStarVisitor.getEigenKernels() 

858 

859 eSum = np.sum(eigenValues) 

860 if eSum == 0.0: 

861 raise RuntimeError("Eigenvalues sum to zero") 

862 trace_logger = getTraceLogger(self.log.getChild("_solve"), 5) 

863 for j in range(len(eigenValues)): 

864 trace_logger.debug("Eigenvalue %d : %f (%f)", j, eigenValues[j], eigenValues[j]/eSum) 

865 

866 nToUse = min(nComponents, len(eigenValues)) 

867 trimBasisList = [] 

868 for j in range(nToUse): 

869 # Check for NaNs? 

870 kimage = afwImage.ImageD(pcaBasisList[j].getDimensions()) 

871 pcaBasisList[j].computeImage(kimage, False) 

872 if not (True in np.isnan(kimage.array)): 

873 trimBasisList.append(pcaBasisList[j]) 

874 

875 # Put all the power in the first kernel, which will not vary spatially 

876 spatialBasisList = diffimLib.renormalizeKernelList(trimBasisList) 

877 

878 # New Kernel visitor for this new basis list (no regularization explicitly) 

879 singlekvPca = diffimLib.BuildSingleKernelVisitorF(spatialBasisList, ps) 

880 singlekvPca.setSkipBuilt(False) 

881 kernelCellSet.visitCandidates(singlekvPca, nStarPerCell) 

882 singlekvPca.setSkipBuilt(True) 

883 nRejectedPca = singlekvPca.getNRejected() 

884 

885 return nRejectedPca, spatialBasisList 

886 

887 @abc.abstractmethod 

888 def _buildCellSet(self, *args): 

889 """Fill a SpatialCellSet with KernelCandidates for the Psf-matching process; 

890 override in derived classes""" 

891 return 

892 

893 @timeMethod 

894 def _solve(self, kernelCellSet, basisList, returnOnExcept=False): 

895 """Solve for the PSF matching kernel 

896 

897 Parameters 

898 ---------- 

899 kernelCellSet : `lsst.afw.math.SpatialCellSet` 

900 a SpatialCellSet to use in determining the matching kernel 

901 (typically as provided by _buildCellSet) 

902 basisList : `list` of `lsst.afw.math.kernel.FixedKernel` 

903 list of Kernels to be used in the decomposition of the spatially varying kernel 

904 (typically as provided by makeKernelBasisList) 

905 returnOnExcept : `bool`, optional 

906 if True then return (None, None) if an error occurs, else raise the exception 

907 

908 Returns 

909 ------- 

910 psfMatchingKernel : `lsst.afw.math.LinearCombinationKernel` 

911 Spatially varying Psf-matching kernel 

912 backgroundModel : `lsst.afw.math.Function2D` 

913 Spatially varying background-matching function 

914 

915 Raises 

916 ------ 

917 RuntimeError : 

918 If unable to determine PSF matching kernel and ``returnOnExcept==False``. 

919 """ 

920 

921 import lsstDebug 

922 display = lsstDebug.Info(__name__).display 

923 

924 maxSpatialIterations = self.kConfig.maxSpatialIterations 

925 nStarPerCell = self.kConfig.nStarPerCell 

926 usePcaForSpatialKernel = self.kConfig.usePcaForSpatialKernel 

927 

928 # Visitor for the single kernel fit 

929 ps = pexConfig.makePropertySet(self.kConfig) 

930 if self.useRegularization: 

931 singlekv = diffimLib.BuildSingleKernelVisitorF(basisList, ps, self.hMat) 

932 else: 

933 singlekv = diffimLib.BuildSingleKernelVisitorF(basisList, ps) 

934 

935 # Visitor for the kernel sum rejection 

936 ksv = diffimLib.KernelSumVisitorF(ps) 

937 

938 # Main loop 

939 trace_loggers = [getTraceLogger(self.log.getChild("_solve"), i) for i in range(5)] 

940 t0 = time.time() 

941 totalIterations = 0 

942 thisIteration = 0 

943 while (thisIteration < maxSpatialIterations): 

944 

945 # Make sure there are no uninitialized candidates as active occupants of Cell 

946 nRejectedSkf = -1 

947 while (nRejectedSkf != 0): 

948 trace_loggers[1].debug("Building single kernels...") 

949 kernelCellSet.visitCandidates(singlekv, nStarPerCell, ignoreExceptions=True) 

950 nRejectedSkf = singlekv.getNRejected() 

951 trace_loggers[1].debug( 

952 "Iteration %d, rejected %d candidates due to initial kernel fit", 

953 thisIteration, nRejectedSkf 

954 ) 

955 

956 # Reject outliers in kernel sum 

957 ksv.resetKernelSum() 

958 ksv.setMode(diffimLib.KernelSumVisitorF.AGGREGATE) 

959 kernelCellSet.visitCandidates(ksv, nStarPerCell, ignoreExceptions=True) 

960 ksv.processKsumDistribution() 

961 ksv.setMode(diffimLib.KernelSumVisitorF.REJECT) 

962 kernelCellSet.visitCandidates(ksv, nStarPerCell, ignoreExceptions=True) 

963 

964 nRejectedKsum = ksv.getNRejected() 

965 trace_loggers[1].debug( 

966 "Iteration %d, rejected %d candidates due to kernel sum", 

967 thisIteration, nRejectedKsum 

968 ) 

969 

970 # Do we jump back to the top without incrementing thisIteration? 

971 if nRejectedKsum > 0: 

972 totalIterations += 1 

973 continue 

974 

975 # At this stage we can either apply the spatial fit to 

976 # the kernels, or we run a PCA, use these as a *new* 

977 # basis set with lower dimensionality, and then apply 

978 # the spatial fit to these kernels 

979 

980 if (usePcaForSpatialKernel): 

981 trace_loggers[0].debug("Building Pca basis") 

982 

983 nRejectedPca, spatialBasisList = self._createPcaBasis(kernelCellSet, nStarPerCell, ps) 

984 trace_loggers[1].debug( 

985 "Iteration %d, rejected %d candidates due to Pca kernel fit", 

986 thisIteration, nRejectedPca 

987 ) 

988 

989 # We don't want to continue on (yet) with the 

990 # spatial modeling, because we have bad objects 

991 # contributing to the Pca basis. We basically 

992 # need to restart from the beginning of this loop, 

993 # since the cell-mates of those objects that were 

994 # rejected need their original Kernels built by 

995 # singleKernelFitter. 

996 

997 # Don't count against thisIteration 

998 if (nRejectedPca > 0): 

999 totalIterations += 1 

1000 continue 

1001 else: 

1002 spatialBasisList = basisList 

1003 

1004 # We have gotten on to the spatial modeling part 

1005 regionBBox = kernelCellSet.getBBox() 

1006 spatialkv = diffimLib.BuildSpatialKernelVisitorF(spatialBasisList, regionBBox, ps) 

1007 kernelCellSet.visitCandidates(spatialkv, nStarPerCell) 

1008 spatialkv.solveLinearEquation() 

1009 trace_loggers[2].debug("Spatial kernel built with %d candidates", spatialkv.getNCandidates()) 

1010 spatialKernel, spatialBackground = spatialkv.getSolutionPair() 

1011 

1012 # Check the quality of the spatial fit (look at residuals) 

1013 assesskv = diffimLib.AssessSpatialKernelVisitorF(spatialKernel, spatialBackground, ps) 

1014 kernelCellSet.visitCandidates(assesskv, nStarPerCell) 

1015 nRejectedSpatial = assesskv.getNRejected() 

1016 nGoodSpatial = assesskv.getNGood() 

1017 trace_loggers[1].debug( 

1018 "Iteration %d, rejected %d candidates due to spatial kernel fit", 

1019 thisIteration, nRejectedSpatial 

1020 ) 

1021 trace_loggers[1].debug("%d candidates used in fit", nGoodSpatial) 

1022 

1023 # If only nGoodSpatial == 0, might be other candidates in the cells 

1024 if nGoodSpatial == 0 and nRejectedSpatial == 0: 

1025 raise RuntimeError("No kernel candidates for spatial fit") 

1026 

1027 if nRejectedSpatial == 0: 

1028 # Nothing rejected, finished with spatial fit 

1029 break 

1030 

1031 # Otherwise, iterate on... 

1032 thisIteration += 1 

1033 

1034 # Final fit if above did not converge 

1035 if (nRejectedSpatial > 0) and (thisIteration == maxSpatialIterations): 

1036 trace_loggers[1].debug("Final spatial fit") 

1037 if (usePcaForSpatialKernel): 

1038 nRejectedPca, spatialBasisList = self._createPcaBasis(kernelCellSet, nStarPerCell, ps) 

1039 regionBBox = kernelCellSet.getBBox() 

1040 spatialkv = diffimLib.BuildSpatialKernelVisitorF(spatialBasisList, regionBBox, ps) 

1041 kernelCellSet.visitCandidates(spatialkv, nStarPerCell) 

1042 spatialkv.solveLinearEquation() 

1043 trace_loggers[2].debug("Spatial kernel built with %d candidates", spatialkv.getNCandidates()) 

1044 spatialKernel, spatialBackground = spatialkv.getSolutionPair() 

1045 

1046 spatialSolution = spatialkv.getKernelSolution() 

1047 

1048 t1 = time.time() 

1049 trace_loggers[0].debug("Total time to compute the spatial kernel : %.2f s", (t1 - t0)) 

1050 

1051 if display: 

1052 self._displayDebug(kernelCellSet, spatialKernel, spatialBackground) 

1053 

1054 self._diagnostic(kernelCellSet, spatialSolution, spatialKernel, spatialBackground) 

1055 

1056 return spatialSolution, spatialKernel, spatialBackground 

1057 

1058 

1059PsfMatch = PsfMatchTask