Coverage for python/lsst/ip/diffim/psfMatch.py: 29%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

295 statements  

1# This file is part of ip_diffim. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["DetectionConfig", "PsfMatchConfig", "PsfMatchConfigAL", "PsfMatchConfigDF", "PsfMatchTask"] 

23 

24import time 

25 

26import numpy as np 

27 

28import lsst.afw.image as afwImage 

29import lsst.pex.config as pexConfig 

30import lsst.afw.math as afwMath 

31import lsst.afw.display as afwDisplay 

32import lsst.log as log 

33import lsst.pipe.base as pipeBase 

34from lsst.meas.algorithms import SubtractBackgroundConfig 

35from lsst.utils.timer import timeMethod 

36from . import utils as diutils 

37from . import diffimLib 

38 

39 

40class DetectionConfig(pexConfig.Config): 

41 """Configuration for detecting sources on images for building a 

42 PSF-matching kernel 

43 

44 Configuration for turning detected lsst.afw.detection.FootPrints into an 

45 acceptable (unmasked, high signal-to-noise, not too large or not too small) 

46 list of `lsst.ip.diffim.KernelSources` that are used to build the 

47 Psf-matching kernel""" 

48 

49 detThreshold = pexConfig.Field( 49 ↛ exitline 49 didn't jump to the function exit

50 dtype=float, 

51 doc="Value of footprint detection threshold", 

52 default=10.0, 

53 check=lambda x: x >= 3.0 

54 ) 

55 detThresholdType = pexConfig.ChoiceField( 

56 dtype=str, 

57 doc="Type of detection threshold", 

58 default="pixel_stdev", 

59 allowed={ 

60 "value": "Use counts as the detection threshold type", 

61 "stdev": "Use standard deviation of image plane", 

62 "variance": "Use variance of image plane", 

63 "pixel_stdev": "Use stdev derived from variance plane" 

64 } 

65 ) 

66 detOnTemplate = pexConfig.Field( 

67 dtype=bool, 

68 doc="""If true run detection on the template (image to convolve); 

69 if false run detection on the science image""", 

70 default=True 

71 ) 

72 badMaskPlanes = pexConfig.ListField( 

73 dtype=str, 

74 doc="""Mask planes that lead to an invalid detection. 

75 Options: NO_DATA EDGE SAT BAD CR INTRP""", 

76 default=("NO_DATA", "EDGE", "SAT") 

77 ) 

78 fpNpixMin = pexConfig.Field( 78 ↛ exitline 78 didn't jump to the function exit

79 dtype=int, 

80 doc="Minimum number of pixels in an acceptable Footprint", 

81 default=5, 

82 check=lambda x: x >= 5 

83 ) 

84 fpNpixMax = pexConfig.Field( 84 ↛ exitline 84 didn't jump to the function exit

85 dtype=int, 

86 doc="""Maximum number of pixels in an acceptable Footprint; 

87 too big and the subsequent convolutions become unwieldy""", 

88 default=500, 

89 check=lambda x: x <= 500 

90 ) 

91 fpGrowKernelScaling = pexConfig.Field( 91 ↛ exitline 91 didn't jump to the function exit

92 dtype=float, 

93 doc="""If config.scaleByFwhm, grow the footprint based on 

94 the final kernelSize. Each footprint will be 

95 2*fpGrowKernelScaling*kernelSize x 

96 2*fpGrowKernelScaling*kernelSize. With the value 

97 of 1.0, the remaining pixels in each KernelCandiate 

98 after convolution by the basis functions will be 

99 equal to the kernel size itself.""", 

100 default=1.0, 

101 check=lambda x: x >= 1.0 

102 ) 

103 fpGrowPix = pexConfig.Field( 103 ↛ exitline 103 didn't jump to the function exit

104 dtype=int, 

105 doc="""Growing radius (in pixels) for each raw detection 

106 footprint. The smaller the faster; however the 

107 kernel sum does not converge if the stamp is too 

108 small; and the kernel is not constrained at all if 

109 the stamp is the size of the kernel. The grown stamp 

110 is 2 * fpGrowPix pixels larger in each dimension. 

111 This is overridden by fpGrowKernelScaling if scaleByFwhm""", 

112 default=30, 

113 check=lambda x: x >= 10 

114 ) 

115 scaleByFwhm = pexConfig.Field( 

116 dtype=bool, 

117 doc="Scale fpGrowPix by input Fwhm?", 

118 default=True, 

119 ) 

120 

121 

122class PsfMatchConfig(pexConfig.Config): 

123 """Base configuration for Psf-matching 

124 

125 The base configuration of the Psf-matching kernel, and of the warping, detection, 

126 and background modeling subTasks.""" 

127 

128 warpingConfig = pexConfig.ConfigField("Config for warping exposures to a common alignment", 

129 afwMath.WarperConfig) 

130 detectionConfig = pexConfig.ConfigField("Controlling the detection of sources for kernel building", 

131 DetectionConfig) 

132 afwBackgroundConfig = pexConfig.ConfigField("Controlling the Afw background fitting", 

133 SubtractBackgroundConfig) 

134 

135 useAfwBackground = pexConfig.Field( 

136 dtype=bool, 

137 doc="Use afw background subtraction instead of ip_diffim", 

138 default=False, 

139 ) 

140 fitForBackground = pexConfig.Field( 

141 dtype=bool, 

142 doc="Include terms (including kernel cross terms) for background in ip_diffim", 

143 default=False, 

144 ) 

145 kernelBasisSet = pexConfig.ChoiceField( 

146 dtype=str, 

147 doc="Type of basis set for PSF matching kernel.", 

148 default="alard-lupton", 

149 allowed={ 

150 "alard-lupton": """Alard-Lupton sum-of-gaussians basis set, 

151 * The first term has no spatial variation 

152 * The kernel sum is conserved 

153 * You may want to turn off 'usePcaForSpatialKernel'""", 

154 "delta-function": """Delta-function kernel basis set, 

155 * You may enable the option useRegularization 

156 * You should seriously consider usePcaForSpatialKernel, which will also 

157 enable kernel sum conservation for the delta function kernels""" 

158 } 

159 ) 

160 kernelSize = pexConfig.Field( 

161 dtype=int, 

162 doc="""Number of rows/columns in the convolution kernel; should be odd-valued. 

163 Modified by kernelSizeFwhmScaling if scaleByFwhm = true""", 

164 default=21, 

165 ) 

166 scaleByFwhm = pexConfig.Field( 

167 dtype=bool, 

168 doc="Scale kernelSize, alardGaussians by input Fwhm", 

169 default=True, 

170 ) 

171 kernelSizeFwhmScaling = pexConfig.Field( 171 ↛ exitline 171 didn't jump to the function exit

172 dtype=float, 

173 doc="Multiplier of the largest AL Gaussian basis sigma to get the kernel bbox (pixel) size.", 

174 default=6.0, 

175 check=lambda x: x >= 1.0 

176 ) 

177 kernelSizeMin = pexConfig.Field( 

178 dtype=int, 

179 doc="Minimum kernel bbox (pixel) size.", 

180 default=21, 

181 ) 

182 kernelSizeMax = pexConfig.Field( 

183 dtype=int, 

184 doc="Maximum kernel bbox (pixel) size.", 

185 default=35, 

186 ) 

187 spatialModelType = pexConfig.ChoiceField( 

188 dtype=str, 

189 doc="Type of spatial functions for kernel and background", 

190 default="chebyshev1", 

191 allowed={ 

192 "chebyshev1": "Chebyshev polynomial of the first kind", 

193 "polynomial": "Standard x,y polynomial", 

194 } 

195 ) 

196 spatialKernelOrder = pexConfig.Field( 196 ↛ exitline 196 didn't jump to the function exit

197 dtype=int, 

198 doc="Spatial order of convolution kernel variation", 

199 default=2, 

200 check=lambda x: x >= 0 

201 ) 

202 spatialBgOrder = pexConfig.Field( 202 ↛ exitline 202 didn't jump to the function exit

203 dtype=int, 

204 doc="Spatial order of differential background variation", 

205 default=1, 

206 check=lambda x: x >= 0 

207 ) 

208 sizeCellX = pexConfig.Field( 208 ↛ exitline 208 didn't jump to the function exit

209 dtype=int, 

210 doc="Size (rows) in pixels of each SpatialCell for spatial modeling", 

211 default=128, 

212 check=lambda x: x >= 32 

213 ) 

214 sizeCellY = pexConfig.Field( 214 ↛ exitline 214 didn't jump to the function exit

215 dtype=int, 

216 doc="Size (columns) in pixels of each SpatialCell for spatial modeling", 

217 default=128, 

218 check=lambda x: x >= 32 

219 ) 

220 nStarPerCell = pexConfig.Field( 220 ↛ exitline 220 didn't jump to the function exit

221 dtype=int, 

222 doc="Number of KernelCandidates in each SpatialCell to use in the spatial fitting", 

223 default=3, 

224 check=lambda x: x >= 1 

225 ) 

226 maxSpatialIterations = pexConfig.Field( 226 ↛ exitline 226 didn't jump to the function exit

227 dtype=int, 

228 doc="Maximum number of iterations for rejecting bad KernelCandidates in spatial fitting", 

229 default=3, 

230 check=lambda x: x >= 1 and x <= 5 

231 ) 

232 usePcaForSpatialKernel = pexConfig.Field( 

233 dtype=bool, 

234 doc="""Use Pca to reduce the dimensionality of the kernel basis sets. 

235 This is particularly useful for delta-function kernels. 

236 Functionally, after all Cells have their raw kernels determined, we run 

237 a Pca on these Kernels, re-fit the Cells using the eigenKernels and then 

238 fit those for spatial variation using the same technique as for Alard-Lupton kernels. 

239 If this option is used, the first term will have no spatial variation and the 

240 kernel sum will be conserved.""", 

241 default=False, 

242 ) 

243 subtractMeanForPca = pexConfig.Field( 

244 dtype=bool, 

245 doc="Subtract off the mean feature before doing the Pca", 

246 default=True, 

247 ) 

248 numPrincipalComponents = pexConfig.Field( 248 ↛ exitline 248 didn't jump to the function exit

249 dtype=int, 

250 doc="""Number of principal components to use for Pca basis, including the 

251 mean kernel if requested.""", 

252 default=5, 

253 check=lambda x: x >= 3 

254 ) 

255 singleKernelClipping = pexConfig.Field( 

256 dtype=bool, 

257 doc="Do sigma clipping on each raw kernel candidate", 

258 default=True, 

259 ) 

260 kernelSumClipping = pexConfig.Field( 

261 dtype=bool, 

262 doc="Do sigma clipping on the ensemble of kernel sums", 

263 default=True, 

264 ) 

265 spatialKernelClipping = pexConfig.Field( 

266 dtype=bool, 

267 doc="Do sigma clipping after building the spatial model", 

268 default=True, 

269 ) 

270 checkConditionNumber = pexConfig.Field( 

271 dtype=bool, 

272 doc="""Test for maximum condition number when inverting a kernel matrix. 

273 Anything above maxConditionNumber is not used and the candidate is set as BAD. 

274 Also used to truncate inverse matrix in estimateBiasedRisk. However, 

275 if you are doing any deconvolution you will want to turn this off, or use 

276 a large maxConditionNumber""", 

277 default=False, 

278 ) 

279 badMaskPlanes = pexConfig.ListField( 

280 dtype=str, 

281 doc="""Mask planes to ignore when calculating diffim statistics 

282 Options: NO_DATA EDGE SAT BAD CR INTRP""", 

283 default=("NO_DATA", "EDGE", "SAT") 

284 ) 

285 candidateResidualMeanMax = pexConfig.Field( 285 ↛ exitline 285 didn't jump to the function exit

286 dtype=float, 

287 doc="""Rejects KernelCandidates yielding bad difference image quality. 

288 Used by BuildSingleKernelVisitor, AssessSpatialKernelVisitor. 

289 Represents average over pixels of (image/sqrt(variance)).""", 

290 default=0.25, 

291 check=lambda x: x >= 0.0 

292 ) 

293 candidateResidualStdMax = pexConfig.Field( 293 ↛ exitline 293 didn't jump to the function exit

294 dtype=float, 

295 doc="""Rejects KernelCandidates yielding bad difference image quality. 

296 Used by BuildSingleKernelVisitor, AssessSpatialKernelVisitor. 

297 Represents stddev over pixels of (image/sqrt(variance)).""", 

298 default=1.50, 

299 check=lambda x: x >= 0.0 

300 ) 

301 useCoreStats = pexConfig.Field( 

302 dtype=bool, 

303 doc="""Use the core of the footprint for the quality statistics, instead of the entire footprint. 

304 WARNING: if there is deconvolution we probably will need to turn this off""", 

305 default=False, 

306 ) 

307 candidateCoreRadius = pexConfig.Field( 307 ↛ exitline 307 didn't jump to the function exit

308 dtype=int, 

309 doc="""Radius for calculation of stats in 'core' of KernelCandidate diffim. 

310 Total number of pixels used will be (2*radius)**2. 

311 This is used both for 'core' diffim quality as well as ranking of 

312 KernelCandidates by their total flux in this core""", 

313 default=3, 

314 check=lambda x: x >= 1 

315 ) 

316 maxKsumSigma = pexConfig.Field( 316 ↛ exitline 316 didn't jump to the function exit

317 dtype=float, 

318 doc="""Maximum allowed sigma for outliers from kernel sum distribution. 

319 Used to reject variable objects from the kernel model""", 

320 default=3.0, 

321 check=lambda x: x >= 0.0 

322 ) 

323 maxConditionNumber = pexConfig.Field( 323 ↛ exitline 323 didn't jump to the function exit

324 dtype=float, 

325 doc="Maximum condition number for a well conditioned matrix", 

326 default=5.0e7, 

327 check=lambda x: x >= 0.0 

328 ) 

329 conditionNumberType = pexConfig.ChoiceField( 

330 dtype=str, 

331 doc="Use singular values (SVD) or eigen values (EIGENVALUE) to determine condition number", 

332 default="EIGENVALUE", 

333 allowed={ 

334 "SVD": "Use singular values", 

335 "EIGENVALUE": "Use eigen values (faster)", 

336 } 

337 ) 

338 maxSpatialConditionNumber = pexConfig.Field( 338 ↛ exitline 338 didn't jump to the function exit

339 dtype=float, 

340 doc="Maximum condition number for a well conditioned spatial matrix", 

341 default=1.0e10, 

342 check=lambda x: x >= 0.0 

343 ) 

344 iterateSingleKernel = pexConfig.Field( 

345 dtype=bool, 

346 doc="""Remake KernelCandidate using better variance estimate after first pass? 

347 Primarily useful when convolving a single-depth image, otherwise not necessary.""", 

348 default=False, 

349 ) 

350 constantVarianceWeighting = pexConfig.Field( 

351 dtype=bool, 

352 doc="""Use constant variance weighting in single kernel fitting? 

353 In some cases this is better for bright star residuals.""", 

354 default=True, 

355 ) 

356 calculateKernelUncertainty = pexConfig.Field( 

357 dtype=bool, 

358 doc="""Calculate kernel and background uncertainties for each kernel candidate? 

359 This comes from the inverse of the covariance matrix. 

360 Warning: regularization can cause problems for this step.""", 

361 default=False, 

362 ) 

363 useBicForKernelBasis = pexConfig.Field( 

364 dtype=bool, 

365 doc="""Use Bayesian Information Criterion to select the number of bases going into the kernel""", 

366 default=False, 

367 ) 

368 

369 

370class PsfMatchConfigAL(PsfMatchConfig): 

371 """The parameters specific to the "Alard-Lupton" (sum-of-Gaussian) Psf-matching basis""" 

372 

373 def setDefaults(self): 

374 PsfMatchConfig.setDefaults(self) 

375 self.kernelBasisSet = "alard-lupton" 

376 self.maxConditionNumber = 5.0e7 

377 

378 alardNGauss = pexConfig.Field( 378 ↛ exitline 378 didn't jump to the function exit

379 dtype=int, 

380 doc="Number of base Gaussians in alard-lupton kernel basis function generation.", 

381 default=3, 

382 check=lambda x: x >= 1 

383 ) 

384 alardDegGauss = pexConfig.ListField( 

385 dtype=int, 

386 doc="Polynomial order of spatial modification of base Gaussians. " 

387 "List length must be `alardNGauss`.", 

388 default=(4, 2, 2), 

389 ) 

390 alardSigGauss = pexConfig.ListField( 

391 dtype=float, 

392 doc="Default sigma values in pixels of base Gaussians. " 

393 "List length must be `alardNGauss`.", 

394 default=(0.7, 1.5, 3.0), 

395 ) 

396 alardGaussBeta = pexConfig.Field( 396 ↛ exitline 396 didn't jump to the function exit

397 dtype=float, 

398 doc="Used if `scaleByFwhm==True`, scaling multiplier of base " 

399 "Gaussian sigmas for automated sigma determination", 

400 default=2.0, 

401 check=lambda x: x >= 0.0, 

402 ) 

403 alardMinSig = pexConfig.Field( 403 ↛ exitline 403 didn't jump to the function exit

404 dtype=float, 

405 doc="Used if `scaleByFwhm==True`, minimum sigma (pixels) for base Gaussians", 

406 default=0.7, 

407 check=lambda x: x >= 0.25 

408 ) 

409 alardDegGaussDeconv = pexConfig.Field( 409 ↛ exitline 409 didn't jump to the function exit

410 dtype=int, 

411 doc="Used if `scaleByFwhm==True`, degree of spatial modification of ALL base Gaussians " 

412 "in AL basis during deconvolution", 

413 default=3, 

414 check=lambda x: x >= 1 

415 ) 

416 alardMinSigDeconv = pexConfig.Field( 416 ↛ exitline 416 didn't jump to the function exit

417 dtype=float, 

418 doc="Used if `scaleByFwhm==True`, minimum sigma (pixels) for base Gaussians during deconvolution; " 

419 "make smaller than `alardMinSig` as this is only indirectly used", 

420 default=0.4, 

421 check=lambda x: x >= 0.25 

422 ) 

423 alardNGaussDeconv = pexConfig.Field( 423 ↛ exitline 423 didn't jump to the function exit

424 dtype=int, 

425 doc="Used if `scaleByFwhm==True`, number of base Gaussians in AL basis during deconvolution", 

426 default=3, 

427 check=lambda x: x >= 1 

428 ) 

429 

430 

431class PsfMatchConfigDF(PsfMatchConfig): 

432 """The parameters specific to the delta-function (one basis per-pixel) Psf-matching basis""" 

433 

434 def setDefaults(self): 

435 PsfMatchConfig.setDefaults(self) 

436 self.kernelBasisSet = "delta-function" 

437 self.maxConditionNumber = 5.0e6 

438 self.usePcaForSpatialKernel = True 

439 self.subtractMeanForPca = True 

440 self.useBicForKernelBasis = False 

441 

442 useRegularization = pexConfig.Field( 

443 dtype=bool, 

444 doc="Use regularization to smooth the delta function kernels", 

445 default=True, 

446 ) 

447 regularizationType = pexConfig.ChoiceField( 

448 dtype=str, 

449 doc="Type of regularization.", 

450 default="centralDifference", 

451 allowed={ 

452 "centralDifference": "Penalize second derivative using 2-D stencil of central finite difference", 

453 "forwardDifference": "Penalize first, second, third derivatives using forward finite differeces" 

454 } 

455 ) 

456 centralRegularizationStencil = pexConfig.ChoiceField( 

457 dtype=int, 

458 doc="Type of stencil to approximate central derivative (for centralDifference only)", 

459 default=9, 

460 allowed={ 

461 5: "5-point stencil including only adjacent-in-x,y elements", 

462 9: "9-point stencil including diagonal elements" 

463 } 

464 ) 

465 forwardRegularizationOrders = pexConfig.ListField( 465 ↛ exitline 465 didn't jump to the function exit

466 dtype=int, 

467 doc="Array showing which order derivatives to penalize (for forwardDifference only)", 

468 default=(1, 2), 

469 itemCheck=lambda x: (x > 0) and (x < 4) 

470 ) 

471 regularizationBorderPenalty = pexConfig.Field( 471 ↛ exitline 471 didn't jump to the function exit

472 dtype=float, 

473 doc="Value of the penalty for kernel border pixels", 

474 default=3.0, 

475 check=lambda x: x >= 0.0 

476 ) 

477 lambdaType = pexConfig.ChoiceField( 

478 dtype=str, 

479 doc="How to choose the value of the regularization strength", 

480 default="absolute", 

481 allowed={ 

482 "absolute": "Use lambdaValue as the value of regularization strength", 

483 "relative": "Use lambdaValue as fraction of the default regularization strength (N.R. 18.5.8)", 

484 "minimizeBiasedRisk": "Minimize biased risk estimate", 

485 "minimizeUnbiasedRisk": "Minimize unbiased risk estimate", 

486 } 

487 ) 

488 lambdaValue = pexConfig.Field( 

489 dtype=float, 

490 doc="Value used for absolute determinations of regularization strength", 

491 default=0.2, 

492 ) 

493 lambdaScaling = pexConfig.Field( 

494 dtype=float, 

495 doc="Fraction of the default lambda strength (N.R. 18.5.8) to use. 1e-4 or 1e-5", 

496 default=1e-4, 

497 ) 

498 lambdaStepType = pexConfig.ChoiceField( 

499 dtype=str, 

500 doc="""If a scan through lambda is needed (minimizeBiasedRisk, minimizeUnbiasedRisk), 

501 use log or linear steps""", 

502 default="log", 

503 allowed={ 

504 "log": "Step in log intervals; e.g. lambdaMin, lambdaMax, lambdaStep = -1.0, 2.0, 0.1", 

505 "linear": "Step in linear intervals; e.g. lambdaMin, lambdaMax, lambdaStep = 0.1, 100, 0.1", 

506 } 

507 ) 

508 lambdaMin = pexConfig.Field( 

509 dtype=float, 

510 doc="""If scan through lambda needed (minimizeBiasedRisk, minimizeUnbiasedRisk), 

511 start at this value. If lambdaStepType = log:linear, suggest -1:0.1""", 

512 default=-1.0, 

513 ) 

514 lambdaMax = pexConfig.Field( 

515 dtype=float, 

516 doc="""If scan through lambda needed (minimizeBiasedRisk, minimizeUnbiasedRisk), 

517 stop at this value. If lambdaStepType = log:linear, suggest 2:100""", 

518 default=2.0, 

519 ) 

520 lambdaStep = pexConfig.Field( 

521 dtype=float, 

522 doc="""If scan through lambda needed (minimizeBiasedRisk, minimizeUnbiasedRisk), 

523 step in these increments. If lambdaStepType = log:linear, suggest 0.1:0.1""", 

524 default=0.1, 

525 ) 

526 

527 

528class PsfMatchTask(pipeBase.Task): 

529 """Base class for Psf Matching; should not be called directly 

530 

531 Notes 

532 ----- 

533 PsfMatchTask is a base class that implements the core functionality for matching the 

534 Psfs of two images using a spatially varying Psf-matching `lsst.afw.math.LinearCombinationKernel`. 

535 The Task requires the user to provide an instance of an `lsst.afw.math.SpatialCellSet`, 

536 filled with `lsst.ip.diffim.KernelCandidate` instances, and a list of `lsst.afw.math.Kernels` 

537 of basis shapes that will be used for the decomposition. If requested, the Task 

538 also performs background matching and returns the differential background model as an 

539 `lsst.afw.math.Kernel.SpatialFunction`. 

540 

541 **Invoking the Task** 

542 

543 As a base class, this Task is not directly invoked. However, ``run()`` methods that are 

544 implemented on derived classes will make use of the core ``_solve()`` functionality, 

545 which defines a sequence of `lsst.afw.math.CandidateVisitor` classes that iterate 

546 through the KernelCandidates, first building up a per-candidate solution and then 

547 building up a spatial model from the ensemble of candidates. Sigma clipping is 

548 performed using the mean and standard deviation of all kernel sums (to reject 

549 variable objects), on the per-candidate substamp diffim residuals 

550 (to indicate a bad choice of kernel basis shapes for that particular object), 

551 and on the substamp diffim residuals using the spatial kernel fit (to indicate a bad 

552 choice of spatial kernel order, or poor constraints on the spatial model). The 

553 ``_diagnostic()`` method logs information on the quality of the spatial fit, and also 

554 modifies the Task metadata. 

555 

556 .. list-table:: Quantities set in Metadata 

557 :header-rows: 1 

558 

559 * - Parameter 

560 - Description 

561 * - ``spatialConditionNum`` 

562 - Condition number of the spatial kernel fit 

563 * - ``spatialKernelSum`` 

564 - Kernel sum (10^{-0.4 * ``Delta``; zeropoint}) of the spatial Psf-matching kernel 

565 * - ``ALBasisNGauss`` 

566 - If using sum-of-Gaussian basis, the number of gaussians used 

567 * - ``ALBasisDegGauss`` 

568 - If using sum-of-Gaussian basis, the deg of spatial variation of the Gaussians 

569 * - ``ALBasisSigGauss`` 

570 - If using sum-of-Gaussian basis, the widths (sigma) of the Gaussians 

571 * - ``ALKernelSize`` 

572 - If using sum-of-Gaussian basis, the kernel size 

573 * - ``NFalsePositivesTotal`` 

574 - Total number of diaSources 

575 * - ``NFalsePositivesRefAssociated`` 

576 - Number of diaSources that associate with the reference catalog 

577 * - ``NFalsePositivesRefAssociated`` 

578 - Number of diaSources that associate with the source catalog 

579 * - ``NFalsePositivesUnassociated`` 

580 - Number of diaSources that are orphans 

581 * - ``metric_MEAN`` 

582 - Mean value of substamp diffim quality metrics across all KernelCandidates, 

583 for both the per-candidate (LOCAL) and SPATIAL residuals 

584 * - ``metric_MEDIAN`` 

585 - Median value of substamp diffim quality metrics across all KernelCandidates, 

586 for both the per-candidate (LOCAL) and SPATIAL residuals 

587 * - ``metric_STDEV`` 

588 - Standard deviation of substamp diffim quality metrics across all KernelCandidates, 

589 for both the per-candidate (LOCAL) and SPATIAL residuals 

590 

591 **Debug variables** 

592 

593 The `lsst.pipe.base.cmdLineTask.CmdLineTask` command line task interface supports a 

594 flag -d/--debug to import @b debug.py from your PYTHONPATH. The relevant contents of debug.py 

595 for this Task include: 

596 

597 .. code-block:: py 

598 

599 import sys 

600 import lsstDebug 

601 def DebugInfo(name): 

602 di = lsstDebug.getInfo(name) 

603 if name == "lsst.ip.diffim.psfMatch": 

604 # enable debug output 

605 di.display = True 

606 # display mask transparency 

607 di.maskTransparency = 80 

608 # show all the candidates and residuals 

609 di.displayCandidates = True 

610 # show kernel basis functions 

611 di.displayKernelBasis = False 

612 # show kernel realized across the image 

613 di.displayKernelMosaic = True 

614 # show coefficients of spatial model 

615 di.plotKernelSpatialModel = False 

616 # show fixed and spatial coefficients and coefficient histograms 

617 di.plotKernelCoefficients = True 

618 # show the bad candidates (red) along with good (green) 

619 di.showBadCandidates = True 

620 return di 

621 lsstDebug.Info = DebugInfo 

622 lsstDebug.frame = 1 

623 

624 Note that if you want additional logging info, you may add to your scripts: 

625 

626 .. code-block:: py 

627 

628 import lsst.log.utils as logUtils 

629 logUtils.traceSetAt("ip.diffim", 4) 

630 """ 

631 ConfigClass = PsfMatchConfig 

632 _DefaultName = "psfMatch" 

633 

634 def __init__(self, *args, **kwargs): 

635 """Create the psf-matching Task 

636 

637 Parameters 

638 ---------- 

639 *args 

640 Arguments to be passed to ``lsst.pipe.base.task.Task.__init__`` 

641 **kwargs 

642 Keyword arguments to be passed to ``lsst.pipe.base.task.Task.__init__`` 

643 

644 Notes 

645 ----- 

646 The initialization sets the Psf-matching kernel configuration using the value of 

647 self.config.kernel.active. If the kernel is requested with regularization to moderate 

648 the bias/variance tradeoff, currently only used when a delta function kernel basis 

649 is provided, it creates a regularization matrix stored as member variable 

650 self.hMat. 

651 """ 

652 pipeBase.Task.__init__(self, *args, **kwargs) 

653 self.kConfig = self.config.kernel.active 

654 

655 if 'useRegularization' in self.kConfig: 

656 self.useRegularization = self.kConfig.useRegularization 

657 else: 

658 self.useRegularization = False 

659 

660 if self.useRegularization: 

661 self.hMat = diffimLib.makeRegularizationMatrix(pexConfig.makePropertySet(self.kConfig)) 

662 

663 def _diagnostic(self, kernelCellSet, spatialSolution, spatialKernel, spatialBg): 

664 """Provide logging diagnostics on quality of spatial kernel fit 

665 

666 Parameters 

667 ---------- 

668 kernelCellSet : `lsst.afw.math.SpatialCellSet` 

669 Cellset that contains the KernelCandidates used in the fitting 

670 spatialSolution : `lsst.ip.diffim.SpatialKernelSolution` 

671 KernelSolution of best-fit 

672 spatialKernel : `lsst.afw.math.LinearCombinationKernel` 

673 Best-fit spatial Kernel model 

674 spatialBg : `lsst.afw.math.Function2D` 

675 Best-fit spatial background model 

676 """ 

677 # What is the final kernel sum 

678 kImage = afwImage.ImageD(spatialKernel.getDimensions()) 

679 kSum = spatialKernel.computeImage(kImage, False) 

680 self.log.info("Final spatial kernel sum %.3f", kSum) 

681 

682 # Look at how well conditioned the matrix is 

683 conditionNum = spatialSolution.getConditionNumber( 

684 getattr(diffimLib.KernelSolution, self.kConfig.conditionNumberType)) 

685 self.log.info("Spatial model condition number %.3e", conditionNum) 

686 

687 if conditionNum < 0.0: 

688 self.log.warning("Condition number is negative (%.3e)", conditionNum) 

689 if conditionNum > self.kConfig.maxSpatialConditionNumber: 

690 self.log.warning("Spatial solution exceeds max condition number (%.3e > %.3e)", 

691 conditionNum, self.kConfig.maxSpatialConditionNumber) 

692 

693 self.metadata.set("spatialConditionNum", conditionNum) 

694 self.metadata.set("spatialKernelSum", kSum) 

695 

696 # Look at how well the solution is constrained 

697 nBasisKernels = spatialKernel.getNBasisKernels() 

698 nKernelTerms = spatialKernel.getNSpatialParameters() 

699 if nKernelTerms == 0: # order 0 

700 nKernelTerms = 1 

701 

702 # Not fit for 

703 nBgTerms = spatialBg.getNParameters() 

704 if nBgTerms == 1: 

705 if spatialBg.getParameters()[0] == 0.0: 

706 nBgTerms = 0 

707 

708 nGood = 0 

709 nBad = 0 

710 nTot = 0 

711 for cell in kernelCellSet.getCellList(): 

712 for cand in cell.begin(False): # False = include bad candidates 

713 nTot += 1 

714 if cand.getStatus() == afwMath.SpatialCellCandidate.GOOD: 

715 nGood += 1 

716 if cand.getStatus() == afwMath.SpatialCellCandidate.BAD: 

717 nBad += 1 

718 

719 self.log.info("Doing stats of kernel candidates used in the spatial fit.") 

720 

721 # Counting statistics 

722 if nBad > 2*nGood: 

723 self.log.warning("Many more candidates rejected than accepted; %d total, %d rejected, %d used", 

724 nTot, nBad, nGood) 

725 else: 

726 self.log.info("%d candidates total, %d rejected, %d used", nTot, nBad, nGood) 

727 

728 # Some judgements on the quality of the spatial models 

729 if nGood < nKernelTerms: 

730 self.log.warning("Spatial kernel model underconstrained; %d candidates, %d terms, %d bases", 

731 nGood, nKernelTerms, nBasisKernels) 

732 self.log.warning("Consider lowering the spatial order") 

733 elif nGood <= 2*nKernelTerms: 

734 self.log.warning("Spatial kernel model poorly constrained; %d candidates, %d terms, %d bases", 

735 nGood, nKernelTerms, nBasisKernels) 

736 self.log.warning("Consider lowering the spatial order") 

737 else: 

738 self.log.info("Spatial kernel model well constrained; %d candidates, %d terms, %d bases", 

739 nGood, nKernelTerms, nBasisKernels) 

740 

741 if nGood < nBgTerms: 

742 self.log.warning("Spatial background model underconstrained; %d candidates, %d terms", 

743 nGood, nBgTerms) 

744 self.log.warning("Consider lowering the spatial order") 

745 elif nGood <= 2*nBgTerms: 

746 self.log.warning("Spatial background model poorly constrained; %d candidates, %d terms", 

747 nGood, nBgTerms) 

748 self.log.warning("Consider lowering the spatial order") 

749 else: 

750 self.log.info("Spatial background model appears well constrained; %d candidates, %d terms", 

751 nGood, nBgTerms) 

752 

753 def _displayDebug(self, kernelCellSet, spatialKernel, spatialBackground): 

754 """Provide visualization of the inputs and ouputs to the Psf-matching code 

755 

756 Parameters 

757 ---------- 

758 kernelCellSet : `lsst.afw.math.SpatialCellSet` 

759 The SpatialCellSet used in determining the matching kernel and background 

760 spatialKernel : `lsst.afw.math.LinearCombinationKernel` 

761 Spatially varying Psf-matching kernel 

762 spatialBackground : `lsst.afw.math.Function2D` 

763 Spatially varying background-matching function 

764 """ 

765 import lsstDebug 

766 displayCandidates = lsstDebug.Info(__name__).displayCandidates 

767 displayKernelBasis = lsstDebug.Info(__name__).displayKernelBasis 

768 displayKernelMosaic = lsstDebug.Info(__name__).displayKernelMosaic 

769 plotKernelSpatialModel = lsstDebug.Info(__name__).plotKernelSpatialModel 

770 plotKernelCoefficients = lsstDebug.Info(__name__).plotKernelCoefficients 

771 showBadCandidates = lsstDebug.Info(__name__).showBadCandidates 

772 maskTransparency = lsstDebug.Info(__name__).maskTransparency 

773 if not maskTransparency: 

774 maskTransparency = 0 

775 afwDisplay.setDefaultMaskTransparency(maskTransparency) 

776 

777 if displayCandidates: 

778 diutils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground, 

779 frame=lsstDebug.frame, 

780 showBadCandidates=showBadCandidates) 

781 lsstDebug.frame += 1 

782 diutils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground, 

783 frame=lsstDebug.frame, 

784 showBadCandidates=showBadCandidates, 

785 kernels=True) 

786 lsstDebug.frame += 1 

787 diutils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground, 

788 frame=lsstDebug.frame, 

789 showBadCandidates=showBadCandidates, 

790 resids=True) 

791 lsstDebug.frame += 1 

792 

793 if displayKernelBasis: 

794 diutils.showKernelBasis(spatialKernel, frame=lsstDebug.frame) 

795 lsstDebug.frame += 1 

796 

797 if displayKernelMosaic: 

798 diutils.showKernelMosaic(kernelCellSet.getBBox(), spatialKernel, frame=lsstDebug.frame) 

799 lsstDebug.frame += 1 

800 

801 if plotKernelSpatialModel: 

802 diutils.plotKernelSpatialModel(spatialKernel, kernelCellSet, showBadCandidates=showBadCandidates) 

803 

804 if plotKernelCoefficients: 

805 diutils.plotKernelCoefficients(spatialKernel, kernelCellSet) 

806 

807 def _createPcaBasis(self, kernelCellSet, nStarPerCell, ps): 

808 """Create Principal Component basis 

809 

810 If a principal component analysis is requested, typically when using a delta function basis, 

811 perform the PCA here and return a new basis list containing the new principal components. 

812 

813 Parameters 

814 ---------- 

815 kernelCellSet : `lsst.afw.math.SpatialCellSet` 

816 a SpatialCellSet containing KernelCandidates, from which components are derived 

817 nStarPerCell : `int` 

818 the number of stars per cell to visit when doing the PCA 

819 ps : `lsst.daf.base.PropertySet` 

820 input property set controlling the single kernel visitor 

821 

822 Returns 

823 ------- 

824 nRejectedPca : `int` 

825 number of KernelCandidates rejected during PCA loop 

826 spatialBasisList : `list` of `lsst.afw.math.kernel.FixedKernel` 

827 basis list containing the principal shapes as Kernels 

828 

829 Raises 

830 ------ 

831 RuntimeError 

832 If the Eigenvalues sum to zero. 

833 """ 

834 nComponents = self.kConfig.numPrincipalComponents 

835 imagePca = diffimLib.KernelPcaD() 

836 importStarVisitor = diffimLib.KernelPcaVisitorF(imagePca) 

837 kernelCellSet.visitCandidates(importStarVisitor, nStarPerCell) 

838 if self.kConfig.subtractMeanForPca: 

839 importStarVisitor.subtractMean() 

840 imagePca.analyze() 

841 

842 eigenValues = imagePca.getEigenValues() 

843 pcaBasisList = importStarVisitor.getEigenKernels() 

844 

845 eSum = np.sum(eigenValues) 

846 if eSum == 0.0: 

847 raise RuntimeError("Eigenvalues sum to zero") 

848 for j in range(len(eigenValues)): 

849 log.log("TRACE5." + self.log.name + "._solve", log.DEBUG, 

850 "Eigenvalue %d : %f (%f)", j, eigenValues[j], eigenValues[j]/eSum) 

851 

852 nToUse = min(nComponents, len(eigenValues)) 

853 trimBasisList = [] 

854 for j in range(nToUse): 

855 # Check for NaNs? 

856 kimage = afwImage.ImageD(pcaBasisList[j].getDimensions()) 

857 pcaBasisList[j].computeImage(kimage, False) 

858 if not (True in np.isnan(kimage.getArray())): 

859 trimBasisList.append(pcaBasisList[j]) 

860 

861 # Put all the power in the first kernel, which will not vary spatially 

862 spatialBasisList = diffimLib.renormalizeKernelList(trimBasisList) 

863 

864 # New Kernel visitor for this new basis list (no regularization explicitly) 

865 singlekvPca = diffimLib.BuildSingleKernelVisitorF(spatialBasisList, ps) 

866 singlekvPca.setSkipBuilt(False) 

867 kernelCellSet.visitCandidates(singlekvPca, nStarPerCell) 

868 singlekvPca.setSkipBuilt(True) 

869 nRejectedPca = singlekvPca.getNRejected() 

870 

871 return nRejectedPca, spatialBasisList 

872 

873 def _buildCellSet(self, *args): 

874 """Fill a SpatialCellSet with KernelCandidates for the Psf-matching process; 

875 override in derived classes""" 

876 return 

877 

878 @timeMethod 

879 def _solve(self, kernelCellSet, basisList, returnOnExcept=False): 

880 """Solve for the PSF matching kernel 

881 

882 Parameters 

883 ---------- 

884 kernelCellSet : `lsst.afw.math.SpatialCellSet` 

885 a SpatialCellSet to use in determining the matching kernel 

886 (typically as provided by _buildCellSet) 

887 basisList : `list` of `lsst.afw.math.kernel.FixedKernel` 

888 list of Kernels to be used in the decomposition of the spatially varying kernel 

889 (typically as provided by makeKernelBasisList) 

890 returnOnExcept : `bool`, optional 

891 if True then return (None, None) if an error occurs, else raise the exception 

892 

893 Returns 

894 ------- 

895 psfMatchingKernel : `lsst.afw.math.LinearCombinationKernel` 

896 Spatially varying Psf-matching kernel 

897 backgroundModel : `lsst.afw.math.Function2D` 

898 Spatially varying background-matching function 

899 

900 Raises 

901 ------ 

902 RuntimeError : 

903 If unable to determine PSF matching kernel and ``returnOnExcept==False``. 

904 """ 

905 

906 import lsstDebug 

907 display = lsstDebug.Info(__name__).display 

908 

909 maxSpatialIterations = self.kConfig.maxSpatialIterations 

910 nStarPerCell = self.kConfig.nStarPerCell 

911 usePcaForSpatialKernel = self.kConfig.usePcaForSpatialKernel 

912 

913 # Visitor for the single kernel fit 

914 ps = pexConfig.makePropertySet(self.kConfig) 

915 if self.useRegularization: 

916 singlekv = diffimLib.BuildSingleKernelVisitorF(basisList, ps, self.hMat) 

917 else: 

918 singlekv = diffimLib.BuildSingleKernelVisitorF(basisList, ps) 

919 

920 # Visitor for the kernel sum rejection 

921 ksv = diffimLib.KernelSumVisitorF(ps) 

922 

923 # Main loop 

924 t0 = time.time() 

925 try: 

926 totalIterations = 0 

927 thisIteration = 0 

928 while (thisIteration < maxSpatialIterations): 

929 

930 # Make sure there are no uninitialized candidates as active occupants of Cell 

931 nRejectedSkf = -1 

932 while (nRejectedSkf != 0): 

933 log.log("TRACE1." + self.log.name + "._solve", log.DEBUG, 

934 "Building single kernels...") 

935 kernelCellSet.visitCandidates(singlekv, nStarPerCell) 

936 nRejectedSkf = singlekv.getNRejected() 

937 log.log("TRACE1." + self.log.name + "._solve", log.DEBUG, 

938 "Iteration %d, rejected %d candidates due to initial kernel fit", 

939 thisIteration, nRejectedSkf) 

940 

941 # Reject outliers in kernel sum 

942 ksv.resetKernelSum() 

943 ksv.setMode(diffimLib.KernelSumVisitorF.AGGREGATE) 

944 kernelCellSet.visitCandidates(ksv, nStarPerCell) 

945 ksv.processKsumDistribution() 

946 ksv.setMode(diffimLib.KernelSumVisitorF.REJECT) 

947 kernelCellSet.visitCandidates(ksv, nStarPerCell) 

948 

949 nRejectedKsum = ksv.getNRejected() 

950 log.log("TRACE1." + self.log.name + "._solve", log.DEBUG, 

951 "Iteration %d, rejected %d candidates due to kernel sum", 

952 thisIteration, nRejectedKsum) 

953 

954 # Do we jump back to the top without incrementing thisIteration? 

955 if nRejectedKsum > 0: 

956 totalIterations += 1 

957 continue 

958 

959 # At this stage we can either apply the spatial fit to 

960 # the kernels, or we run a PCA, use these as a *new* 

961 # basis set with lower dimensionality, and then apply 

962 # the spatial fit to these kernels 

963 

964 if (usePcaForSpatialKernel): 

965 log.log("TRACE0." + self.log.name + "._solve", log.DEBUG, 

966 "Building Pca basis") 

967 

968 nRejectedPca, spatialBasisList = self._createPcaBasis(kernelCellSet, nStarPerCell, ps) 

969 log.log("TRACE1." + self.log.name + "._solve", log.DEBUG, 

970 "Iteration %d, rejected %d candidates due to Pca kernel fit", 

971 thisIteration, nRejectedPca) 

972 

973 # We don't want to continue on (yet) with the 

974 # spatial modeling, because we have bad objects 

975 # contributing to the Pca basis. We basically 

976 # need to restart from the beginning of this loop, 

977 # since the cell-mates of those objects that were 

978 # rejected need their original Kernels built by 

979 # singleKernelFitter. 

980 

981 # Don't count against thisIteration 

982 if (nRejectedPca > 0): 

983 totalIterations += 1 

984 continue 

985 else: 

986 spatialBasisList = basisList 

987 

988 # We have gotten on to the spatial modeling part 

989 regionBBox = kernelCellSet.getBBox() 

990 spatialkv = diffimLib.BuildSpatialKernelVisitorF(spatialBasisList, regionBBox, ps) 

991 kernelCellSet.visitCandidates(spatialkv, nStarPerCell) 

992 spatialkv.solveLinearEquation() 

993 log.log("TRACE2." + self.log.name + "._solve", log.DEBUG, 

994 "Spatial kernel built with %d candidates", spatialkv.getNCandidates()) 

995 spatialKernel, spatialBackground = spatialkv.getSolutionPair() 

996 

997 # Check the quality of the spatial fit (look at residuals) 

998 assesskv = diffimLib.AssessSpatialKernelVisitorF(spatialKernel, spatialBackground, ps) 

999 kernelCellSet.visitCandidates(assesskv, nStarPerCell) 

1000 nRejectedSpatial = assesskv.getNRejected() 

1001 nGoodSpatial = assesskv.getNGood() 

1002 log.log("TRACE1." + self.log.name + "._solve", log.DEBUG, 

1003 "Iteration %d, rejected %d candidates due to spatial kernel fit", 

1004 thisIteration, nRejectedSpatial) 

1005 log.log("TRACE1." + self.log.name + "._solve", log.DEBUG, 

1006 "%d candidates used in fit", nGoodSpatial) 

1007 

1008 # If only nGoodSpatial == 0, might be other candidates in the cells 

1009 if nGoodSpatial == 0 and nRejectedSpatial == 0: 

1010 raise RuntimeError("No kernel candidates for spatial fit") 

1011 

1012 if nRejectedSpatial == 0: 

1013 # Nothing rejected, finished with spatial fit 

1014 break 

1015 

1016 # Otherwise, iterate on... 

1017 thisIteration += 1 

1018 

1019 # Final fit if above did not converge 

1020 if (nRejectedSpatial > 0) and (thisIteration == maxSpatialIterations): 

1021 log.log("TRACE1." + self.log.name + "._solve", log.DEBUG, "Final spatial fit") 

1022 if (usePcaForSpatialKernel): 

1023 nRejectedPca, spatialBasisList = self._createPcaBasis(kernelCellSet, nStarPerCell, ps) 

1024 regionBBox = kernelCellSet.getBBox() 

1025 spatialkv = diffimLib.BuildSpatialKernelVisitorF(spatialBasisList, regionBBox, ps) 

1026 kernelCellSet.visitCandidates(spatialkv, nStarPerCell) 

1027 spatialkv.solveLinearEquation() 

1028 log.log("TRACE2." + self.log.name + "._solve", log.DEBUG, 

1029 "Spatial kernel built with %d candidates", spatialkv.getNCandidates()) 

1030 spatialKernel, spatialBackground = spatialkv.getSolutionPair() 

1031 

1032 spatialSolution = spatialkv.getKernelSolution() 

1033 

1034 except Exception as e: 

1035 self.log.error("ERROR: Unable to calculate psf matching kernel") 

1036 

1037 log.log("TRACE1." + self.log.name + "._solve", log.DEBUG, "%s", e) 

1038 raise e 

1039 

1040 t1 = time.time() 

1041 log.log("TRACE0." + self.log.name + "._solve", log.DEBUG, 

1042 "Total time to compute the spatial kernel : %.2f s", (t1 - t0)) 

1043 

1044 if display: 

1045 self._displayDebug(kernelCellSet, spatialKernel, spatialBackground) 

1046 

1047 self._diagnostic(kernelCellSet, spatialSolution, spatialKernel, spatialBackground) 

1048 

1049 return spatialSolution, spatialKernel, spatialBackground 

1050 

1051 

1052PsfMatch = PsfMatchTask