Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ip_diffim. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["DetectionConfig", "PsfMatchConfig", "PsfMatchConfigAL", "PsfMatchConfigDF", "PsfMatchTask"] 

23 

24import time 

25 

26import numpy as np 

27 

28import lsst.afw.image as afwImage 

29import lsst.pex.config as pexConfig 

30import lsst.afw.math as afwMath 

31import lsst.afw.display as afwDisplay 

32import lsst.log as log 

33import lsst.pipe.base as pipeBase 

34from lsst.meas.algorithms import SubtractBackgroundConfig 

35from . import utils as diutils 

36from . import diffimLib 

37 

38 

39class DetectionConfig(pexConfig.Config): 

40 """Configuration for detecting sources on images for building a 

41 PSF-matching kernel 

42 

43 Configuration for turning detected lsst.afw.detection.FootPrints into an 

44 acceptable (unmasked, high signal-to-noise, not too large or not too small) 

45 list of `lsst.ip.diffim.KernelSources` that are used to build the 

46 Psf-matching kernel""" 

47 

48 detThreshold = pexConfig.Field( 48 ↛ exitline 48 didn't jump to the function exit

49 dtype=float, 

50 doc="Value of footprint detection threshold", 

51 default=10.0, 

52 check=lambda x: x >= 3.0 

53 ) 

54 detThresholdType = pexConfig.ChoiceField( 

55 dtype=str, 

56 doc="Type of detection threshold", 

57 default="pixel_stdev", 

58 allowed={ 

59 "value": "Use counts as the detection threshold type", 

60 "stdev": "Use standard deviation of image plane", 

61 "variance": "Use variance of image plane", 

62 "pixel_stdev": "Use stdev derived from variance plane" 

63 } 

64 ) 

65 detOnTemplate = pexConfig.Field( 

66 dtype=bool, 

67 doc="""If true run detection on the template (image to convolve); 

68 if false run detection on the science image""", 

69 default=True 

70 ) 

71 badMaskPlanes = pexConfig.ListField( 

72 dtype=str, 

73 doc="""Mask planes that lead to an invalid detection. 

74 Options: NO_DATA EDGE SAT BAD CR INTRP""", 

75 default=("NO_DATA", "EDGE", "SAT") 

76 ) 

77 fpNpixMin = pexConfig.Field( 77 ↛ exitline 77 didn't jump to the function exit

78 dtype=int, 

79 doc="Minimum number of pixels in an acceptable Footprint", 

80 default=5, 

81 check=lambda x: x >= 5 

82 ) 

83 fpNpixMax = pexConfig.Field( 83 ↛ exitline 83 didn't jump to the function exit

84 dtype=int, 

85 doc="""Maximum number of pixels in an acceptable Footprint; 

86 too big and the subsequent convolutions become unwieldy""", 

87 default=500, 

88 check=lambda x: x <= 500 

89 ) 

90 fpGrowKernelScaling = pexConfig.Field( 90 ↛ exitline 90 didn't jump to the function exit

91 dtype=float, 

92 doc="""If config.scaleByFwhm, grow the footprint based on 

93 the final kernelSize. Each footprint will be 

94 2*fpGrowKernelScaling*kernelSize x 

95 2*fpGrowKernelScaling*kernelSize. With the value 

96 of 1.0, the remaining pixels in each KernelCandiate 

97 after convolution by the basis functions will be 

98 equal to the kernel size itself.""", 

99 default=1.0, 

100 check=lambda x: x >= 1.0 

101 ) 

102 fpGrowPix = pexConfig.Field( 102 ↛ exitline 102 didn't jump to the function exit

103 dtype=int, 

104 doc="""Growing radius (in pixels) for each raw detection 

105 footprint. The smaller the faster; however the 

106 kernel sum does not converge if the stamp is too 

107 small; and the kernel is not constrained at all if 

108 the stamp is the size of the kernel. The grown stamp 

109 is 2 * fpGrowPix pixels larger in each dimension. 

110 This is overridden by fpGrowKernelScaling if scaleByFwhm""", 

111 default=30, 

112 check=lambda x: x >= 10 

113 ) 

114 scaleByFwhm = pexConfig.Field( 

115 dtype=bool, 

116 doc="Scale fpGrowPix by input Fwhm?", 

117 default=True, 

118 ) 

119 

120 

121class PsfMatchConfig(pexConfig.Config): 

122 """Base configuration for Psf-matching 

123 

124 The base configuration of the Psf-matching kernel, and of the warping, detection, 

125 and background modeling subTasks.""" 

126 

127 warpingConfig = pexConfig.ConfigField("Config for warping exposures to a common alignment", 

128 afwMath.warper.WarperConfig) 

129 detectionConfig = pexConfig.ConfigField("Controlling the detection of sources for kernel building", 

130 DetectionConfig) 

131 afwBackgroundConfig = pexConfig.ConfigField("Controlling the Afw background fitting", 

132 SubtractBackgroundConfig) 

133 

134 useAfwBackground = pexConfig.Field( 

135 dtype=bool, 

136 doc="Use afw background subtraction instead of ip_diffim", 

137 default=False, 

138 ) 

139 fitForBackground = pexConfig.Field( 

140 dtype=bool, 

141 doc="Include terms (including kernel cross terms) for background in ip_diffim", 

142 default=False, 

143 ) 

144 kernelBasisSet = pexConfig.ChoiceField( 

145 dtype=str, 

146 doc="Type of basis set for PSF matching kernel.", 

147 default="alard-lupton", 

148 allowed={ 

149 "alard-lupton": """Alard-Lupton sum-of-gaussians basis set, 

150 * The first term has no spatial variation 

151 * The kernel sum is conserved 

152 * You may want to turn off 'usePcaForSpatialKernel'""", 

153 "delta-function": """Delta-function kernel basis set, 

154 * You may enable the option useRegularization 

155 * You should seriously consider usePcaForSpatialKernel, which will also 

156 enable kernel sum conservation for the delta function kernels""" 

157 } 

158 ) 

159 kernelSize = pexConfig.Field( 

160 dtype=int, 

161 doc="""Number of rows/columns in the convolution kernel; should be odd-valued. 

162 Modified by kernelSizeFwhmScaling if scaleByFwhm = true""", 

163 default=21, 

164 ) 

165 scaleByFwhm = pexConfig.Field( 

166 dtype=bool, 

167 doc="Scale kernelSize, alardGaussians by input Fwhm", 

168 default=True, 

169 ) 

170 kernelSizeFwhmScaling = pexConfig.Field( 170 ↛ exitline 170 didn't jump to the function exit

171 dtype=float, 

172 doc="Multiplier of the largest AL Gaussian basis sigma to get the kernel bbox (pixel) size.", 

173 default=6.0, 

174 check=lambda x: x >= 1.0 

175 ) 

176 kernelSizeMin = pexConfig.Field( 

177 dtype=int, 

178 doc="Minimum kernel bbox (pixel) size.", 

179 default=21, 

180 ) 

181 kernelSizeMax = pexConfig.Field( 

182 dtype=int, 

183 doc="Maximum kernel bbox (pixel) size.", 

184 default=35, 

185 ) 

186 spatialModelType = pexConfig.ChoiceField( 

187 dtype=str, 

188 doc="Type of spatial functions for kernel and background", 

189 default="chebyshev1", 

190 allowed={ 

191 "chebyshev1": "Chebyshev polynomial of the first kind", 

192 "polynomial": "Standard x,y polynomial", 

193 } 

194 ) 

195 spatialKernelOrder = pexConfig.Field( 195 ↛ exitline 195 didn't jump to the function exit

196 dtype=int, 

197 doc="Spatial order of convolution kernel variation", 

198 default=2, 

199 check=lambda x: x >= 0 

200 ) 

201 spatialBgOrder = pexConfig.Field( 201 ↛ exitline 201 didn't jump to the function exit

202 dtype=int, 

203 doc="Spatial order of differential background variation", 

204 default=1, 

205 check=lambda x: x >= 0 

206 ) 

207 sizeCellX = pexConfig.Field( 207 ↛ exitline 207 didn't jump to the function exit

208 dtype=int, 

209 doc="Size (rows) in pixels of each SpatialCell for spatial modeling", 

210 default=128, 

211 check=lambda x: x >= 32 

212 ) 

213 sizeCellY = pexConfig.Field( 213 ↛ exitline 213 didn't jump to the function exit

214 dtype=int, 

215 doc="Size (columns) in pixels of each SpatialCell for spatial modeling", 

216 default=128, 

217 check=lambda x: x >= 32 

218 ) 

219 nStarPerCell = pexConfig.Field( 219 ↛ exitline 219 didn't jump to the function exit

220 dtype=int, 

221 doc="Number of KernelCandidates in each SpatialCell to use in the spatial fitting", 

222 default=3, 

223 check=lambda x: x >= 1 

224 ) 

225 maxSpatialIterations = pexConfig.Field( 225 ↛ exitline 225 didn't jump to the function exit

226 dtype=int, 

227 doc="Maximum number of iterations for rejecting bad KernelCandidates in spatial fitting", 

228 default=3, 

229 check=lambda x: x >= 1 and x <= 5 

230 ) 

231 usePcaForSpatialKernel = pexConfig.Field( 

232 dtype=bool, 

233 doc="""Use Pca to reduce the dimensionality of the kernel basis sets. 

234 This is particularly useful for delta-function kernels. 

235 Functionally, after all Cells have their raw kernels determined, we run 

236 a Pca on these Kernels, re-fit the Cells using the eigenKernels and then 

237 fit those for spatial variation using the same technique as for Alard-Lupton kernels. 

238 If this option is used, the first term will have no spatial variation and the 

239 kernel sum will be conserved.""", 

240 default=False, 

241 ) 

242 subtractMeanForPca = pexConfig.Field( 

243 dtype=bool, 

244 doc="Subtract off the mean feature before doing the Pca", 

245 default=True, 

246 ) 

247 numPrincipalComponents = pexConfig.Field( 247 ↛ exitline 247 didn't jump to the function exit

248 dtype=int, 

249 doc="""Number of principal components to use for Pca basis, including the 

250 mean kernel if requested.""", 

251 default=5, 

252 check=lambda x: x >= 3 

253 ) 

254 singleKernelClipping = pexConfig.Field( 

255 dtype=bool, 

256 doc="Do sigma clipping on each raw kernel candidate", 

257 default=True, 

258 ) 

259 kernelSumClipping = pexConfig.Field( 

260 dtype=bool, 

261 doc="Do sigma clipping on the ensemble of kernel sums", 

262 default=True, 

263 ) 

264 spatialKernelClipping = pexConfig.Field( 

265 dtype=bool, 

266 doc="Do sigma clipping after building the spatial model", 

267 default=True, 

268 ) 

269 checkConditionNumber = pexConfig.Field( 

270 dtype=bool, 

271 doc="""Test for maximum condition number when inverting a kernel matrix. 

272 Anything above maxConditionNumber is not used and the candidate is set as BAD. 

273 Also used to truncate inverse matrix in estimateBiasedRisk. However, 

274 if you are doing any deconvolution you will want to turn this off, or use 

275 a large maxConditionNumber""", 

276 default=False, 

277 ) 

278 badMaskPlanes = pexConfig.ListField( 

279 dtype=str, 

280 doc="""Mask planes to ignore when calculating diffim statistics 

281 Options: NO_DATA EDGE SAT BAD CR INTRP""", 

282 default=("NO_DATA", "EDGE", "SAT") 

283 ) 

284 candidateResidualMeanMax = pexConfig.Field( 284 ↛ exitline 284 didn't jump to the function exit

285 dtype=float, 

286 doc="""Rejects KernelCandidates yielding bad difference image quality. 

287 Used by BuildSingleKernelVisitor, AssessSpatialKernelVisitor. 

288 Represents average over pixels of (image/sqrt(variance)).""", 

289 default=0.25, 

290 check=lambda x: x >= 0.0 

291 ) 

292 candidateResidualStdMax = pexConfig.Field( 292 ↛ exitline 292 didn't jump to the function exit

293 dtype=float, 

294 doc="""Rejects KernelCandidates yielding bad difference image quality. 

295 Used by BuildSingleKernelVisitor, AssessSpatialKernelVisitor. 

296 Represents stddev over pixels of (image/sqrt(variance)).""", 

297 default=1.50, 

298 check=lambda x: x >= 0.0 

299 ) 

300 useCoreStats = pexConfig.Field( 

301 dtype=bool, 

302 doc="""Use the core of the footprint for the quality statistics, instead of the entire footprint. 

303 WARNING: if there is deconvolution we probably will need to turn this off""", 

304 default=False, 

305 ) 

306 candidateCoreRadius = pexConfig.Field( 306 ↛ exitline 306 didn't jump to the function exit

307 dtype=int, 

308 doc="""Radius for calculation of stats in 'core' of KernelCandidate diffim. 

309 Total number of pixels used will be (2*radius)**2. 

310 This is used both for 'core' diffim quality as well as ranking of 

311 KernelCandidates by their total flux in this core""", 

312 default=3, 

313 check=lambda x: x >= 1 

314 ) 

315 maxKsumSigma = pexConfig.Field( 315 ↛ exitline 315 didn't jump to the function exit

316 dtype=float, 

317 doc="""Maximum allowed sigma for outliers from kernel sum distribution. 

318 Used to reject variable objects from the kernel model""", 

319 default=3.0, 

320 check=lambda x: x >= 0.0 

321 ) 

322 maxConditionNumber = pexConfig.Field( 322 ↛ exitline 322 didn't jump to the function exit

323 dtype=float, 

324 doc="Maximum condition number for a well conditioned matrix", 

325 default=5.0e7, 

326 check=lambda x: x >= 0.0 

327 ) 

328 conditionNumberType = pexConfig.ChoiceField( 

329 dtype=str, 

330 doc="Use singular values (SVD) or eigen values (EIGENVALUE) to determine condition number", 

331 default="EIGENVALUE", 

332 allowed={ 

333 "SVD": "Use singular values", 

334 "EIGENVALUE": "Use eigen values (faster)", 

335 } 

336 ) 

337 maxSpatialConditionNumber = pexConfig.Field( 337 ↛ exitline 337 didn't jump to the function exit

338 dtype=float, 

339 doc="Maximum condition number for a well conditioned spatial matrix", 

340 default=1.0e10, 

341 check=lambda x: x >= 0.0 

342 ) 

343 iterateSingleKernel = pexConfig.Field( 

344 dtype=bool, 

345 doc="""Remake KernelCandidate using better variance estimate after first pass? 

346 Primarily useful when convolving a single-depth image, otherwise not necessary.""", 

347 default=False, 

348 ) 

349 constantVarianceWeighting = pexConfig.Field( 

350 dtype=bool, 

351 doc="""Use constant variance weighting in single kernel fitting? 

352 In some cases this is better for bright star residuals.""", 

353 default=True, 

354 ) 

355 calculateKernelUncertainty = pexConfig.Field( 

356 dtype=bool, 

357 doc="""Calculate kernel and background uncertainties for each kernel candidate? 

358 This comes from the inverse of the covariance matrix. 

359 Warning: regularization can cause problems for this step.""", 

360 default=False, 

361 ) 

362 useBicForKernelBasis = pexConfig.Field( 

363 dtype=bool, 

364 doc="""Use Bayesian Information Criterion to select the number of bases going into the kernel""", 

365 default=False, 

366 ) 

367 

368 

369class PsfMatchConfigAL(PsfMatchConfig): 

370 """The parameters specific to the "Alard-Lupton" (sum-of-Gaussian) Psf-matching basis""" 

371 

372 def setDefaults(self): 

373 PsfMatchConfig.setDefaults(self) 

374 self.kernelBasisSet = "alard-lupton" 

375 self.maxConditionNumber = 5.0e7 

376 

377 alardNGauss = pexConfig.Field( 377 ↛ exitline 377 didn't jump to the function exit

378 dtype=int, 

379 doc="Number of base Gaussians in alard-lupton kernel basis function generation.", 

380 default=3, 

381 check=lambda x: x >= 1 

382 ) 

383 alardDegGauss = pexConfig.ListField( 

384 dtype=int, 

385 doc="Polynomial order of spatial modification of base Gaussians. " 

386 "List length must be `alardNGauss`.", 

387 default=(4, 2, 2), 

388 ) 

389 alardSigGauss = pexConfig.ListField( 

390 dtype=float, 

391 doc="Default sigma values in pixels of base Gaussians. " 

392 "List length must be `alardNGauss`.", 

393 default=(0.7, 1.5, 3.0), 

394 ) 

395 alardGaussBeta = pexConfig.Field( 395 ↛ exitline 395 didn't jump to the function exit

396 dtype=float, 

397 doc="Used if `scaleByFwhm==True`, scaling multiplier of base " 

398 "Gaussian sigmas for automated sigma determination", 

399 default=2.0, 

400 check=lambda x: x >= 0.0, 

401 ) 

402 alardMinSig = pexConfig.Field( 402 ↛ exitline 402 didn't jump to the function exit

403 dtype=float, 

404 doc="Used if `scaleByFwhm==True`, minimum sigma (pixels) for base Gaussians", 

405 default=0.7, 

406 check=lambda x: x >= 0.25 

407 ) 

408 alardDegGaussDeconv = pexConfig.Field( 408 ↛ exitline 408 didn't jump to the function exit

409 dtype=int, 

410 doc="Used if `scaleByFwhm==True`, degree of spatial modification of ALL base Gaussians " 

411 "in AL basis during deconvolution", 

412 default=3, 

413 check=lambda x: x >= 1 

414 ) 

415 alardMinSigDeconv = pexConfig.Field( 415 ↛ exitline 415 didn't jump to the function exit

416 dtype=float, 

417 doc="Used if `scaleByFwhm==True`, minimum sigma (pixels) for base Gaussians during deconvolution; " 

418 "make smaller than `alardMinSig` as this is only indirectly used", 

419 default=0.4, 

420 check=lambda x: x >= 0.25 

421 ) 

422 alardNGaussDeconv = pexConfig.Field( 422 ↛ exitline 422 didn't jump to the function exit

423 dtype=int, 

424 doc="Used if `scaleByFwhm==True`, number of base Gaussians in AL basis during deconvolution", 

425 default=3, 

426 check=lambda x: x >= 1 

427 ) 

428 

429 

430class PsfMatchConfigDF(PsfMatchConfig): 

431 """The parameters specific to the delta-function (one basis per-pixel) Psf-matching basis""" 

432 

433 def setDefaults(self): 

434 PsfMatchConfig.setDefaults(self) 

435 self.kernelBasisSet = "delta-function" 

436 self.maxConditionNumber = 5.0e6 

437 self.usePcaForSpatialKernel = True 

438 self.subtractMeanForPca = True 

439 self.useBicForKernelBasis = False 

440 

441 useRegularization = pexConfig.Field( 

442 dtype=bool, 

443 doc="Use regularization to smooth the delta function kernels", 

444 default=True, 

445 ) 

446 regularizationType = pexConfig.ChoiceField( 

447 dtype=str, 

448 doc="Type of regularization.", 

449 default="centralDifference", 

450 allowed={ 

451 "centralDifference": "Penalize second derivative using 2-D stencil of central finite difference", 

452 "forwardDifference": "Penalize first, second, third derivatives using forward finite differeces" 

453 } 

454 ) 

455 centralRegularizationStencil = pexConfig.ChoiceField( 

456 dtype=int, 

457 doc="Type of stencil to approximate central derivative (for centralDifference only)", 

458 default=9, 

459 allowed={ 

460 5: "5-point stencil including only adjacent-in-x,y elements", 

461 9: "9-point stencil including diagonal elements" 

462 } 

463 ) 

464 forwardRegularizationOrders = pexConfig.ListField( 464 ↛ exitline 464 didn't jump to the function exit

465 dtype=int, 

466 doc="Array showing which order derivatives to penalize (for forwardDifference only)", 

467 default=(1, 2), 

468 itemCheck=lambda x: (x > 0) and (x < 4) 

469 ) 

470 regularizationBorderPenalty = pexConfig.Field( 470 ↛ exitline 470 didn't jump to the function exit

471 dtype=float, 

472 doc="Value of the penalty for kernel border pixels", 

473 default=3.0, 

474 check=lambda x: x >= 0.0 

475 ) 

476 lambdaType = pexConfig.ChoiceField( 

477 dtype=str, 

478 doc="How to choose the value of the regularization strength", 

479 default="absolute", 

480 allowed={ 

481 "absolute": "Use lambdaValue as the value of regularization strength", 

482 "relative": "Use lambdaValue as fraction of the default regularization strength (N.R. 18.5.8)", 

483 "minimizeBiasedRisk": "Minimize biased risk estimate", 

484 "minimizeUnbiasedRisk": "Minimize unbiased risk estimate", 

485 } 

486 ) 

487 lambdaValue = pexConfig.Field( 

488 dtype=float, 

489 doc="Value used for absolute determinations of regularization strength", 

490 default=0.2, 

491 ) 

492 lambdaScaling = pexConfig.Field( 

493 dtype=float, 

494 doc="Fraction of the default lambda strength (N.R. 18.5.8) to use. 1e-4 or 1e-5", 

495 default=1e-4, 

496 ) 

497 lambdaStepType = pexConfig.ChoiceField( 

498 dtype=str, 

499 doc="""If a scan through lambda is needed (minimizeBiasedRisk, minimizeUnbiasedRisk), 

500 use log or linear steps""", 

501 default="log", 

502 allowed={ 

503 "log": "Step in log intervals; e.g. lambdaMin, lambdaMax, lambdaStep = -1.0, 2.0, 0.1", 

504 "linear": "Step in linear intervals; e.g. lambdaMin, lambdaMax, lambdaStep = 0.1, 100, 0.1", 

505 } 

506 ) 

507 lambdaMin = pexConfig.Field( 

508 dtype=float, 

509 doc="""If scan through lambda needed (minimizeBiasedRisk, minimizeUnbiasedRisk), 

510 start at this value. If lambdaStepType = log:linear, suggest -1:0.1""", 

511 default=-1.0, 

512 ) 

513 lambdaMax = pexConfig.Field( 

514 dtype=float, 

515 doc="""If scan through lambda needed (minimizeBiasedRisk, minimizeUnbiasedRisk), 

516 stop at this value. If lambdaStepType = log:linear, suggest 2:100""", 

517 default=2.0, 

518 ) 

519 lambdaStep = pexConfig.Field( 

520 dtype=float, 

521 doc="""If scan through lambda needed (minimizeBiasedRisk, minimizeUnbiasedRisk), 

522 step in these increments. If lambdaStepType = log:linear, suggest 0.1:0.1""", 

523 default=0.1, 

524 ) 

525 

526 

527class PsfMatchTask(pipeBase.Task): 

528 """Base class for Psf Matching; should not be called directly 

529 

530 Notes 

531 ----- 

532 PsfMatchTask is a base class that implements the core functionality for matching the 

533 Psfs of two images using a spatially varying Psf-matching `lsst.afw.math.LinearCombinationKernel`. 

534 The Task requires the user to provide an instance of an `lsst.afw.math.SpatialCellSet`, 

535 filled with `lsst.ip.diffim.KernelCandidate` instances, and a list of `lsst.afw.math.Kernels` 

536 of basis shapes that will be used for the decomposition. If requested, the Task 

537 also performs background matching and returns the differential background model as an 

538 `lsst.afw.math.Kernel.SpatialFunction`. 

539 

540 Invoking the Task 

541 ----------------- 

542 

543 As a base class, this Task is not directly invoked. However, ``run()`` methods that are 

544 implemented on derived classes will make use of the core ``_solve()`` functionality, 

545 which defines a sequence of `lsst.afw.math.CandidateVisitor` classes that iterate 

546 through the KernelCandidates, first building up a per-candidate solution and then 

547 building up a spatial model from the ensemble of candidates. Sigma clipping is 

548 performed using the mean and standard deviation of all kernel sums (to reject 

549 variable objects), on the per-candidate substamp diffim residuals 

550 (to indicate a bad choice of kernel basis shapes for that particular object), 

551 and on the substamp diffim residuals using the spatial kernel fit (to indicate a bad 

552 choice of spatial kernel order, or poor constraints on the spatial model). The 

553 ``_diagnostic()`` method logs information on the quality of the spatial fit, and also 

554 modifies the Task metadata. 

555 

556 .. list-table:: Quantities set in Metadata 

557 :header-rows: 1 

558 

559 * - Parameter 

560 - Description 

561 * - ``spatialConditionNum`` 

562 - Condition number of the spatial kernel fit 

563 * - ``spatialKernelSum`` 

564 - Kernel sum (10^{-0.4 * ``Delta``; zeropoint}) of the spatial Psf-matching kernel 

565 * - ``ALBasisNGauss`` 

566 - If using sum-of-Gaussian basis, the number of gaussians used 

567 * - ``ALBasisDegGauss`` 

568 - If using sum-of-Gaussian basis, the deg of spatial variation of the Gaussians 

569 * - ``ALBasisSigGauss`` 

570 - If using sum-of-Gaussian basis, the widths (sigma) of the Gaussians 

571 * - ``ALKernelSize`` 

572 - If using sum-of-Gaussian basis, the kernel size 

573 * - ``NFalsePositivesTotal`` 

574 - Total number of diaSources 

575 * - ``NFalsePositivesRefAssociated`` 

576 - Number of diaSources that associate with the reference catalog 

577 * - ``NFalsePositivesRefAssociated`` 

578 - Number of diaSources that associate with the source catalog 

579 * - ``NFalsePositivesUnassociated`` 

580 - Number of diaSources that are orphans 

581 * - ``metric_MEAN`` 

582 - Mean value of substamp diffim quality metrics across all KernelCandidates, 

583 for both the per-candidate (LOCAL) and SPATIAL residuals 

584 * - ``metric_MEDIAN`` 

585 - Median value of substamp diffim quality metrics across all KernelCandidates, 

586 for both the per-candidate (LOCAL) and SPATIAL residuals 

587 * - ``metric_STDEV`` 

588 - Standard deviation of substamp diffim quality metrics across all KernelCandidates, 

589 for both the per-candidate (LOCAL) and SPATIAL residuals 

590 

591 Debug variables 

592 --------------- 

593 

594 The `lsst.pipe.base.cmdLineTask.CmdLineTask` command line task interface supports a 

595 flag -d/--debug to import @b debug.py from your PYTHONPATH. The relevant contents of debug.py 

596 for this Task include: 

597 

598 .. code-block:: py 

599 

600 import sys 

601 import lsstDebug 

602 def DebugInfo(name): 

603 di = lsstDebug.getInfo(name) 

604 if name == "lsst.ip.diffim.psfMatch": 

605 # enable debug output 

606 di.display = True 

607 # display mask transparency 

608 di.maskTransparency = 80 

609 # show all the candidates and residuals 

610 di.displayCandidates = True 

611 # show kernel basis functions 

612 di.displayKernelBasis = False 

613 # show kernel realized across the image 

614 di.displayKernelMosaic = True 

615 # show coefficients of spatial model 

616 di.plotKernelSpatialModel = False 

617 # show fixed and spatial coefficients and coefficient histograms 

618 di.plotKernelCoefficients = True 

619 # show the bad candidates (red) along with good (green) 

620 di.showBadCandidates = True 

621 return di 

622 lsstDebug.Info = DebugInfo 

623 lsstDebug.frame = 1 

624 

625 Note that if you want additional logging info, you may add to your scripts: 

626 

627 .. code-block:: py 

628 

629 import lsst.log.utils as logUtils 

630 logUtils.traceSetAt("ip.diffim", 4) 

631 """ 

632 ConfigClass = PsfMatchConfig 

633 _DefaultName = "psfMatch" 

634 

635 def __init__(self, *args, **kwargs): 

636 """Create the psf-matching Task 

637 

638 Parameters 

639 ---------- 

640 *args 

641 Arguments to be passed to ``lsst.pipe.base.task.Task.__init__`` 

642 **kwargs 

643 Keyword arguments to be passed to ``lsst.pipe.base.task.Task.__init__`` 

644 

645 Notes 

646 ----- 

647 The initialization sets the Psf-matching kernel configuration using the value of 

648 self.config.kernel.active. If the kernel is requested with regularization to moderate 

649 the bias/variance tradeoff, currently only used when a delta function kernel basis 

650 is provided, it creates a regularization matrix stored as member variable 

651 self.hMat. 

652 """ 

653 pipeBase.Task.__init__(self, *args, **kwargs) 

654 self.kConfig = self.config.kernel.active 

655 

656 if 'useRegularization' in self.kConfig: 

657 self.useRegularization = self.kConfig.useRegularization 

658 else: 

659 self.useRegularization = False 

660 

661 if self.useRegularization: 

662 self.hMat = diffimLib.makeRegularizationMatrix(pexConfig.makePropertySet(self.kConfig)) 

663 

664 def _diagnostic(self, kernelCellSet, spatialSolution, spatialKernel, spatialBg): 

665 """Provide logging diagnostics on quality of spatial kernel fit 

666 

667 Parameters 

668 ---------- 

669 kernelCellSet : `lsst.afw.math.SpatialCellSet` 

670 Cellset that contains the KernelCandidates used in the fitting 

671 spatialSolution : `lsst.ip.diffim.SpatialKernelSolution` 

672 KernelSolution of best-fit 

673 spatialKernel : `lsst.afw.math.LinearCombinationKernel` 

674 Best-fit spatial Kernel model 

675 spatialBg : `lsst.afw.math.Function2D` 

676 Best-fit spatial background model 

677 """ 

678 # What is the final kernel sum 

679 kImage = afwImage.ImageD(spatialKernel.getDimensions()) 

680 kSum = spatialKernel.computeImage(kImage, False) 

681 self.log.info("Final spatial kernel sum %.3f" % (kSum)) 

682 

683 # Look at how well conditioned the matrix is 

684 conditionNum = spatialSolution.getConditionNumber( 

685 getattr(diffimLib.KernelSolution, self.kConfig.conditionNumberType)) 

686 self.log.info("Spatial model condition number %.3e" % (conditionNum)) 

687 

688 if conditionNum < 0.0: 

689 self.log.warn("Condition number is negative (%.3e)" % (conditionNum)) 

690 if conditionNum > self.kConfig.maxSpatialConditionNumber: 

691 self.log.warn("Spatial solution exceeds max condition number (%.3e > %.3e)" % ( 

692 conditionNum, self.kConfig.maxSpatialConditionNumber)) 

693 

694 self.metadata.set("spatialConditionNum", conditionNum) 

695 self.metadata.set("spatialKernelSum", kSum) 

696 

697 # Look at how well the solution is constrained 

698 nBasisKernels = spatialKernel.getNBasisKernels() 

699 nKernelTerms = spatialKernel.getNSpatialParameters() 

700 if nKernelTerms == 0: # order 0 

701 nKernelTerms = 1 

702 

703 # Not fit for 

704 nBgTerms = spatialBg.getNParameters() 

705 if nBgTerms == 1: 

706 if spatialBg.getParameters()[0] == 0.0: 

707 nBgTerms = 0 

708 

709 nGood = 0 

710 nBad = 0 

711 nTot = 0 

712 for cell in kernelCellSet.getCellList(): 

713 for cand in cell.begin(False): # False = include bad candidates 

714 nTot += 1 

715 if cand.getStatus() == afwMath.SpatialCellCandidate.GOOD: 

716 nGood += 1 

717 if cand.getStatus() == afwMath.SpatialCellCandidate.BAD: 

718 nBad += 1 

719 

720 self.log.info("Doing stats of kernel candidates used in the spatial fit.") 

721 

722 # Counting statistics 

723 if nBad > 2*nGood: 

724 self.log.warn("Many more candidates rejected than accepted; %d total, %d rejected, %d used" % ( 

725 nTot, nBad, nGood)) 

726 else: 

727 self.log.info("%d candidates total, %d rejected, %d used" % (nTot, nBad, nGood)) 

728 

729 # Some judgements on the quality of the spatial models 

730 if nGood < nKernelTerms: 

731 self.log.warn("Spatial kernel model underconstrained; %d candidates, %d terms, %d bases" % ( 

732 nGood, nKernelTerms, nBasisKernels)) 

733 self.log.warn("Consider lowering the spatial order") 

734 elif nGood <= 2*nKernelTerms: 

735 self.log.warn("Spatial kernel model poorly constrained; %d candidates, %d terms, %d bases" % ( 

736 nGood, nKernelTerms, nBasisKernels)) 

737 self.log.warn("Consider lowering the spatial order") 

738 else: 

739 self.log.info("Spatial kernel model well constrained; %d candidates, %d terms, %d bases" % ( 

740 nGood, nKernelTerms, nBasisKernels)) 

741 

742 if nGood < nBgTerms: 

743 self.log.warn("Spatial background model underconstrained; %d candidates, %d terms" % ( 

744 nGood, nBgTerms)) 

745 self.log.warn("Consider lowering the spatial order") 

746 elif nGood <= 2*nBgTerms: 

747 self.log.warn("Spatial background model poorly constrained; %d candidates, %d terms" % ( 

748 nGood, nBgTerms)) 

749 self.log.warn("Consider lowering the spatial order") 

750 else: 

751 self.log.info("Spatial background model appears well constrained; %d candidates, %d terms" % ( 

752 nGood, nBgTerms)) 

753 

754 def _displayDebug(self, kernelCellSet, spatialKernel, spatialBackground): 

755 """Provide visualization of the inputs and ouputs to the Psf-matching code 

756 

757 Parameters 

758 ---------- 

759 kernelCellSet : `lsst.afw.math.SpatialCellSet` 

760 The SpatialCellSet used in determining the matching kernel and background 

761 spatialKernel : `lsst.afw.math.LinearCombinationKernel` 

762 Spatially varying Psf-matching kernel 

763 spatialBackground : `lsst.afw.math.Function2D` 

764 Spatially varying background-matching function 

765 """ 

766 import lsstDebug 

767 displayCandidates = lsstDebug.Info(__name__).displayCandidates 

768 displayKernelBasis = lsstDebug.Info(__name__).displayKernelBasis 

769 displayKernelMosaic = lsstDebug.Info(__name__).displayKernelMosaic 

770 plotKernelSpatialModel = lsstDebug.Info(__name__).plotKernelSpatialModel 

771 plotKernelCoefficients = lsstDebug.Info(__name__).plotKernelCoefficients 

772 showBadCandidates = lsstDebug.Info(__name__).showBadCandidates 

773 maskTransparency = lsstDebug.Info(__name__).maskTransparency 

774 if not maskTransparency: 

775 maskTransparency = 0 

776 afwDisplay.setDefaultMaskTransparency(maskTransparency) 

777 

778 if displayCandidates: 

779 diutils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground, 

780 frame=lsstDebug.frame, 

781 showBadCandidates=showBadCandidates) 

782 lsstDebug.frame += 1 

783 diutils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground, 

784 frame=lsstDebug.frame, 

785 showBadCandidates=showBadCandidates, 

786 kernels=True) 

787 lsstDebug.frame += 1 

788 diutils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground, 

789 frame=lsstDebug.frame, 

790 showBadCandidates=showBadCandidates, 

791 resids=True) 

792 lsstDebug.frame += 1 

793 

794 if displayKernelBasis: 

795 diutils.showKernelBasis(spatialKernel, frame=lsstDebug.frame) 

796 lsstDebug.frame += 1 

797 

798 if displayKernelMosaic: 

799 diutils.showKernelMosaic(kernelCellSet.getBBox(), spatialKernel, frame=lsstDebug.frame) 

800 lsstDebug.frame += 1 

801 

802 if plotKernelSpatialModel: 

803 diutils.plotKernelSpatialModel(spatialKernel, kernelCellSet, showBadCandidates=showBadCandidates) 

804 

805 if plotKernelCoefficients: 

806 diutils.plotKernelCoefficients(spatialKernel, kernelCellSet) 

807 

808 def _createPcaBasis(self, kernelCellSet, nStarPerCell, ps): 

809 """Create Principal Component basis 

810 

811 If a principal component analysis is requested, typically when using a delta function basis, 

812 perform the PCA here and return a new basis list containing the new principal components. 

813 

814 Parameters 

815 ---------- 

816 kernelCellSet : `lsst.afw.math.SpatialCellSet` 

817 a SpatialCellSet containing KernelCandidates, from which components are derived 

818 nStarPerCell : `int` 

819 the number of stars per cell to visit when doing the PCA 

820 ps : `lsst.daf.base.PropertySet` 

821 input property set controlling the single kernel visitor 

822 

823 Returns 

824 ------- 

825 nRejectedPca : `int` 

826 number of KernelCandidates rejected during PCA loop 

827 spatialBasisList : `list` of `lsst.afw.math.kernel.FixedKernel` 

828 basis list containing the principal shapes as Kernels 

829 

830 Raises 

831 ------ 

832 RuntimeError 

833 If the Eigenvalues sum to zero. 

834 """ 

835 nComponents = self.kConfig.numPrincipalComponents 

836 imagePca = diffimLib.KernelPcaD() 

837 importStarVisitor = diffimLib.KernelPcaVisitorF(imagePca) 

838 kernelCellSet.visitCandidates(importStarVisitor, nStarPerCell) 

839 if self.kConfig.subtractMeanForPca: 

840 importStarVisitor.subtractMean() 

841 imagePca.analyze() 

842 

843 eigenValues = imagePca.getEigenValues() 

844 pcaBasisList = importStarVisitor.getEigenKernels() 

845 

846 eSum = np.sum(eigenValues) 

847 if eSum == 0.0: 

848 raise RuntimeError("Eigenvalues sum to zero") 

849 for j in range(len(eigenValues)): 

850 log.log("TRACE5." + self.log.getName() + "._solve", log.DEBUG, 

851 "Eigenvalue %d : %f (%f)", j, eigenValues[j], eigenValues[j]/eSum) 

852 

853 nToUse = min(nComponents, len(eigenValues)) 

854 trimBasisList = [] 

855 for j in range(nToUse): 

856 # Check for NaNs? 

857 kimage = afwImage.ImageD(pcaBasisList[j].getDimensions()) 

858 pcaBasisList[j].computeImage(kimage, False) 

859 if not (True in np.isnan(kimage.getArray())): 

860 trimBasisList.append(pcaBasisList[j]) 

861 

862 # Put all the power in the first kernel, which will not vary spatially 

863 spatialBasisList = diffimLib.renormalizeKernelList(trimBasisList) 

864 

865 # New Kernel visitor for this new basis list (no regularization explicitly) 

866 singlekvPca = diffimLib.BuildSingleKernelVisitorF(spatialBasisList, ps) 

867 singlekvPca.setSkipBuilt(False) 

868 kernelCellSet.visitCandidates(singlekvPca, nStarPerCell) 

869 singlekvPca.setSkipBuilt(True) 

870 nRejectedPca = singlekvPca.getNRejected() 

871 

872 return nRejectedPca, spatialBasisList 

873 

874 def _buildCellSet(self, *args): 

875 """Fill a SpatialCellSet with KernelCandidates for the Psf-matching process; 

876 override in derived classes""" 

877 return 

878 

879 @pipeBase.timeMethod 

880 def _solve(self, kernelCellSet, basisList, returnOnExcept=False): 

881 """Solve for the PSF matching kernel 

882 

883 Parameters 

884 ---------- 

885 kernelCellSet : `lsst.afw.math.SpatialCellSet` 

886 a SpatialCellSet to use in determining the matching kernel 

887 (typically as provided by _buildCellSet) 

888 basisList : `list` of `lsst.afw.math.kernel.FixedKernel` 

889 list of Kernels to be used in the decomposition of the spatially varying kernel 

890 (typically as provided by makeKernelBasisList) 

891 returnOnExcept : `bool`, optional 

892 if True then return (None, None) if an error occurs, else raise the exception 

893 

894 Returns 

895 ------- 

896 psfMatchingKernel : `lsst.afw.math.LinearCombinationKernel` 

897 Spatially varying Psf-matching kernel 

898 backgroundModel : `lsst.afw.math.Function2D` 

899 Spatially varying background-matching function 

900 

901 Raises 

902 ------ 

903 RuntimeError : 

904 If unable to determine PSF matching kernel and ``returnOnExcept==False``. 

905 """ 

906 

907 import lsstDebug 

908 display = lsstDebug.Info(__name__).display 

909 

910 maxSpatialIterations = self.kConfig.maxSpatialIterations 

911 nStarPerCell = self.kConfig.nStarPerCell 

912 usePcaForSpatialKernel = self.kConfig.usePcaForSpatialKernel 

913 

914 # Visitor for the single kernel fit 

915 ps = pexConfig.makePropertySet(self.kConfig) 

916 if self.useRegularization: 

917 singlekv = diffimLib.BuildSingleKernelVisitorF(basisList, ps, self.hMat) 

918 else: 

919 singlekv = diffimLib.BuildSingleKernelVisitorF(basisList, ps) 

920 

921 # Visitor for the kernel sum rejection 

922 ksv = diffimLib.KernelSumVisitorF(ps) 

923 

924 # Main loop 

925 t0 = time.time() 

926 try: 

927 totalIterations = 0 

928 thisIteration = 0 

929 while (thisIteration < maxSpatialIterations): 

930 

931 # Make sure there are no uninitialized candidates as active occupants of Cell 

932 nRejectedSkf = -1 

933 while (nRejectedSkf != 0): 

934 log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG, 

935 "Building single kernels...") 

936 kernelCellSet.visitCandidates(singlekv, nStarPerCell) 

937 nRejectedSkf = singlekv.getNRejected() 

938 log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG, 

939 "Iteration %d, rejected %d candidates due to initial kernel fit", 

940 thisIteration, nRejectedSkf) 

941 

942 # Reject outliers in kernel sum 

943 ksv.resetKernelSum() 

944 ksv.setMode(diffimLib.KernelSumVisitorF.AGGREGATE) 

945 kernelCellSet.visitCandidates(ksv, nStarPerCell) 

946 ksv.processKsumDistribution() 

947 ksv.setMode(diffimLib.KernelSumVisitorF.REJECT) 

948 kernelCellSet.visitCandidates(ksv, nStarPerCell) 

949 

950 nRejectedKsum = ksv.getNRejected() 

951 log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG, 

952 "Iteration %d, rejected %d candidates due to kernel sum", 

953 thisIteration, nRejectedKsum) 

954 

955 # Do we jump back to the top without incrementing thisIteration? 

956 if nRejectedKsum > 0: 

957 totalIterations += 1 

958 continue 

959 

960 # At this stage we can either apply the spatial fit to 

961 # the kernels, or we run a PCA, use these as a *new* 

962 # basis set with lower dimensionality, and then apply 

963 # the spatial fit to these kernels 

964 

965 if (usePcaForSpatialKernel): 

966 log.log("TRACE0." + self.log.getName() + "._solve", log.DEBUG, 

967 "Building Pca basis") 

968 

969 nRejectedPca, spatialBasisList = self._createPcaBasis(kernelCellSet, nStarPerCell, ps) 

970 log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG, 

971 "Iteration %d, rejected %d candidates due to Pca kernel fit", 

972 thisIteration, nRejectedPca) 

973 

974 # We don't want to continue on (yet) with the 

975 # spatial modeling, because we have bad objects 

976 # contributing to the Pca basis. We basically 

977 # need to restart from the beginning of this loop, 

978 # since the cell-mates of those objects that were 

979 # rejected need their original Kernels built by 

980 # singleKernelFitter. 

981 

982 # Don't count against thisIteration 

983 if (nRejectedPca > 0): 

984 totalIterations += 1 

985 continue 

986 else: 

987 spatialBasisList = basisList 

988 

989 # We have gotten on to the spatial modeling part 

990 regionBBox = kernelCellSet.getBBox() 

991 spatialkv = diffimLib.BuildSpatialKernelVisitorF(spatialBasisList, regionBBox, ps) 

992 kernelCellSet.visitCandidates(spatialkv, nStarPerCell) 

993 spatialkv.solveLinearEquation() 

994 log.log("TRACE2." + self.log.getName() + "._solve", log.DEBUG, 

995 "Spatial kernel built with %d candidates", spatialkv.getNCandidates()) 

996 spatialKernel, spatialBackground = spatialkv.getSolutionPair() 

997 

998 # Check the quality of the spatial fit (look at residuals) 

999 assesskv = diffimLib.AssessSpatialKernelVisitorF(spatialKernel, spatialBackground, ps) 

1000 kernelCellSet.visitCandidates(assesskv, nStarPerCell) 

1001 nRejectedSpatial = assesskv.getNRejected() 

1002 nGoodSpatial = assesskv.getNGood() 

1003 log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG, 

1004 "Iteration %d, rejected %d candidates due to spatial kernel fit", 

1005 thisIteration, nRejectedSpatial) 

1006 log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG, 

1007 "%d candidates used in fit", nGoodSpatial) 

1008 

1009 # If only nGoodSpatial == 0, might be other candidates in the cells 

1010 if nGoodSpatial == 0 and nRejectedSpatial == 0: 

1011 raise RuntimeError("No kernel candidates for spatial fit") 

1012 

1013 if nRejectedSpatial == 0: 

1014 # Nothing rejected, finished with spatial fit 

1015 break 

1016 

1017 # Otherwise, iterate on... 

1018 thisIteration += 1 

1019 

1020 # Final fit if above did not converge 

1021 if (nRejectedSpatial > 0) and (thisIteration == maxSpatialIterations): 

1022 log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG, "Final spatial fit") 

1023 if (usePcaForSpatialKernel): 

1024 nRejectedPca, spatialBasisList = self._createPcaBasis(kernelCellSet, nStarPerCell, ps) 

1025 regionBBox = kernelCellSet.getBBox() 

1026 spatialkv = diffimLib.BuildSpatialKernelVisitorF(spatialBasisList, regionBBox, ps) 

1027 kernelCellSet.visitCandidates(spatialkv, nStarPerCell) 

1028 spatialkv.solveLinearEquation() 

1029 log.log("TRACE2." + self.log.getName() + "._solve", log.DEBUG, 

1030 "Spatial kernel built with %d candidates", spatialkv.getNCandidates()) 

1031 spatialKernel, spatialBackground = spatialkv.getSolutionPair() 

1032 

1033 spatialSolution = spatialkv.getKernelSolution() 

1034 

1035 except Exception as e: 

1036 self.log.error("ERROR: Unable to calculate psf matching kernel") 

1037 

1038 log.log("TRACE1." + self.log.getName() + "._solve", log.DEBUG, str(e)) 

1039 raise e 

1040 

1041 t1 = time.time() 

1042 log.log("TRACE0." + self.log.getName() + "._solve", log.DEBUG, 

1043 "Total time to compute the spatial kernel : %.2f s", (t1 - t0)) 

1044 

1045 if display: 

1046 self._displayDebug(kernelCellSet, spatialKernel, spatialBackground) 

1047 

1048 self._diagnostic(kernelCellSet, spatialSolution, spatialKernel, spatialBackground) 

1049 

1050 return spatialSolution, spatialKernel, spatialBackground 

1051 

1052 

1053PsfMatch = PsfMatchTask