Coverage for python/lsst/ip/diffim/imageDecorrelation.py: 14%

286 statements  

« prev     ^ index     » next       coverage.py v7.1.0, created at 2023-02-05 18:32 -0800

1# 

2# LSST Data Management System 

3# Copyright 2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23import numpy as np 

24 

25import lsst.afw.image as afwImage 

26import lsst.afw.math as afwMath 

27import lsst.geom as geom 

28import lsst.log 

29import lsst.meas.algorithms as measAlg 

30import lsst.pex.config as pexConfig 

31import lsst.pipe.base as pipeBase 

32 

33 

34from .imageMapReduce import (ImageMapReduceConfig, ImageMapReduceTask, 

35 ImageMapper) 

36 

37__all__ = ("DecorrelateALKernelTask", "DecorrelateALKernelConfig", 

38 "DecorrelateALKernelMapper", "DecorrelateALKernelMapReduceConfig", 

39 "DecorrelateALKernelSpatialConfig", "DecorrelateALKernelSpatialTask") 

40 

41 

42class DecorrelateALKernelConfig(pexConfig.Config): 

43 """Configuration parameters for the DecorrelateALKernelTask 

44 """ 

45 

46 ignoreMaskPlanes = pexConfig.ListField( 

47 dtype=str, 

48 doc="""Mask planes to ignore for sigma-clipped statistics""", 

49 default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE") 

50 ) 

51 completeVarPlanePropagation = pexConfig.Field( 

52 dtype=bool, 

53 default=False, 

54 doc="Compute the full effect of the decorrelated matching kernel on the variance plane." 

55 " Otherwise use a model weighed sum of the input variances." 

56 ) 

57 

58 

59class DecorrelateALKernelTask(pipeBase.Task): 

60 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference 

61 

62 Notes 

63 ----- 

64 

65 Pipe-task that removes the neighboring-pixel covariance in an 

66 image difference that are added when the template image is 

67 convolved with the Alard-Lupton PSF matching kernel. 

68 

69 The image differencing pipeline task @link 

70 ip.diffim.psfMatch.PsfMatchTask PSFMatchTask@endlink and @link 

71 ip.diffim.psfMatch.PsfMatchConfigAL PSFMatchConfigAL@endlink uses 

72 the Alard and Lupton (1998) method for matching the PSFs of the 

73 template and science exposures prior to subtraction. The 

74 Alard-Lupton method identifies a matching kernel, which is then 

75 (typically) convolved with the template image to perform PSF 

76 matching. This convolution has the effect of adding covariance 

77 between neighboring pixels in the template image, which is then 

78 added to the image difference by subtraction. 

79 

80 The pixel covariance may be corrected by whitening the noise of 

81 the image difference. This task performs such a decorrelation by 

82 computing a decorrelation kernel (based upon the A&L matching 

83 kernel and variances in the template and science images) and 

84 convolving the image difference with it. This process is described 

85 in detail in [DMTN-021](http://dmtn-021.lsst.io). 

86 

87 This task has no standalone example, however it is applied as a 

88 subtask of pipe.tasks.imageDifference.ImageDifferenceTask. 

89 """ 

90 ConfigClass = DecorrelateALKernelConfig 

91 _DefaultName = "ip_diffim_decorrelateALKernel" 

92 

93 def __init__(self, *args, **kwargs): 

94 """Create the image decorrelation Task 

95 

96 Parameters 

97 ---------- 

98 args : 

99 arguments to be passed to ``lsst.pipe.base.task.Task.__init__`` 

100 kwargs : 

101 keyword arguments to be passed to ``lsst.pipe.base.task.Task.__init__`` 

102 """ 

103 pipeBase.Task.__init__(self, *args, **kwargs) 

104 

105 self.statsControl = afwMath.StatisticsControl() 

106 self.statsControl.setNumSigmaClip(3.) 

107 self.statsControl.setNumIter(3) 

108 self.statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes)) 

109 

110 def computeVarianceMean(self, exposure): 

111 statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(), 

112 exposure.getMaskedImage().getMask(), 

113 afwMath.MEANCLIP, self.statsControl) 

114 var = statObj.getValue(afwMath.MEANCLIP) 

115 return var 

116 

117 @pipeBase.timeMethod 

118 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel, 

119 preConvKernel=None, xcen=None, ycen=None, svar=None, tvar=None, 

120 templateMatched=True, preConvMode=False, **kwargs): 

121 """Perform decorrelation of an image difference or of a score difference exposure. 

122 

123 Corrects the difference or score image due to the convolution of the 

124 templateExposure with the A&L PSF matching kernel. 

125 See [DMTN-021, Equation 1](http://dmtn-021.lsst.io/#equation-1) and 

126 [DMTN-179](http://dmtn-179.lsst.io/) for details. 

127 

128 Parameters 

129 ---------- 

130 scienceExposure : `lsst.afw.image.Exposure` 

131 The original science exposure (before pre-convolution, if ``preConvMode==True``). 

132 templateExposure : `lsst.afw.image.Exposure` 

133 The original template exposure warped into the science exposure dimensions. 

134 subtractedExposure : `lsst.afw.image.Exposure` 

135 the subtracted exposure produced by 

136 `ip_diffim.ImagePsfMatchTask.subtractExposures()`. The `subtractedExposure` must 

137 inherit its PSF from `exposure`, see notes below. 

138 psfMatchingKernel : `lsst.afw.detection.Psf` 

139 An (optionally spatially-varying) PSF matching kernel produced 

140 by `ip_diffim.ImagePsfMatchTask.subtractExposures()`. 

141 preConvKernel : `lsst.afw.math.Kernel`, optional 

142 If not `None`, then the `scienceExposure` was pre-convolved with (the reflection of) 

143 this kernel. Must be normalized to sum to 1. 

144 Allowed only if ``templateMatched==True`` and ``preConvMode==True``. 

145 Defaults to the PSF of the science exposure at the image center. 

146 xcen : `float`, optional 

147 X-pixel coordinate to use for computing constant matching kernel to use 

148 If `None` (default), then use the center of the image. 

149 ycen : `float`, optional 

150 Y-pixel coordinate to use for computing constant matching kernel to use 

151 If `None` (default), then use the center of the image. 

152 svar : `float`, optional 

153 Image variance for science image 

154 If `None` (default) then compute the variance over the entire input science image. 

155 tvar : `float`, optional 

156 Image variance for template image 

157 If `None` (default) then compute the variance over the entire input template image. 

158 templateMatched : `bool`, optional 

159 If True, the template exposure was matched (convolved) to the science exposure. 

160 See also notes below. 

161 preConvMode : `bool`, optional 

162 If True, ``subtractedExposure`` is assumed to be a likelihood difference image 

163 and will be noise corrected as a likelihood image. 

164 **kwargs 

165 Additional keyword arguments propagated from DecorrelateALKernelSpatialTask. 

166 

167 Returns 

168 ------- 

169 result : `lsst.pipe.base.Struct` 

170 - ``correctedExposure`` : the decorrelated diffim 

171 

172 Notes 

173 ----- 

174 If ``preConvMode==True``, ``subtractedExposure`` is assumed to be a 

175 score image and the noise correction for likelihood images 

176 is applied. The resulting image is an optimal detection likelihood image 

177 when the templateExposure has noise. (See DMTN-179) If ``preConvKernel`` is 

178 not specified, the PSF of ``scienceExposure`` is assumed as pre-convolution kernel. 

179 

180 The ``subtractedExposure`` is NOT updated. The returned ``correctedExposure`` 

181 has an updated but spatially fixed PSF. It is calculated as the center of 

182 image PSF corrected by the center of image matching kernel. 

183 

184 If ``templateMatched==True``, the templateExposure was matched (convolved) 

185 to the ``scienceExposure`` by ``psfMatchingKernel``. Otherwise the ``scienceExposure`` 

186 was matched (convolved) by ``psfMatchingKernel``. 

187 

188 This task discards the variance plane of ``subtractedExposure`` and re-computes 

189 it from the variance planes of ``scienceExposure`` and ``templateExposure``. 

190 The image plane of ``subtractedExposure`` must be at the photometric level 

191 set by the AL PSF matching in `ImagePsfMatchTask.subtractExposures`. 

192 The assumptions about the photometric level are controlled by the 

193 `templateMatched` option in this task. 

194 

195 Here we currently convert a spatially-varying matching kernel into a constant kernel, 

196 just by computing it at the center of the image (tickets DM-6243, DM-6244). 

197 

198 We are also using a constant accross-the-image measure of sigma (sqrt(variance)) to compute 

199 the decorrelation kernel. 

200 

201 TODO DM-23857 As part of the spatially varying correction implementation 

202 consider whether returning a Struct is still necessary. 

203 """ 

204 if preConvKernel is not None and not (templateMatched and preConvMode): 

205 raise ValueError("Pre-convolution kernel is allowed only if " 

206 "preConvMode==True and templateMatched==True.") 

207 

208 spatialKernel = psfMatchingKernel 

209 kimg = afwImage.ImageD(spatialKernel.getDimensions()) 

210 bbox = subtractedExposure.getBBox() 

211 if xcen is None: 

212 xcen = (bbox.getBeginX() + bbox.getEndX()) / 2. 

213 if ycen is None: 

214 ycen = (bbox.getBeginY() + bbox.getEndY()) / 2. 

215 self.log.info("Using matching kernel computed at (%d, %d)", xcen, ycen) 

216 spatialKernel.computeImage(kimg, False, xcen, ycen) 

217 

218 preConvImg = None 

219 if preConvMode: 

220 if preConvKernel is None: 

221 preConvKernel = scienceExposure.getPsf().getLocalKernel() # at average position 

222 preConvImg = afwImage.ImageD(preConvKernel.getDimensions()) 

223 preConvKernel.computeImage(preConvImg, True) 

224 

225 if svar is None: 

226 svar = self.computeVarianceMean(scienceExposure) 

227 if tvar is None: 

228 tvar = self.computeVarianceMean(templateExposure) 

229 self.log.info("Original variance plane means. Science:%.5e, warped template:%.5e)", 

230 svar, tvar) 

231 

232 if templateMatched: 

233 # Regular subtraction, we convolved the template 

234 self.log.info("Decorrelation after template image convolution") 

235 expVar = svar 

236 matchedVar = tvar 

237 exposure = scienceExposure 

238 matchedExposure = templateExposure 

239 else: 

240 # We convolved the science image 

241 self.log.info("Decorrelation after science image convolution") 

242 expVar = tvar 

243 matchedVar = svar 

244 exposure = templateExposure 

245 matchedExposure = scienceExposure 

246 

247 # Should not happen unless entire image has been masked, which could happen 

248 # if this is a small subimage of the main exposure. In this case, just return a full NaN 

249 # exposure 

250 if np.isnan(expVar) or np.isnan(matchedVar): 

251 # Double check that one of the exposures is all NaNs 

252 if (np.all(np.isnan(exposure.image.array)) 

253 or np.all(np.isnan(matchedExposure.image.array))): 

254 self.log.warning('Template or science image is entirely NaNs: skipping decorrelation.') 

255 outExposure = subtractedExposure.clone() 

256 return pipeBase.Struct(correctedExposure=outExposure, ) 

257 

258 # The maximal correction value converges to sqrt(matchedVar/expVar). 

259 # Correction divergence warning if the correction exceeds 4 orders of magnitude. 

260 mOverExpVar = matchedVar/expVar 

261 if mOverExpVar > 1e8: 

262 self.log.warning("Diverging correction: matched image variance is " 

263 " much larger than the unconvolved one's" 

264 ", matchedVar/expVar:%.2e", mOverExpVar) 

265 

266 oldVarMean = self.computeVarianceMean(subtractedExposure) 

267 self.log.info("Variance plane mean of uncorrected diffim: %f", oldVarMean) 

268 

269 kArr = kimg.array 

270 diffExpArr = subtractedExposure.image.array 

271 psfImg = subtractedExposure.getPsf().computeKernelImage(geom.Point2D(xcen, ycen)) 

272 psfDim = psfImg.getDimensions() 

273 psfArr = psfImg.array 

274 

275 # Determine the common shape 

276 kSum = np.sum(kArr) 

277 kSumSq = kSum*kSum 

278 self.log.debug("Matching kernel sum: %.3e", kSum) 

279 

280 if preConvMode: 

281 self.log.info("Decorrelation of likelihood image") 

282 self.computeCommonShape(preConvImg.array.shape, kArr.shape, 

283 psfArr.shape, diffExpArr.shape) 

284 corr = self.computeScoreCorrection(kArr, expVar, matchedVar, preConvImg.array) 

285 else: 

286 self.log.info("Decorrelation of difference image") 

287 self.computeCommonShape(kArr.shape, psfArr.shape, diffExpArr.shape) 

288 corr = self.computeDiffimCorrection(kArr, expVar, matchedVar) 

289 

290 diffExpArr = self.computeCorrectedImage(corr.corrft, diffExpArr) 

291 

292 corrPsfArr = self.computeCorrectedDiffimPsf(corr.corrft, psfArr) 

293 psfcI = afwImage.ImageD(psfDim) 

294 psfcI.array = corrPsfArr 

295 psfcK = afwMath.FixedKernel(psfcI) 

296 psfNew = measAlg.KernelPsf(psfcK) 

297 

298 correctedExposure = subtractedExposure.clone() 

299 correctedExposure.image.array[...] = diffExpArr # Allow for numpy type casting 

300 # The subtracted exposure variance plane is already correlated, we cannot propagate 

301 # it through another convolution; instead we need to use the uncorrelated originals 

302 # The whitening should scale it to expVar + matchedVar on average 

303 if self.config.completeVarPlanePropagation: 

304 self.log.debug("Using full variance plane calculation in decorrelation") 

305 newVarArr = self.calculateVariancePlane( 

306 exposure.variance.array, matchedExposure.variance.array, 

307 expVar, matchedVar, corr.cnft, corr.crft) 

308 else: 

309 self.log.debug("Using estimated variance plane calculation in decorrelation") 

310 newVarArr = self.estimateVariancePlane( 

311 exposure.variance.array, matchedExposure.variance.array, 

312 corr.cnft, corr.crft) 

313 

314 corrExpVarArr = correctedExposure.variance.array 

315 corrExpVarArr[...] = newVarArr # Allow for numpy type casting 

316 

317 if not templateMatched: 

318 # ImagePsfMatch.subtractExposures re-scales the difference in 

319 # the science image convolution mode 

320 corrExpVarArr /= kSumSq 

321 correctedExposure.setPsf(psfNew) 

322 

323 newVarMean = self.computeVarianceMean(correctedExposure) 

324 self.log.info("Variance plane mean of corrected diffim: %.5e", newVarMean) 

325 

326 # TODO DM-23857 As part of the spatially varying correction implementation 

327 # consider whether returning a Struct is still necessary. 

328 return pipeBase.Struct(correctedExposure=correctedExposure, ) 

329 

330 def computeCommonShape(self, *shapes): 

331 """Calculate the common shape for FFT operations. Set `self.freqSpaceShape` 

332 internally. 

333 

334 Parameters 

335 ---------- 

336 shapes : one or more `tuple` of `int` 

337 Shapes of the arrays. All must have the same dimensionality. 

338 At least one shape must be provided. 

339 

340 Returns 

341 ------- 

342 None. 

343 

344 Notes 

345 ----- 

346 For each dimension, gets the smallest even number greater than or equal to 

347 `N1+N2-1` where `N1` and `N2` are the two largest values. 

348 In case of only one shape given, rounds up to even each dimension value. 

349 """ 

350 S = np.array(shapes, dtype=int) 

351 if len(shapes) > 2: 

352 S.sort(axis=0) 

353 S = S[-2:] 

354 if len(shapes) > 1: 

355 commonShape = np.sum(S, axis=0) - 1 

356 else: 

357 commonShape = S[0] 

358 commonShape[commonShape % 2 != 0] += 1 

359 self.freqSpaceShape = tuple(commonShape) 

360 self.log.info("Common frequency space shape %s", self.freqSpaceShape) 

361 

362 @staticmethod 

363 def padCenterOriginArray(A, newShape: tuple, useInverse=False): 

364 """Zero pad an image where the origin is at the center and replace the 

365 origin to the corner as required by the periodic input of FFT. Implement also 

366 the inverse operation, crop the padding and re-center data. 

367 

368 Parameters 

369 ---------- 

370 A : `numpy.ndarray` 

371 An array to copy from. 

372 newShape : `tuple` of `int` 

373 The dimensions of the resulting array. For padding, the resulting array 

374 must be larger than A in each dimension. For the inverse operation this 

375 must be the original, before padding size of the array. 

376 useInverse : bool, optional 

377 Selector of forward, add padding, operation (False) 

378 or its inverse, crop padding, operation (True). 

379 

380 Returns 

381 ------- 

382 R : `numpy.ndarray` 

383 The padded or unpadded array with shape of `newShape` and the same dtype as A. 

384 

385 Notes 

386 ----- 

387 For odd dimensions, the splitting is rounded to 

388 put the center pixel into the new corner origin (0,0). This is to be consistent 

389 e.g. for a dirac delta kernel that is originally located at the center pixel. 

390 """ 

391 

392 # The forward and inverse operations should round odd dimension halves at the opposite 

393 # sides to get the pixels back to their original positions. 

394 if not useInverse: 

395 # Forward operation: First and second halves with respect to the axes of A. 

396 firstHalves = [x//2 for x in A.shape] 

397 secondHalves = [x-y for x, y in zip(A.shape, firstHalves)] 

398 else: 

399 # Inverse operation: Opposite rounding 

400 secondHalves = [x//2 for x in newShape] 

401 firstHalves = [x-y for x, y in zip(newShape, secondHalves)] 

402 

403 R = np.zeros_like(A, shape=newShape) 

404 R[-firstHalves[0]:, -firstHalves[1]:] = A[:firstHalves[0], :firstHalves[1]] 

405 R[:secondHalves[0], -firstHalves[1]:] = A[-secondHalves[0]:, :firstHalves[1]] 

406 R[:secondHalves[0], :secondHalves[1]] = A[-secondHalves[0]:, -secondHalves[1]:] 

407 R[-firstHalves[0]:, :secondHalves[1]] = A[:firstHalves[0], -secondHalves[1]:] 

408 return R 

409 

410 def computeDiffimCorrection(self, kappa, svar, tvar): 

411 """Compute the Lupton decorrelation post-convolution kernel for decorrelating an 

412 image difference, based on the PSF-matching kernel. 

413 

414 Parameters 

415 ---------- 

416 kappa : `numpy.ndarray` of `float` 

417 A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching. 

418 svar : `float` > 0. 

419 Average variance of science image used for PSF matching. 

420 tvar : `float` > 0. 

421 Average variance of the template (matched) image used for PSF matching. 

422 

423 Returns 

424 ------- 

425 corrft : `numpy.ndarray` of `float` 

426 The frequency space representation of the correction. The array is real (dtype float). 

427 Shape is `self.freqSpaceShape`. 

428 

429 cnft, crft : `numpy.ndarray` of `complex` 

430 The overall convolution (pre-conv, PSF matching, noise correction) kernel 

431 for the science and template images, respectively for the variance plane 

432 calculations. These are intermediate results in frequency space. 

433 

434 Notes 

435 ----- 

436 The maximum correction factor converges to `sqrt(tvar/svar)` towards high frequencies. 

437 This should be a plausible value. 

438 """ 

439 kSum = np.sum(kappa) # We scale the decorrelation to preserve fluxes 

440 kappa = self.padCenterOriginArray(kappa, self.freqSpaceShape) 

441 kft = np.fft.fft2(kappa) 

442 kftAbsSq = np.real(np.conj(kft) * kft) 

443 

444 denom = svar + tvar * kftAbsSq 

445 corrft = np.sqrt((svar + tvar * kSum*kSum) / denom) 

446 cnft = corrft 

447 crft = kft*corrft 

448 return pipeBase.Struct(corrft=corrft, cnft=cnft, crft=crft) 

449 

450 def computeScoreCorrection(self, kappa, svar, tvar, preConvArr): 

451 """Compute the correction kernel for a score image. 

452 

453 Parameters 

454 ---------- 

455 kappa : `numpy.ndarray` 

456 A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching. 

457 svar : `float` 

458 Average variance of science image used for PSF matching (before pre-convolution). 

459 tvar : `float` 

460 Average variance of the template (matched) image used for PSF matching. 

461 preConvArr : `numpy.ndarray` 

462 The pre-convolution kernel of the science image. It should be the PSF 

463 of the science image or an approximation of it. It must be normed to sum 1. 

464 

465 Returns 

466 ------- 

467 corrft : `numpy.ndarray` of `float` 

468 The frequency space representation of the correction. The array is real (dtype float). 

469 Shape is `self.freqSpaceShape`. 

470 cnft, crft : `numpy.ndarray` of `complex` 

471 The overall convolution (pre-conv, PSF matching, noise correction) kernel 

472 for the science and template images, respectively for the variance plane 

473 calculations. These are intermediate results in frequency space. 

474 

475 Notes 

476 ----- 

477 To be precise, the science image should be _correlated_ by ``preConvArray`` but this 

478 does not matter for this calculation. 

479 

480 ``cnft``, ``crft`` contain the scaling factor as well. 

481 

482 """ 

483 kSum = np.sum(kappa) 

484 kappa = self.padCenterOriginArray(kappa, self.freqSpaceShape) 

485 kft = np.fft.fft2(kappa) 

486 preConvArr = self.padCenterOriginArray(preConvArr, self.freqSpaceShape) 

487 preFt = np.fft.fft2(preConvArr) 

488 preFtAbsSq = np.real(np.conj(preFt) * preFt) 

489 kftAbsSq = np.real(np.conj(kft) * kft) 

490 # Avoid zero division, though we don't normally approach `tiny`. 

491 # We have numerical noise instead. 

492 tiny = np.finfo(preFtAbsSq.dtype).tiny * 1000. 

493 flt = preFtAbsSq < tiny 

494 # If we pre-convolve to avoid deconvolution in AL, then kftAbsSq / preFtAbsSq 

495 # theoretically expected to diverge to +inf. But we don't care about the convergence 

496 # properties here, S goes to 0 at these frequencies anyway. 

497 preFtAbsSq[flt] = tiny 

498 denom = svar + tvar * kftAbsSq / preFtAbsSq 

499 corrft = (svar + tvar * kSum*kSum) / denom 

500 cnft = np.conj(preFt)*corrft 

501 crft = kft*corrft 

502 return pipeBase.Struct(corrft=corrft, cnft=cnft, crft=crft) 

503 

504 @staticmethod 

505 def estimateVariancePlane(vplane1, vplane2, c1ft, c2ft): 

506 """Estimate the variance planes. 

507 

508 The estimation assumes that around each pixel the surrounding 

509 pixels' sigmas within the convolution kernel are the same. 

510 

511 Parameters 

512 ---------- 

513 vplane1, vplane2 : `numpy.ndarray` of `float` 

514 Variance planes of the original (before pre-convolution or matching) 

515 exposures. 

516 c1ft, c2ft : `numpy.ndarray` of `complex` 

517 The overall convolution that includes the matching and the 

518 afterburner in frequency space. The result of either 

519 ``computeScoreCorrection`` or ``computeDiffimCorrection``. 

520 

521 Returns 

522 ------- 

523 vplaneD : `numpy.ndarray` of `float` 

524 The estimated variance plane of the difference/score image 

525 as a weighted sum of the input variances. 

526 

527 Notes 

528 ------ 

529 See DMTN-179 Section 5 about the variance plane calculations. 

530 """ 

531 w1 = np.sum(np.real(np.conj(c1ft)*c1ft)) / c1ft.size 

532 w2 = np.sum(np.real(np.conj(c2ft)*c2ft)) / c2ft.size 

533 # w1, w2: the frequency space sum of abs(c1)^2 is the same as in image 

534 # space. 

535 return vplane1*w1 + vplane2*w2 

536 

537 def calculateVariancePlane(self, vplane1, vplane2, varMean1, varMean2, c1ft, c2ft): 

538 """Full propagation of the variance planes of the original exposures. 

539 

540 The original variance planes of independent pixels are convolved with the 

541 image space square of the overall kernels. 

542 

543 Parameters 

544 ---------- 

545 vplane1, vplane2 : `numpy.ndarray` of `float` 

546 Variance planes of the original (before pre-convolution or matching) 

547 exposures. 

548 varMean1, varMean2 : `float` 

549 Replacement average values for non-finite ``vplane1`` and ``vplane2`` values respectively. 

550 

551 c1ft, c2ft : `numpy.ndarray` of `complex` 

552 The overall convolution that includes the matching and the 

553 afterburner in frequency space. The result of either 

554 ``computeScoreCorrection`` or ``computeDiffimCorrection``. 

555 

556 Returns 

557 ------- 

558 vplaneD : `numpy.ndarray` of `float` 

559 The variance plane of the difference/score images. 

560 

561 Notes 

562 ------ 

563 See DMTN-179 Section 5 about the variance plane calculations. 

564 

565 Infs and NaNs are allowed and kept in the returned array. 

566 """ 

567 D = np.real(np.fft.ifft2(c1ft)) 

568 c1SqFt = np.fft.fft2(D*D) 

569 

570 v1shape = vplane1.shape 

571 filtInf = np.isinf(vplane1) 

572 filtNan = np.isnan(vplane1) 

573 # This copy could be eliminated if inf/nan handling were go into padCenterOriginArray 

574 vplane1 = np.copy(vplane1) 

575 vplane1[filtInf | filtNan] = varMean1 

576 D = self.padCenterOriginArray(vplane1, self.freqSpaceShape) 

577 v1 = np.real(np.fft.ifft2(np.fft.fft2(D) * c1SqFt)) 

578 v1 = self.padCenterOriginArray(v1, v1shape, useInverse=True) 

579 v1[filtNan] = np.nan 

580 v1[filtInf] = np.inf 

581 

582 D = np.real(np.fft.ifft2(c2ft)) 

583 c2ft = np.fft.fft2(D*D) 

584 

585 v2shape = vplane2.shape 

586 filtInf = np.isinf(vplane2) 

587 filtNan = np.isnan(vplane2) 

588 vplane2 = np.copy(vplane2) 

589 vplane2[filtInf | filtNan] = varMean2 

590 D = self.padCenterOriginArray(vplane2, self.freqSpaceShape) 

591 v2 = np.real(np.fft.ifft2(np.fft.fft2(D) * c2ft)) 

592 v2 = self.padCenterOriginArray(v2, v2shape, useInverse=True) 

593 v2[filtNan] = np.nan 

594 v2[filtInf] = np.inf 

595 

596 return v1 + v2 

597 

598 def computeCorrectedDiffimPsf(self, corrft, psfOld): 

599 """Compute the (decorrelated) difference image's new PSF. 

600 

601 Parameters 

602 ---------- 

603 corrft : `numpy.ndarray` 

604 The frequency space representation of the correction calculated by 

605 `computeCorrection`. Shape must be `self.freqSpaceShape`. 

606 psfOld : `numpy.ndarray` 

607 The psf of the difference image to be corrected. 

608 

609 Returns 

610 ------- 

611 psfNew : `numpy.ndarray` 

612 The corrected psf, same shape as `psfOld`, sum normed to 1. 

613 

614 Notes 

615 ----- 

616 There is no algorithmic guarantee that the corrected psf can 

617 meaningfully fit to the same size as the original one. 

618 """ 

619 psfShape = psfOld.shape 

620 psfNew = self.padCenterOriginArray(psfOld, self.freqSpaceShape) 

621 psfNew = np.fft.fft2(psfNew) 

622 psfNew *= corrft 

623 psfNew = np.fft.ifft2(psfNew) 

624 psfNew = psfNew.real 

625 psfNew = self.padCenterOriginArray(psfNew, psfShape, useInverse=True) 

626 psfNew = psfNew/psfNew.sum() 

627 return psfNew 

628 

629 def computeCorrectedImage(self, corrft, imgOld): 

630 """Compute the decorrelated difference image. 

631 

632 Parameters 

633 ---------- 

634 corrft : `numpy.ndarray` 

635 The frequency space representation of the correction calculated by 

636 `computeCorrection`. Shape must be `self.freqSpaceShape`. 

637 imgOld : `numpy.ndarray` 

638 The difference image to be corrected. 

639 

640 Returns 

641 ------- 

642 imgNew : `numpy.ndarray` 

643 The corrected image, same size as the input. 

644 """ 

645 expShape = imgOld.shape 

646 imgNew = np.copy(imgOld) 

647 filtInf = np.isinf(imgNew) 

648 filtNan = np.isnan(imgNew) 

649 imgNew[filtInf] = np.nan 

650 imgNew[filtInf | filtNan] = np.nanmean(imgNew) 

651 imgNew = self.padCenterOriginArray(imgNew, self.freqSpaceShape) 

652 imgNew = np.fft.fft2(imgNew) 

653 imgNew *= corrft 

654 imgNew = np.fft.ifft2(imgNew) 

655 imgNew = imgNew.real 

656 imgNew = self.padCenterOriginArray(imgNew, expShape, useInverse=True) 

657 imgNew[filtNan] = np.nan 

658 imgNew[filtInf] = np.inf 

659 return imgNew 

660 

661 

662class DecorrelateALKernelMapper(DecorrelateALKernelTask, ImageMapper): 

663 """Task to be used as an ImageMapper for performing 

664 A&L decorrelation on subimages on a grid across a A&L difference image. 

665 

666 This task subclasses DecorrelateALKernelTask in order to implement 

667 all of that task's configuration parameters, as well as its `run` method. 

668 """ 

669 

670 ConfigClass = DecorrelateALKernelConfig 

671 _DefaultName = 'ip_diffim_decorrelateALKernelMapper' 

672 

673 def __init__(self, *args, **kwargs): 

674 DecorrelateALKernelTask.__init__(self, *args, **kwargs) 

675 

676 def run(self, subExposure, expandedSubExposure, fullBBox, 

677 template, science, alTaskResult=None, psfMatchingKernel=None, 

678 preConvKernel=None, **kwargs): 

679 """Perform decorrelation operation on `subExposure`, using 

680 `expandedSubExposure` to allow for invalid edge pixels arising from 

681 convolutions. 

682 

683 This method performs A&L decorrelation on `subExposure` using 

684 local measures for image variances and PSF. `subExposure` is a 

685 sub-exposure of the non-decorrelated A&L diffim. It also 

686 requires the corresponding sub-exposures of the template 

687 (`template`) and science (`science`) exposures. 

688 

689 Parameters 

690 ---------- 

691 subExposure : `lsst.afw.image.Exposure` 

692 the sub-exposure of the diffim 

693 expandedSubExposure : `lsst.afw.image.Exposure` 

694 the expanded sub-exposure upon which to operate 

695 fullBBox : `lsst.geom.Box2I` 

696 the bounding box of the original exposure 

697 template : `lsst.afw.image.Exposure` 

698 the corresponding sub-exposure of the template exposure 

699 science : `lsst.afw.image.Exposure` 

700 the corresponding sub-exposure of the science exposure 

701 alTaskResult : `lsst.pipe.base.Struct` 

702 the result of A&L image differencing on `science` and 

703 `template`, importantly containing the resulting 

704 `psfMatchingKernel`. Can be `None`, only if 

705 `psfMatchingKernel` is not `None`. 

706 psfMatchingKernel : Alternative parameter for passing the 

707 A&L `psfMatchingKernel` directly. 

708 preConvKernel : If not None, then pre-filtering was applied 

709 to science exposure, and this is the pre-convolution 

710 kernel. 

711 kwargs : 

712 additional keyword arguments propagated from 

713 `ImageMapReduceTask.run`. 

714 

715 Returns 

716 ------- 

717 A `pipeBase.Struct` containing: 

718 

719 - ``subExposure`` : the result of the `subExposure` processing. 

720 - ``decorrelationKernel`` : the decorrelation kernel, currently 

721 not used. 

722 

723 Notes 

724 ----- 

725 This `run` method accepts parameters identical to those of 

726 `ImageMapper.run`, since it is called from the 

727 `ImageMapperTask`. See that class for more information. 

728 """ 

729 templateExposure = template # input template 

730 scienceExposure = science # input science image 

731 if alTaskResult is None and psfMatchingKernel is None: 

732 raise RuntimeError('Both alTaskResult and psfMatchingKernel cannot be None') 

733 psfMatchingKernel = alTaskResult.psfMatchingKernel if alTaskResult is not None else psfMatchingKernel 

734 

735 # subExp and expandedSubExp are subimages of the (un-decorrelated) diffim! 

736 # So here we compute corresponding subimages of templateExposure and scienceExposure 

737 subExp2 = scienceExposure.Factory(scienceExposure, expandedSubExposure.getBBox()) 

738 subExp1 = templateExposure.Factory(templateExposure, expandedSubExposure.getBBox()) 

739 

740 # Prevent too much log INFO verbosity from DecorrelateALKernelTask.run 

741 logLevel = self.log.getLevel() 

742 self.log.setLevel(lsst.log.WARN) 

743 res = DecorrelateALKernelTask.run(self, subExp2, subExp1, expandedSubExposure, 

744 psfMatchingKernel, preConvKernel, **kwargs) 

745 self.log.setLevel(logLevel) # reset the log level 

746 

747 diffim = res.correctedExposure.Factory(res.correctedExposure, subExposure.getBBox()) 

748 out = pipeBase.Struct(subExposure=diffim, ) 

749 return out 

750 

751 

752class DecorrelateALKernelMapReduceConfig(ImageMapReduceConfig): 

753 """Configuration parameters for the ImageMapReduceTask to direct it to use 

754 DecorrelateALKernelMapper as its mapper for A&L decorrelation. 

755 """ 

756 mapper = pexConfig.ConfigurableField( 

757 doc='A&L decorrelation task to run on each sub-image', 

758 target=DecorrelateALKernelMapper 

759 ) 

760 

761 

762class DecorrelateALKernelSpatialConfig(pexConfig.Config): 

763 """Configuration parameters for the DecorrelateALKernelSpatialTask. 

764 """ 

765 decorrelateConfig = pexConfig.ConfigField( 

766 dtype=DecorrelateALKernelConfig, 

767 doc='DecorrelateALKernel config to use when running on complete exposure (non spatially-varying)', 

768 ) 

769 

770 decorrelateMapReduceConfig = pexConfig.ConfigField( 

771 dtype=DecorrelateALKernelMapReduceConfig, 

772 doc='DecorrelateALKernelMapReduce config to use when running on each sub-image (spatially-varying)', 

773 ) 

774 

775 ignoreMaskPlanes = pexConfig.ListField( 

776 dtype=str, 

777 doc="""Mask planes to ignore for sigma-clipped statistics""", 

778 default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE") 

779 ) 

780 

781 def setDefaults(self): 

782 self.decorrelateMapReduceConfig.gridStepX = self.decorrelateMapReduceConfig.gridStepY = 40 

783 self.decorrelateMapReduceConfig.cellSizeX = self.decorrelateMapReduceConfig.cellSizeY = 41 

784 self.decorrelateMapReduceConfig.borderSizeX = self.decorrelateMapReduceConfig.borderSizeY = 8 

785 self.decorrelateMapReduceConfig.reducer.reduceOperation = 'average' 

786 

787 

788class DecorrelateALKernelSpatialTask(pipeBase.Task): 

789 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference 

790 

791 Notes 

792 ----- 

793 

794 Pipe-task that removes the neighboring-pixel covariance in an 

795 image difference that are added when the template image is 

796 convolved with the Alard-Lupton PSF matching kernel. 

797 

798 This task is a simple wrapper around @ref DecorrelateALKernelTask, 

799 which takes a `spatiallyVarying` parameter in its `run` method. If 

800 it is `False`, then it simply calls the `run` method of @ref 

801 DecorrelateALKernelTask. If it is True, then it uses the @ref 

802 ImageMapReduceTask framework to break the exposures into 

803 subExposures on a grid, and performs the `run` method of @ref 

804 DecorrelateALKernelTask on each subExposure. This enables it to 

805 account for spatially-varying PSFs and noise in the exposures when 

806 performing the decorrelation. 

807 

808 This task has no standalone example, however it is applied as a 

809 subtask of pipe.tasks.imageDifference.ImageDifferenceTask. 

810 There is also an example of its use in `tests/testImageDecorrelation.py`. 

811 """ 

812 ConfigClass = DecorrelateALKernelSpatialConfig 

813 _DefaultName = "ip_diffim_decorrelateALKernelSpatial" 

814 

815 def __init__(self, *args, **kwargs): 

816 """Create the image decorrelation Task 

817 

818 Parameters 

819 ---------- 

820 args : 

821 arguments to be passed to 

822 `lsst.pipe.base.task.Task.__init__` 

823 kwargs : 

824 additional keyword arguments to be passed to 

825 `lsst.pipe.base.task.Task.__init__` 

826 """ 

827 pipeBase.Task.__init__(self, *args, **kwargs) 

828 

829 self.statsControl = afwMath.StatisticsControl() 

830 self.statsControl.setNumSigmaClip(3.) 

831 self.statsControl.setNumIter(3) 

832 self.statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes)) 

833 

834 def computeVarianceMean(self, exposure): 

835 """Compute the mean of the variance plane of `exposure`. 

836 """ 

837 statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(), 

838 exposure.getMaskedImage().getMask(), 

839 afwMath.MEANCLIP, self.statsControl) 

840 var = statObj.getValue(afwMath.MEANCLIP) 

841 return var 

842 

843 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel, 

844 spatiallyVarying=True, preConvKernel=None, templateMatched=True, preConvMode=False): 

845 """Perform decorrelation of an image difference exposure. 

846 

847 Decorrelates the diffim due to the convolution of the 

848 templateExposure with the A&L psfMatchingKernel. If 

849 `spatiallyVarying` is True, it utilizes the spatially varying 

850 matching kernel via the `imageMapReduce` framework to perform 

851 spatially-varying decorrelation on a grid of subExposures. 

852 

853 Parameters 

854 ---------- 

855 scienceExposure : `lsst.afw.image.Exposure` 

856 the science Exposure used for PSF matching 

857 templateExposure : `lsst.afw.image.Exposure` 

858 the template Exposure used for PSF matching 

859 subtractedExposure : `lsst.afw.image.Exposure` 

860 the subtracted Exposure produced by `ip_diffim.ImagePsfMatchTask.subtractExposures()` 

861 psfMatchingKernel : an (optionally spatially-varying) PSF matching kernel produced 

862 by `ip_diffim.ImagePsfMatchTask.subtractExposures()` 

863 spatiallyVarying : `bool` 

864 if True, perform the spatially-varying operation 

865 preConvKernel : `lsst.meas.algorithms.Psf` 

866 if not none, the scienceExposure has been pre-filtered with this kernel. (Currently 

867 this option is experimental.) 

868 templateMatched : `bool`, optional 

869 If True, the template exposure was matched (convolved) to the science exposure. 

870 preConvMode : `bool`, optional 

871 If True, ``subtractedExposure`` is assumed to be a likelihood difference image 

872 and will be noise corrected as a likelihood image. 

873 

874 Returns 

875 ------- 

876 results : `lsst.pipe.base.Struct` 

877 a structure containing: 

878 - ``correctedExposure`` : the decorrelated diffim 

879 """ 

880 self.log.info('Running A&L decorrelation: spatiallyVarying=%r', spatiallyVarying) 

881 

882 svar = self.computeVarianceMean(scienceExposure) 

883 tvar = self.computeVarianceMean(templateExposure) 

884 if np.isnan(svar) or np.isnan(tvar): # Should not happen unless entire image has been masked. 

885 # Double check that one of the exposures is all NaNs 

886 if (np.all(np.isnan(scienceExposure.image.array)) 

887 or np.all(np.isnan(templateExposure.image.array))): 

888 self.log.warning('Template or science image is entirely NaNs: skipping decorrelation.') 

889 if np.isnan(svar): 

890 svar = 1e-9 

891 if np.isnan(tvar): 

892 tvar = 1e-9 

893 

894 var = self.computeVarianceMean(subtractedExposure) 

895 

896 if spatiallyVarying: 

897 self.log.info("Variance (science, template): (%f, %f)", svar, tvar) 

898 self.log.info("Variance (uncorrected diffim): %f", var) 

899 config = self.config.decorrelateMapReduceConfig 

900 task = ImageMapReduceTask(config=config) 

901 results = task.run(subtractedExposure, science=scienceExposure, 

902 template=templateExposure, psfMatchingKernel=psfMatchingKernel, 

903 preConvKernel=preConvKernel, forceEvenSized=True, 

904 templateMatched=templateMatched, preConvMode=preConvMode) 

905 results.correctedExposure = results.exposure 

906 

907 # Make sure masks of input image are propagated to diffim 

908 def gm(exp): 

909 return exp.getMaskedImage().getMask() 

910 gm(results.correctedExposure)[:, :] = gm(subtractedExposure) 

911 

912 var = self.computeVarianceMean(results.correctedExposure) 

913 self.log.info("Variance (corrected diffim): %f", var) 

914 

915 else: 

916 config = self.config.decorrelateConfig 

917 task = DecorrelateALKernelTask(config=config) 

918 results = task.run(scienceExposure, templateExposure, 

919 subtractedExposure, psfMatchingKernel, preConvKernel=preConvKernel, 

920 templateMatched=templateMatched, preConvMode=preConvMode) 

921 

922 return results