Coverage for python/lsst/ip/diffim/imageDecorrelation.py: 17%

283 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-08-20 02:24 -0700

1# 

2# LSST Data Management System 

3# Copyright 2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23import numpy as np 

24 

25import lsst.afw.image as afwImage 

26import lsst.afw.math as afwMath 

27import lsst.geom as geom 

28import lsst.meas.algorithms as measAlg 

29import lsst.pex.config as pexConfig 

30import lsst.pipe.base as pipeBase 

31from lsst.utils.timer import timeMethod 

32 

33from .imageMapReduce import (ImageMapReduceConfig, ImageMapReduceTask, 

34 ImageMapper) 

35 

36__all__ = ("DecorrelateALKernelTask", "DecorrelateALKernelConfig", 

37 "DecorrelateALKernelMapper", "DecorrelateALKernelMapReduceConfig", 

38 "DecorrelateALKernelSpatialConfig", "DecorrelateALKernelSpatialTask") 

39 

40 

41class DecorrelateALKernelConfig(pexConfig.Config): 

42 """Configuration parameters for the DecorrelateALKernelTask 

43 """ 

44 

45 ignoreMaskPlanes = pexConfig.ListField( 

46 dtype=str, 

47 doc="""Mask planes to ignore for sigma-clipped statistics""", 

48 default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE") 

49 ) 

50 completeVarPlanePropagation = pexConfig.Field( 

51 dtype=bool, 

52 default=False, 

53 doc="Compute the full effect of the decorrelated matching kernel on the variance plane." 

54 " Otherwise use a model weighed sum of the input variances." 

55 ) 

56 

57 

58class DecorrelateALKernelTask(pipeBase.Task): 

59 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference 

60 

61 Notes 

62 ----- 

63 

64 Pipe-task that removes the neighboring-pixel covariance in an 

65 image difference that are added when the template image is 

66 convolved with the Alard-Lupton PSF matching kernel. 

67 

68 The image differencing pipeline task @link 

69 ip.diffim.psfMatch.PsfMatchTask PSFMatchTask@endlink and @link 

70 ip.diffim.psfMatch.PsfMatchConfigAL PSFMatchConfigAL@endlink uses 

71 the Alard and Lupton (1998) method for matching the PSFs of the 

72 template and science exposures prior to subtraction. The 

73 Alard-Lupton method identifies a matching kernel, which is then 

74 (typically) convolved with the template image to perform PSF 

75 matching. This convolution has the effect of adding covariance 

76 between neighboring pixels in the template image, which is then 

77 added to the image difference by subtraction. 

78 

79 The pixel covariance may be corrected by whitening the noise of 

80 the image difference. This task performs such a decorrelation by 

81 computing a decorrelation kernel (based upon the A&L matching 

82 kernel and variances in the template and science images) and 

83 convolving the image difference with it. This process is described 

84 in detail in [DMTN-021](http://dmtn-021.lsst.io). 

85 

86 This task has no standalone example, however it is applied as a 

87 subtask of pipe.tasks.imageDifference.ImageDifferenceTask. 

88 """ 

89 ConfigClass = DecorrelateALKernelConfig 

90 _DefaultName = "ip_diffim_decorrelateALKernel" 

91 

92 def __init__(self, *args, **kwargs): 

93 """Create the image decorrelation Task 

94 

95 Parameters 

96 ---------- 

97 args : 

98 arguments to be passed to ``lsst.pipe.base.task.Task.__init__`` 

99 kwargs : 

100 keyword arguments to be passed to ``lsst.pipe.base.task.Task.__init__`` 

101 """ 

102 pipeBase.Task.__init__(self, *args, **kwargs) 

103 

104 self.statsControl = afwMath.StatisticsControl() 

105 self.statsControl.setNumSigmaClip(3.) 

106 self.statsControl.setNumIter(3) 

107 self.statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes)) 

108 

109 def computeVarianceMean(self, exposure): 

110 statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(), 

111 exposure.getMaskedImage().getMask(), 

112 afwMath.MEANCLIP, self.statsControl) 

113 var = statObj.getValue(afwMath.MEANCLIP) 

114 return var 

115 

116 @timeMethod 

117 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel, 

118 preConvKernel=None, xcen=None, ycen=None, svar=None, tvar=None, 

119 templateMatched=True, preConvMode=False, **kwargs): 

120 """Perform decorrelation of an image difference or of a score difference exposure. 

121 

122 Corrects the difference or score image due to the convolution of the 

123 templateExposure with the A&L PSF matching kernel. 

124 See [DMTN-021, Equation 1](http://dmtn-021.lsst.io/#equation-1) and 

125 [DMTN-179](http://dmtn-179.lsst.io/) for details. 

126 

127 Parameters 

128 ---------- 

129 scienceExposure : `lsst.afw.image.Exposure` 

130 The original science exposure (before pre-convolution, if ``preConvMode==True``). 

131 templateExposure : `lsst.afw.image.Exposure` 

132 The original template exposure warped, but not psf-matched, to the science exposure. 

133 subtractedExposure : `lsst.afw.image.Exposure` 

134 the subtracted exposure produced by 

135 `ip_diffim.ImagePsfMatchTask.subtractExposures()`. The `subtractedExposure` must 

136 inherit its PSF from `exposure`, see notes below. 

137 psfMatchingKernel : `lsst.afw.detection.Psf` 

138 An (optionally spatially-varying) PSF matching kernel produced 

139 by `ip_diffim.ImagePsfMatchTask.subtractExposures()`. 

140 preConvKernel : `lsst.afw.math.Kernel`, optional 

141 If not `None`, then the `scienceExposure` was pre-convolved with (the reflection of) 

142 this kernel. Must be normalized to sum to 1. 

143 Allowed only if ``templateMatched==True`` and ``preConvMode==True``. 

144 Defaults to the PSF of the science exposure at the image center. 

145 xcen : `float`, optional 

146 X-pixel coordinate to use for computing constant matching kernel to use 

147 If `None` (default), then use the center of the image. 

148 ycen : `float`, optional 

149 Y-pixel coordinate to use for computing constant matching kernel to use 

150 If `None` (default), then use the center of the image. 

151 svar : `float`, optional 

152 Image variance for science image 

153 If `None` (default) then compute the variance over the entire input science image. 

154 tvar : `float`, optional 

155 Image variance for template image 

156 If `None` (default) then compute the variance over the entire input template image. 

157 templateMatched : `bool`, optional 

158 If True, the template exposure was matched (convolved) to the science exposure. 

159 See also notes below. 

160 preConvMode : `bool`, optional 

161 If True, ``subtractedExposure`` is assumed to be a likelihood difference image 

162 and will be noise corrected as a likelihood image. 

163 **kwargs 

164 Additional keyword arguments propagated from DecorrelateALKernelSpatialTask. 

165 

166 Returns 

167 ------- 

168 result : `lsst.pipe.base.Struct` 

169 - ``correctedExposure`` : the decorrelated diffim 

170 

171 Notes 

172 ----- 

173 If ``preConvMode==True``, ``subtractedExposure`` is assumed to be a 

174 score image and the noise correction for likelihood images 

175 is applied. The resulting image is an optimal detection likelihood image 

176 when the templateExposure has noise. (See DMTN-179) If ``preConvKernel`` is 

177 not specified, the PSF of ``scienceExposure`` is assumed as pre-convolution kernel. 

178 

179 The ``subtractedExposure`` is NOT updated. The returned ``correctedExposure`` 

180 has an updated but spatially fixed PSF. It is calculated as the center of 

181 image PSF corrected by the center of image matching kernel. 

182 

183 If ``templateMatched==True``, the templateExposure was matched (convolved) 

184 to the ``scienceExposure`` by ``psfMatchingKernel`` during image differencing. 

185 Otherwise the ``scienceExposure`` was matched (convolved) by ``psfMatchingKernel``. 

186 In either case, note that the original template and science images are required, 

187 not the psf-matched version. 

188 

189 This task discards the variance plane of ``subtractedExposure`` and re-computes 

190 it from the variance planes of ``scienceExposure`` and ``templateExposure``. 

191 The image plane of ``subtractedExposure`` must be at the photometric level 

192 set by the AL PSF matching in `ImagePsfMatchTask.subtractExposures`. 

193 The assumptions about the photometric level are controlled by the 

194 `templateMatched` option in this task. 

195 

196 Here we currently convert a spatially-varying matching kernel into a constant kernel, 

197 just by computing it at the center of the image (tickets DM-6243, DM-6244). 

198 

199 We are also using a constant accross-the-image measure of sigma (sqrt(variance)) to compute 

200 the decorrelation kernel. 

201 

202 TODO DM-23857 As part of the spatially varying correction implementation 

203 consider whether returning a Struct is still necessary. 

204 """ 

205 if preConvKernel is not None and not (templateMatched and preConvMode): 

206 raise ValueError("Pre-convolution kernel is allowed only if " 

207 "preConvMode==True and templateMatched==True.") 

208 

209 spatialKernel = psfMatchingKernel 

210 kimg = afwImage.ImageD(spatialKernel.getDimensions()) 

211 bbox = subtractedExposure.getBBox() 

212 if xcen is None: 

213 xcen = (bbox.getBeginX() + bbox.getEndX()) / 2. 

214 if ycen is None: 

215 ycen = (bbox.getBeginY() + bbox.getEndY()) / 2. 

216 self.log.info("Using matching kernel computed at (%d, %d)", xcen, ycen) 

217 spatialKernel.computeImage(kimg, False, xcen, ycen) 

218 

219 preConvImg = None 

220 if preConvMode: 

221 if preConvKernel is None: 

222 preConvKernel = scienceExposure.getPsf().getLocalKernel() # at average position 

223 preConvImg = afwImage.ImageD(preConvKernel.getDimensions()) 

224 preConvKernel.computeImage(preConvImg, True) 

225 

226 if svar is None: 

227 svar = self.computeVarianceMean(scienceExposure) 

228 if tvar is None: 

229 tvar = self.computeVarianceMean(templateExposure) 

230 self.log.info("Original variance plane means. Science:%.5e, warped template:%.5e)", 

231 svar, tvar) 

232 

233 # Should not happen unless entire image has been masked, which could happen 

234 # if this is a small subimage of the main exposure. In this case, just return a full NaN 

235 # exposure 

236 if np.isnan(svar) or np.isnan(tvar): 

237 # Double check that one of the exposures is all NaNs 

238 if (np.all(np.isnan(scienceExposure.image.array)) 

239 or np.all(np.isnan(templateExposure.image.array))): 

240 self.log.warning('Template or science image is entirely NaNs: skipping decorrelation.') 

241 outExposure = subtractedExposure.clone() 

242 return pipeBase.Struct(correctedExposure=outExposure, ) 

243 

244 if templateMatched: 

245 # Regular subtraction, we convolved the template 

246 self.log.info("Decorrelation after template image convolution") 

247 varianceMean = svar 

248 targetVarianceMean = tvar 

249 # variance plane of the image that is not convolved 

250 variance = scienceExposure.variance.array 

251 # Variance plane of the convolved image, before convolution. 

252 targetVariance = templateExposure.variance.array 

253 else: 

254 # We convolved the science image 

255 self.log.info("Decorrelation after science image convolution") 

256 varianceMean = tvar 

257 targetVarianceMean = svar 

258 # variance plane of the image that is not convolved 

259 variance = templateExposure.variance.array 

260 # Variance plane of the convolved image, before convolution. 

261 targetVariance = scienceExposure.variance.array 

262 

263 # The maximal correction value converges to sqrt(targetVarianceMean/varianceMean). 

264 # Correction divergence warning if the correction exceeds 4 orders of magnitude. 

265 mOverExpVar = targetVarianceMean/varianceMean 

266 if mOverExpVar > 1e8: 

267 self.log.warning("Diverging correction: matched image variance is " 

268 " much larger than the unconvolved one's" 

269 ", targetVarianceMean/varianceMean:%.2e", mOverExpVar) 

270 

271 oldVarMean = self.computeVarianceMean(subtractedExposure) 

272 self.log.info("Variance plane mean of uncorrected diffim: %f", oldVarMean) 

273 

274 kArr = kimg.array 

275 diffimShape = subtractedExposure.image.array.shape 

276 psfImg = subtractedExposure.getPsf().computeKernelImage(geom.Point2D(xcen, ycen)) 

277 psfShape = psfImg.array.shape 

278 

279 if preConvMode: 

280 self.log.info("Decorrelation of likelihood image") 

281 self.computeCommonShape(preConvImg.array.shape, kArr.shape, 

282 psfShape, diffimShape) 

283 corr = self.computeScoreCorrection(kArr, varianceMean, targetVarianceMean, preConvImg.array) 

284 else: 

285 self.log.info("Decorrelation of difference image") 

286 self.computeCommonShape(kArr.shape, psfShape, diffimShape) 

287 corr = self.computeDiffimCorrection(kArr, varianceMean, targetVarianceMean) 

288 

289 correctedImage = self.computeCorrectedImage(corr.corrft, subtractedExposure.image.array) 

290 correctedPsf = self.computeCorrectedDiffimPsf(corr.corrft, psfImg.array) 

291 

292 # The subtracted exposure variance plane is already correlated, we cannot propagate 

293 # it through another convolution; instead we need to use the uncorrelated originals 

294 # The whitening should scale it to varianceMean + targetVarianceMean on average 

295 if self.config.completeVarPlanePropagation: 

296 self.log.debug("Using full variance plane calculation in decorrelation") 

297 correctedVariance = self.calculateVariancePlane( 

298 variance, targetVariance, 

299 varianceMean, targetVarianceMean, corr.cnft, corr.crft) 

300 else: 

301 self.log.debug("Using estimated variance plane calculation in decorrelation") 

302 correctedVariance = self.estimateVariancePlane( 

303 variance, targetVariance, 

304 corr.cnft, corr.crft) 

305 

306 # Determine the common shape 

307 kSum = np.sum(kArr) 

308 kSumSq = kSum*kSum 

309 self.log.debug("Matching kernel sum: %.3e", kSum) 

310 if not templateMatched: 

311 # ImagePsfMatch.subtractExposures re-scales the difference in 

312 # the science image convolution mode 

313 correctedVariance /= kSumSq 

314 subtractedExposure.image.array[...] = correctedImage # Allow for numpy type casting 

315 subtractedExposure.variance.array[...] = correctedVariance 

316 subtractedExposure.setPsf(correctedPsf) 

317 

318 newVarMean = self.computeVarianceMean(subtractedExposure) 

319 self.log.info("Variance plane mean of corrected diffim: %.5e", newVarMean) 

320 

321 # TODO DM-23857 As part of the spatially varying correction implementation 

322 # consider whether returning a Struct is still necessary. 

323 return pipeBase.Struct(correctedExposure=subtractedExposure, ) 

324 

325 def computeCommonShape(self, *shapes): 

326 """Calculate the common shape for FFT operations. Set `self.freqSpaceShape` 

327 internally. 

328 

329 Parameters 

330 ---------- 

331 shapes : one or more `tuple` of `int` 

332 Shapes of the arrays. All must have the same dimensionality. 

333 At least one shape must be provided. 

334 

335 Returns 

336 ------- 

337 None. 

338 

339 Notes 

340 ----- 

341 For each dimension, gets the smallest even number greater than or equal to 

342 `N1+N2-1` where `N1` and `N2` are the two largest values. 

343 In case of only one shape given, rounds up to even each dimension value. 

344 """ 

345 S = np.array(shapes, dtype=int) 

346 if len(shapes) > 2: 

347 S.sort(axis=0) 

348 S = S[-2:] 

349 if len(shapes) > 1: 

350 commonShape = np.sum(S, axis=0) - 1 

351 else: 

352 commonShape = S[0] 

353 commonShape[commonShape % 2 != 0] += 1 

354 self.freqSpaceShape = tuple(commonShape) 

355 self.log.info("Common frequency space shape %s", self.freqSpaceShape) 

356 

357 @staticmethod 

358 def padCenterOriginArray(A, newShape: tuple, useInverse=False): 

359 """Zero pad an image where the origin is at the center and replace the 

360 origin to the corner as required by the periodic input of FFT. Implement also 

361 the inverse operation, crop the padding and re-center data. 

362 

363 Parameters 

364 ---------- 

365 A : `numpy.ndarray` 

366 An array to copy from. 

367 newShape : `tuple` of `int` 

368 The dimensions of the resulting array. For padding, the resulting array 

369 must be larger than A in each dimension. For the inverse operation this 

370 must be the original, before padding size of the array. 

371 useInverse : bool, optional 

372 Selector of forward, add padding, operation (False) 

373 or its inverse, crop padding, operation (True). 

374 

375 Returns 

376 ------- 

377 R : `numpy.ndarray` 

378 The padded or unpadded array with shape of `newShape` and the same dtype as A. 

379 

380 Notes 

381 ----- 

382 For odd dimensions, the splitting is rounded to 

383 put the center pixel into the new corner origin (0,0). This is to be consistent 

384 e.g. for a dirac delta kernel that is originally located at the center pixel. 

385 """ 

386 

387 # The forward and inverse operations should round odd dimension halves at the opposite 

388 # sides to get the pixels back to their original positions. 

389 if not useInverse: 

390 # Forward operation: First and second halves with respect to the axes of A. 

391 firstHalves = [x//2 for x in A.shape] 

392 secondHalves = [x-y for x, y in zip(A.shape, firstHalves)] 

393 else: 

394 # Inverse operation: Opposite rounding 

395 secondHalves = [x//2 for x in newShape] 

396 firstHalves = [x-y for x, y in zip(newShape, secondHalves)] 

397 

398 R = np.zeros_like(A, shape=newShape) 

399 R[-firstHalves[0]:, -firstHalves[1]:] = A[:firstHalves[0], :firstHalves[1]] 

400 R[:secondHalves[0], -firstHalves[1]:] = A[-secondHalves[0]:, :firstHalves[1]] 

401 R[:secondHalves[0], :secondHalves[1]] = A[-secondHalves[0]:, -secondHalves[1]:] 

402 R[-firstHalves[0]:, :secondHalves[1]] = A[:firstHalves[0], -secondHalves[1]:] 

403 return R 

404 

405 def computeDiffimCorrection(self, kappa, svar, tvar): 

406 """Compute the Lupton decorrelation post-convolution kernel for decorrelating an 

407 image difference, based on the PSF-matching kernel. 

408 

409 Parameters 

410 ---------- 

411 kappa : `numpy.ndarray` of `float` 

412 A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching. 

413 svar : `float` > 0. 

414 Average variance of science image used for PSF matching. 

415 tvar : `float` > 0. 

416 Average variance of the template (matched) image used for PSF matching. 

417 

418 Returns 

419 ------- 

420 corrft : `numpy.ndarray` of `float` 

421 The frequency space representation of the correction. The array is real (dtype float). 

422 Shape is `self.freqSpaceShape`. 

423 

424 cnft, crft : `numpy.ndarray` of `complex` 

425 The overall convolution (pre-conv, PSF matching, noise correction) kernel 

426 for the science and template images, respectively for the variance plane 

427 calculations. These are intermediate results in frequency space. 

428 

429 Notes 

430 ----- 

431 The maximum correction factor converges to `sqrt(tvar/svar)` towards high frequencies. 

432 This should be a plausible value. 

433 """ 

434 kSum = np.sum(kappa) # We scale the decorrelation to preserve fluxes 

435 kappa = self.padCenterOriginArray(kappa, self.freqSpaceShape) 

436 kft = np.fft.fft2(kappa) 

437 kftAbsSq = np.real(np.conj(kft) * kft) 

438 

439 denom = svar + tvar * kftAbsSq 

440 corrft = np.sqrt((svar + tvar * kSum*kSum) / denom) 

441 cnft = corrft 

442 crft = kft*corrft 

443 return pipeBase.Struct(corrft=corrft, cnft=cnft, crft=crft) 

444 

445 def computeScoreCorrection(self, kappa, svar, tvar, preConvArr): 

446 """Compute the correction kernel for a score image. 

447 

448 Parameters 

449 ---------- 

450 kappa : `numpy.ndarray` 

451 A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching. 

452 svar : `float` 

453 Average variance of science image used for PSF matching (before pre-convolution). 

454 tvar : `float` 

455 Average variance of the template (matched) image used for PSF matching. 

456 preConvArr : `numpy.ndarray` 

457 The pre-convolution kernel of the science image. It should be the PSF 

458 of the science image or an approximation of it. It must be normed to sum 1. 

459 

460 Returns 

461 ------- 

462 corrft : `numpy.ndarray` of `float` 

463 The frequency space representation of the correction. The array is real (dtype float). 

464 Shape is `self.freqSpaceShape`. 

465 cnft, crft : `numpy.ndarray` of `complex` 

466 The overall convolution (pre-conv, PSF matching, noise correction) kernel 

467 for the science and template images, respectively for the variance plane 

468 calculations. These are intermediate results in frequency space. 

469 

470 Notes 

471 ----- 

472 To be precise, the science image should be _correlated_ by ``preConvArray`` but this 

473 does not matter for this calculation. 

474 

475 ``cnft``, ``crft`` contain the scaling factor as well. 

476 

477 """ 

478 kSum = np.sum(kappa) 

479 kappa = self.padCenterOriginArray(kappa, self.freqSpaceShape) 

480 kft = np.fft.fft2(kappa) 

481 preConvArr = self.padCenterOriginArray(preConvArr, self.freqSpaceShape) 

482 preFt = np.fft.fft2(preConvArr) 

483 preFtAbsSq = np.real(np.conj(preFt) * preFt) 

484 kftAbsSq = np.real(np.conj(kft) * kft) 

485 # Avoid zero division, though we don't normally approach `tiny`. 

486 # We have numerical noise instead. 

487 tiny = np.finfo(preFtAbsSq.dtype).tiny * 1000. 

488 flt = preFtAbsSq < tiny 

489 # If we pre-convolve to avoid deconvolution in AL, then kftAbsSq / preFtAbsSq 

490 # theoretically expected to diverge to +inf. But we don't care about the convergence 

491 # properties here, S goes to 0 at these frequencies anyway. 

492 preFtAbsSq[flt] = tiny 

493 denom = svar + tvar * kftAbsSq / preFtAbsSq 

494 corrft = (svar + tvar * kSum*kSum) / denom 

495 cnft = np.conj(preFt)*corrft 

496 crft = kft*corrft 

497 return pipeBase.Struct(corrft=corrft, cnft=cnft, crft=crft) 

498 

499 @staticmethod 

500 def estimateVariancePlane(vplane1, vplane2, c1ft, c2ft): 

501 """Estimate the variance planes. 

502 

503 The estimation assumes that around each pixel the surrounding 

504 pixels' sigmas within the convolution kernel are the same. 

505 

506 Parameters 

507 ---------- 

508 vplane1, vplane2 : `numpy.ndarray` of `float` 

509 Variance planes of the original (before pre-convolution or matching) 

510 exposures. 

511 c1ft, c2ft : `numpy.ndarray` of `complex` 

512 The overall convolution that includes the matching and the 

513 afterburner in frequency space. The result of either 

514 ``computeScoreCorrection`` or ``computeDiffimCorrection``. 

515 

516 Returns 

517 ------- 

518 vplaneD : `numpy.ndarray` of `float` 

519 The estimated variance plane of the difference/score image 

520 as a weighted sum of the input variances. 

521 

522 Notes 

523 ------ 

524 See DMTN-179 Section 5 about the variance plane calculations. 

525 """ 

526 w1 = np.sum(np.real(np.conj(c1ft)*c1ft)) / c1ft.size 

527 w2 = np.sum(np.real(np.conj(c2ft)*c2ft)) / c2ft.size 

528 # w1, w2: the frequency space sum of abs(c1)^2 is the same as in image 

529 # space. 

530 return vplane1*w1 + vplane2*w2 

531 

532 def calculateVariancePlane(self, vplane1, vplane2, varMean1, varMean2, c1ft, c2ft): 

533 """Full propagation of the variance planes of the original exposures. 

534 

535 The original variance planes of independent pixels are convolved with the 

536 image space square of the overall kernels. 

537 

538 Parameters 

539 ---------- 

540 vplane1, vplane2 : `numpy.ndarray` of `float` 

541 Variance planes of the original (before pre-convolution or matching) 

542 exposures. 

543 varMean1, varMean2 : `float` 

544 Replacement average values for non-finite ``vplane1`` and ``vplane2`` values respectively. 

545 

546 c1ft, c2ft : `numpy.ndarray` of `complex` 

547 The overall convolution that includes the matching and the 

548 afterburner in frequency space. The result of either 

549 ``computeScoreCorrection`` or ``computeDiffimCorrection``. 

550 

551 Returns 

552 ------- 

553 vplaneD : `numpy.ndarray` of `float` 

554 The variance plane of the difference/score images. 

555 

556 Notes 

557 ------ 

558 See DMTN-179 Section 5 about the variance plane calculations. 

559 

560 Infs and NaNs are allowed and kept in the returned array. 

561 """ 

562 D = np.real(np.fft.ifft2(c1ft)) 

563 c1SqFt = np.fft.fft2(D*D) 

564 

565 v1shape = vplane1.shape 

566 filtInf = np.isinf(vplane1) 

567 filtNan = np.isnan(vplane1) 

568 # This copy could be eliminated if inf/nan handling were go into padCenterOriginArray 

569 vplane1 = np.copy(vplane1) 

570 vplane1[filtInf | filtNan] = varMean1 

571 D = self.padCenterOriginArray(vplane1, self.freqSpaceShape) 

572 v1 = np.real(np.fft.ifft2(np.fft.fft2(D) * c1SqFt)) 

573 v1 = self.padCenterOriginArray(v1, v1shape, useInverse=True) 

574 v1[filtNan] = np.nan 

575 v1[filtInf] = np.inf 

576 

577 D = np.real(np.fft.ifft2(c2ft)) 

578 c2SqFt = np.fft.fft2(D*D) 

579 

580 v2shape = vplane2.shape 

581 filtInf = np.isinf(vplane2) 

582 filtNan = np.isnan(vplane2) 

583 vplane2 = np.copy(vplane2) 

584 vplane2[filtInf | filtNan] = varMean2 

585 D = self.padCenterOriginArray(vplane2, self.freqSpaceShape) 

586 v2 = np.real(np.fft.ifft2(np.fft.fft2(D) * c2SqFt)) 

587 v2 = self.padCenterOriginArray(v2, v2shape, useInverse=True) 

588 v2[filtNan] = np.nan 

589 v2[filtInf] = np.inf 

590 

591 return v1 + v2 

592 

593 def computeCorrectedDiffimPsf(self, corrft, psfOld): 

594 """Compute the (decorrelated) difference image's new PSF. 

595 

596 Parameters 

597 ---------- 

598 corrft : `numpy.ndarray` 

599 The frequency space representation of the correction calculated by 

600 `computeCorrection`. Shape must be `self.freqSpaceShape`. 

601 psfOld : `numpy.ndarray` 

602 The psf of the difference image to be corrected. 

603 

604 Returns 

605 ------- 

606 correctedPsf : `lsst.meas.algorithms.KernelPsf` 

607 The corrected psf, same shape as `psfOld`, sum normed to 1. 

608 

609 Notes 

610 ----- 

611 There is no algorithmic guarantee that the corrected psf can 

612 meaningfully fit to the same size as the original one. 

613 """ 

614 psfShape = psfOld.shape 

615 psfNew = self.padCenterOriginArray(psfOld, self.freqSpaceShape) 

616 psfNew = np.fft.fft2(psfNew) 

617 psfNew *= corrft 

618 psfNew = np.fft.ifft2(psfNew) 

619 psfNew = psfNew.real 

620 psfNew = self.padCenterOriginArray(psfNew, psfShape, useInverse=True) 

621 psfNew = psfNew/psfNew.sum() 

622 

623 psfcI = afwImage.ImageD(geom.Extent2I(psfShape[1], psfShape[0])) 

624 psfcI.array = psfNew 

625 psfcK = afwMath.FixedKernel(psfcI) 

626 correctedPsf = measAlg.KernelPsf(psfcK) 

627 return correctedPsf 

628 

629 def computeCorrectedImage(self, corrft, imgOld): 

630 """Compute the decorrelated difference image. 

631 

632 Parameters 

633 ---------- 

634 corrft : `numpy.ndarray` 

635 The frequency space representation of the correction calculated by 

636 `computeCorrection`. Shape must be `self.freqSpaceShape`. 

637 imgOld : `numpy.ndarray` 

638 The difference image to be corrected. 

639 

640 Returns 

641 ------- 

642 imgNew : `numpy.ndarray` 

643 The corrected image, same size as the input. 

644 """ 

645 expShape = imgOld.shape 

646 imgNew = np.copy(imgOld) 

647 filtInf = np.isinf(imgNew) 

648 filtNan = np.isnan(imgNew) 

649 imgNew[filtInf] = np.nan 

650 imgNew[filtInf | filtNan] = np.nanmean(imgNew) 

651 imgNew = self.padCenterOriginArray(imgNew, self.freqSpaceShape) 

652 imgNew = np.fft.fft2(imgNew) 

653 imgNew *= corrft 

654 imgNew = np.fft.ifft2(imgNew) 

655 imgNew = imgNew.real 

656 imgNew = self.padCenterOriginArray(imgNew, expShape, useInverse=True) 

657 imgNew[filtNan] = np.nan 

658 imgNew[filtInf] = np.inf 

659 return imgNew 

660 

661 

662class DecorrelateALKernelMapper(DecorrelateALKernelTask, ImageMapper): 

663 """Task to be used as an ImageMapper for performing 

664 A&L decorrelation on subimages on a grid across a A&L difference image. 

665 

666 This task subclasses DecorrelateALKernelTask in order to implement 

667 all of that task's configuration parameters, as well as its `run` method. 

668 """ 

669 

670 ConfigClass = DecorrelateALKernelConfig 

671 _DefaultName = 'ip_diffim_decorrelateALKernelMapper' 

672 

673 def __init__(self, *args, **kwargs): 

674 DecorrelateALKernelTask.__init__(self, *args, **kwargs) 

675 

676 def run(self, subExposure, expandedSubExposure, fullBBox, 

677 template, science, alTaskResult=None, psfMatchingKernel=None, 

678 preConvKernel=None, **kwargs): 

679 """Perform decorrelation operation on `subExposure`, using 

680 `expandedSubExposure` to allow for invalid edge pixels arising from 

681 convolutions. 

682 

683 This method performs A&L decorrelation on `subExposure` using 

684 local measures for image variances and PSF. `subExposure` is a 

685 sub-exposure of the non-decorrelated A&L diffim. It also 

686 requires the corresponding sub-exposures of the template 

687 (`template`) and science (`science`) exposures. 

688 

689 Parameters 

690 ---------- 

691 subExposure : `lsst.afw.image.Exposure` 

692 the sub-exposure of the diffim 

693 expandedSubExposure : `lsst.afw.image.Exposure` 

694 the expanded sub-exposure upon which to operate 

695 fullBBox : `lsst.geom.Box2I` 

696 the bounding box of the original exposure 

697 template : `lsst.afw.image.Exposure` 

698 the corresponding sub-exposure of the template exposure 

699 science : `lsst.afw.image.Exposure` 

700 the corresponding sub-exposure of the science exposure 

701 alTaskResult : `lsst.pipe.base.Struct` 

702 the result of A&L image differencing on `science` and 

703 `template`, importantly containing the resulting 

704 `psfMatchingKernel`. Can be `None`, only if 

705 `psfMatchingKernel` is not `None`. 

706 psfMatchingKernel : Alternative parameter for passing the 

707 A&L `psfMatchingKernel` directly. 

708 preConvKernel : If not None, then pre-filtering was applied 

709 to science exposure, and this is the pre-convolution 

710 kernel. 

711 kwargs : 

712 additional keyword arguments propagated from 

713 `ImageMapReduceTask.run`. 

714 

715 Returns 

716 ------- 

717 A `pipeBase.Struct` containing: 

718 

719 - ``subExposure`` : the result of the `subExposure` processing. 

720 - ``decorrelationKernel`` : the decorrelation kernel, currently 

721 not used. 

722 

723 Notes 

724 ----- 

725 This `run` method accepts parameters identical to those of 

726 `ImageMapper.run`, since it is called from the 

727 `ImageMapperTask`. See that class for more information. 

728 """ 

729 templateExposure = template # input template 

730 scienceExposure = science # input science image 

731 if alTaskResult is None and psfMatchingKernel is None: 

732 raise RuntimeError('Both alTaskResult and psfMatchingKernel cannot be None') 

733 psfMatchingKernel = alTaskResult.psfMatchingKernel if alTaskResult is not None else psfMatchingKernel 

734 

735 # subExp and expandedSubExp are subimages of the (un-decorrelated) diffim! 

736 # So here we compute corresponding subimages of templateExposure and scienceExposure 

737 subExp2 = scienceExposure.Factory(scienceExposure, expandedSubExposure.getBBox()) 

738 subExp1 = templateExposure.Factory(templateExposure, expandedSubExposure.getBBox()) 

739 

740 # Prevent too much log INFO verbosity from DecorrelateALKernelTask.run 

741 logLevel = self.log.level 

742 self.log.setLevel(self.log.WARNING) 

743 res = DecorrelateALKernelTask.run(self, subExp2, subExp1, expandedSubExposure, 

744 psfMatchingKernel, preConvKernel, **kwargs) 

745 self.log.setLevel(logLevel) # reset the log level 

746 

747 diffim = res.correctedExposure.Factory(res.correctedExposure, subExposure.getBBox()) 

748 out = pipeBase.Struct(subExposure=diffim, ) 

749 return out 

750 

751 

752class DecorrelateALKernelMapReduceConfig(ImageMapReduceConfig): 

753 """Configuration parameters for the ImageMapReduceTask to direct it to use 

754 DecorrelateALKernelMapper as its mapper for A&L decorrelation. 

755 """ 

756 mapper = pexConfig.ConfigurableField( 

757 doc='A&L decorrelation task to run on each sub-image', 

758 target=DecorrelateALKernelMapper 

759 ) 

760 

761 

762class DecorrelateALKernelSpatialConfig(pexConfig.Config): 

763 """Configuration parameters for the DecorrelateALKernelSpatialTask. 

764 """ 

765 decorrelateConfig = pexConfig.ConfigField( 

766 dtype=DecorrelateALKernelConfig, 

767 doc='DecorrelateALKernel config to use when running on complete exposure (non spatially-varying)', 

768 ) 

769 

770 decorrelateMapReduceConfig = pexConfig.ConfigField( 

771 dtype=DecorrelateALKernelMapReduceConfig, 

772 doc='DecorrelateALKernelMapReduce config to use when running on each sub-image (spatially-varying)', 

773 ) 

774 

775 ignoreMaskPlanes = pexConfig.ListField( 

776 dtype=str, 

777 doc="""Mask planes to ignore for sigma-clipped statistics""", 

778 default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE") 

779 ) 

780 

781 def setDefaults(self): 

782 self.decorrelateMapReduceConfig.gridStepX = self.decorrelateMapReduceConfig.gridStepY = 40 

783 self.decorrelateMapReduceConfig.cellSizeX = self.decorrelateMapReduceConfig.cellSizeY = 41 

784 self.decorrelateMapReduceConfig.borderSizeX = self.decorrelateMapReduceConfig.borderSizeY = 8 

785 self.decorrelateMapReduceConfig.reducer.reduceOperation = 'average' 

786 

787 

788class DecorrelateALKernelSpatialTask(pipeBase.Task): 

789 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference 

790 

791 Notes 

792 ----- 

793 

794 Pipe-task that removes the neighboring-pixel covariance in an 

795 image difference that are added when the template image is 

796 convolved with the Alard-Lupton PSF matching kernel. 

797 

798 This task is a simple wrapper around @ref DecorrelateALKernelTask, 

799 which takes a `spatiallyVarying` parameter in its `run` method. If 

800 it is `False`, then it simply calls the `run` method of @ref 

801 DecorrelateALKernelTask. If it is True, then it uses the @ref 

802 ImageMapReduceTask framework to break the exposures into 

803 subExposures on a grid, and performs the `run` method of @ref 

804 DecorrelateALKernelTask on each subExposure. This enables it to 

805 account for spatially-varying PSFs and noise in the exposures when 

806 performing the decorrelation. 

807 

808 This task has no standalone example, however it is applied as a 

809 subtask of pipe.tasks.imageDifference.ImageDifferenceTask. 

810 There is also an example of its use in `tests/testImageDecorrelation.py`. 

811 """ 

812 ConfigClass = DecorrelateALKernelSpatialConfig 

813 _DefaultName = "ip_diffim_decorrelateALKernelSpatial" 

814 

815 def __init__(self, *args, **kwargs): 

816 """Create the image decorrelation Task 

817 

818 Parameters 

819 ---------- 

820 args : 

821 arguments to be passed to 

822 `lsst.pipe.base.task.Task.__init__` 

823 kwargs : 

824 additional keyword arguments to be passed to 

825 `lsst.pipe.base.task.Task.__init__` 

826 """ 

827 pipeBase.Task.__init__(self, *args, **kwargs) 

828 

829 self.statsControl = afwMath.StatisticsControl() 

830 self.statsControl.setNumSigmaClip(3.) 

831 self.statsControl.setNumIter(3) 

832 self.statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes)) 

833 

834 def computeVarianceMean(self, exposure): 

835 """Compute the mean of the variance plane of `exposure`. 

836 """ 

837 statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(), 

838 exposure.getMaskedImage().getMask(), 

839 afwMath.MEANCLIP, self.statsControl) 

840 var = statObj.getValue(afwMath.MEANCLIP) 

841 return var 

842 

843 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel, 

844 spatiallyVarying=True, preConvKernel=None, templateMatched=True, preConvMode=False): 

845 """Perform decorrelation of an image difference exposure. 

846 

847 Decorrelates the diffim due to the convolution of the 

848 templateExposure with the A&L psfMatchingKernel. If 

849 `spatiallyVarying` is True, it utilizes the spatially varying 

850 matching kernel via the `imageMapReduce` framework to perform 

851 spatially-varying decorrelation on a grid of subExposures. 

852 

853 Parameters 

854 ---------- 

855 scienceExposure : `lsst.afw.image.Exposure` 

856 the science Exposure used for PSF matching 

857 templateExposure : `lsst.afw.image.Exposure` 

858 the template Exposure used for PSF matching 

859 subtractedExposure : `lsst.afw.image.Exposure` 

860 the subtracted Exposure produced by `ip_diffim.ImagePsfMatchTask.subtractExposures()` 

861 psfMatchingKernel : an (optionally spatially-varying) PSF matching kernel produced 

862 by `ip_diffim.ImagePsfMatchTask.subtractExposures()` 

863 spatiallyVarying : `bool` 

864 if True, perform the spatially-varying operation 

865 preConvKernel : `lsst.meas.algorithms.Psf` 

866 if not none, the scienceExposure has been pre-filtered with this kernel. (Currently 

867 this option is experimental.) 

868 templateMatched : `bool`, optional 

869 If True, the template exposure was matched (convolved) to the science exposure. 

870 preConvMode : `bool`, optional 

871 If True, ``subtractedExposure`` is assumed to be a likelihood difference image 

872 and will be noise corrected as a likelihood image. 

873 

874 Returns 

875 ------- 

876 results : `lsst.pipe.base.Struct` 

877 a structure containing: 

878 - ``correctedExposure`` : the decorrelated diffim 

879 """ 

880 self.log.info('Running A&L decorrelation: spatiallyVarying=%r', spatiallyVarying) 

881 

882 svar = self.computeVarianceMean(scienceExposure) 

883 tvar = self.computeVarianceMean(templateExposure) 

884 if np.isnan(svar) or np.isnan(tvar): # Should not happen unless entire image has been masked. 

885 # Double check that one of the exposures is all NaNs 

886 if (np.all(np.isnan(scienceExposure.image.array)) 

887 or np.all(np.isnan(templateExposure.image.array))): 

888 self.log.warning('Template or science image is entirely NaNs: skipping decorrelation.') 

889 if np.isnan(svar): 

890 svar = 1e-9 

891 if np.isnan(tvar): 

892 tvar = 1e-9 

893 

894 var = self.computeVarianceMean(subtractedExposure) 

895 

896 if spatiallyVarying: 

897 self.log.info("Variance (science, template): (%f, %f)", svar, tvar) 

898 self.log.info("Variance (uncorrected diffim): %f", var) 

899 config = self.config.decorrelateMapReduceConfig 

900 task = ImageMapReduceTask(config=config) 

901 results = task.run(subtractedExposure, science=scienceExposure, 

902 template=templateExposure, psfMatchingKernel=psfMatchingKernel, 

903 preConvKernel=preConvKernel, forceEvenSized=True, 

904 templateMatched=templateMatched, preConvMode=preConvMode) 

905 results.correctedExposure = results.exposure 

906 

907 # Make sure masks of input image are propagated to diffim 

908 def gm(exp): 

909 return exp.getMaskedImage().getMask() 

910 gm(results.correctedExposure)[:, :] = gm(subtractedExposure) 

911 

912 var = self.computeVarianceMean(results.correctedExposure) 

913 self.log.info("Variance (corrected diffim): %f", var) 

914 

915 else: 

916 config = self.config.decorrelateConfig 

917 task = DecorrelateALKernelTask(config=config) 

918 results = task.run(scienceExposure, templateExposure, 

919 subtractedExposure, psfMatchingKernel, preConvKernel=preConvKernel, 

920 templateMatched=templateMatched, preConvMode=preConvMode) 

921 

922 return results