Coverage for python/lsst/ip/diffim/imageDecorrelation.py: 17%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

286 statements  

1# 

2# LSST Data Management System 

3# Copyright 2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23import numpy as np 

24 

25import lsst.afw.image as afwImage 

26import lsst.afw.math as afwMath 

27import lsst.geom as geom 

28import lsst.meas.algorithms as measAlg 

29import lsst.pex.config as pexConfig 

30import lsst.pipe.base as pipeBase 

31from lsst.utils.timer import timeMethod 

32 

33from .imageMapReduce import (ImageMapReduceConfig, ImageMapReduceTask, 

34 ImageMapper) 

35 

36__all__ = ("DecorrelateALKernelTask", "DecorrelateALKernelConfig", 

37 "DecorrelateALKernelMapper", "DecorrelateALKernelMapReduceConfig", 

38 "DecorrelateALKernelSpatialConfig", "DecorrelateALKernelSpatialTask") 

39 

40 

41class DecorrelateALKernelConfig(pexConfig.Config): 

42 """Configuration parameters for the DecorrelateALKernelTask 

43 """ 

44 

45 ignoreMaskPlanes = pexConfig.ListField( 

46 dtype=str, 

47 doc="""Mask planes to ignore for sigma-clipped statistics""", 

48 default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE") 

49 ) 

50 completeVarPlanePropagation = pexConfig.Field( 

51 dtype=bool, 

52 default=False, 

53 doc="Compute the full effect of the decorrelated matching kernel on the variance plane." 

54 " Otherwise use a model weighed sum of the input variances." 

55 ) 

56 

57 

58class DecorrelateALKernelTask(pipeBase.Task): 

59 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference 

60 

61 Notes 

62 ----- 

63 

64 Pipe-task that removes the neighboring-pixel covariance in an 

65 image difference that are added when the template image is 

66 convolved with the Alard-Lupton PSF matching kernel. 

67 

68 The image differencing pipeline task @link 

69 ip.diffim.psfMatch.PsfMatchTask PSFMatchTask@endlink and @link 

70 ip.diffim.psfMatch.PsfMatchConfigAL PSFMatchConfigAL@endlink uses 

71 the Alard and Lupton (1998) method for matching the PSFs of the 

72 template and science exposures prior to subtraction. The 

73 Alard-Lupton method identifies a matching kernel, which is then 

74 (typically) convolved with the template image to perform PSF 

75 matching. This convolution has the effect of adding covariance 

76 between neighboring pixels in the template image, which is then 

77 added to the image difference by subtraction. 

78 

79 The pixel covariance may be corrected by whitening the noise of 

80 the image difference. This task performs such a decorrelation by 

81 computing a decorrelation kernel (based upon the A&L matching 

82 kernel and variances in the template and science images) and 

83 convolving the image difference with it. This process is described 

84 in detail in [DMTN-021](http://dmtn-021.lsst.io). 

85 

86 This task has no standalone example, however it is applied as a 

87 subtask of pipe.tasks.imageDifference.ImageDifferenceTask. 

88 """ 

89 ConfigClass = DecorrelateALKernelConfig 

90 _DefaultName = "ip_diffim_decorrelateALKernel" 

91 

92 def __init__(self, *args, **kwargs): 

93 """Create the image decorrelation Task 

94 

95 Parameters 

96 ---------- 

97 args : 

98 arguments to be passed to ``lsst.pipe.base.task.Task.__init__`` 

99 kwargs : 

100 keyword arguments to be passed to ``lsst.pipe.base.task.Task.__init__`` 

101 """ 

102 pipeBase.Task.__init__(self, *args, **kwargs) 

103 

104 self.statsControl = afwMath.StatisticsControl() 

105 self.statsControl.setNumSigmaClip(3.) 

106 self.statsControl.setNumIter(3) 

107 self.statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes)) 

108 

109 def computeVarianceMean(self, exposure): 

110 statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(), 

111 exposure.getMaskedImage().getMask(), 

112 afwMath.MEANCLIP, self.statsControl) 

113 var = statObj.getValue(afwMath.MEANCLIP) 

114 return var 

115 

116 @timeMethod 

117 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel, 

118 preConvKernel=None, xcen=None, ycen=None, svar=None, tvar=None, 

119 templateMatched=True, preConvMode=False, **kwargs): 

120 """Perform decorrelation of an image difference or of a score difference exposure. 

121 

122 Corrects the difference or score image due to the convolution of the 

123 templateExposure with the A&L PSF matching kernel. 

124 See [DMTN-021, Equation 1](http://dmtn-021.lsst.io/#equation-1) and 

125 [DMTN-179](http://dmtn-179.lsst.io/) for details. 

126 

127 Parameters 

128 ---------- 

129 scienceExposure : `lsst.afw.image.Exposure` 

130 The original science exposure (before pre-convolution, if ``preConvMode==True``). 

131 templateExposure : `lsst.afw.image.Exposure` 

132 The original template exposure warped into the science exposure dimensions. 

133 subtractedExposure : `lsst.afw.image.Exposure` 

134 the subtracted exposure produced by 

135 `ip_diffim.ImagePsfMatchTask.subtractExposures()`. The `subtractedExposure` must 

136 inherit its PSF from `exposure`, see notes below. 

137 psfMatchingKernel : `lsst.afw.detection.Psf` 

138 An (optionally spatially-varying) PSF matching kernel produced 

139 by `ip_diffim.ImagePsfMatchTask.subtractExposures()`. 

140 preConvKernel : `lsst.afw.math.Kernel`, optional 

141 If not `None`, then the `scienceExposure` was pre-convolved with (the reflection of) 

142 this kernel. Must be normalized to sum to 1. 

143 Allowed only if ``templateMatched==True`` and ``preConvMode==True``. 

144 Defaults to the PSF of the science exposure at the image center. 

145 xcen : `float`, optional 

146 X-pixel coordinate to use for computing constant matching kernel to use 

147 If `None` (default), then use the center of the image. 

148 ycen : `float`, optional 

149 Y-pixel coordinate to use for computing constant matching kernel to use 

150 If `None` (default), then use the center of the image. 

151 svar : `float`, optional 

152 Image variance for science image 

153 If `None` (default) then compute the variance over the entire input science image. 

154 tvar : `float`, optional 

155 Image variance for template image 

156 If `None` (default) then compute the variance over the entire input template image. 

157 templateMatched : `bool`, optional 

158 If True, the template exposure was matched (convolved) to the science exposure. 

159 See also notes below. 

160 preConvMode : `bool`, optional 

161 If True, ``subtractedExposure`` is assumed to be a likelihood difference image 

162 and will be noise corrected as a likelihood image. 

163 **kwargs 

164 Additional keyword arguments propagated from DecorrelateALKernelSpatialTask. 

165 

166 Returns 

167 ------- 

168 result : `lsst.pipe.base.Struct` 

169 - ``correctedExposure`` : the decorrelated diffim 

170 

171 Notes 

172 ----- 

173 If ``preConvMode==True``, ``subtractedExposure`` is assumed to be a 

174 score image and the noise correction for likelihood images 

175 is applied. The resulting image is an optimal detection likelihood image 

176 when the templateExposure has noise. (See DMTN-179) If ``preConvKernel`` is 

177 not specified, the PSF of ``scienceExposure`` is assumed as pre-convolution kernel. 

178 

179 The ``subtractedExposure`` is NOT updated. The returned ``correctedExposure`` 

180 has an updated but spatially fixed PSF. It is calculated as the center of 

181 image PSF corrected by the center of image matching kernel. 

182 

183 If ``templateMatched==True``, the templateExposure was matched (convolved) 

184 to the ``scienceExposure`` by ``psfMatchingKernel``. Otherwise the ``scienceExposure`` 

185 was matched (convolved) by ``psfMatchingKernel``. 

186 

187 This task discards the variance plane of ``subtractedExposure`` and re-computes 

188 it from the variance planes of ``scienceExposure`` and ``templateExposure``. 

189 The image plane of ``subtractedExposure`` must be at the photometric level 

190 set by the AL PSF matching in `ImagePsfMatchTask.subtractExposures`. 

191 The assumptions about the photometric level are controlled by the 

192 `templateMatched` option in this task. 

193 

194 Here we currently convert a spatially-varying matching kernel into a constant kernel, 

195 just by computing it at the center of the image (tickets DM-6243, DM-6244). 

196 

197 We are also using a constant accross-the-image measure of sigma (sqrt(variance)) to compute 

198 the decorrelation kernel. 

199 

200 TODO DM-23857 As part of the spatially varying correction implementation 

201 consider whether returning a Struct is still necessary. 

202 """ 

203 if preConvKernel is not None and not (templateMatched and preConvMode): 

204 raise ValueError("Pre-convolution kernel is allowed only if " 

205 "preConvMode==True and templateMatched==True.") 

206 

207 spatialKernel = psfMatchingKernel 

208 kimg = afwImage.ImageD(spatialKernel.getDimensions()) 

209 bbox = subtractedExposure.getBBox() 

210 if xcen is None: 

211 xcen = (bbox.getBeginX() + bbox.getEndX()) / 2. 

212 if ycen is None: 

213 ycen = (bbox.getBeginY() + bbox.getEndY()) / 2. 

214 self.log.info("Using matching kernel computed at (%d, %d)", xcen, ycen) 

215 spatialKernel.computeImage(kimg, False, xcen, ycen) 

216 

217 preConvImg = None 

218 if preConvMode: 

219 if preConvKernel is None: 

220 preConvKernel = scienceExposure.getPsf().getLocalKernel() # at average position 

221 preConvImg = afwImage.ImageD(preConvKernel.getDimensions()) 

222 preConvKernel.computeImage(preConvImg, True) 

223 

224 if svar is None: 

225 svar = self.computeVarianceMean(scienceExposure) 

226 if tvar is None: 

227 tvar = self.computeVarianceMean(templateExposure) 

228 self.log.info("Original variance plane means. Science:%.5e, warped template:%.5e)", 

229 svar, tvar) 

230 

231 if templateMatched: 

232 # Regular subtraction, we convolved the template 

233 self.log.info("Decorrelation after template image convolution") 

234 expVar = svar 

235 matchedVar = tvar 

236 exposure = scienceExposure 

237 matchedExposure = templateExposure 

238 else: 

239 # We convolved the science image 

240 self.log.info("Decorrelation after science image convolution") 

241 expVar = tvar 

242 matchedVar = svar 

243 exposure = templateExposure 

244 matchedExposure = scienceExposure 

245 

246 # Should not happen unless entire image has been masked, which could happen 

247 # if this is a small subimage of the main exposure. In this case, just return a full NaN 

248 # exposure 

249 if np.isnan(expVar) or np.isnan(matchedVar): 

250 # Double check that one of the exposures is all NaNs 

251 if (np.all(np.isnan(exposure.image.array)) 

252 or np.all(np.isnan(matchedExposure.image.array))): 

253 self.log.warning('Template or science image is entirely NaNs: skipping decorrelation.') 

254 outExposure = subtractedExposure.clone() 

255 return pipeBase.Struct(correctedExposure=outExposure, ) 

256 

257 # The maximal correction value converges to sqrt(matchedVar/expVar). 

258 # Correction divergence warning if the correction exceeds 4 orders of magnitude. 

259 mOverExpVar = matchedVar/expVar 

260 if mOverExpVar > 1e8: 

261 self.log.warning("Diverging correction: matched image variance is " 

262 " much larger than the unconvolved one's" 

263 ", matchedVar/expVar:%.2e", mOverExpVar) 

264 

265 oldVarMean = self.computeVarianceMean(subtractedExposure) 

266 self.log.info("Variance plane mean of uncorrected diffim: %f", oldVarMean) 

267 

268 kArr = kimg.array 

269 diffExpArr = subtractedExposure.image.array 

270 psfImg = subtractedExposure.getPsf().computeKernelImage(geom.Point2D(xcen, ycen)) 

271 psfDim = psfImg.getDimensions() 

272 psfArr = psfImg.array 

273 

274 # Determine the common shape 

275 kSum = np.sum(kArr) 

276 kSumSq = kSum*kSum 

277 self.log.debug("Matching kernel sum: %.3e", kSum) 

278 

279 if preConvMode: 

280 self.log.info("Decorrelation of likelihood image") 

281 self.computeCommonShape(preConvImg.array.shape, kArr.shape, 

282 psfArr.shape, diffExpArr.shape) 

283 corr = self.computeScoreCorrection(kArr, expVar, matchedVar, preConvImg.array) 

284 else: 

285 self.log.info("Decorrelation of difference image") 

286 self.computeCommonShape(kArr.shape, psfArr.shape, diffExpArr.shape) 

287 corr = self.computeDiffimCorrection(kArr, expVar, matchedVar) 

288 

289 diffExpArr = self.computeCorrectedImage(corr.corrft, diffExpArr) 

290 

291 corrPsfArr = self.computeCorrectedDiffimPsf(corr.corrft, psfArr) 

292 psfcI = afwImage.ImageD(psfDim) 

293 psfcI.array = corrPsfArr 

294 psfcK = afwMath.FixedKernel(psfcI) 

295 psfNew = measAlg.KernelPsf(psfcK) 

296 

297 correctedExposure = subtractedExposure.clone() 

298 correctedExposure.image.array[...] = diffExpArr # Allow for numpy type casting 

299 # The subtracted exposure variance plane is already correlated, we cannot propagate 

300 # it through another convolution; instead we need to use the uncorrelated originals 

301 # The whitening should scale it to expVar + matchedVar on average 

302 if self.config.completeVarPlanePropagation: 

303 self.log.debug("Using full variance plane calculation in decorrelation") 

304 newVarArr = self.calculateVariancePlane( 

305 exposure.variance.array, matchedExposure.variance.array, 

306 expVar, matchedVar, corr.cnft, corr.crft) 

307 else: 

308 self.log.debug("Using estimated variance plane calculation in decorrelation") 

309 newVarArr = self.estimateVariancePlane( 

310 exposure.variance.array, matchedExposure.variance.array, 

311 corr.cnft, corr.crft) 

312 

313 corrExpVarArr = correctedExposure.variance.array 

314 corrExpVarArr[...] = newVarArr # Allow for numpy type casting 

315 

316 if not templateMatched: 

317 # ImagePsfMatch.subtractExposures re-scales the difference in 

318 # the science image convolution mode 

319 corrExpVarArr /= kSumSq 

320 correctedExposure.setPsf(psfNew) 

321 

322 newVarMean = self.computeVarianceMean(correctedExposure) 

323 self.log.info("Variance plane mean of corrected diffim: %.5e", newVarMean) 

324 

325 # TODO DM-23857 As part of the spatially varying correction implementation 

326 # consider whether returning a Struct is still necessary. 

327 return pipeBase.Struct(correctedExposure=correctedExposure, ) 

328 

329 def computeCommonShape(self, *shapes): 

330 """Calculate the common shape for FFT operations. Set `self.freqSpaceShape` 

331 internally. 

332 

333 Parameters 

334 ---------- 

335 shapes : one or more `tuple` of `int` 

336 Shapes of the arrays. All must have the same dimensionality. 

337 At least one shape must be provided. 

338 

339 Returns 

340 ------- 

341 None. 

342 

343 Notes 

344 ----- 

345 For each dimension, gets the smallest even number greater than or equal to 

346 `N1+N2-1` where `N1` and `N2` are the two largest values. 

347 In case of only one shape given, rounds up to even each dimension value. 

348 """ 

349 S = np.array(shapes, dtype=int) 

350 if len(shapes) > 2: 

351 S.sort(axis=0) 

352 S = S[-2:] 

353 if len(shapes) > 1: 

354 commonShape = np.sum(S, axis=0) - 1 

355 else: 

356 commonShape = S[0] 

357 commonShape[commonShape % 2 != 0] += 1 

358 self.freqSpaceShape = tuple(commonShape) 

359 self.log.info("Common frequency space shape %s", self.freqSpaceShape) 

360 

361 @staticmethod 

362 def padCenterOriginArray(A, newShape: tuple, useInverse=False): 

363 """Zero pad an image where the origin is at the center and replace the 

364 origin to the corner as required by the periodic input of FFT. Implement also 

365 the inverse operation, crop the padding and re-center data. 

366 

367 Parameters 

368 ---------- 

369 A : `numpy.ndarray` 

370 An array to copy from. 

371 newShape : `tuple` of `int` 

372 The dimensions of the resulting array. For padding, the resulting array 

373 must be larger than A in each dimension. For the inverse operation this 

374 must be the original, before padding size of the array. 

375 useInverse : bool, optional 

376 Selector of forward, add padding, operation (False) 

377 or its inverse, crop padding, operation (True). 

378 

379 Returns 

380 ------- 

381 R : `numpy.ndarray` 

382 The padded or unpadded array with shape of `newShape` and the same dtype as A. 

383 

384 Notes 

385 ----- 

386 For odd dimensions, the splitting is rounded to 

387 put the center pixel into the new corner origin (0,0). This is to be consistent 

388 e.g. for a dirac delta kernel that is originally located at the center pixel. 

389 """ 

390 

391 # The forward and inverse operations should round odd dimension halves at the opposite 

392 # sides to get the pixels back to their original positions. 

393 if not useInverse: 

394 # Forward operation: First and second halves with respect to the axes of A. 

395 firstHalves = [x//2 for x in A.shape] 

396 secondHalves = [x-y for x, y in zip(A.shape, firstHalves)] 

397 else: 

398 # Inverse operation: Opposite rounding 

399 secondHalves = [x//2 for x in newShape] 

400 firstHalves = [x-y for x, y in zip(newShape, secondHalves)] 

401 

402 R = np.zeros_like(A, shape=newShape) 

403 R[-firstHalves[0]:, -firstHalves[1]:] = A[:firstHalves[0], :firstHalves[1]] 

404 R[:secondHalves[0], -firstHalves[1]:] = A[-secondHalves[0]:, :firstHalves[1]] 

405 R[:secondHalves[0], :secondHalves[1]] = A[-secondHalves[0]:, -secondHalves[1]:] 

406 R[-firstHalves[0]:, :secondHalves[1]] = A[:firstHalves[0], -secondHalves[1]:] 

407 return R 

408 

409 def computeDiffimCorrection(self, kappa, svar, tvar): 

410 """Compute the Lupton decorrelation post-convolution kernel for decorrelating an 

411 image difference, based on the PSF-matching kernel. 

412 

413 Parameters 

414 ---------- 

415 kappa : `numpy.ndarray` of `float` 

416 A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching. 

417 svar : `float` > 0. 

418 Average variance of science image used for PSF matching. 

419 tvar : `float` > 0. 

420 Average variance of the template (matched) image used for PSF matching. 

421 

422 Returns 

423 ------- 

424 corrft : `numpy.ndarray` of `float` 

425 The frequency space representation of the correction. The array is real (dtype float). 

426 Shape is `self.freqSpaceShape`. 

427 

428 cnft, crft : `numpy.ndarray` of `complex` 

429 The overall convolution (pre-conv, PSF matching, noise correction) kernel 

430 for the science and template images, respectively for the variance plane 

431 calculations. These are intermediate results in frequency space. 

432 

433 Notes 

434 ----- 

435 The maximum correction factor converges to `sqrt(tvar/svar)` towards high frequencies. 

436 This should be a plausible value. 

437 """ 

438 kSum = np.sum(kappa) # We scale the decorrelation to preserve fluxes 

439 kappa = self.padCenterOriginArray(kappa, self.freqSpaceShape) 

440 kft = np.fft.fft2(kappa) 

441 kftAbsSq = np.real(np.conj(kft) * kft) 

442 

443 denom = svar + tvar * kftAbsSq 

444 corrft = np.sqrt((svar + tvar * kSum*kSum) / denom) 

445 cnft = corrft 

446 crft = kft*corrft 

447 return pipeBase.Struct(corrft=corrft, cnft=cnft, crft=crft) 

448 

449 def computeScoreCorrection(self, kappa, svar, tvar, preConvArr): 

450 """Compute the correction kernel for a score image. 

451 

452 Parameters 

453 ---------- 

454 kappa : `numpy.ndarray` 

455 A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching. 

456 svar : `float` 

457 Average variance of science image used for PSF matching (before pre-convolution). 

458 tvar : `float` 

459 Average variance of the template (matched) image used for PSF matching. 

460 preConvArr : `numpy.ndarray` 

461 The pre-convolution kernel of the science image. It should be the PSF 

462 of the science image or an approximation of it. It must be normed to sum 1. 

463 

464 Returns 

465 ------- 

466 corrft : `numpy.ndarray` of `float` 

467 The frequency space representation of the correction. The array is real (dtype float). 

468 Shape is `self.freqSpaceShape`. 

469 cnft, crft : `numpy.ndarray` of `complex` 

470 The overall convolution (pre-conv, PSF matching, noise correction) kernel 

471 for the science and template images, respectively for the variance plane 

472 calculations. These are intermediate results in frequency space. 

473 

474 Notes 

475 ----- 

476 To be precise, the science image should be _correlated_ by ``preConvArray`` but this 

477 does not matter for this calculation. 

478 

479 ``cnft``, ``crft`` contain the scaling factor as well. 

480 

481 """ 

482 kSum = np.sum(kappa) 

483 kappa = self.padCenterOriginArray(kappa, self.freqSpaceShape) 

484 kft = np.fft.fft2(kappa) 

485 preConvArr = self.padCenterOriginArray(preConvArr, self.freqSpaceShape) 

486 preFt = np.fft.fft2(preConvArr) 

487 preFtAbsSq = np.real(np.conj(preFt) * preFt) 

488 kftAbsSq = np.real(np.conj(kft) * kft) 

489 # Avoid zero division, though we don't normally approach `tiny`. 

490 # We have numerical noise instead. 

491 tiny = np.finfo(preFtAbsSq.dtype).tiny * 1000. 

492 flt = preFtAbsSq < tiny 

493 # If we pre-convolve to avoid deconvolution in AL, then kftAbsSq / preFtAbsSq 

494 # theoretically expected to diverge to +inf. But we don't care about the convergence 

495 # properties here, S goes to 0 at these frequencies anyway. 

496 preFtAbsSq[flt] = tiny 

497 denom = svar + tvar * kftAbsSq / preFtAbsSq 

498 corrft = (svar + tvar * kSum*kSum) / denom 

499 cnft = np.conj(preFt)*corrft 

500 crft = kft*corrft 

501 return pipeBase.Struct(corrft=corrft, cnft=cnft, crft=crft) 

502 

503 @staticmethod 

504 def estimateVariancePlane(vplane1, vplane2, c1ft, c2ft): 

505 """Estimate the variance planes. 

506 

507 The estimation assumes that around each pixel the surrounding 

508 pixels' sigmas within the convolution kernel are the same. 

509 

510 Parameters 

511 ---------- 

512 vplane1, vplane2 : `numpy.ndarray` of `float` 

513 Variance planes of the original (before pre-convolution or matching) 

514 exposures. 

515 c1ft, c2ft : `numpy.ndarray` of `complex` 

516 The overall convolution that includes the matching and the 

517 afterburner in frequency space. The result of either 

518 ``computeScoreCorrection`` or ``computeDiffimCorrection``. 

519 

520 Returns 

521 ------- 

522 vplaneD : `numpy.ndarray` of `float` 

523 The estimated variance plane of the difference/score image 

524 as a weighted sum of the input variances. 

525 

526 Notes 

527 ------ 

528 See DMTN-179 Section 5 about the variance plane calculations. 

529 """ 

530 w1 = np.sum(np.real(np.conj(c1ft)*c1ft)) / c1ft.size 

531 w2 = np.sum(np.real(np.conj(c2ft)*c2ft)) / c2ft.size 

532 # w1, w2: the frequency space sum of abs(c1)^2 is the same as in image 

533 # space. 

534 return vplane1*w1 + vplane2*w2 

535 

536 def calculateVariancePlane(self, vplane1, vplane2, varMean1, varMean2, c1ft, c2ft): 

537 """Full propagation of the variance planes of the original exposures. 

538 

539 The original variance planes of independent pixels are convolved with the 

540 image space square of the overall kernels. 

541 

542 Parameters 

543 ---------- 

544 vplane1, vplane2 : `numpy.ndarray` of `float` 

545 Variance planes of the original (before pre-convolution or matching) 

546 exposures. 

547 varMean1, varMean2 : `float` 

548 Replacement average values for non-finite ``vplane1`` and ``vplane2`` values respectively. 

549 

550 c1ft, c2ft : `numpy.ndarray` of `complex` 

551 The overall convolution that includes the matching and the 

552 afterburner in frequency space. The result of either 

553 ``computeScoreCorrection`` or ``computeDiffimCorrection``. 

554 

555 Returns 

556 ------- 

557 vplaneD : `numpy.ndarray` of `float` 

558 The variance plane of the difference/score images. 

559 

560 Notes 

561 ------ 

562 See DMTN-179 Section 5 about the variance plane calculations. 

563 

564 Infs and NaNs are allowed and kept in the returned array. 

565 """ 

566 D = np.real(np.fft.ifft2(c1ft)) 

567 c1SqFt = np.fft.fft2(D*D) 

568 

569 v1shape = vplane1.shape 

570 filtInf = np.isinf(vplane1) 

571 filtNan = np.isnan(vplane1) 

572 # This copy could be eliminated if inf/nan handling were go into padCenterOriginArray 

573 vplane1 = np.copy(vplane1) 

574 vplane1[filtInf | filtNan] = varMean1 

575 D = self.padCenterOriginArray(vplane1, self.freqSpaceShape) 

576 v1 = np.real(np.fft.ifft2(np.fft.fft2(D) * c1SqFt)) 

577 v1 = self.padCenterOriginArray(v1, v1shape, useInverse=True) 

578 v1[filtNan] = np.nan 

579 v1[filtInf] = np.inf 

580 

581 D = np.real(np.fft.ifft2(c2ft)) 

582 c2ft = np.fft.fft2(D*D) 

583 

584 v2shape = vplane2.shape 

585 filtInf = np.isinf(vplane2) 

586 filtNan = np.isnan(vplane2) 

587 vplane2 = np.copy(vplane2) 

588 vplane2[filtInf | filtNan] = varMean2 

589 D = self.padCenterOriginArray(vplane2, self.freqSpaceShape) 

590 v2 = np.real(np.fft.ifft2(np.fft.fft2(D) * c2ft)) 

591 v2 = self.padCenterOriginArray(v2, v2shape, useInverse=True) 

592 v2[filtNan] = np.nan 

593 v2[filtInf] = np.inf 

594 

595 return v1 + v2 

596 

597 def computeCorrectedDiffimPsf(self, corrft, psfOld): 

598 """Compute the (decorrelated) difference image's new PSF. 

599 

600 Parameters 

601 ---------- 

602 corrft : `numpy.ndarray` 

603 The frequency space representation of the correction calculated by 

604 `computeCorrection`. Shape must be `self.freqSpaceShape`. 

605 psfOld : `numpy.ndarray` 

606 The psf of the difference image to be corrected. 

607 

608 Returns 

609 ------- 

610 psfNew : `numpy.ndarray` 

611 The corrected psf, same shape as `psfOld`, sum normed to 1. 

612 

613 Notes 

614 ----- 

615 There is no algorithmic guarantee that the corrected psf can 

616 meaningfully fit to the same size as the original one. 

617 """ 

618 psfShape = psfOld.shape 

619 psfNew = self.padCenterOriginArray(psfOld, self.freqSpaceShape) 

620 psfNew = np.fft.fft2(psfNew) 

621 psfNew *= corrft 

622 psfNew = np.fft.ifft2(psfNew) 

623 psfNew = psfNew.real 

624 psfNew = self.padCenterOriginArray(psfNew, psfShape, useInverse=True) 

625 psfNew = psfNew/psfNew.sum() 

626 return psfNew 

627 

628 def computeCorrectedImage(self, corrft, imgOld): 

629 """Compute the decorrelated difference image. 

630 

631 Parameters 

632 ---------- 

633 corrft : `numpy.ndarray` 

634 The frequency space representation of the correction calculated by 

635 `computeCorrection`. Shape must be `self.freqSpaceShape`. 

636 imgOld : `numpy.ndarray` 

637 The difference image to be corrected. 

638 

639 Returns 

640 ------- 

641 imgNew : `numpy.ndarray` 

642 The corrected image, same size as the input. 

643 """ 

644 expShape = imgOld.shape 

645 imgNew = np.copy(imgOld) 

646 filtInf = np.isinf(imgNew) 

647 filtNan = np.isnan(imgNew) 

648 imgNew[filtInf] = np.nan 

649 imgNew[filtInf | filtNan] = np.nanmean(imgNew) 

650 imgNew = self.padCenterOriginArray(imgNew, self.freqSpaceShape) 

651 imgNew = np.fft.fft2(imgNew) 

652 imgNew *= corrft 

653 imgNew = np.fft.ifft2(imgNew) 

654 imgNew = imgNew.real 

655 imgNew = self.padCenterOriginArray(imgNew, expShape, useInverse=True) 

656 imgNew[filtNan] = np.nan 

657 imgNew[filtInf] = np.inf 

658 return imgNew 

659 

660 

661class DecorrelateALKernelMapper(DecorrelateALKernelTask, ImageMapper): 

662 """Task to be used as an ImageMapper for performing 

663 A&L decorrelation on subimages on a grid across a A&L difference image. 

664 

665 This task subclasses DecorrelateALKernelTask in order to implement 

666 all of that task's configuration parameters, as well as its `run` method. 

667 """ 

668 

669 ConfigClass = DecorrelateALKernelConfig 

670 _DefaultName = 'ip_diffim_decorrelateALKernelMapper' 

671 

672 def __init__(self, *args, **kwargs): 

673 DecorrelateALKernelTask.__init__(self, *args, **kwargs) 

674 

675 def run(self, subExposure, expandedSubExposure, fullBBox, 

676 template, science, alTaskResult=None, psfMatchingKernel=None, 

677 preConvKernel=None, **kwargs): 

678 """Perform decorrelation operation on `subExposure`, using 

679 `expandedSubExposure` to allow for invalid edge pixels arising from 

680 convolutions. 

681 

682 This method performs A&L decorrelation on `subExposure` using 

683 local measures for image variances and PSF. `subExposure` is a 

684 sub-exposure of the non-decorrelated A&L diffim. It also 

685 requires the corresponding sub-exposures of the template 

686 (`template`) and science (`science`) exposures. 

687 

688 Parameters 

689 ---------- 

690 subExposure : `lsst.afw.image.Exposure` 

691 the sub-exposure of the diffim 

692 expandedSubExposure : `lsst.afw.image.Exposure` 

693 the expanded sub-exposure upon which to operate 

694 fullBBox : `lsst.geom.Box2I` 

695 the bounding box of the original exposure 

696 template : `lsst.afw.image.Exposure` 

697 the corresponding sub-exposure of the template exposure 

698 science : `lsst.afw.image.Exposure` 

699 the corresponding sub-exposure of the science exposure 

700 alTaskResult : `lsst.pipe.base.Struct` 

701 the result of A&L image differencing on `science` and 

702 `template`, importantly containing the resulting 

703 `psfMatchingKernel`. Can be `None`, only if 

704 `psfMatchingKernel` is not `None`. 

705 psfMatchingKernel : Alternative parameter for passing the 

706 A&L `psfMatchingKernel` directly. 

707 preConvKernel : If not None, then pre-filtering was applied 

708 to science exposure, and this is the pre-convolution 

709 kernel. 

710 kwargs : 

711 additional keyword arguments propagated from 

712 `ImageMapReduceTask.run`. 

713 

714 Returns 

715 ------- 

716 A `pipeBase.Struct` containing: 

717 

718 - ``subExposure`` : the result of the `subExposure` processing. 

719 - ``decorrelationKernel`` : the decorrelation kernel, currently 

720 not used. 

721 

722 Notes 

723 ----- 

724 This `run` method accepts parameters identical to those of 

725 `ImageMapper.run`, since it is called from the 

726 `ImageMapperTask`. See that class for more information. 

727 """ 

728 templateExposure = template # input template 

729 scienceExposure = science # input science image 

730 if alTaskResult is None and psfMatchingKernel is None: 

731 raise RuntimeError('Both alTaskResult and psfMatchingKernel cannot be None') 

732 psfMatchingKernel = alTaskResult.psfMatchingKernel if alTaskResult is not None else psfMatchingKernel 

733 

734 # subExp and expandedSubExp are subimages of the (un-decorrelated) diffim! 

735 # So here we compute corresponding subimages of templateExposure and scienceExposure 

736 subExp2 = scienceExposure.Factory(scienceExposure, expandedSubExposure.getBBox()) 

737 subExp1 = templateExposure.Factory(templateExposure, expandedSubExposure.getBBox()) 

738 

739 # Prevent too much log INFO verbosity from DecorrelateALKernelTask.run 

740 logLevel = self.log.getLevel() 

741 self.log.setLevel(self.log.WARNING) 

742 res = DecorrelateALKernelTask.run(self, subExp2, subExp1, expandedSubExposure, 

743 psfMatchingKernel, preConvKernel, **kwargs) 

744 self.log.setLevel(logLevel) # reset the log level 

745 

746 diffim = res.correctedExposure.Factory(res.correctedExposure, subExposure.getBBox()) 

747 out = pipeBase.Struct(subExposure=diffim, ) 

748 return out 

749 

750 

751class DecorrelateALKernelMapReduceConfig(ImageMapReduceConfig): 

752 """Configuration parameters for the ImageMapReduceTask to direct it to use 

753 DecorrelateALKernelMapper as its mapper for A&L decorrelation. 

754 """ 

755 mapper = pexConfig.ConfigurableField( 

756 doc='A&L decorrelation task to run on each sub-image', 

757 target=DecorrelateALKernelMapper 

758 ) 

759 

760 

761class DecorrelateALKernelSpatialConfig(pexConfig.Config): 

762 """Configuration parameters for the DecorrelateALKernelSpatialTask. 

763 """ 

764 decorrelateConfig = pexConfig.ConfigField( 

765 dtype=DecorrelateALKernelConfig, 

766 doc='DecorrelateALKernel config to use when running on complete exposure (non spatially-varying)', 

767 ) 

768 

769 decorrelateMapReduceConfig = pexConfig.ConfigField( 

770 dtype=DecorrelateALKernelMapReduceConfig, 

771 doc='DecorrelateALKernelMapReduce config to use when running on each sub-image (spatially-varying)', 

772 ) 

773 

774 ignoreMaskPlanes = pexConfig.ListField( 

775 dtype=str, 

776 doc="""Mask planes to ignore for sigma-clipped statistics""", 

777 default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE") 

778 ) 

779 

780 def setDefaults(self): 

781 self.decorrelateMapReduceConfig.gridStepX = self.decorrelateMapReduceConfig.gridStepY = 40 

782 self.decorrelateMapReduceConfig.cellSizeX = self.decorrelateMapReduceConfig.cellSizeY = 41 

783 self.decorrelateMapReduceConfig.borderSizeX = self.decorrelateMapReduceConfig.borderSizeY = 8 

784 self.decorrelateMapReduceConfig.reducer.reduceOperation = 'average' 

785 

786 

787class DecorrelateALKernelSpatialTask(pipeBase.Task): 

788 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference 

789 

790 Notes 

791 ----- 

792 

793 Pipe-task that removes the neighboring-pixel covariance in an 

794 image difference that are added when the template image is 

795 convolved with the Alard-Lupton PSF matching kernel. 

796 

797 This task is a simple wrapper around @ref DecorrelateALKernelTask, 

798 which takes a `spatiallyVarying` parameter in its `run` method. If 

799 it is `False`, then it simply calls the `run` method of @ref 

800 DecorrelateALKernelTask. If it is True, then it uses the @ref 

801 ImageMapReduceTask framework to break the exposures into 

802 subExposures on a grid, and performs the `run` method of @ref 

803 DecorrelateALKernelTask on each subExposure. This enables it to 

804 account for spatially-varying PSFs and noise in the exposures when 

805 performing the decorrelation. 

806 

807 This task has no standalone example, however it is applied as a 

808 subtask of pipe.tasks.imageDifference.ImageDifferenceTask. 

809 There is also an example of its use in `tests/testImageDecorrelation.py`. 

810 """ 

811 ConfigClass = DecorrelateALKernelSpatialConfig 

812 _DefaultName = "ip_diffim_decorrelateALKernelSpatial" 

813 

814 def __init__(self, *args, **kwargs): 

815 """Create the image decorrelation Task 

816 

817 Parameters 

818 ---------- 

819 args : 

820 arguments to be passed to 

821 `lsst.pipe.base.task.Task.__init__` 

822 kwargs : 

823 additional keyword arguments to be passed to 

824 `lsst.pipe.base.task.Task.__init__` 

825 """ 

826 pipeBase.Task.__init__(self, *args, **kwargs) 

827 

828 self.statsControl = afwMath.StatisticsControl() 

829 self.statsControl.setNumSigmaClip(3.) 

830 self.statsControl.setNumIter(3) 

831 self.statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes)) 

832 

833 def computeVarianceMean(self, exposure): 

834 """Compute the mean of the variance plane of `exposure`. 

835 """ 

836 statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(), 

837 exposure.getMaskedImage().getMask(), 

838 afwMath.MEANCLIP, self.statsControl) 

839 var = statObj.getValue(afwMath.MEANCLIP) 

840 return var 

841 

842 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel, 

843 spatiallyVarying=True, preConvKernel=None, templateMatched=True, preConvMode=False): 

844 """Perform decorrelation of an image difference exposure. 

845 

846 Decorrelates the diffim due to the convolution of the 

847 templateExposure with the A&L psfMatchingKernel. If 

848 `spatiallyVarying` is True, it utilizes the spatially varying 

849 matching kernel via the `imageMapReduce` framework to perform 

850 spatially-varying decorrelation on a grid of subExposures. 

851 

852 Parameters 

853 ---------- 

854 scienceExposure : `lsst.afw.image.Exposure` 

855 the science Exposure used for PSF matching 

856 templateExposure : `lsst.afw.image.Exposure` 

857 the template Exposure used for PSF matching 

858 subtractedExposure : `lsst.afw.image.Exposure` 

859 the subtracted Exposure produced by `ip_diffim.ImagePsfMatchTask.subtractExposures()` 

860 psfMatchingKernel : an (optionally spatially-varying) PSF matching kernel produced 

861 by `ip_diffim.ImagePsfMatchTask.subtractExposures()` 

862 spatiallyVarying : `bool` 

863 if True, perform the spatially-varying operation 

864 preConvKernel : `lsst.meas.algorithms.Psf` 

865 if not none, the scienceExposure has been pre-filtered with this kernel. (Currently 

866 this option is experimental.) 

867 templateMatched : `bool`, optional 

868 If True, the template exposure was matched (convolved) to the science exposure. 

869 preConvMode : `bool`, optional 

870 If True, ``subtractedExposure`` is assumed to be a likelihood difference image 

871 and will be noise corrected as a likelihood image. 

872 

873 Returns 

874 ------- 

875 results : `lsst.pipe.base.Struct` 

876 a structure containing: 

877 - ``correctedExposure`` : the decorrelated diffim 

878 """ 

879 self.log.info('Running A&L decorrelation: spatiallyVarying=%r', spatiallyVarying) 

880 

881 svar = self.computeVarianceMean(scienceExposure) 

882 tvar = self.computeVarianceMean(templateExposure) 

883 if np.isnan(svar) or np.isnan(tvar): # Should not happen unless entire image has been masked. 

884 # Double check that one of the exposures is all NaNs 

885 if (np.all(np.isnan(scienceExposure.image.array)) 

886 or np.all(np.isnan(templateExposure.image.array))): 

887 self.log.warning('Template or science image is entirely NaNs: skipping decorrelation.') 

888 if np.isnan(svar): 

889 svar = 1e-9 

890 if np.isnan(tvar): 

891 tvar = 1e-9 

892 

893 var = self.computeVarianceMean(subtractedExposure) 

894 

895 if spatiallyVarying: 

896 self.log.info("Variance (science, template): (%f, %f)", svar, tvar) 

897 self.log.info("Variance (uncorrected diffim): %f", var) 

898 config = self.config.decorrelateMapReduceConfig 

899 task = ImageMapReduceTask(config=config) 

900 results = task.run(subtractedExposure, science=scienceExposure, 

901 template=templateExposure, psfMatchingKernel=psfMatchingKernel, 

902 preConvKernel=preConvKernel, forceEvenSized=True, 

903 templateMatched=templateMatched, preConvMode=preConvMode) 

904 results.correctedExposure = results.exposure 

905 

906 # Make sure masks of input image are propagated to diffim 

907 def gm(exp): 

908 return exp.getMaskedImage().getMask() 

909 gm(results.correctedExposure)[:, :] = gm(subtractedExposure) 

910 

911 var = self.computeVarianceMean(results.correctedExposure) 

912 self.log.info("Variance (corrected diffim): %f", var) 

913 

914 else: 

915 config = self.config.decorrelateConfig 

916 task = DecorrelateALKernelTask(config=config) 

917 results = task.run(scienceExposure, templateExposure, 

918 subtractedExposure, psfMatchingKernel, preConvKernel=preConvKernel, 

919 templateMatched=templateMatched, preConvMode=preConvMode) 

920 

921 return results