Coverage for python/lsst/ip/diffim/imageDecorrelation.py: 14%

284 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-04-04 02:42 -0700

1# 

2# LSST Data Management System 

3# Copyright 2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23import numpy as np 

24 

25import lsst.afw.image as afwImage 

26import lsst.afw.math as afwMath 

27import lsst.geom as geom 

28import lsst.meas.algorithms as measAlg 

29import lsst.pex.config as pexConfig 

30import lsst.pipe.base as pipeBase 

31from lsst.utils.timer import timeMethod 

32 

33from .imageMapReduce import (ImageMapReduceConfig, ImageMapReduceTask, 

34 ImageMapper) 

35 

36__all__ = ("DecorrelateALKernelTask", "DecorrelateALKernelConfig", 

37 "DecorrelateALKernelMapper", "DecorrelateALKernelMapReduceConfig", 

38 "DecorrelateALKernelSpatialConfig", "DecorrelateALKernelSpatialTask") 

39 

40 

41class DecorrelateALKernelConfig(pexConfig.Config): 

42 """Configuration parameters for the DecorrelateALKernelTask 

43 """ 

44 

45 ignoreMaskPlanes = pexConfig.ListField( 

46 dtype=str, 

47 doc="""Mask planes to ignore for sigma-clipped statistics""", 

48 default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE") 

49 ) 

50 completeVarPlanePropagation = pexConfig.Field( 

51 dtype=bool, 

52 default=False, 

53 doc="Compute the full effect of the decorrelated matching kernel on the variance plane." 

54 " Otherwise use a model weighed sum of the input variances." 

55 ) 

56 

57 

58class DecorrelateALKernelTask(pipeBase.Task): 

59 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference 

60 

61 Notes 

62 ----- 

63 

64 Pipe-task that removes the neighboring-pixel covariance in an 

65 image difference that are added when the template image is 

66 convolved with the Alard-Lupton PSF matching kernel. 

67 

68 The image differencing pipeline task @link 

69 ip.diffim.psfMatch.PsfMatchTask PSFMatchTask@endlink and @link 

70 ip.diffim.psfMatch.PsfMatchConfigAL PSFMatchConfigAL@endlink uses 

71 the Alard and Lupton (1998) method for matching the PSFs of the 

72 template and science exposures prior to subtraction. The 

73 Alard-Lupton method identifies a matching kernel, which is then 

74 (typically) convolved with the template image to perform PSF 

75 matching. This convolution has the effect of adding covariance 

76 between neighboring pixels in the template image, which is then 

77 added to the image difference by subtraction. 

78 

79 The pixel covariance may be corrected by whitening the noise of 

80 the image difference. This task performs such a decorrelation by 

81 computing a decorrelation kernel (based upon the A&L matching 

82 kernel and variances in the template and science images) and 

83 convolving the image difference with it. This process is described 

84 in detail in [DMTN-021](http://dmtn-021.lsst.io). 

85 

86 This task has no standalone example, however it is applied as a 

87 subtask of pipe.tasks.imageDifference.ImageDifferenceTask. 

88 """ 

89 ConfigClass = DecorrelateALKernelConfig 

90 _DefaultName = "ip_diffim_decorrelateALKernel" 

91 

92 def __init__(self, *args, **kwargs): 

93 """Create the image decorrelation Task 

94 

95 Parameters 

96 ---------- 

97 args : 

98 arguments to be passed to ``lsst.pipe.base.task.Task.__init__`` 

99 kwargs : 

100 keyword arguments to be passed to ``lsst.pipe.base.task.Task.__init__`` 

101 """ 

102 pipeBase.Task.__init__(self, *args, **kwargs) 

103 

104 self.statsControl = afwMath.StatisticsControl() 

105 self.statsControl.setNumSigmaClip(3.) 

106 self.statsControl.setNumIter(3) 

107 self.statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes)) 

108 

109 def computeVarianceMean(self, exposure): 

110 statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(), 

111 exposure.getMaskedImage().getMask(), 

112 afwMath.MEANCLIP, self.statsControl) 

113 var = statObj.getValue(afwMath.MEANCLIP) 

114 return var 

115 

116 @timeMethod 

117 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel, 

118 preConvKernel=None, xcen=None, ycen=None, svar=None, tvar=None, 

119 templateMatched=True, preConvMode=False, **kwargs): 

120 """Perform decorrelation of an image difference or of a score difference exposure. 

121 

122 Corrects the difference or score image due to the convolution of the 

123 templateExposure with the A&L PSF matching kernel. 

124 See [DMTN-021, Equation 1](http://dmtn-021.lsst.io/#equation-1) and 

125 [DMTN-179](http://dmtn-179.lsst.io/) for details. 

126 

127 Parameters 

128 ---------- 

129 scienceExposure : `lsst.afw.image.Exposure` 

130 The original science exposure (before pre-convolution, if ``preConvMode==True``). 

131 templateExposure : `lsst.afw.image.Exposure` 

132 The original template exposure warped, but not psf-matched, to the science exposure. 

133 subtractedExposure : `lsst.afw.image.Exposure` 

134 the subtracted exposure produced by 

135 `ip_diffim.ImagePsfMatchTask.subtractExposures()`. The `subtractedExposure` must 

136 inherit its PSF from `exposure`, see notes below. 

137 psfMatchingKernel : `lsst.afw.detection.Psf` 

138 An (optionally spatially-varying) PSF matching kernel produced 

139 by `ip_diffim.ImagePsfMatchTask.subtractExposures()`. 

140 preConvKernel : `lsst.afw.math.Kernel`, optional 

141 If not `None`, then the `scienceExposure` was pre-convolved with (the reflection of) 

142 this kernel. Must be normalized to sum to 1. 

143 Allowed only if ``templateMatched==True`` and ``preConvMode==True``. 

144 Defaults to the PSF of the science exposure at the image center. 

145 xcen : `float`, optional 

146 X-pixel coordinate to use for computing constant matching kernel to use 

147 If `None` (default), then use the center of the image. 

148 ycen : `float`, optional 

149 Y-pixel coordinate to use for computing constant matching kernel to use 

150 If `None` (default), then use the center of the image. 

151 svar : `float`, optional 

152 Image variance for science image 

153 If `None` (default) then compute the variance over the entire input science image. 

154 tvar : `float`, optional 

155 Image variance for template image 

156 If `None` (default) then compute the variance over the entire input template image. 

157 templateMatched : `bool`, optional 

158 If True, the template exposure was matched (convolved) to the science exposure. 

159 See also notes below. 

160 preConvMode : `bool`, optional 

161 If True, ``subtractedExposure`` is assumed to be a likelihood difference image 

162 and will be noise corrected as a likelihood image. 

163 **kwargs 

164 Additional keyword arguments propagated from DecorrelateALKernelSpatialTask. 

165 

166 Returns 

167 ------- 

168 result : `lsst.pipe.base.Struct` 

169 - ``correctedExposure`` : the decorrelated diffim 

170 

171 Notes 

172 ----- 

173 If ``preConvMode==True``, ``subtractedExposure`` is assumed to be a 

174 score image and the noise correction for likelihood images 

175 is applied. The resulting image is an optimal detection likelihood image 

176 when the templateExposure has noise. (See DMTN-179) If ``preConvKernel`` is 

177 not specified, the PSF of ``scienceExposure`` is assumed as pre-convolution kernel. 

178 

179 The ``subtractedExposure`` is NOT updated. The returned ``correctedExposure`` 

180 has an updated but spatially fixed PSF. It is calculated as the center of 

181 image PSF corrected by the center of image matching kernel. 

182 

183 If ``templateMatched==True``, the templateExposure was matched (convolved) 

184 to the ``scienceExposure`` by ``psfMatchingKernel`` during image differencing. 

185 Otherwise the ``scienceExposure`` was matched (convolved) by ``psfMatchingKernel``. 

186 In either case, note that the original template and science images are required, 

187 not the psf-matched version. 

188 

189 This task discards the variance plane of ``subtractedExposure`` and re-computes 

190 it from the variance planes of ``scienceExposure`` and ``templateExposure``. 

191 The image plane of ``subtractedExposure`` must be at the photometric level 

192 set by the AL PSF matching in `ImagePsfMatchTask.subtractExposures`. 

193 The assumptions about the photometric level are controlled by the 

194 `templateMatched` option in this task. 

195 

196 Here we currently convert a spatially-varying matching kernel into a constant kernel, 

197 just by computing it at the center of the image (tickets DM-6243, DM-6244). 

198 

199 We are also using a constant accross-the-image measure of sigma (sqrt(variance)) to compute 

200 the decorrelation kernel. 

201 

202 TODO DM-23857 As part of the spatially varying correction implementation 

203 consider whether returning a Struct is still necessary. 

204 """ 

205 if preConvKernel is not None and not (templateMatched and preConvMode): 

206 raise ValueError("Pre-convolution kernel is allowed only if " 

207 "preConvMode==True and templateMatched==True.") 

208 

209 spatialKernel = psfMatchingKernel 

210 kimg = afwImage.ImageD(spatialKernel.getDimensions()) 

211 bbox = subtractedExposure.getBBox() 

212 if xcen is None: 

213 xcen = (bbox.getBeginX() + bbox.getEndX()) / 2. 

214 if ycen is None: 

215 ycen = (bbox.getBeginY() + bbox.getEndY()) / 2. 

216 self.log.info("Using matching kernel computed at (%d, %d)", xcen, ycen) 

217 spatialKernel.computeImage(kimg, False, xcen, ycen) 

218 

219 preConvImg = None 

220 if preConvMode: 

221 if preConvKernel is None: 

222 pos = scienceExposure.getPsf().getAveragePosition() 

223 preConvKernel = scienceExposure.getPsf().getLocalKernel(pos) 

224 preConvImg = afwImage.ImageD(preConvKernel.getDimensions()) 

225 preConvKernel.computeImage(preConvImg, True) 

226 

227 if svar is None: 

228 svar = self.computeVarianceMean(scienceExposure) 

229 if tvar is None: 

230 tvar = self.computeVarianceMean(templateExposure) 

231 self.log.info("Original variance plane means. Science:%.5e, warped template:%.5e)", 

232 svar, tvar) 

233 

234 # Should not happen unless entire image has been masked, which could happen 

235 # if this is a small subimage of the main exposure. In this case, just return a full NaN 

236 # exposure 

237 if np.isnan(svar) or np.isnan(tvar): 

238 # Double check that one of the exposures is all NaNs 

239 if (np.all(np.isnan(scienceExposure.image.array)) 

240 or np.all(np.isnan(templateExposure.image.array))): 

241 self.log.warning('Template or science image is entirely NaNs: skipping decorrelation.') 

242 outExposure = subtractedExposure.clone() 

243 return pipeBase.Struct(correctedExposure=outExposure, ) 

244 

245 if templateMatched: 

246 # Regular subtraction, we convolved the template 

247 self.log.info("Decorrelation after template image convolution") 

248 varianceMean = svar 

249 targetVarianceMean = tvar 

250 # variance plane of the image that is not convolved 

251 variance = scienceExposure.variance.array 

252 # Variance plane of the convolved image, before convolution. 

253 targetVariance = templateExposure.variance.array 

254 else: 

255 # We convolved the science image 

256 self.log.info("Decorrelation after science image convolution") 

257 varianceMean = tvar 

258 targetVarianceMean = svar 

259 # variance plane of the image that is not convolved 

260 variance = templateExposure.variance.array 

261 # Variance plane of the convolved image, before convolution. 

262 targetVariance = scienceExposure.variance.array 

263 

264 # The maximal correction value converges to sqrt(targetVarianceMean/varianceMean). 

265 # Correction divergence warning if the correction exceeds 4 orders of magnitude. 

266 mOverExpVar = targetVarianceMean/varianceMean 

267 if mOverExpVar > 1e8: 

268 self.log.warning("Diverging correction: matched image variance is " 

269 " much larger than the unconvolved one's" 

270 ", targetVarianceMean/varianceMean:%.2e", mOverExpVar) 

271 

272 oldVarMean = self.computeVarianceMean(subtractedExposure) 

273 self.log.info("Variance plane mean of uncorrected diffim: %f", oldVarMean) 

274 

275 kArr = kimg.array 

276 diffimShape = subtractedExposure.image.array.shape 

277 psfImg = subtractedExposure.getPsf().computeKernelImage(geom.Point2D(xcen, ycen)) 

278 psfShape = psfImg.array.shape 

279 

280 if preConvMode: 

281 self.log.info("Decorrelation of likelihood image") 

282 self.computeCommonShape(preConvImg.array.shape, kArr.shape, 

283 psfShape, diffimShape) 

284 corr = self.computeScoreCorrection(kArr, varianceMean, targetVarianceMean, preConvImg.array) 

285 else: 

286 self.log.info("Decorrelation of difference image") 

287 self.computeCommonShape(kArr.shape, psfShape, diffimShape) 

288 corr = self.computeDiffimCorrection(kArr, varianceMean, targetVarianceMean) 

289 

290 correctedImage = self.computeCorrectedImage(corr.corrft, subtractedExposure.image.array) 

291 correctedPsf = self.computeCorrectedDiffimPsf(corr.corrft, psfImg.array) 

292 

293 # The subtracted exposure variance plane is already correlated, we cannot propagate 

294 # it through another convolution; instead we need to use the uncorrelated originals 

295 # The whitening should scale it to varianceMean + targetVarianceMean on average 

296 if self.config.completeVarPlanePropagation: 

297 self.log.debug("Using full variance plane calculation in decorrelation") 

298 correctedVariance = self.calculateVariancePlane( 

299 variance, targetVariance, 

300 varianceMean, targetVarianceMean, corr.cnft, corr.crft) 

301 else: 

302 self.log.debug("Using estimated variance plane calculation in decorrelation") 

303 correctedVariance = self.estimateVariancePlane( 

304 variance, targetVariance, 

305 corr.cnft, corr.crft) 

306 

307 # Determine the common shape 

308 kSum = np.sum(kArr) 

309 kSumSq = kSum*kSum 

310 self.log.debug("Matching kernel sum: %.3e", kSum) 

311 if not templateMatched: 

312 # ImagePsfMatch.subtractExposures re-scales the difference in 

313 # the science image convolution mode 

314 correctedVariance /= kSumSq 

315 subtractedExposure.image.array[...] = correctedImage # Allow for numpy type casting 

316 subtractedExposure.variance.array[...] = correctedVariance 

317 subtractedExposure.setPsf(correctedPsf) 

318 

319 newVarMean = self.computeVarianceMean(subtractedExposure) 

320 self.log.info("Variance plane mean of corrected diffim: %.5e", newVarMean) 

321 

322 # TODO DM-23857 As part of the spatially varying correction implementation 

323 # consider whether returning a Struct is still necessary. 

324 return pipeBase.Struct(correctedExposure=subtractedExposure, ) 

325 

326 def computeCommonShape(self, *shapes): 

327 """Calculate the common shape for FFT operations. Set `self.freqSpaceShape` 

328 internally. 

329 

330 Parameters 

331 ---------- 

332 shapes : one or more `tuple` of `int` 

333 Shapes of the arrays. All must have the same dimensionality. 

334 At least one shape must be provided. 

335 

336 Returns 

337 ------- 

338 None. 

339 

340 Notes 

341 ----- 

342 For each dimension, gets the smallest even number greater than or equal to 

343 `N1+N2-1` where `N1` and `N2` are the two largest values. 

344 In case of only one shape given, rounds up to even each dimension value. 

345 """ 

346 S = np.array(shapes, dtype=int) 

347 if len(shapes) > 2: 

348 S.sort(axis=0) 

349 S = S[-2:] 

350 if len(shapes) > 1: 

351 commonShape = np.sum(S, axis=0) - 1 

352 else: 

353 commonShape = S[0] 

354 commonShape[commonShape % 2 != 0] += 1 

355 self.freqSpaceShape = tuple(commonShape) 

356 self.log.info("Common frequency space shape %s", self.freqSpaceShape) 

357 

358 @staticmethod 

359 def padCenterOriginArray(A, newShape: tuple, useInverse=False): 

360 """Zero pad an image where the origin is at the center and replace the 

361 origin to the corner as required by the periodic input of FFT. Implement also 

362 the inverse operation, crop the padding and re-center data. 

363 

364 Parameters 

365 ---------- 

366 A : `numpy.ndarray` 

367 An array to copy from. 

368 newShape : `tuple` of `int` 

369 The dimensions of the resulting array. For padding, the resulting array 

370 must be larger than A in each dimension. For the inverse operation this 

371 must be the original, before padding size of the array. 

372 useInverse : bool, optional 

373 Selector of forward, add padding, operation (False) 

374 or its inverse, crop padding, operation (True). 

375 

376 Returns 

377 ------- 

378 R : `numpy.ndarray` 

379 The padded or unpadded array with shape of `newShape` and the same dtype as A. 

380 

381 Notes 

382 ----- 

383 For odd dimensions, the splitting is rounded to 

384 put the center pixel into the new corner origin (0,0). This is to be consistent 

385 e.g. for a dirac delta kernel that is originally located at the center pixel. 

386 """ 

387 

388 # The forward and inverse operations should round odd dimension halves at the opposite 

389 # sides to get the pixels back to their original positions. 

390 if not useInverse: 

391 # Forward operation: First and second halves with respect to the axes of A. 

392 firstHalves = [x//2 for x in A.shape] 

393 secondHalves = [x-y for x, y in zip(A.shape, firstHalves)] 

394 else: 

395 # Inverse operation: Opposite rounding 

396 secondHalves = [x//2 for x in newShape] 

397 firstHalves = [x-y for x, y in zip(newShape, secondHalves)] 

398 

399 R = np.zeros_like(A, shape=newShape) 

400 R[-firstHalves[0]:, -firstHalves[1]:] = A[:firstHalves[0], :firstHalves[1]] 

401 R[:secondHalves[0], -firstHalves[1]:] = A[-secondHalves[0]:, :firstHalves[1]] 

402 R[:secondHalves[0], :secondHalves[1]] = A[-secondHalves[0]:, -secondHalves[1]:] 

403 R[-firstHalves[0]:, :secondHalves[1]] = A[:firstHalves[0], -secondHalves[1]:] 

404 return R 

405 

406 def computeDiffimCorrection(self, kappa, svar, tvar): 

407 """Compute the Lupton decorrelation post-convolution kernel for decorrelating an 

408 image difference, based on the PSF-matching kernel. 

409 

410 Parameters 

411 ---------- 

412 kappa : `numpy.ndarray` of `float` 

413 A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching. 

414 svar : `float` > 0. 

415 Average variance of science image used for PSF matching. 

416 tvar : `float` > 0. 

417 Average variance of the template (matched) image used for PSF matching. 

418 

419 Returns 

420 ------- 

421 corrft : `numpy.ndarray` of `float` 

422 The frequency space representation of the correction. The array is real (dtype float). 

423 Shape is `self.freqSpaceShape`. 

424 

425 cnft, crft : `numpy.ndarray` of `complex` 

426 The overall convolution (pre-conv, PSF matching, noise correction) kernel 

427 for the science and template images, respectively for the variance plane 

428 calculations. These are intermediate results in frequency space. 

429 

430 Notes 

431 ----- 

432 The maximum correction factor converges to `sqrt(tvar/svar)` towards high frequencies. 

433 This should be a plausible value. 

434 """ 

435 kSum = np.sum(kappa) # We scale the decorrelation to preserve fluxes 

436 kappa = self.padCenterOriginArray(kappa, self.freqSpaceShape) 

437 kft = np.fft.fft2(kappa) 

438 kftAbsSq = np.real(np.conj(kft) * kft) 

439 

440 denom = svar + tvar * kftAbsSq 

441 corrft = np.sqrt((svar + tvar * kSum*kSum) / denom) 

442 cnft = corrft 

443 crft = kft*corrft 

444 return pipeBase.Struct(corrft=corrft, cnft=cnft, crft=crft) 

445 

446 def computeScoreCorrection(self, kappa, svar, tvar, preConvArr): 

447 """Compute the correction kernel for a score image. 

448 

449 Parameters 

450 ---------- 

451 kappa : `numpy.ndarray` 

452 A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching. 

453 svar : `float` 

454 Average variance of science image used for PSF matching (before pre-convolution). 

455 tvar : `float` 

456 Average variance of the template (matched) image used for PSF matching. 

457 preConvArr : `numpy.ndarray` 

458 The pre-convolution kernel of the science image. It should be the PSF 

459 of the science image or an approximation of it. It must be normed to sum 1. 

460 

461 Returns 

462 ------- 

463 corrft : `numpy.ndarray` of `float` 

464 The frequency space representation of the correction. The array is real (dtype float). 

465 Shape is `self.freqSpaceShape`. 

466 cnft, crft : `numpy.ndarray` of `complex` 

467 The overall convolution (pre-conv, PSF matching, noise correction) kernel 

468 for the science and template images, respectively for the variance plane 

469 calculations. These are intermediate results in frequency space. 

470 

471 Notes 

472 ----- 

473 To be precise, the science image should be _correlated_ by ``preConvArray`` but this 

474 does not matter for this calculation. 

475 

476 ``cnft``, ``crft`` contain the scaling factor as well. 

477 

478 """ 

479 kSum = np.sum(kappa) 

480 kappa = self.padCenterOriginArray(kappa, self.freqSpaceShape) 

481 kft = np.fft.fft2(kappa) 

482 preConvArr = self.padCenterOriginArray(preConvArr, self.freqSpaceShape) 

483 preFt = np.fft.fft2(preConvArr) 

484 preFtAbsSq = np.real(np.conj(preFt) * preFt) 

485 kftAbsSq = np.real(np.conj(kft) * kft) 

486 # Avoid zero division, though we don't normally approach `tiny`. 

487 # We have numerical noise instead. 

488 tiny = np.finfo(preFtAbsSq.dtype).tiny * 1000. 

489 flt = preFtAbsSq < tiny 

490 # If we pre-convolve to avoid deconvolution in AL, then kftAbsSq / preFtAbsSq 

491 # theoretically expected to diverge to +inf. But we don't care about the convergence 

492 # properties here, S goes to 0 at these frequencies anyway. 

493 preFtAbsSq[flt] = tiny 

494 denom = svar + tvar * kftAbsSq / preFtAbsSq 

495 corrft = (svar + tvar * kSum*kSum) / denom 

496 cnft = np.conj(preFt)*corrft 

497 crft = kft*corrft 

498 return pipeBase.Struct(corrft=corrft, cnft=cnft, crft=crft) 

499 

500 @staticmethod 

501 def estimateVariancePlane(vplane1, vplane2, c1ft, c2ft): 

502 """Estimate the variance planes. 

503 

504 The estimation assumes that around each pixel the surrounding 

505 pixels' sigmas within the convolution kernel are the same. 

506 

507 Parameters 

508 ---------- 

509 vplane1, vplane2 : `numpy.ndarray` of `float` 

510 Variance planes of the original (before pre-convolution or matching) 

511 exposures. 

512 c1ft, c2ft : `numpy.ndarray` of `complex` 

513 The overall convolution that includes the matching and the 

514 afterburner in frequency space. The result of either 

515 ``computeScoreCorrection`` or ``computeDiffimCorrection``. 

516 

517 Returns 

518 ------- 

519 vplaneD : `numpy.ndarray` of `float` 

520 The estimated variance plane of the difference/score image 

521 as a weighted sum of the input variances. 

522 

523 Notes 

524 ------ 

525 See DMTN-179 Section 5 about the variance plane calculations. 

526 """ 

527 w1 = np.sum(np.real(np.conj(c1ft)*c1ft)) / c1ft.size 

528 w2 = np.sum(np.real(np.conj(c2ft)*c2ft)) / c2ft.size 

529 # w1, w2: the frequency space sum of abs(c1)^2 is the same as in image 

530 # space. 

531 return vplane1*w1 + vplane2*w2 

532 

533 def calculateVariancePlane(self, vplane1, vplane2, varMean1, varMean2, c1ft, c2ft): 

534 """Full propagation of the variance planes of the original exposures. 

535 

536 The original variance planes of independent pixels are convolved with the 

537 image space square of the overall kernels. 

538 

539 Parameters 

540 ---------- 

541 vplane1, vplane2 : `numpy.ndarray` of `float` 

542 Variance planes of the original (before pre-convolution or matching) 

543 exposures. 

544 varMean1, varMean2 : `float` 

545 Replacement average values for non-finite ``vplane1`` and ``vplane2`` values respectively. 

546 

547 c1ft, c2ft : `numpy.ndarray` of `complex` 

548 The overall convolution that includes the matching and the 

549 afterburner in frequency space. The result of either 

550 ``computeScoreCorrection`` or ``computeDiffimCorrection``. 

551 

552 Returns 

553 ------- 

554 vplaneD : `numpy.ndarray` of `float` 

555 The variance plane of the difference/score images. 

556 

557 Notes 

558 ------ 

559 See DMTN-179 Section 5 about the variance plane calculations. 

560 

561 Infs and NaNs are allowed and kept in the returned array. 

562 """ 

563 D = np.real(np.fft.ifft2(c1ft)) 

564 c1SqFt = np.fft.fft2(D*D) 

565 

566 v1shape = vplane1.shape 

567 filtInf = np.isinf(vplane1) 

568 filtNan = np.isnan(vplane1) 

569 # This copy could be eliminated if inf/nan handling were go into padCenterOriginArray 

570 vplane1 = np.copy(vplane1) 

571 vplane1[filtInf | filtNan] = varMean1 

572 D = self.padCenterOriginArray(vplane1, self.freqSpaceShape) 

573 v1 = np.real(np.fft.ifft2(np.fft.fft2(D) * c1SqFt)) 

574 v1 = self.padCenterOriginArray(v1, v1shape, useInverse=True) 

575 v1[filtNan] = np.nan 

576 v1[filtInf] = np.inf 

577 

578 D = np.real(np.fft.ifft2(c2ft)) 

579 c2SqFt = np.fft.fft2(D*D) 

580 

581 v2shape = vplane2.shape 

582 filtInf = np.isinf(vplane2) 

583 filtNan = np.isnan(vplane2) 

584 vplane2 = np.copy(vplane2) 

585 vplane2[filtInf | filtNan] = varMean2 

586 D = self.padCenterOriginArray(vplane2, self.freqSpaceShape) 

587 v2 = np.real(np.fft.ifft2(np.fft.fft2(D) * c2SqFt)) 

588 v2 = self.padCenterOriginArray(v2, v2shape, useInverse=True) 

589 v2[filtNan] = np.nan 

590 v2[filtInf] = np.inf 

591 

592 return v1 + v2 

593 

594 def computeCorrectedDiffimPsf(self, corrft, psfOld): 

595 """Compute the (decorrelated) difference image's new PSF. 

596 

597 Parameters 

598 ---------- 

599 corrft : `numpy.ndarray` 

600 The frequency space representation of the correction calculated by 

601 `computeCorrection`. Shape must be `self.freqSpaceShape`. 

602 psfOld : `numpy.ndarray` 

603 The psf of the difference image to be corrected. 

604 

605 Returns 

606 ------- 

607 correctedPsf : `lsst.meas.algorithms.KernelPsf` 

608 The corrected psf, same shape as `psfOld`, sum normed to 1. 

609 

610 Notes 

611 ----- 

612 There is no algorithmic guarantee that the corrected psf can 

613 meaningfully fit to the same size as the original one. 

614 """ 

615 psfShape = psfOld.shape 

616 psfNew = self.padCenterOriginArray(psfOld, self.freqSpaceShape) 

617 psfNew = np.fft.fft2(psfNew) 

618 psfNew *= corrft 

619 psfNew = np.fft.ifft2(psfNew) 

620 psfNew = psfNew.real 

621 psfNew = self.padCenterOriginArray(psfNew, psfShape, useInverse=True) 

622 psfNew = psfNew/psfNew.sum() 

623 

624 psfcI = afwImage.ImageD(geom.Extent2I(psfShape[1], psfShape[0])) 

625 psfcI.array = psfNew 

626 psfcK = afwMath.FixedKernel(psfcI) 

627 correctedPsf = measAlg.KernelPsf(psfcK) 

628 return correctedPsf 

629 

630 def computeCorrectedImage(self, corrft, imgOld): 

631 """Compute the decorrelated difference image. 

632 

633 Parameters 

634 ---------- 

635 corrft : `numpy.ndarray` 

636 The frequency space representation of the correction calculated by 

637 `computeCorrection`. Shape must be `self.freqSpaceShape`. 

638 imgOld : `numpy.ndarray` 

639 The difference image to be corrected. 

640 

641 Returns 

642 ------- 

643 imgNew : `numpy.ndarray` 

644 The corrected image, same size as the input. 

645 """ 

646 expShape = imgOld.shape 

647 imgNew = np.copy(imgOld) 

648 filtInf = np.isinf(imgNew) 

649 filtNan = np.isnan(imgNew) 

650 imgNew[filtInf] = np.nan 

651 imgNew[filtInf | filtNan] = np.nanmean(imgNew) 

652 imgNew = self.padCenterOriginArray(imgNew, self.freqSpaceShape) 

653 imgNew = np.fft.fft2(imgNew) 

654 imgNew *= corrft 

655 imgNew = np.fft.ifft2(imgNew) 

656 imgNew = imgNew.real 

657 imgNew = self.padCenterOriginArray(imgNew, expShape, useInverse=True) 

658 imgNew[filtNan] = np.nan 

659 imgNew[filtInf] = np.inf 

660 return imgNew 

661 

662 

663class DecorrelateALKernelMapper(DecorrelateALKernelTask, ImageMapper): 

664 """Task to be used as an ImageMapper for performing 

665 A&L decorrelation on subimages on a grid across a A&L difference image. 

666 

667 This task subclasses DecorrelateALKernelTask in order to implement 

668 all of that task's configuration parameters, as well as its `run` method. 

669 """ 

670 

671 ConfigClass = DecorrelateALKernelConfig 

672 _DefaultName = 'ip_diffim_decorrelateALKernelMapper' 

673 

674 def __init__(self, *args, **kwargs): 

675 DecorrelateALKernelTask.__init__(self, *args, **kwargs) 

676 

677 def run(self, subExposure, expandedSubExposure, fullBBox, 

678 template, science, alTaskResult=None, psfMatchingKernel=None, 

679 preConvKernel=None, **kwargs): 

680 """Perform decorrelation operation on `subExposure`, using 

681 `expandedSubExposure` to allow for invalid edge pixels arising from 

682 convolutions. 

683 

684 This method performs A&L decorrelation on `subExposure` using 

685 local measures for image variances and PSF. `subExposure` is a 

686 sub-exposure of the non-decorrelated A&L diffim. It also 

687 requires the corresponding sub-exposures of the template 

688 (`template`) and science (`science`) exposures. 

689 

690 Parameters 

691 ---------- 

692 subExposure : `lsst.afw.image.Exposure` 

693 the sub-exposure of the diffim 

694 expandedSubExposure : `lsst.afw.image.Exposure` 

695 the expanded sub-exposure upon which to operate 

696 fullBBox : `lsst.geom.Box2I` 

697 the bounding box of the original exposure 

698 template : `lsst.afw.image.Exposure` 

699 the corresponding sub-exposure of the template exposure 

700 science : `lsst.afw.image.Exposure` 

701 the corresponding sub-exposure of the science exposure 

702 alTaskResult : `lsst.pipe.base.Struct` 

703 the result of A&L image differencing on `science` and 

704 `template`, importantly containing the resulting 

705 `psfMatchingKernel`. Can be `None`, only if 

706 `psfMatchingKernel` is not `None`. 

707 psfMatchingKernel : Alternative parameter for passing the 

708 A&L `psfMatchingKernel` directly. 

709 preConvKernel : If not None, then pre-filtering was applied 

710 to science exposure, and this is the pre-convolution 

711 kernel. 

712 kwargs : 

713 additional keyword arguments propagated from 

714 `ImageMapReduceTask.run`. 

715 

716 Returns 

717 ------- 

718 A `pipeBase.Struct` containing: 

719 

720 - ``subExposure`` : the result of the `subExposure` processing. 

721 - ``decorrelationKernel`` : the decorrelation kernel, currently 

722 not used. 

723 

724 Notes 

725 ----- 

726 This `run` method accepts parameters identical to those of 

727 `ImageMapper.run`, since it is called from the 

728 `ImageMapperTask`. See that class for more information. 

729 """ 

730 templateExposure = template # input template 

731 scienceExposure = science # input science image 

732 if alTaskResult is None and psfMatchingKernel is None: 

733 raise RuntimeError('Both alTaskResult and psfMatchingKernel cannot be None') 

734 psfMatchingKernel = alTaskResult.psfMatchingKernel if alTaskResult is not None else psfMatchingKernel 

735 

736 # subExp and expandedSubExp are subimages of the (un-decorrelated) diffim! 

737 # So here we compute corresponding subimages of templateExposure and scienceExposure 

738 subExp2 = scienceExposure.Factory(scienceExposure, expandedSubExposure.getBBox()) 

739 subExp1 = templateExposure.Factory(templateExposure, expandedSubExposure.getBBox()) 

740 

741 # Prevent too much log INFO verbosity from DecorrelateALKernelTask.run 

742 logLevel = self.log.level 

743 self.log.setLevel(self.log.WARNING) 

744 res = DecorrelateALKernelTask.run(self, subExp2, subExp1, expandedSubExposure, 

745 psfMatchingKernel, preConvKernel, **kwargs) 

746 self.log.setLevel(logLevel) # reset the log level 

747 

748 diffim = res.correctedExposure.Factory(res.correctedExposure, subExposure.getBBox()) 

749 out = pipeBase.Struct(subExposure=diffim, ) 

750 return out 

751 

752 

753class DecorrelateALKernelMapReduceConfig(ImageMapReduceConfig): 

754 """Configuration parameters for the ImageMapReduceTask to direct it to use 

755 DecorrelateALKernelMapper as its mapper for A&L decorrelation. 

756 """ 

757 mapper = pexConfig.ConfigurableField( 

758 doc='A&L decorrelation task to run on each sub-image', 

759 target=DecorrelateALKernelMapper 

760 ) 

761 

762 

763class DecorrelateALKernelSpatialConfig(pexConfig.Config): 

764 """Configuration parameters for the DecorrelateALKernelSpatialTask. 

765 """ 

766 decorrelateConfig = pexConfig.ConfigField( 

767 dtype=DecorrelateALKernelConfig, 

768 doc='DecorrelateALKernel config to use when running on complete exposure (non spatially-varying)', 

769 ) 

770 

771 decorrelateMapReduceConfig = pexConfig.ConfigField( 

772 dtype=DecorrelateALKernelMapReduceConfig, 

773 doc='DecorrelateALKernelMapReduce config to use when running on each sub-image (spatially-varying)', 

774 ) 

775 

776 ignoreMaskPlanes = pexConfig.ListField( 

777 dtype=str, 

778 doc="""Mask planes to ignore for sigma-clipped statistics""", 

779 default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE") 

780 ) 

781 

782 def setDefaults(self): 

783 self.decorrelateMapReduceConfig.gridStepX = self.decorrelateMapReduceConfig.gridStepY = 40 

784 self.decorrelateMapReduceConfig.cellSizeX = self.decorrelateMapReduceConfig.cellSizeY = 41 

785 self.decorrelateMapReduceConfig.borderSizeX = self.decorrelateMapReduceConfig.borderSizeY = 8 

786 self.decorrelateMapReduceConfig.reducer.reduceOperation = 'average' 

787 

788 

789class DecorrelateALKernelSpatialTask(pipeBase.Task): 

790 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference 

791 

792 Notes 

793 ----- 

794 

795 Pipe-task that removes the neighboring-pixel covariance in an 

796 image difference that are added when the template image is 

797 convolved with the Alard-Lupton PSF matching kernel. 

798 

799 This task is a simple wrapper around @ref DecorrelateALKernelTask, 

800 which takes a `spatiallyVarying` parameter in its `run` method. If 

801 it is `False`, then it simply calls the `run` method of @ref 

802 DecorrelateALKernelTask. If it is True, then it uses the @ref 

803 ImageMapReduceTask framework to break the exposures into 

804 subExposures on a grid, and performs the `run` method of @ref 

805 DecorrelateALKernelTask on each subExposure. This enables it to 

806 account for spatially-varying PSFs and noise in the exposures when 

807 performing the decorrelation. 

808 

809 This task has no standalone example, however it is applied as a 

810 subtask of pipe.tasks.imageDifference.ImageDifferenceTask. 

811 There is also an example of its use in `tests/testImageDecorrelation.py`. 

812 """ 

813 ConfigClass = DecorrelateALKernelSpatialConfig 

814 _DefaultName = "ip_diffim_decorrelateALKernelSpatial" 

815 

816 def __init__(self, *args, **kwargs): 

817 """Create the image decorrelation Task 

818 

819 Parameters 

820 ---------- 

821 args : 

822 arguments to be passed to 

823 `lsst.pipe.base.task.Task.__init__` 

824 kwargs : 

825 additional keyword arguments to be passed to 

826 `lsst.pipe.base.task.Task.__init__` 

827 """ 

828 pipeBase.Task.__init__(self, *args, **kwargs) 

829 

830 self.statsControl = afwMath.StatisticsControl() 

831 self.statsControl.setNumSigmaClip(3.) 

832 self.statsControl.setNumIter(3) 

833 self.statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes)) 

834 

835 def computeVarianceMean(self, exposure): 

836 """Compute the mean of the variance plane of `exposure`. 

837 """ 

838 statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(), 

839 exposure.getMaskedImage().getMask(), 

840 afwMath.MEANCLIP, self.statsControl) 

841 var = statObj.getValue(afwMath.MEANCLIP) 

842 return var 

843 

844 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel, 

845 spatiallyVarying=True, preConvKernel=None, templateMatched=True, preConvMode=False): 

846 """Perform decorrelation of an image difference exposure. 

847 

848 Decorrelates the diffim due to the convolution of the 

849 templateExposure with the A&L psfMatchingKernel. If 

850 `spatiallyVarying` is True, it utilizes the spatially varying 

851 matching kernel via the `imageMapReduce` framework to perform 

852 spatially-varying decorrelation on a grid of subExposures. 

853 

854 Parameters 

855 ---------- 

856 scienceExposure : `lsst.afw.image.Exposure` 

857 the science Exposure used for PSF matching 

858 templateExposure : `lsst.afw.image.Exposure` 

859 the template Exposure used for PSF matching 

860 subtractedExposure : `lsst.afw.image.Exposure` 

861 the subtracted Exposure produced by `ip_diffim.ImagePsfMatchTask.subtractExposures()` 

862 psfMatchingKernel : an (optionally spatially-varying) PSF matching kernel produced 

863 by `ip_diffim.ImagePsfMatchTask.subtractExposures()` 

864 spatiallyVarying : `bool` 

865 if True, perform the spatially-varying operation 

866 preConvKernel : `lsst.meas.algorithms.Psf` 

867 if not none, the scienceExposure has been pre-filtered with this kernel. (Currently 

868 this option is experimental.) 

869 templateMatched : `bool`, optional 

870 If True, the template exposure was matched (convolved) to the science exposure. 

871 preConvMode : `bool`, optional 

872 If True, ``subtractedExposure`` is assumed to be a likelihood difference image 

873 and will be noise corrected as a likelihood image. 

874 

875 Returns 

876 ------- 

877 results : `lsst.pipe.base.Struct` 

878 a structure containing: 

879 - ``correctedExposure`` : the decorrelated diffim 

880 """ 

881 self.log.info('Running A&L decorrelation: spatiallyVarying=%r', spatiallyVarying) 

882 

883 svar = self.computeVarianceMean(scienceExposure) 

884 tvar = self.computeVarianceMean(templateExposure) 

885 if np.isnan(svar) or np.isnan(tvar): # Should not happen unless entire image has been masked. 

886 # Double check that one of the exposures is all NaNs 

887 if (np.all(np.isnan(scienceExposure.image.array)) 

888 or np.all(np.isnan(templateExposure.image.array))): 

889 self.log.warning('Template or science image is entirely NaNs: skipping decorrelation.') 

890 if np.isnan(svar): 

891 svar = 1e-9 

892 if np.isnan(tvar): 

893 tvar = 1e-9 

894 

895 var = self.computeVarianceMean(subtractedExposure) 

896 

897 if spatiallyVarying: 

898 self.log.info("Variance (science, template): (%f, %f)", svar, tvar) 

899 self.log.info("Variance (uncorrected diffim): %f", var) 

900 config = self.config.decorrelateMapReduceConfig 

901 task = ImageMapReduceTask(config=config) 

902 results = task.run(subtractedExposure, science=scienceExposure, 

903 template=templateExposure, psfMatchingKernel=psfMatchingKernel, 

904 preConvKernel=preConvKernel, forceEvenSized=True, 

905 templateMatched=templateMatched, preConvMode=preConvMode) 

906 results.correctedExposure = results.exposure 

907 

908 # Make sure masks of input image are propagated to diffim 

909 def gm(exp): 

910 return exp.getMaskedImage().getMask() 

911 gm(results.correctedExposure)[:, :] = gm(subtractedExposure) 

912 

913 var = self.computeVarianceMean(results.correctedExposure) 

914 self.log.info("Variance (corrected diffim): %f", var) 

915 

916 else: 

917 config = self.config.decorrelateConfig 

918 task = DecorrelateALKernelTask(config=config) 

919 results = task.run(scienceExposure, templateExposure, 

920 subtractedExposure, psfMatchingKernel, preConvKernel=preConvKernel, 

921 templateMatched=templateMatched, preConvMode=preConvMode) 

922 

923 return results