Coverage for python/lsst/ip/diffim/imageDecorrelation.py: 14%

284 statements  

« prev     ^ index     » next       coverage.py v7.2.5, created at 2023-05-08 01:36 -0700

1# 

2# LSST Data Management System 

3# Copyright 2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23import numpy as np 

24 

25import lsst.afw.image as afwImage 

26import lsst.afw.math as afwMath 

27import lsst.geom as geom 

28import lsst.meas.algorithms as measAlg 

29import lsst.pex.config as pexConfig 

30import lsst.pipe.base as pipeBase 

31from lsst.utils.timer import timeMethod 

32 

33from .imageMapReduce import (ImageMapReduceConfig, ImageMapReduceTask, 

34 ImageMapper) 

35 

36__all__ = ("DecorrelateALKernelTask", "DecorrelateALKernelConfig", 

37 "DecorrelateALKernelMapper", "DecorrelateALKernelMapReduceConfig", 

38 "DecorrelateALKernelSpatialConfig", "DecorrelateALKernelSpatialTask") 

39 

40 

41class DecorrelateALKernelConfig(pexConfig.Config): 

42 """Configuration parameters for the DecorrelateALKernelTask 

43 """ 

44 

45 ignoreMaskPlanes = pexConfig.ListField( 

46 dtype=str, 

47 doc="""Mask planes to ignore for sigma-clipped statistics""", 

48 default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE") 

49 ) 

50 completeVarPlanePropagation = pexConfig.Field( 

51 dtype=bool, 

52 default=False, 

53 doc="Compute the full effect of the decorrelated matching kernel on the variance plane." 

54 " Otherwise use a model weighed sum of the input variances." 

55 ) 

56 

57 

58class DecorrelateALKernelTask(pipeBase.Task): 

59 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference 

60 

61 """ 

62 ConfigClass = DecorrelateALKernelConfig 

63 _DefaultName = "ip_diffim_decorrelateALKernel" 

64 

65 def __init__(self, *args, **kwargs): 

66 """Create the image decorrelation Task 

67 

68 Parameters 

69 ---------- 

70 args : 

71 arguments to be passed to ``lsst.pipe.base.task.Task.__init__`` 

72 kwargs : 

73 keyword arguments to be passed to ``lsst.pipe.base.task.Task.__init__`` 

74 """ 

75 pipeBase.Task.__init__(self, *args, **kwargs) 

76 

77 self.statsControl = afwMath.StatisticsControl() 

78 self.statsControl.setNumSigmaClip(3.) 

79 self.statsControl.setNumIter(3) 

80 self.statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes)) 

81 

82 def computeVarianceMean(self, exposure): 

83 statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(), 

84 exposure.getMaskedImage().getMask(), 

85 afwMath.MEANCLIP, self.statsControl) 

86 var = statObj.getValue(afwMath.MEANCLIP) 

87 return var 

88 

89 @timeMethod 

90 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel, 

91 preConvKernel=None, xcen=None, ycen=None, svar=None, tvar=None, 

92 templateMatched=True, preConvMode=False, **kwargs): 

93 """Perform decorrelation of an image difference or of a score difference exposure. 

94 

95 Corrects the difference or score image due to the convolution of the 

96 templateExposure with the A&L PSF matching kernel. 

97 See [DMTN-021, Equation 1](http://dmtn-021.lsst.io/#equation-1) and 

98 [DMTN-179](http://dmtn-179.lsst.io/) for details. 

99 

100 Parameters 

101 ---------- 

102 scienceExposure : `lsst.afw.image.Exposure` 

103 The original science exposure (before pre-convolution, if ``preConvMode==True``). 

104 templateExposure : `lsst.afw.image.Exposure` 

105 The original template exposure warped, but not psf-matched, to the science exposure. 

106 subtractedExposure : `lsst.afw.image.Exposure` 

107 the subtracted exposure produced by 

108 `ip_diffim.ImagePsfMatchTask.subtractExposures()`. The `subtractedExposure` must 

109 inherit its PSF from `exposure`, see notes below. 

110 psfMatchingKernel : `lsst.afw.detection.Psf` 

111 An (optionally spatially-varying) PSF matching kernel produced 

112 by `ip_diffim.ImagePsfMatchTask.subtractExposures()`. 

113 preConvKernel : `lsst.afw.math.Kernel`, optional 

114 If not `None`, then the `scienceExposure` was pre-convolved with (the reflection of) 

115 this kernel. Must be normalized to sum to 1. 

116 Allowed only if ``templateMatched==True`` and ``preConvMode==True``. 

117 Defaults to the PSF of the science exposure at the image center. 

118 xcen : `float`, optional 

119 X-pixel coordinate to use for computing constant matching kernel to use 

120 If `None` (default), then use the center of the image. 

121 ycen : `float`, optional 

122 Y-pixel coordinate to use for computing constant matching kernel to use 

123 If `None` (default), then use the center of the image. 

124 svar : `float`, optional 

125 Image variance for science image 

126 If `None` (default) then compute the variance over the entire input science image. 

127 tvar : `float`, optional 

128 Image variance for template image 

129 If `None` (default) then compute the variance over the entire input template image. 

130 templateMatched : `bool`, optional 

131 If True, the template exposure was matched (convolved) to the science exposure. 

132 See also notes below. 

133 preConvMode : `bool`, optional 

134 If True, ``subtractedExposure`` is assumed to be a likelihood difference image 

135 and will be noise corrected as a likelihood image. 

136 **kwargs 

137 Additional keyword arguments propagated from DecorrelateALKernelSpatialTask. 

138 

139 Returns 

140 ------- 

141 result : `lsst.pipe.base.Struct` 

142 - ``correctedExposure`` : the decorrelated diffim 

143 

144 Notes 

145 ----- 

146 If ``preConvMode==True``, ``subtractedExposure`` is assumed to be a 

147 score image and the noise correction for likelihood images 

148 is applied. The resulting image is an optimal detection likelihood image 

149 when the templateExposure has noise. (See DMTN-179) If ``preConvKernel`` is 

150 not specified, the PSF of ``scienceExposure`` is assumed as pre-convolution kernel. 

151 

152 The ``subtractedExposure`` is NOT updated. The returned ``correctedExposure`` 

153 has an updated but spatially fixed PSF. It is calculated as the center of 

154 image PSF corrected by the center of image matching kernel. 

155 

156 If ``templateMatched==True``, the templateExposure was matched (convolved) 

157 to the ``scienceExposure`` by ``psfMatchingKernel`` during image differencing. 

158 Otherwise the ``scienceExposure`` was matched (convolved) by ``psfMatchingKernel``. 

159 In either case, note that the original template and science images are required, 

160 not the psf-matched version. 

161 

162 This task discards the variance plane of ``subtractedExposure`` and re-computes 

163 it from the variance planes of ``scienceExposure`` and ``templateExposure``. 

164 The image plane of ``subtractedExposure`` must be at the photometric level 

165 set by the AL PSF matching in `ImagePsfMatchTask.subtractExposures`. 

166 The assumptions about the photometric level are controlled by the 

167 `templateMatched` option in this task. 

168 

169 Here we currently convert a spatially-varying matching kernel into a constant kernel, 

170 just by computing it at the center of the image (tickets DM-6243, DM-6244). 

171 

172 We are also using a constant accross-the-image measure of sigma (sqrt(variance)) to compute 

173 the decorrelation kernel. 

174 

175 TODO DM-23857 As part of the spatially varying correction implementation 

176 consider whether returning a Struct is still necessary. 

177 """ 

178 if preConvKernel is not None and not (templateMatched and preConvMode): 

179 raise ValueError("Pre-convolution kernel is allowed only if " 

180 "preConvMode==True and templateMatched==True.") 

181 

182 spatialKernel = psfMatchingKernel 

183 kimg = afwImage.ImageD(spatialKernel.getDimensions()) 

184 bbox = subtractedExposure.getBBox() 

185 if xcen is None: 

186 xcen = (bbox.getBeginX() + bbox.getEndX()) / 2. 

187 if ycen is None: 

188 ycen = (bbox.getBeginY() + bbox.getEndY()) / 2. 

189 self.log.info("Using matching kernel computed at (%d, %d)", xcen, ycen) 

190 spatialKernel.computeImage(kimg, False, xcen, ycen) 

191 

192 preConvImg = None 

193 if preConvMode: 

194 if preConvKernel is None: 

195 pos = scienceExposure.getPsf().getAveragePosition() 

196 preConvKernel = scienceExposure.getPsf().getLocalKernel(pos) 

197 preConvImg = afwImage.ImageD(preConvKernel.getDimensions()) 

198 preConvKernel.computeImage(preConvImg, True) 

199 

200 if svar is None: 

201 svar = self.computeVarianceMean(scienceExposure) 

202 if tvar is None: 

203 tvar = self.computeVarianceMean(templateExposure) 

204 self.log.info("Original variance plane means. Science:%.5e, warped template:%.5e)", 

205 svar, tvar) 

206 

207 # Should not happen unless entire image has been masked, which could happen 

208 # if this is a small subimage of the main exposure. In this case, just return a full NaN 

209 # exposure 

210 if np.isnan(svar) or np.isnan(tvar): 

211 # Double check that one of the exposures is all NaNs 

212 if (np.all(np.isnan(scienceExposure.image.array)) 

213 or np.all(np.isnan(templateExposure.image.array))): 

214 self.log.warning('Template or science image is entirely NaNs: skipping decorrelation.') 

215 outExposure = subtractedExposure.clone() 

216 return pipeBase.Struct(correctedExposure=outExposure, ) 

217 

218 if templateMatched: 

219 # Regular subtraction, we convolved the template 

220 self.log.info("Decorrelation after template image convolution") 

221 varianceMean = svar 

222 targetVarianceMean = tvar 

223 # variance plane of the image that is not convolved 

224 variance = scienceExposure.variance.array 

225 # Variance plane of the convolved image, before convolution. 

226 targetVariance = templateExposure.variance.array 

227 else: 

228 # We convolved the science image 

229 self.log.info("Decorrelation after science image convolution") 

230 varianceMean = tvar 

231 targetVarianceMean = svar 

232 # variance plane of the image that is not convolved 

233 variance = templateExposure.variance.array 

234 # Variance plane of the convolved image, before convolution. 

235 targetVariance = scienceExposure.variance.array 

236 

237 # The maximal correction value converges to sqrt(targetVarianceMean/varianceMean). 

238 # Correction divergence warning if the correction exceeds 4 orders of magnitude. 

239 mOverExpVar = targetVarianceMean/varianceMean 

240 if mOverExpVar > 1e8: 

241 self.log.warning("Diverging correction: matched image variance is " 

242 " much larger than the unconvolved one's" 

243 ", targetVarianceMean/varianceMean:%.2e", mOverExpVar) 

244 

245 oldVarMean = self.computeVarianceMean(subtractedExposure) 

246 self.log.info("Variance plane mean of uncorrected diffim: %f", oldVarMean) 

247 

248 kArr = kimg.array 

249 diffimShape = subtractedExposure.image.array.shape 

250 psfImg = subtractedExposure.getPsf().computeKernelImage(geom.Point2D(xcen, ycen)) 

251 psfShape = psfImg.array.shape 

252 

253 if preConvMode: 

254 self.log.info("Decorrelation of likelihood image") 

255 self.computeCommonShape(preConvImg.array.shape, kArr.shape, 

256 psfShape, diffimShape) 

257 corr = self.computeScoreCorrection(kArr, varianceMean, targetVarianceMean, preConvImg.array) 

258 else: 

259 self.log.info("Decorrelation of difference image") 

260 self.computeCommonShape(kArr.shape, psfShape, diffimShape) 

261 corr = self.computeDiffimCorrection(kArr, varianceMean, targetVarianceMean) 

262 

263 correctedImage = self.computeCorrectedImage(corr.corrft, subtractedExposure.image.array) 

264 correctedPsf = self.computeCorrectedDiffimPsf(corr.corrft, psfImg.array) 

265 

266 # The subtracted exposure variance plane is already correlated, we cannot propagate 

267 # it through another convolution; instead we need to use the uncorrelated originals 

268 # The whitening should scale it to varianceMean + targetVarianceMean on average 

269 if self.config.completeVarPlanePropagation: 

270 self.log.debug("Using full variance plane calculation in decorrelation") 

271 correctedVariance = self.calculateVariancePlane( 

272 variance, targetVariance, 

273 varianceMean, targetVarianceMean, corr.cnft, corr.crft) 

274 else: 

275 self.log.debug("Using estimated variance plane calculation in decorrelation") 

276 correctedVariance = self.estimateVariancePlane( 

277 variance, targetVariance, 

278 corr.cnft, corr.crft) 

279 

280 # Determine the common shape 

281 kSum = np.sum(kArr) 

282 kSumSq = kSum*kSum 

283 self.log.debug("Matching kernel sum: %.3e", kSum) 

284 if not templateMatched: 

285 # ImagePsfMatch.subtractExposures re-scales the difference in 

286 # the science image convolution mode 

287 correctedVariance /= kSumSq 

288 subtractedExposure.image.array[...] = correctedImage # Allow for numpy type casting 

289 subtractedExposure.variance.array[...] = correctedVariance 

290 subtractedExposure.setPsf(correctedPsf) 

291 

292 newVarMean = self.computeVarianceMean(subtractedExposure) 

293 self.log.info("Variance plane mean of corrected diffim: %.5e", newVarMean) 

294 

295 # TODO DM-23857 As part of the spatially varying correction implementation 

296 # consider whether returning a Struct is still necessary. 

297 return pipeBase.Struct(correctedExposure=subtractedExposure, ) 

298 

299 def computeCommonShape(self, *shapes): 

300 """Calculate the common shape for FFT operations. Set `self.freqSpaceShape` 

301 internally. 

302 

303 Parameters 

304 ---------- 

305 shapes : one or more `tuple` of `int` 

306 Shapes of the arrays. All must have the same dimensionality. 

307 At least one shape must be provided. 

308 

309 Returns 

310 ------- 

311 None. 

312 

313 Notes 

314 ----- 

315 For each dimension, gets the smallest even number greater than or equal to 

316 `N1+N2-1` where `N1` and `N2` are the two largest values. 

317 In case of only one shape given, rounds up to even each dimension value. 

318 """ 

319 S = np.array(shapes, dtype=int) 

320 if len(shapes) > 2: 

321 S.sort(axis=0) 

322 S = S[-2:] 

323 if len(shapes) > 1: 

324 commonShape = np.sum(S, axis=0) - 1 

325 else: 

326 commonShape = S[0] 

327 commonShape[commonShape % 2 != 0] += 1 

328 self.freqSpaceShape = tuple(commonShape) 

329 self.log.info("Common frequency space shape %s", self.freqSpaceShape) 

330 

331 @staticmethod 

332 def padCenterOriginArray(A, newShape: tuple, useInverse=False): 

333 """Zero pad an image where the origin is at the center and replace the 

334 origin to the corner as required by the periodic input of FFT. Implement also 

335 the inverse operation, crop the padding and re-center data. 

336 

337 Parameters 

338 ---------- 

339 A : `numpy.ndarray` 

340 An array to copy from. 

341 newShape : `tuple` of `int` 

342 The dimensions of the resulting array. For padding, the resulting array 

343 must be larger than A in each dimension. For the inverse operation this 

344 must be the original, before padding size of the array. 

345 useInverse : bool, optional 

346 Selector of forward, add padding, operation (False) 

347 or its inverse, crop padding, operation (True). 

348 

349 Returns 

350 ------- 

351 R : `numpy.ndarray` 

352 The padded or unpadded array with shape of `newShape` and the same dtype as A. 

353 

354 Notes 

355 ----- 

356 For odd dimensions, the splitting is rounded to 

357 put the center pixel into the new corner origin (0,0). This is to be consistent 

358 e.g. for a dirac delta kernel that is originally located at the center pixel. 

359 """ 

360 

361 # The forward and inverse operations should round odd dimension halves at the opposite 

362 # sides to get the pixels back to their original positions. 

363 if not useInverse: 

364 # Forward operation: First and second halves with respect to the axes of A. 

365 firstHalves = [x//2 for x in A.shape] 

366 secondHalves = [x-y for x, y in zip(A.shape, firstHalves)] 

367 else: 

368 # Inverse operation: Opposite rounding 

369 secondHalves = [x//2 for x in newShape] 

370 firstHalves = [x-y for x, y in zip(newShape, secondHalves)] 

371 

372 R = np.zeros_like(A, shape=newShape) 

373 R[-firstHalves[0]:, -firstHalves[1]:] = A[:firstHalves[0], :firstHalves[1]] 

374 R[:secondHalves[0], -firstHalves[1]:] = A[-secondHalves[0]:, :firstHalves[1]] 

375 R[:secondHalves[0], :secondHalves[1]] = A[-secondHalves[0]:, -secondHalves[1]:] 

376 R[-firstHalves[0]:, :secondHalves[1]] = A[:firstHalves[0], -secondHalves[1]:] 

377 return R 

378 

379 def computeDiffimCorrection(self, kappa, svar, tvar): 

380 """Compute the Lupton decorrelation post-convolution kernel for decorrelating an 

381 image difference, based on the PSF-matching kernel. 

382 

383 Parameters 

384 ---------- 

385 kappa : `numpy.ndarray` of `float` 

386 A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching. 

387 svar : `float` > 0. 

388 Average variance of science image used for PSF matching. 

389 tvar : `float` > 0. 

390 Average variance of the template (matched) image used for PSF matching. 

391 

392 Returns 

393 ------- 

394 corrft : `numpy.ndarray` of `float` 

395 The frequency space representation of the correction. The array is real (dtype float). 

396 Shape is `self.freqSpaceShape`. 

397 

398 cnft, crft : `numpy.ndarray` of `complex` 

399 The overall convolution (pre-conv, PSF matching, noise correction) kernel 

400 for the science and template images, respectively for the variance plane 

401 calculations. These are intermediate results in frequency space. 

402 

403 Notes 

404 ----- 

405 The maximum correction factor converges to `sqrt(tvar/svar)` towards high frequencies. 

406 This should be a plausible value. 

407 """ 

408 kSum = np.sum(kappa) # We scale the decorrelation to preserve fluxes 

409 kappa = self.padCenterOriginArray(kappa, self.freqSpaceShape) 

410 kft = np.fft.fft2(kappa) 

411 kftAbsSq = np.real(np.conj(kft) * kft) 

412 

413 denom = svar + tvar * kftAbsSq 

414 corrft = np.sqrt((svar + tvar * kSum*kSum) / denom) 

415 cnft = corrft 

416 crft = kft*corrft 

417 return pipeBase.Struct(corrft=corrft, cnft=cnft, crft=crft) 

418 

419 def computeScoreCorrection(self, kappa, svar, tvar, preConvArr): 

420 """Compute the correction kernel for a score image. 

421 

422 Parameters 

423 ---------- 

424 kappa : `numpy.ndarray` 

425 A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching. 

426 svar : `float` 

427 Average variance of science image used for PSF matching (before pre-convolution). 

428 tvar : `float` 

429 Average variance of the template (matched) image used for PSF matching. 

430 preConvArr : `numpy.ndarray` 

431 The pre-convolution kernel of the science image. It should be the PSF 

432 of the science image or an approximation of it. It must be normed to sum 1. 

433 

434 Returns 

435 ------- 

436 corrft : `numpy.ndarray` of `float` 

437 The frequency space representation of the correction. The array is real (dtype float). 

438 Shape is `self.freqSpaceShape`. 

439 cnft, crft : `numpy.ndarray` of `complex` 

440 The overall convolution (pre-conv, PSF matching, noise correction) kernel 

441 for the science and template images, respectively for the variance plane 

442 calculations. These are intermediate results in frequency space. 

443 

444 Notes 

445 ----- 

446 To be precise, the science image should be _correlated_ by ``preConvArray`` but this 

447 does not matter for this calculation. 

448 

449 ``cnft``, ``crft`` contain the scaling factor as well. 

450 

451 """ 

452 kSum = np.sum(kappa) 

453 kappa = self.padCenterOriginArray(kappa, self.freqSpaceShape) 

454 kft = np.fft.fft2(kappa) 

455 preConvArr = self.padCenterOriginArray(preConvArr, self.freqSpaceShape) 

456 preFt = np.fft.fft2(preConvArr) 

457 preFtAbsSq = np.real(np.conj(preFt) * preFt) 

458 kftAbsSq = np.real(np.conj(kft) * kft) 

459 # Avoid zero division, though we don't normally approach `tiny`. 

460 # We have numerical noise instead. 

461 tiny = np.finfo(preFtAbsSq.dtype).tiny * 1000. 

462 flt = preFtAbsSq < tiny 

463 # If we pre-convolve to avoid deconvolution in AL, then kftAbsSq / preFtAbsSq 

464 # theoretically expected to diverge to +inf. But we don't care about the convergence 

465 # properties here, S goes to 0 at these frequencies anyway. 

466 preFtAbsSq[flt] = tiny 

467 denom = svar + tvar * kftAbsSq / preFtAbsSq 

468 corrft = (svar + tvar * kSum*kSum) / denom 

469 cnft = np.conj(preFt)*corrft 

470 crft = kft*corrft 

471 return pipeBase.Struct(corrft=corrft, cnft=cnft, crft=crft) 

472 

473 @staticmethod 

474 def estimateVariancePlane(vplane1, vplane2, c1ft, c2ft): 

475 """Estimate the variance planes. 

476 

477 The estimation assumes that around each pixel the surrounding 

478 pixels' sigmas within the convolution kernel are the same. 

479 

480 Parameters 

481 ---------- 

482 vplane1, vplane2 : `numpy.ndarray` of `float` 

483 Variance planes of the original (before pre-convolution or matching) 

484 exposures. 

485 c1ft, c2ft : `numpy.ndarray` of `complex` 

486 The overall convolution that includes the matching and the 

487 afterburner in frequency space. The result of either 

488 ``computeScoreCorrection`` or ``computeDiffimCorrection``. 

489 

490 Returns 

491 ------- 

492 vplaneD : `numpy.ndarray` of `float` 

493 The estimated variance plane of the difference/score image 

494 as a weighted sum of the input variances. 

495 

496 Notes 

497 ------ 

498 See DMTN-179 Section 5 about the variance plane calculations. 

499 """ 

500 w1 = np.sum(np.real(np.conj(c1ft)*c1ft)) / c1ft.size 

501 w2 = np.sum(np.real(np.conj(c2ft)*c2ft)) / c2ft.size 

502 # w1, w2: the frequency space sum of abs(c1)^2 is the same as in image 

503 # space. 

504 return vplane1*w1 + vplane2*w2 

505 

506 def calculateVariancePlane(self, vplane1, vplane2, varMean1, varMean2, c1ft, c2ft): 

507 """Full propagation of the variance planes of the original exposures. 

508 

509 The original variance planes of independent pixels are convolved with the 

510 image space square of the overall kernels. 

511 

512 Parameters 

513 ---------- 

514 vplane1, vplane2 : `numpy.ndarray` of `float` 

515 Variance planes of the original (before pre-convolution or matching) 

516 exposures. 

517 varMean1, varMean2 : `float` 

518 Replacement average values for non-finite ``vplane1`` and ``vplane2`` values respectively. 

519 

520 c1ft, c2ft : `numpy.ndarray` of `complex` 

521 The overall convolution that includes the matching and the 

522 afterburner in frequency space. The result of either 

523 ``computeScoreCorrection`` or ``computeDiffimCorrection``. 

524 

525 Returns 

526 ------- 

527 vplaneD : `numpy.ndarray` of `float` 

528 The variance plane of the difference/score images. 

529 

530 Notes 

531 ------ 

532 See DMTN-179 Section 5 about the variance plane calculations. 

533 

534 Infs and NaNs are allowed and kept in the returned array. 

535 """ 

536 D = np.real(np.fft.ifft2(c1ft)) 

537 c1SqFt = np.fft.fft2(D*D) 

538 

539 v1shape = vplane1.shape 

540 filtInf = np.isinf(vplane1) 

541 filtNan = np.isnan(vplane1) 

542 # This copy could be eliminated if inf/nan handling were go into padCenterOriginArray 

543 vplane1 = np.copy(vplane1) 

544 vplane1[filtInf | filtNan] = varMean1 

545 D = self.padCenterOriginArray(vplane1, self.freqSpaceShape) 

546 v1 = np.real(np.fft.ifft2(np.fft.fft2(D) * c1SqFt)) 

547 v1 = self.padCenterOriginArray(v1, v1shape, useInverse=True) 

548 v1[filtNan] = np.nan 

549 v1[filtInf] = np.inf 

550 

551 D = np.real(np.fft.ifft2(c2ft)) 

552 c2SqFt = np.fft.fft2(D*D) 

553 

554 v2shape = vplane2.shape 

555 filtInf = np.isinf(vplane2) 

556 filtNan = np.isnan(vplane2) 

557 vplane2 = np.copy(vplane2) 

558 vplane2[filtInf | filtNan] = varMean2 

559 D = self.padCenterOriginArray(vplane2, self.freqSpaceShape) 

560 v2 = np.real(np.fft.ifft2(np.fft.fft2(D) * c2SqFt)) 

561 v2 = self.padCenterOriginArray(v2, v2shape, useInverse=True) 

562 v2[filtNan] = np.nan 

563 v2[filtInf] = np.inf 

564 

565 return v1 + v2 

566 

567 def computeCorrectedDiffimPsf(self, corrft, psfOld): 

568 """Compute the (decorrelated) difference image's new PSF. 

569 

570 Parameters 

571 ---------- 

572 corrft : `numpy.ndarray` 

573 The frequency space representation of the correction calculated by 

574 `computeCorrection`. Shape must be `self.freqSpaceShape`. 

575 psfOld : `numpy.ndarray` 

576 The psf of the difference image to be corrected. 

577 

578 Returns 

579 ------- 

580 correctedPsf : `lsst.meas.algorithms.KernelPsf` 

581 The corrected psf, same shape as `psfOld`, sum normed to 1. 

582 

583 Notes 

584 ----- 

585 There is no algorithmic guarantee that the corrected psf can 

586 meaningfully fit to the same size as the original one. 

587 """ 

588 psfShape = psfOld.shape 

589 psfNew = self.padCenterOriginArray(psfOld, self.freqSpaceShape) 

590 psfNew = np.fft.fft2(psfNew) 

591 psfNew *= corrft 

592 psfNew = np.fft.ifft2(psfNew) 

593 psfNew = psfNew.real 

594 psfNew = self.padCenterOriginArray(psfNew, psfShape, useInverse=True) 

595 psfNew = psfNew/psfNew.sum() 

596 

597 psfcI = afwImage.ImageD(geom.Extent2I(psfShape[1], psfShape[0])) 

598 psfcI.array = psfNew 

599 psfcK = afwMath.FixedKernel(psfcI) 

600 correctedPsf = measAlg.KernelPsf(psfcK) 

601 return correctedPsf 

602 

603 def computeCorrectedImage(self, corrft, imgOld): 

604 """Compute the decorrelated difference image. 

605 

606 Parameters 

607 ---------- 

608 corrft : `numpy.ndarray` 

609 The frequency space representation of the correction calculated by 

610 `computeCorrection`. Shape must be `self.freqSpaceShape`. 

611 imgOld : `numpy.ndarray` 

612 The difference image to be corrected. 

613 

614 Returns 

615 ------- 

616 imgNew : `numpy.ndarray` 

617 The corrected image, same size as the input. 

618 """ 

619 expShape = imgOld.shape 

620 imgNew = np.copy(imgOld) 

621 filtInf = np.isinf(imgNew) 

622 filtNan = np.isnan(imgNew) 

623 imgNew[filtInf] = np.nan 

624 imgNew[filtInf | filtNan] = np.nanmean(imgNew) 

625 imgNew = self.padCenterOriginArray(imgNew, self.freqSpaceShape) 

626 imgNew = np.fft.fft2(imgNew) 

627 imgNew *= corrft 

628 imgNew = np.fft.ifft2(imgNew) 

629 imgNew = imgNew.real 

630 imgNew = self.padCenterOriginArray(imgNew, expShape, useInverse=True) 

631 imgNew[filtNan] = np.nan 

632 imgNew[filtInf] = np.inf 

633 return imgNew 

634 

635 

636class DecorrelateALKernelMapper(DecorrelateALKernelTask, ImageMapper): 

637 """Task to be used as an ImageMapper for performing 

638 A&L decorrelation on subimages on a grid across a A&L difference image. 

639 

640 This task subclasses DecorrelateALKernelTask in order to implement 

641 all of that task's configuration parameters, as well as its `run` method. 

642 """ 

643 

644 ConfigClass = DecorrelateALKernelConfig 

645 _DefaultName = 'ip_diffim_decorrelateALKernelMapper' 

646 

647 def __init__(self, *args, **kwargs): 

648 DecorrelateALKernelTask.__init__(self, *args, **kwargs) 

649 

650 def run(self, subExposure, expandedSubExposure, fullBBox, 

651 template, science, alTaskResult=None, psfMatchingKernel=None, 

652 preConvKernel=None, **kwargs): 

653 """Perform decorrelation operation on `subExposure`, using 

654 `expandedSubExposure` to allow for invalid edge pixels arising from 

655 convolutions. 

656 

657 This method performs A&L decorrelation on `subExposure` using 

658 local measures for image variances and PSF. `subExposure` is a 

659 sub-exposure of the non-decorrelated A&L diffim. It also 

660 requires the corresponding sub-exposures of the template 

661 (`template`) and science (`science`) exposures. 

662 

663 Parameters 

664 ---------- 

665 subExposure : `lsst.afw.image.Exposure` 

666 the sub-exposure of the diffim 

667 expandedSubExposure : `lsst.afw.image.Exposure` 

668 the expanded sub-exposure upon which to operate 

669 fullBBox : `lsst.geom.Box2I` 

670 the bounding box of the original exposure 

671 template : `lsst.afw.image.Exposure` 

672 the corresponding sub-exposure of the template exposure 

673 science : `lsst.afw.image.Exposure` 

674 the corresponding sub-exposure of the science exposure 

675 alTaskResult : `lsst.pipe.base.Struct` 

676 the result of A&L image differencing on `science` and 

677 `template`, importantly containing the resulting 

678 `psfMatchingKernel`. Can be `None`, only if 

679 `psfMatchingKernel` is not `None`. 

680 psfMatchingKernel : Alternative parameter for passing the 

681 A&L `psfMatchingKernel` directly. 

682 preConvKernel : If not None, then pre-filtering was applied 

683 to science exposure, and this is the pre-convolution 

684 kernel. 

685 kwargs : 

686 additional keyword arguments propagated from 

687 `ImageMapReduceTask.run`. 

688 

689 Returns 

690 ------- 

691 A `pipeBase.Struct` containing: 

692 

693 - ``subExposure`` : the result of the `subExposure` processing. 

694 - ``decorrelationKernel`` : the decorrelation kernel, currently 

695 not used. 

696 

697 Notes 

698 ----- 

699 This `run` method accepts parameters identical to those of 

700 `ImageMapper.run`, since it is called from the 

701 `ImageMapperTask`. See that class for more information. 

702 """ 

703 templateExposure = template # input template 

704 scienceExposure = science # input science image 

705 if alTaskResult is None and psfMatchingKernel is None: 

706 raise RuntimeError('Both alTaskResult and psfMatchingKernel cannot be None') 

707 psfMatchingKernel = alTaskResult.psfMatchingKernel if alTaskResult is not None else psfMatchingKernel 

708 

709 # subExp and expandedSubExp are subimages of the (un-decorrelated) diffim! 

710 # So here we compute corresponding subimages of templateExposure and scienceExposure 

711 subExp2 = scienceExposure.Factory(scienceExposure, expandedSubExposure.getBBox()) 

712 subExp1 = templateExposure.Factory(templateExposure, expandedSubExposure.getBBox()) 

713 

714 # Prevent too much log INFO verbosity from DecorrelateALKernelTask.run 

715 logLevel = self.log.level 

716 self.log.setLevel(self.log.WARNING) 

717 res = DecorrelateALKernelTask.run(self, subExp2, subExp1, expandedSubExposure, 

718 psfMatchingKernel, preConvKernel, **kwargs) 

719 self.log.setLevel(logLevel) # reset the log level 

720 

721 diffim = res.correctedExposure.Factory(res.correctedExposure, subExposure.getBBox()) 

722 out = pipeBase.Struct(subExposure=diffim, ) 

723 return out 

724 

725 

726class DecorrelateALKernelMapReduceConfig(ImageMapReduceConfig): 

727 """Configuration parameters for the ImageMapReduceTask to direct it to use 

728 DecorrelateALKernelMapper as its mapper for A&L decorrelation. 

729 """ 

730 mapper = pexConfig.ConfigurableField( 

731 doc='A&L decorrelation task to run on each sub-image', 

732 target=DecorrelateALKernelMapper 

733 ) 

734 

735 

736class DecorrelateALKernelSpatialConfig(pexConfig.Config): 

737 """Configuration parameters for the DecorrelateALKernelSpatialTask. 

738 """ 

739 decorrelateConfig = pexConfig.ConfigField( 

740 dtype=DecorrelateALKernelConfig, 

741 doc='DecorrelateALKernel config to use when running on complete exposure (non spatially-varying)', 

742 ) 

743 

744 decorrelateMapReduceConfig = pexConfig.ConfigField( 

745 dtype=DecorrelateALKernelMapReduceConfig, 

746 doc='DecorrelateALKernelMapReduce config to use when running on each sub-image (spatially-varying)', 

747 ) 

748 

749 ignoreMaskPlanes = pexConfig.ListField( 

750 dtype=str, 

751 doc="""Mask planes to ignore for sigma-clipped statistics""", 

752 default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE") 

753 ) 

754 

755 def setDefaults(self): 

756 self.decorrelateMapReduceConfig.gridStepX = self.decorrelateMapReduceConfig.gridStepY = 40 

757 self.decorrelateMapReduceConfig.cellSizeX = self.decorrelateMapReduceConfig.cellSizeY = 41 

758 self.decorrelateMapReduceConfig.borderSizeX = self.decorrelateMapReduceConfig.borderSizeY = 8 

759 self.decorrelateMapReduceConfig.reducer.reduceOperation = 'average' 

760 

761 

762class DecorrelateALKernelSpatialTask(pipeBase.Task): 

763 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference 

764 

765 """ 

766 ConfigClass = DecorrelateALKernelSpatialConfig 

767 _DefaultName = "ip_diffim_decorrelateALKernelSpatial" 

768 

769 def __init__(self, *args, **kwargs): 

770 """Create the image decorrelation Task 

771 

772 Parameters 

773 ---------- 

774 args : 

775 arguments to be passed to 

776 `lsst.pipe.base.task.Task.__init__` 

777 kwargs : 

778 additional keyword arguments to be passed to 

779 `lsst.pipe.base.task.Task.__init__` 

780 """ 

781 pipeBase.Task.__init__(self, *args, **kwargs) 

782 

783 self.statsControl = afwMath.StatisticsControl() 

784 self.statsControl.setNumSigmaClip(3.) 

785 self.statsControl.setNumIter(3) 

786 self.statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes)) 

787 

788 def computeVarianceMean(self, exposure): 

789 """Compute the mean of the variance plane of `exposure`. 

790 """ 

791 statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(), 

792 exposure.getMaskedImage().getMask(), 

793 afwMath.MEANCLIP, self.statsControl) 

794 var = statObj.getValue(afwMath.MEANCLIP) 

795 return var 

796 

797 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel, 

798 spatiallyVarying=True, preConvKernel=None, templateMatched=True, preConvMode=False): 

799 """Perform decorrelation of an image difference exposure. 

800 

801 Decorrelates the diffim due to the convolution of the 

802 templateExposure with the A&L psfMatchingKernel. If 

803 `spatiallyVarying` is True, it utilizes the spatially varying 

804 matching kernel via the `imageMapReduce` framework to perform 

805 spatially-varying decorrelation on a grid of subExposures. 

806 

807 Parameters 

808 ---------- 

809 scienceExposure : `lsst.afw.image.Exposure` 

810 the science Exposure used for PSF matching 

811 templateExposure : `lsst.afw.image.Exposure` 

812 the template Exposure used for PSF matching 

813 subtractedExposure : `lsst.afw.image.Exposure` 

814 the subtracted Exposure produced by `ip_diffim.ImagePsfMatchTask.subtractExposures()` 

815 psfMatchingKernel : an (optionally spatially-varying) PSF matching kernel produced 

816 by `ip_diffim.ImagePsfMatchTask.subtractExposures()` 

817 spatiallyVarying : `bool` 

818 if True, perform the spatially-varying operation 

819 preConvKernel : `lsst.meas.algorithms.Psf` 

820 if not none, the scienceExposure has been pre-filtered with this kernel. (Currently 

821 this option is experimental.) 

822 templateMatched : `bool`, optional 

823 If True, the template exposure was matched (convolved) to the science exposure. 

824 preConvMode : `bool`, optional 

825 If True, ``subtractedExposure`` is assumed to be a likelihood difference image 

826 and will be noise corrected as a likelihood image. 

827 

828 Returns 

829 ------- 

830 results : `lsst.pipe.base.Struct` 

831 a structure containing: 

832 - ``correctedExposure`` : the decorrelated diffim 

833 """ 

834 self.log.info('Running A&L decorrelation: spatiallyVarying=%r', spatiallyVarying) 

835 

836 svar = self.computeVarianceMean(scienceExposure) 

837 tvar = self.computeVarianceMean(templateExposure) 

838 if np.isnan(svar) or np.isnan(tvar): # Should not happen unless entire image has been masked. 

839 # Double check that one of the exposures is all NaNs 

840 if (np.all(np.isnan(scienceExposure.image.array)) 

841 or np.all(np.isnan(templateExposure.image.array))): 

842 self.log.warning('Template or science image is entirely NaNs: skipping decorrelation.') 

843 if np.isnan(svar): 

844 svar = 1e-9 

845 if np.isnan(tvar): 

846 tvar = 1e-9 

847 

848 var = self.computeVarianceMean(subtractedExposure) 

849 

850 if spatiallyVarying: 

851 self.log.info("Variance (science, template): (%f, %f)", svar, tvar) 

852 self.log.info("Variance (uncorrected diffim): %f", var) 

853 config = self.config.decorrelateMapReduceConfig 

854 task = ImageMapReduceTask(config=config) 

855 results = task.run(subtractedExposure, science=scienceExposure, 

856 template=templateExposure, psfMatchingKernel=psfMatchingKernel, 

857 preConvKernel=preConvKernel, forceEvenSized=True, 

858 templateMatched=templateMatched, preConvMode=preConvMode) 

859 results.correctedExposure = results.exposure 

860 

861 # Make sure masks of input image are propagated to diffim 

862 def gm(exp): 

863 return exp.getMaskedImage().getMask() 

864 gm(results.correctedExposure)[:, :] = gm(subtractedExposure) 

865 

866 var = self.computeVarianceMean(results.correctedExposure) 

867 self.log.info("Variance (corrected diffim): %f", var) 

868 

869 else: 

870 config = self.config.decorrelateConfig 

871 task = DecorrelateALKernelTask(config=config) 

872 results = task.run(scienceExposure, templateExposure, 

873 subtractedExposure, psfMatchingKernel, preConvKernel=preConvKernel, 

874 templateMatched=templateMatched, preConvMode=preConvMode) 

875 

876 return results