Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23import numpy as np 

24 

25import lsst.afw.image as afwImage 

26import lsst.afw.math as afwMath 

27import lsst.geom as geom 

28import lsst.log 

29import lsst.meas.algorithms as measAlg 

30import lsst.pex.config as pexConfig 

31import lsst.pipe.base as pipeBase 

32 

33 

34from .imageMapReduce import (ImageMapReduceConfig, ImageMapReduceTask, 

35 ImageMapper) 

36 

37__all__ = ("DecorrelateALKernelTask", "DecorrelateALKernelConfig", 

38 "DecorrelateALKernelMapper", "DecorrelateALKernelMapReduceConfig", 

39 "DecorrelateALKernelSpatialConfig", "DecorrelateALKernelSpatialTask") 

40 

41 

42class DecorrelateALKernelConfig(pexConfig.Config): 

43 """Configuration parameters for the DecorrelateALKernelTask 

44 """ 

45 

46 ignoreMaskPlanes = pexConfig.ListField( 

47 dtype=str, 

48 doc="""Mask planes to ignore for sigma-clipped statistics""", 

49 default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE") 

50 ) 

51 

52 

53class DecorrelateALKernelTask(pipeBase.Task): 

54 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference 

55 

56 Notes 

57 ----- 

58 

59 Pipe-task that removes the neighboring-pixel covariance in an 

60 image difference that are added when the template image is 

61 convolved with the Alard-Lupton PSF matching kernel. 

62 

63 The image differencing pipeline task @link 

64 ip.diffim.psfMatch.PsfMatchTask PSFMatchTask@endlink and @link 

65 ip.diffim.psfMatch.PsfMatchConfigAL PSFMatchConfigAL@endlink uses 

66 the Alard and Lupton (1998) method for matching the PSFs of the 

67 template and science exposures prior to subtraction. The 

68 Alard-Lupton method identifies a matching kernel, which is then 

69 (typically) convolved with the template image to perform PSF 

70 matching. This convolution has the effect of adding covariance 

71 between neighboring pixels in the template image, which is then 

72 added to the image difference by subtraction. 

73 

74 The pixel covariance may be corrected by whitening the noise of 

75 the image difference. This task performs such a decorrelation by 

76 computing a decorrelation kernel (based upon the A&L matching 

77 kernel and variances in the template and science images) and 

78 convolving the image difference with it. This process is described 

79 in detail in [DMTN-021](http://dmtn-021.lsst.io). 

80 

81 This task has no standalone example, however it is applied as a 

82 subtask of pipe.tasks.imageDifference.ImageDifferenceTask. 

83 """ 

84 ConfigClass = DecorrelateALKernelConfig 

85 _DefaultName = "ip_diffim_decorrelateALKernel" 

86 

87 def __init__(self, *args, **kwargs): 

88 """Create the image decorrelation Task 

89 

90 Parameters 

91 ---------- 

92 args : 

93 arguments to be passed to ``lsst.pipe.base.task.Task.__init__`` 

94 kwargs : 

95 keyword arguments to be passed to ``lsst.pipe.base.task.Task.__init__`` 

96 """ 

97 pipeBase.Task.__init__(self, *args, **kwargs) 

98 

99 self.statsControl = afwMath.StatisticsControl() 

100 self.statsControl.setNumSigmaClip(3.) 

101 self.statsControl.setNumIter(3) 

102 self.statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes)) 

103 

104 def computeVarianceMean(self, exposure): 

105 statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(), 

106 exposure.getMaskedImage().getMask(), 

107 afwMath.MEANCLIP, self.statsControl) 

108 var = statObj.getValue(afwMath.MEANCLIP) 

109 return var 

110 

111 @pipeBase.timeMethod 

112 def run(self, exposure, templateExposure, subtractedExposure, psfMatchingKernel, 

113 preConvKernel=None, xcen=None, ycen=None, svar=None, tvar=None): 

114 """Perform decorrelation of an image difference exposure. 

115 

116 Decorrelates the diffim due to the convolution of the templateExposure with the 

117 A&L PSF matching kernel. Currently can accept a spatially varying matching kernel but in 

118 this case it simply uses a static kernel from the center of the exposure. The decorrelation 

119 is described in [DMTN-021, Equation 1](http://dmtn-021.lsst.io/#equation-1), where 

120 `exposure` is I_1; templateExposure is I_2; `subtractedExposure` is D(k); 

121 `psfMatchingKernel` is kappa; and svar and tvar are their respective 

122 variances (see below). 

123 

124 Parameters 

125 ---------- 

126 exposure : `lsst.afw.image.Exposure` 

127 The original science exposure (before `preConvKernel` applied) used for PSF matching. 

128 templateExposure : `lsst.afw.image.Exposure` 

129 The original template exposure (before matched to the science exposure 

130 by `psfMatchingKernel`) warped into the science exposure dimensions. Always the PSF of the 

131 `templateExposure` should be matched to the PSF of `exposure`, see notes below. 

132 subtractedExposure : 

133 the subtracted exposure produced by 

134 `ip_diffim.ImagePsfMatchTask.subtractExposures()`. The `subtractedExposure` must 

135 inherit its PSF from `exposure`, see notes below. 

136 psfMatchingKernel : 

137 An (optionally spatially-varying) PSF matching kernel produced 

138 by `ip_diffim.ImagePsfMatchTask.subtractExposures()` 

139 preConvKernel : 

140 if not None, then the `exposure` was pre-convolved with this kernel 

141 xcen : `float`, optional 

142 X-pixel coordinate to use for computing constant matching kernel to use 

143 If `None` (default), then use the center of the image. 

144 ycen : `float`, optional 

145 Y-pixel coordinate to use for computing constant matching kernel to use 

146 If `None` (default), then use the center of the image. 

147 svar : `float`, optional 

148 Image variance for science image 

149 If `None` (default) then compute the variance over the entire input science image. 

150 tvar : `float`, optional 

151 Image variance for template image 

152 If `None` (default) then compute the variance over the entire input template image. 

153 

154 Returns 

155 ------- 

156 result : `lsst.pipe.base.Struct` 

157 - ``correctedExposure`` : the decorrelated diffim 

158 

159 Notes 

160 ----- 

161 The `subtractedExposure` is NOT updated. The returned `correctedExposure` has an updated but 

162 spatially fixed PSF. It is calculated as the center of image PSF corrected by the center of 

163 image matching kernel. 

164 

165 In this task, it is _always_ the `templateExposure` that was matched to the `exposure` 

166 by `psfMatchingKernel`. Swap arguments accordingly if actually the science exposure was matched 

167 to a co-added template. In this case, tvar > svar typically occurs. 

168 

169 The `templateExposure` and `exposure` image dimensions must be the same. 

170 

171 Here we currently convert a spatially-varying matching kernel into a constant kernel, 

172 just by computing it at the center of the image (tickets DM-6243, DM-6244). 

173 

174 We are also using a constant accross-the-image measure of sigma (sqrt(variance)) to compute 

175 the decorrelation kernel. 

176 

177 TODO DM-23857 As part of the spatially varying correction implementation 

178 consider whether returning a Struct is still necessary. 

179 """ 

180 spatialKernel = psfMatchingKernel 

181 kimg = afwImage.ImageD(spatialKernel.getDimensions()) 

182 bbox = subtractedExposure.getBBox() 

183 if xcen is None: 

184 xcen = (bbox.getBeginX() + bbox.getEndX()) / 2. 

185 if ycen is None: 

186 ycen = (bbox.getBeginY() + bbox.getEndY()) / 2. 

187 self.log.info("Using matching kernel computed at (%d, %d)", xcen, ycen) 

188 spatialKernel.computeImage(kimg, True, xcen, ycen) 

189 

190 if svar is None: 

191 svar = self.computeVarianceMean(exposure) 

192 if tvar is None: 

193 tvar = self.computeVarianceMean(templateExposure) 

194 self.log.info("Variance (science, template): (%f, %f)", svar, tvar) 

195 

196 # Should not happen unless entire image has been masked, which could happen 

197 # if this is a small subimage of the main exposure. In this case, just return a full NaN 

198 # exposure 

199 if np.isnan(svar) or np.isnan(tvar): 

200 # Double check that one of the exposures is all NaNs 

201 if (np.all(np.isnan(exposure.image.array)) 

202 or np.all(np.isnan(templateExposure.image.array))): 

203 self.log.warn('Template or science image is entirely NaNs: skipping decorrelation.') 

204 outExposure = subtractedExposure.clone() 

205 return pipeBase.Struct(correctedExposure=outExposure, ) 

206 

207 # The maximal correction value converges to sqrt(tvar/svar). 

208 # Correction divergence warning if the correction exceeds 4 orders of magnitude. 

209 tOverSVar = tvar/svar 

210 if tOverSVar > 1e8: 

211 self.log.warn("Diverging correction: science image variance is much smaller than template" 

212 f", tvar/svar:{tOverSVar:.2e}") 

213 

214 oldVarMean = self.computeVarianceMean(subtractedExposure) 

215 self.log.info("Variance (uncorrected diffim): %f", oldVarMean) 

216 

217 if preConvKernel is not None: 

218 self.log.info('Using a pre-convolution kernel as part of decorrelation correction.') 

219 kimg2 = afwImage.ImageD(preConvKernel.getDimensions()) 

220 preConvKernel.computeImage(kimg2, False) 

221 pckArr = kimg2.array 

222 

223 kArr = kimg.array 

224 diffExpArr = subtractedExposure.image.array 

225 psfImg = subtractedExposure.getPsf().computeKernelImage(geom.Point2D(xcen, ycen)) 

226 psfDim = psfImg.getDimensions() 

227 psfArr = psfImg.array 

228 

229 # Determine the common shape 

230 if preConvKernel is None: 

231 self.computeCommonShape(kArr.shape, psfArr.shape, diffExpArr.shape) 

232 corrft = self.computeCorrection(kArr, svar, tvar) 

233 else: 

234 self.computeCommonShape(pckArr.shape, kArr.shape, 

235 psfArr.shape, diffExpArr.shape) 

236 corrft = self.computeCorrection(kArr, svar, tvar, preConvArr=pckArr) 

237 

238 diffExpArr = self.computeCorrectedImage(corrft, diffExpArr) 

239 corrPsfArr = self.computeCorrectedDiffimPsf(corrft, psfArr) 

240 

241 psfcI = afwImage.ImageD(psfDim) 

242 psfcI.array = corrPsfArr 

243 psfcK = afwMath.FixedKernel(psfcI) 

244 psfNew = measAlg.KernelPsf(psfcK) 

245 

246 correctedExposure = subtractedExposure.clone() 

247 correctedExposure.image.array[...] = diffExpArr # Allow for numpy type casting 

248 # The subtracted exposure variance plane is already correlated, we cannot propagate 

249 # it through another convolution; instead we need to use the uncorrelated originals 

250 # The whitening should scale it to svar + tvar on average 

251 varImg = correctedExposure.variance.array 

252 # Allow for numpy type casting 

253 varImg[...] = exposure.variance.array + templateExposure.variance.array 

254 correctedExposure.setPsf(psfNew) 

255 

256 newVarMean = self.computeVarianceMean(correctedExposure) 

257 self.log.info(f"Variance (corrected diffim): {newVarMean:.2e}") 

258 

259 # TODO DM-23857 As part of the spatially varying correction implementation 

260 # consider whether returning a Struct is still necessary. 

261 return pipeBase.Struct(correctedExposure=correctedExposure, ) 

262 

263 def computeCommonShape(self, *shapes): 

264 """Calculate the common shape for FFT operations. Set `self.freqSpaceShape` 

265 internally. 

266 

267 Parameters 

268 ---------- 

269 shapes : one or more `tuple` of `int` 

270 Shapes of the arrays. All must have the same dimensionality. 

271 At least one shape must be provided. 

272 

273 Returns 

274 ------- 

275 None. 

276 

277 Notes 

278 ----- 

279 For each dimension, gets the smallest even number greater than or equal to 

280 `N1+N2-1` where `N1` and `N2` are the two largest values. 

281 In case of only one shape given, rounds up to even each dimension value. 

282 """ 

283 S = np.array(shapes, dtype=int) 

284 if len(shapes) > 2: 

285 S.sort(axis=0) 

286 S = S[-2:] 

287 if len(shapes) > 1: 

288 commonShape = np.sum(S, axis=0) - 1 

289 else: 

290 commonShape = S[0] 

291 commonShape[commonShape % 2 != 0] += 1 

292 self.freqSpaceShape = tuple(commonShape) 

293 self.log.info(f"Common frequency space shape {self.freqSpaceShape}") 

294 

295 @staticmethod 

296 def padCenterOriginArray(A, newShape: tuple, useInverse=False): 

297 """Zero pad an image where the origin is at the center and replace the 

298 origin to the corner as required by the periodic input of FFT. Implement also 

299 the inverse operation, crop the padding and re-center data. 

300 

301 Parameters 

302 ---------- 

303 A : `numpy.ndarray` 

304 An array to copy from. 

305 newShape : `tuple` of `int` 

306 The dimensions of the resulting array. For padding, the resulting array 

307 must be larger than A in each dimension. For the inverse operation this 

308 must be the original, before padding size of the array. 

309 useInverse : bool, optional 

310 Selector of forward, add padding, operation (False) 

311 or its inverse, crop padding, operation (True). 

312 

313 Returns 

314 ------- 

315 R : `numpy.ndarray` 

316 The padded or unpadded array with shape of `newShape` and the same dtype as A. 

317 

318 Notes 

319 ----- 

320 For odd dimensions, the splitting is rounded to 

321 put the center pixel into the new corner origin (0,0). This is to be consistent 

322 e.g. for a dirac delta kernel that is originally located at the center pixel. 

323 """ 

324 

325 # The forward and inverse operations should round odd dimension halves at the opposite 

326 # sides to get the pixels back to their original positions. 

327 if not useInverse: 

328 # Forward operation: First and second halves with respect to the axes of A. 

329 firstHalves = [x//2 for x in A.shape] 

330 secondHalves = [x-y for x, y in zip(A.shape, firstHalves)] 

331 else: 

332 # Inverse operation: Opposite rounding 

333 secondHalves = [x//2 for x in newShape] 

334 firstHalves = [x-y for x, y in zip(newShape, secondHalves)] 

335 

336 R = np.zeros_like(A, shape=newShape) 

337 R[-firstHalves[0]:, -firstHalves[1]:] = A[:firstHalves[0], :firstHalves[1]] 

338 R[:secondHalves[0], -firstHalves[1]:] = A[-secondHalves[0]:, :firstHalves[1]] 

339 R[:secondHalves[0], :secondHalves[1]] = A[-secondHalves[0]:, -secondHalves[1]:] 

340 R[-firstHalves[0]:, :secondHalves[1]] = A[:firstHalves[0], -secondHalves[1]:] 

341 return R 

342 

343 def computeCorrection(self, kappa, svar, tvar, preConvArr=None): 

344 """Compute the Lupton decorrelation post-convolution kernel for decorrelating an 

345 image difference, based on the PSF-matching kernel. 

346 

347 Parameters 

348 ---------- 

349 kappa : `numpy.ndarray` 

350 A matching kernel 2-d numpy.array derived from Alard & Lupton PSF matching. 

351 svar : `float` 

352 Average variance of science image used for PSF matching. 

353 tvar : `float` 

354 Average variance of the template (matched) image used for PSF matching. 

355 preConvArr : `numpy.ndarray`, optional 

356 If not None, then pre-filtering was applied 

357 to science exposure, and this is the pre-convolution kernel. 

358 

359 Returns 

360 ------- 

361 corrft : `numpy.ndarray` of `float` 

362 The frequency space representation of the correction. The array is real (dtype float). 

363 Shape is `self.freqSpaceShape`. 

364 

365 Notes 

366 ----- 

367 The maximum correction factor converges to `sqrt(tvar/svar)` towards high frequencies. 

368 This should be a plausible value. 

369 """ 

370 kSum = np.sum(kappa) 

371 kappa = self.padCenterOriginArray(kappa, self.freqSpaceShape) 

372 kft = np.fft.fft2(kappa) 

373 kftAbsSq = np.real(np.conj(kft) * kft) 

374 # If there is no pre-convolution kernel, use placeholder scalars 

375 if preConvArr is None: 

376 preSum = 1. 

377 preAbsSq = 1. 

378 else: 

379 preSum = np.sum(preConvArr) 

380 preConvArr = self.padCenterOriginArray(preConvArr, self.freqSpaceShape) 

381 preK = np.fft.fft2(preConvArr) 

382 preAbsSq = np.real(np.conj(preK)*preK) 

383 

384 denom = svar * preAbsSq + tvar * kftAbsSq 

385 # Division by zero protection, though we don't expect to hit it 

386 # (rather we'll have numerical noise) 

387 tiny = np.finfo(kftAbsSq.dtype).tiny * 1000. 

388 flt = denom < tiny 

389 sumFlt = np.sum(flt) 

390 if sumFlt > 0: 

391 self.log.warnf("Avoid zero division. Skip decorrelation " 

392 "at {} divergent frequencies.", sumFlt) 

393 denom[flt] = 1. 

394 kft = np.sqrt((svar * preSum*preSum + tvar * kSum*kSum) / denom) 

395 # Don't do any correction at these frequencies 

396 # the difference image should be close to zero anyway, so can't be decorrelated 

397 if sumFlt > 0: 

398 kft[flt] = 1. 

399 return kft 

400 

401 def computeCorrectedDiffimPsf(self, corrft, psfOld): 

402 """Compute the (decorrelated) difference image's new PSF. 

403 

404 Parameters 

405 ---------- 

406 corrft : `numpy.ndarray` 

407 The frequency space representation of the correction calculated by 

408 `computeCorrection`. Shape must be `self.freqSpaceShape`. 

409 psfOld : `numpy.ndarray` 

410 The psf of the difference image to be corrected. 

411 

412 Returns 

413 ------- 

414 psfNew : `numpy.ndarray` 

415 The corrected psf, same shape as `psfOld`, sum normed to 1. 

416 

417 Notes 

418 ---- 

419 There is no algorithmic guarantee that the corrected psf can 

420 meaningfully fit to the same size as the original one. 

421 """ 

422 psfShape = psfOld.shape 

423 psfNew = self.padCenterOriginArray(psfOld, self.freqSpaceShape) 

424 psfNew = np.fft.fft2(psfNew) 

425 psfNew *= corrft 

426 psfNew = np.fft.ifft2(psfNew) 

427 psfNew = psfNew.real 

428 psfNew = self.padCenterOriginArray(psfNew, psfShape, useInverse=True) 

429 psfNew = psfNew/psfNew.sum() 

430 return psfNew 

431 

432 def computeCorrectedImage(self, corrft, imgOld): 

433 """Compute the decorrelated difference image. 

434 

435 Parameters 

436 ---------- 

437 corrft : `numpy.ndarray` 

438 The frequency space representation of the correction calculated by 

439 `computeCorrection`. Shape must be `self.freqSpaceShape`. 

440 imgOld : `numpy.ndarray` 

441 The difference image to be corrected. 

442 

443 Returns 

444 ------- 

445 imgNew : `numpy.ndarray` 

446 The corrected image, same size as the input. 

447 """ 

448 expShape = imgOld.shape 

449 imgNew = np.copy(imgOld) 

450 filtInf = np.isinf(imgNew) 

451 filtNan = np.isnan(imgNew) 

452 imgNew[filtInf] = np.nan 

453 imgNew[filtInf | filtNan] = np.nanmean(imgNew) 

454 imgNew = self.padCenterOriginArray(imgNew, self.freqSpaceShape) 

455 imgNew = np.fft.fft2(imgNew) 

456 imgNew *= corrft 

457 imgNew = np.fft.ifft2(imgNew) 

458 imgNew = imgNew.real 

459 imgNew = self.padCenterOriginArray(imgNew, expShape, useInverse=True) 

460 imgNew[filtNan] = np.nan 

461 imgNew[filtInf] = np.inf 

462 return imgNew 

463 

464 

465class DecorrelateALKernelMapper(DecorrelateALKernelTask, ImageMapper): 

466 """Task to be used as an ImageMapper for performing 

467 A&L decorrelation on subimages on a grid across a A&L difference image. 

468 

469 This task subclasses DecorrelateALKernelTask in order to implement 

470 all of that task's configuration parameters, as well as its `run` method. 

471 """ 

472 

473 ConfigClass = DecorrelateALKernelConfig 

474 _DefaultName = 'ip_diffim_decorrelateALKernelMapper' 

475 

476 def __init__(self, *args, **kwargs): 

477 DecorrelateALKernelTask.__init__(self, *args, **kwargs) 

478 

479 def run(self, subExposure, expandedSubExposure, fullBBox, 

480 template, science, alTaskResult=None, psfMatchingKernel=None, 

481 preConvKernel=None, **kwargs): 

482 """Perform decorrelation operation on `subExposure`, using 

483 `expandedSubExposure` to allow for invalid edge pixels arising from 

484 convolutions. 

485 

486 This method performs A&L decorrelation on `subExposure` using 

487 local measures for image variances and PSF. `subExposure` is a 

488 sub-exposure of the non-decorrelated A&L diffim. It also 

489 requires the corresponding sub-exposures of the template 

490 (`template`) and science (`science`) exposures. 

491 

492 Parameters 

493 ---------- 

494 subExposure : `lsst.afw.image.Exposure` 

495 the sub-exposure of the diffim 

496 expandedSubExposure : `lsst.afw.image.Exposure` 

497 the expanded sub-exposure upon which to operate 

498 fullBBox : `lsst.geom.Box2I` 

499 the bounding box of the original exposure 

500 template : `lsst.afw.image.Exposure` 

501 the corresponding sub-exposure of the template exposure 

502 science : `lsst.afw.image.Exposure` 

503 the corresponding sub-exposure of the science exposure 

504 alTaskResult : `lsst.pipe.base.Struct` 

505 the result of A&L image differencing on `science` and 

506 `template`, importantly containing the resulting 

507 `psfMatchingKernel`. Can be `None`, only if 

508 `psfMatchingKernel` is not `None`. 

509 psfMatchingKernel : Alternative parameter for passing the 

510 A&L `psfMatchingKernel` directly. 

511 preConvKernel : If not None, then pre-filtering was applied 

512 to science exposure, and this is the pre-convolution 

513 kernel. 

514 kwargs : 

515 additional keyword arguments propagated from 

516 `ImageMapReduceTask.run`. 

517 

518 Returns 

519 ------- 

520 A `pipeBase.Struct` containing: 

521 

522 - ``subExposure`` : the result of the `subExposure` processing. 

523 - ``decorrelationKernel`` : the decorrelation kernel, currently 

524 not used. 

525 

526 Notes 

527 ----- 

528 This `run` method accepts parameters identical to those of 

529 `ImageMapper.run`, since it is called from the 

530 `ImageMapperTask`. See that class for more information. 

531 """ 

532 templateExposure = template # input template 

533 scienceExposure = science # input science image 

534 if alTaskResult is None and psfMatchingKernel is None: 

535 raise RuntimeError('Both alTaskResult and psfMatchingKernel cannot be None') 

536 psfMatchingKernel = alTaskResult.psfMatchingKernel if alTaskResult is not None else psfMatchingKernel 

537 

538 # subExp and expandedSubExp are subimages of the (un-decorrelated) diffim! 

539 # So here we compute corresponding subimages of templateExposure and scienceExposure 

540 subExp2 = scienceExposure.Factory(scienceExposure, expandedSubExposure.getBBox()) 

541 subExp1 = templateExposure.Factory(templateExposure, expandedSubExposure.getBBox()) 

542 

543 # Prevent too much log INFO verbosity from DecorrelateALKernelTask.run 

544 logLevel = self.log.getLevel() 

545 self.log.setLevel(lsst.log.WARN) 

546 res = DecorrelateALKernelTask.run(self, subExp2, subExp1, expandedSubExposure, 

547 psfMatchingKernel, preConvKernel) 

548 self.log.setLevel(logLevel) # reset the log level 

549 

550 diffim = res.correctedExposure.Factory(res.correctedExposure, subExposure.getBBox()) 

551 out = pipeBase.Struct(subExposure=diffim, ) 

552 return out 

553 

554 

555class DecorrelateALKernelMapReduceConfig(ImageMapReduceConfig): 

556 """Configuration parameters for the ImageMapReduceTask to direct it to use 

557 DecorrelateALKernelMapper as its mapper for A&L decorrelation. 

558 """ 

559 mapper = pexConfig.ConfigurableField( 

560 doc='A&L decorrelation task to run on each sub-image', 

561 target=DecorrelateALKernelMapper 

562 ) 

563 

564 

565class DecorrelateALKernelSpatialConfig(pexConfig.Config): 

566 """Configuration parameters for the DecorrelateALKernelSpatialTask. 

567 """ 

568 decorrelateConfig = pexConfig.ConfigField( 

569 dtype=DecorrelateALKernelConfig, 

570 doc='DecorrelateALKernel config to use when running on complete exposure (non spatially-varying)', 

571 ) 

572 

573 decorrelateMapReduceConfig = pexConfig.ConfigField( 

574 dtype=DecorrelateALKernelMapReduceConfig, 

575 doc='DecorrelateALKernelMapReduce config to use when running on each sub-image (spatially-varying)', 

576 ) 

577 

578 ignoreMaskPlanes = pexConfig.ListField( 

579 dtype=str, 

580 doc="""Mask planes to ignore for sigma-clipped statistics""", 

581 default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE") 

582 ) 

583 

584 def setDefaults(self): 

585 self.decorrelateMapReduceConfig.gridStepX = self.decorrelateMapReduceConfig.gridStepY = 40 

586 self.decorrelateMapReduceConfig.cellSizeX = self.decorrelateMapReduceConfig.cellSizeY = 41 

587 self.decorrelateMapReduceConfig.borderSizeX = self.decorrelateMapReduceConfig.borderSizeY = 8 

588 self.decorrelateMapReduceConfig.reducer.reduceOperation = 'average' 

589 

590 

591class DecorrelateALKernelSpatialTask(pipeBase.Task): 

592 """Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference 

593 

594 Notes 

595 ----- 

596 

597 Pipe-task that removes the neighboring-pixel covariance in an 

598 image difference that are added when the template image is 

599 convolved with the Alard-Lupton PSF matching kernel. 

600 

601 This task is a simple wrapper around @ref DecorrelateALKernelTask, 

602 which takes a `spatiallyVarying` parameter in its `run` method. If 

603 it is `False`, then it simply calls the `run` method of @ref 

604 DecorrelateALKernelTask. If it is True, then it uses the @ref 

605 ImageMapReduceTask framework to break the exposures into 

606 subExposures on a grid, and performs the `run` method of @ref 

607 DecorrelateALKernelTask on each subExposure. This enables it to 

608 account for spatially-varying PSFs and noise in the exposures when 

609 performing the decorrelation. 

610 

611 This task has no standalone example, however it is applied as a 

612 subtask of pipe.tasks.imageDifference.ImageDifferenceTask. 

613 There is also an example of its use in `tests/testImageDecorrelation.py`. 

614 """ 

615 ConfigClass = DecorrelateALKernelSpatialConfig 

616 _DefaultName = "ip_diffim_decorrelateALKernelSpatial" 

617 

618 def __init__(self, *args, **kwargs): 

619 """Create the image decorrelation Task 

620 

621 Parameters 

622 ---------- 

623 args : 

624 arguments to be passed to 

625 `lsst.pipe.base.task.Task.__init__` 

626 kwargs : 

627 additional keyword arguments to be passed to 

628 `lsst.pipe.base.task.Task.__init__` 

629 """ 

630 pipeBase.Task.__init__(self, *args, **kwargs) 

631 

632 self.statsControl = afwMath.StatisticsControl() 

633 self.statsControl.setNumSigmaClip(3.) 

634 self.statsControl.setNumIter(3) 

635 self.statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.ignoreMaskPlanes)) 

636 

637 def computeVarianceMean(self, exposure): 

638 """Compute the mean of the variance plane of `exposure`. 

639 """ 

640 statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(), 

641 exposure.getMaskedImage().getMask(), 

642 afwMath.MEANCLIP, self.statsControl) 

643 var = statObj.getValue(afwMath.MEANCLIP) 

644 return var 

645 

646 def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel, 

647 spatiallyVarying=True, preConvKernel=None): 

648 """Perform decorrelation of an image difference exposure. 

649 

650 Decorrelates the diffim due to the convolution of the 

651 templateExposure with the A&L psfMatchingKernel. If 

652 `spatiallyVarying` is True, it utilizes the spatially varying 

653 matching kernel via the `imageMapReduce` framework to perform 

654 spatially-varying decorrelation on a grid of subExposures. 

655 

656 Parameters 

657 ---------- 

658 scienceExposure : `lsst.afw.image.Exposure` 

659 the science Exposure used for PSF matching 

660 templateExposure : `lsst.afw.image.Exposure` 

661 the template Exposure used for PSF matching 

662 subtractedExposure : `lsst.afw.image.Exposure` 

663 the subtracted Exposure produced by `ip_diffim.ImagePsfMatchTask.subtractExposures()` 

664 psfMatchingKernel : 

665 an (optionally spatially-varying) PSF matching kernel produced 

666 by `ip_diffim.ImagePsfMatchTask.subtractExposures()` 

667 spatiallyVarying : `bool` 

668 if True, perform the spatially-varying operation 

669 preConvKernel : `lsst.meas.algorithms.Psf` 

670 if not none, the scienceExposure has been pre-filtered with this kernel. (Currently 

671 this option is experimental.) 

672 

673 Returns 

674 ------- 

675 results : `lsst.pipe.base.Struct` 

676 a structure containing: 

677 

678 - ``correctedExposure`` : the decorrelated diffim 

679 

680 """ 

681 self.log.info('Running A&L decorrelation: spatiallyVarying=%r' % spatiallyVarying) 

682 

683 svar = self.computeVarianceMean(scienceExposure) 

684 tvar = self.computeVarianceMean(templateExposure) 

685 if np.isnan(svar) or np.isnan(tvar): # Should not happen unless entire image has been masked. 

686 # Double check that one of the exposures is all NaNs 

687 if (np.all(np.isnan(scienceExposure.image.array)) 

688 or np.all(np.isnan(templateExposure.image.array))): 

689 self.log.warn('Template or science image is entirely NaNs: skipping decorrelation.') 

690 if np.isnan(svar): 

691 svar = 1e-9 

692 if np.isnan(tvar): 

693 tvar = 1e-9 

694 

695 var = self.computeVarianceMean(subtractedExposure) 

696 

697 if spatiallyVarying: 

698 self.log.info("Variance (science, template): (%f, %f)", svar, tvar) 

699 self.log.info("Variance (uncorrected diffim): %f", var) 

700 config = self.config.decorrelateMapReduceConfig 

701 task = ImageMapReduceTask(config=config) 

702 results = task.run(subtractedExposure, science=scienceExposure, 

703 template=templateExposure, psfMatchingKernel=psfMatchingKernel, 

704 preConvKernel=preConvKernel, forceEvenSized=True) 

705 results.correctedExposure = results.exposure 

706 

707 # Make sure masks of input image are propagated to diffim 

708 def gm(exp): 

709 return exp.getMaskedImage().getMask() 

710 gm(results.correctedExposure)[:, :] = gm(subtractedExposure) 

711 

712 var = self.computeVarianceMean(results.correctedExposure) 

713 self.log.info("Variance (corrected diffim): %f", var) 

714 

715 else: 

716 config = self.config.decorrelateConfig 

717 task = DecorrelateALKernelTask(config=config) 

718 results = task.run(scienceExposure, templateExposure, 

719 subtractedExposure, psfMatchingKernel, preConvKernel=preConvKernel) 

720 

721 return results