Coverage for python/lsst/ip/diffim/subtractImages.py: 31%

139 statements  

« prev     ^ index     » next       coverage.py v6.4.1, created at 2022-06-09 03:32 -0700

1# This file is part of ip_diffim. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import numpy as np 

23 

24import lsst.afw.image 

25import lsst.afw.math 

26import lsst.geom 

27from lsst.ip.diffim.utils import getPsfFwhm 

28from lsst.meas.algorithms import ScaleVarianceTask 

29import lsst.pex.config 

30import lsst.pipe.base 

31from lsst.pipe.base import connectionTypes 

32from . import MakeKernelTask, DecorrelateALKernelTask 

33 

34__all__ = ["AlardLuptonSubtractConfig", "AlardLuptonSubtractTask"] 

35 

36_dimensions = ("instrument", "visit", "detector") 

37_defaultTemplates = {"coaddName": "deep", "fakesType": ""} 

38 

39 

40class SubtractInputConnections(lsst.pipe.base.PipelineTaskConnections, 

41 dimensions=_dimensions, 

42 defaultTemplates=_defaultTemplates): 

43 template = connectionTypes.Input( 

44 doc="Input warped template to subtract.", 

45 dimensions=("instrument", "visit", "detector"), 

46 storageClass="ExposureF", 

47 name="{fakesType}{coaddName}Diff_templateExp" 

48 ) 

49 science = connectionTypes.Input( 

50 doc="Input science exposure to subtract from.", 

51 dimensions=("instrument", "visit", "detector"), 

52 storageClass="ExposureF", 

53 name="{fakesType}calexp" 

54 ) 

55 sources = connectionTypes.Input( 

56 doc="Sources measured on the science exposure; " 

57 "used to select sources for making the matching kernel.", 

58 dimensions=("instrument", "visit", "detector"), 

59 storageClass="SourceCatalog", 

60 name="{fakesType}src" 

61 ) 

62 

63 

64class SubtractImageOutputConnections(lsst.pipe.base.PipelineTaskConnections, 

65 dimensions=_dimensions, 

66 defaultTemplates=_defaultTemplates): 

67 difference = connectionTypes.Output( 

68 doc="Result of subtracting convolved template from science image.", 

69 dimensions=("instrument", "visit", "detector"), 

70 storageClass="ExposureF", 

71 name="{fakesType}{coaddName}Diff_differenceTempExp", 

72 ) 

73 matchedTemplate = connectionTypes.Output( 

74 doc="Warped and PSF-matched template used to create `subtractedExposure`.", 

75 dimensions=("instrument", "visit", "detector"), 

76 storageClass="ExposureF", 

77 name="{fakesType}{coaddName}Diff_matchedExp", 

78 ) 

79 

80 

81class AlardLuptonSubtractConnections(SubtractInputConnections, SubtractImageOutputConnections): 

82 pass 

83 

84 

85class AlardLuptonSubtractConfig(lsst.pipe.base.PipelineTaskConfig, 

86 pipelineConnections=AlardLuptonSubtractConnections): 

87 mode = lsst.pex.config.ChoiceField( 

88 dtype=str, 

89 default="auto", 

90 allowed={"auto": "Choose which image to convolve at runtime.", 

91 "convolveScience": "Only convolve the science image.", 

92 "convolveTemplate": "Only convolve the template image."}, 

93 doc="Choose which image to convolve at runtime, or require that a specific image is convolved." 

94 ) 

95 makeKernel = lsst.pex.config.ConfigurableField( 

96 target=MakeKernelTask, 

97 doc="Task to construct a matching kernel for convolution.", 

98 ) 

99 doDecorrelation = lsst.pex.config.Field( 

100 dtype=bool, 

101 default=True, 

102 doc="Perform diffim decorrelation to undo pixel correlation due to A&L " 

103 "kernel convolution? If True, also update the diffim PSF." 

104 ) 

105 decorrelate = lsst.pex.config.ConfigurableField( 

106 target=DecorrelateALKernelTask, 

107 doc="Task to decorrelate the image difference.", 

108 ) 

109 requiredTemplateFraction = lsst.pex.config.Field( 

110 dtype=float, 

111 default=0.1, 

112 doc="Abort task if template covers less than this fraction of pixels." 

113 " Setting to 0 will always attempt image subtraction." 

114 ) 

115 doScaleVariance = lsst.pex.config.Field( 

116 dtype=bool, 

117 default=True, 

118 doc="Scale variance of the image difference?" 

119 ) 

120 scaleVariance = lsst.pex.config.ConfigurableField( 

121 target=ScaleVarianceTask, 

122 doc="Subtask to rescale the variance of the template to the statistically expected level." 

123 ) 

124 doSubtractBackground = lsst.pex.config.Field( 

125 doc="Subtract the background fit when solving the kernel?", 

126 dtype=bool, 

127 default=True, 

128 ) 

129 

130 forceCompatibility = lsst.pex.config.Field( 

131 dtype=bool, 

132 default=False, 

133 doc="Set up and run diffim using settings that ensure the results" 

134 "are compatible with the old version in pipe_tasks.", 

135 deprecated="This option is only for backwards compatibility purposes" 

136 " and will be removed after v24.", 

137 ) 

138 

139 def setDefaults(self): 

140 self.makeKernel.kernel.name = "AL" 

141 self.makeKernel.kernel.active.fitForBackground = self.doSubtractBackground 

142 self.makeKernel.kernel.active.spatialKernelOrder = 1 

143 self.makeKernel.kernel.active.spatialBgOrder = 2 

144 if self.forceCompatibility: 

145 self.mode = "convolveTemplate" 

146 

147 

148class AlardLuptonSubtractTask(lsst.pipe.base.PipelineTask): 

149 """Compute the image difference of a science and template image using 

150 the Alard & Lupton (1998) algorithm. 

151 """ 

152 ConfigClass = AlardLuptonSubtractConfig 

153 _DefaultName = "alardLuptonSubtract" 

154 

155 def __init__(self, **kwargs): 

156 super().__init__(**kwargs) 

157 self.makeSubtask("decorrelate") 

158 self.makeSubtask("makeKernel") 

159 if self.config.doScaleVariance: 

160 self.makeSubtask("scaleVariance") 

161 

162 self.convolutionControl = lsst.afw.math.ConvolutionControl() 

163 # Normalization is an extra, unnecessary, calculation and will result 

164 # in mis-subtraction of the images if there are calibration errors. 

165 self.convolutionControl.setDoNormalize(False) 

166 

167 def run(self, template, science, sources): 

168 """PSF match, subtract, and decorrelate two images. 

169 

170 Parameters 

171 ---------- 

172 template : `lsst.afw.image.ExposureF` 

173 Template exposure, warped to match the science exposure. 

174 science : `lsst.afw.image.ExposureF` 

175 Science exposure to subtract from the template. 

176 sources : `lsst.afw.table.SourceCatalog` 

177 Identified sources on the science exposure. This catalog is used to 

178 select sources in order to perform the AL PSF matching on stamp 

179 images around them. 

180 

181 Returns 

182 ------- 

183 results : `lsst.pipe.base.Struct` 

184 ``difference`` : `lsst.afw.image.ExposureF` 

185 Result of subtracting template and science. 

186 ``matchedTemplate`` : `lsst.afw.image.ExposureF` 

187 Warped and PSF-matched template exposure. 

188 ``backgroundModel`` : `lsst.afw.math.Chebyshev1Function2D` 

189 Background model that was fit while solving for the PSF-matching kernel 

190 ``psfMatchingKernel`` : `lsst.afw.math.Kernel` 

191 Kernel used to PSF-match the convolved image. 

192 

193 Raises 

194 ------ 

195 RuntimeError 

196 If an unsupported convolution mode is supplied. 

197 lsst.pipe.base.NoWorkFound 

198 Raised if fraction of good pixels, defined as not having NO_DATA 

199 set, is less then the configured requiredTemplateFraction 

200 """ 

201 self._validateExposures(template, science) 

202 checkTemplateIsSufficient(template, self.log, 

203 requiredTemplateFraction=self.config.requiredTemplateFraction) 

204 if self.config.forceCompatibility: 

205 # Compatibility option to maintain old functionality 

206 # This should be removed in the future! 

207 sources = None 

208 kernelSources = self.makeKernel.selectKernelSources(template, science, 

209 candidateList=sources, 

210 preconvolved=False) 

211 sciencePsfSize = getPsfFwhm(science.psf) 

212 templatePsfSize = getPsfFwhm(template.psf) 

213 self.log.info("Science PSF size: %f", sciencePsfSize) 

214 self.log.info("Template PSF size: %f", templatePsfSize) 

215 if self.config.mode == "auto": 

216 if sciencePsfSize < templatePsfSize: 

217 self.log.info("Template PSF size is greater: convolving science image.") 

218 convolveTemplate = False 

219 else: 

220 self.log.info("Science PSF size is greater: convolving template image.") 

221 convolveTemplate = True 

222 elif self.config.mode == "convolveTemplate": 

223 self.log.info("`convolveTemplate` is set: convolving template image.") 

224 convolveTemplate = True 

225 elif self.config.mode == "convolveScience": 

226 self.log.info("`convolveScience` is set: convolving science image.") 

227 convolveTemplate = False 

228 else: 

229 raise RuntimeError("Cannot handle AlardLuptonSubtract mode: %s", self.config.mode) 

230 

231 if convolveTemplate: 

232 subtractResults = self.runConvolveTemplate(template, science, kernelSources) 

233 else: 

234 subtractResults = self.runConvolveScience(template, science, kernelSources) 

235 

236 if self.config.doScaleVariance: 

237 diffimVarFactor = self.scaleVariance.run(subtractResults.difference.maskedImage) 

238 self.log.info("Diffim variance scaling factor: %.2f", diffimVarFactor) 

239 self.metadata.add("scaleDiffimVarianceFactor", diffimVarFactor) 

240 

241 return subtractResults 

242 

243 def runConvolveTemplate(self, template, science, sources): 

244 """Convolve the template image with a PSF-matching kernel and subtract 

245 from the science image. 

246 

247 Parameters 

248 ---------- 

249 template : `lsst.afw.image.ExposureF` 

250 Template exposure, warped to match the science exposure. 

251 science : `lsst.afw.image.ExposureF` 

252 Science exposure to subtract from the template. 

253 sources : `lsst.afw.table.SourceCatalog` 

254 Identified sources on the science exposure. This catalog is used to 

255 select sources in order to perform the AL PSF matching on stamp 

256 images around them. 

257 

258 Returns 

259 ------- 

260 results : `lsst.pipe.base.Struct` 

261 

262 ``difference`` : `lsst.afw.image.ExposureF` 

263 Result of subtracting template and science. 

264 ``matchedTemplate`` : `lsst.afw.image.ExposureF` 

265 Warped and PSF-matched template exposure. 

266 ``backgroundModel`` : `lsst.afw.math.Chebyshev1Function2D` 

267 Background model that was fit while solving for the PSF-matching kernel 

268 ``psfMatchingKernel`` : `lsst.afw.math.Kernel` 

269 Kernel used to PSF-match the template to the science image. 

270 """ 

271 if self.config.forceCompatibility: 

272 # Compatibility option to maintain old behavior 

273 # This should be removed in the future! 

274 template = template[science.getBBox()] 

275 kernelResult = self.makeKernel.run(template, science, sources, preconvolved=False) 

276 

277 matchedTemplate = self._convolveExposure(template, kernelResult.psfMatchingKernel, 

278 self.convolutionControl, 

279 bbox=science.getBBox(), 

280 psf=science.psf) 

281 difference = _subtractImages(science, matchedTemplate, 

282 backgroundModel=(kernelResult.backgroundModel 

283 if self.config.doSubtractBackground else None)) 

284 correctedExposure = self.finalize(template, science, difference, kernelResult.psfMatchingKernel, 

285 templateMatched=True) 

286 

287 return lsst.pipe.base.Struct(difference=correctedExposure, 

288 matchedTemplate=matchedTemplate, 

289 backgroundModel=kernelResult.backgroundModel, 

290 psfMatchingKernel=kernelResult.psfMatchingKernel) 

291 

292 def runConvolveScience(self, template, science, sources): 

293 """Convolve the science image with a PSF-matching kernel and subtract the template image. 

294 

295 Parameters 

296 ---------- 

297 template : `lsst.afw.image.ExposureF` 

298 Template exposure, warped to match the science exposure. 

299 science : `lsst.afw.image.ExposureF` 

300 Science exposure to subtract from the template. 

301 sources : `lsst.afw.table.SourceCatalog` 

302 Identified sources on the science exposure. This catalog is used to 

303 select sources in order to perform the AL PSF matching on stamp 

304 images around them. 

305 

306 Returns 

307 ------- 

308 results : `lsst.pipe.base.Struct` 

309 

310 ``difference`` : `lsst.afw.image.ExposureF` 

311 Result of subtracting template and science. 

312 ``matchedTemplate`` : `lsst.afw.image.ExposureF` 

313 Warped template exposure. Note that in this case, the template 

314 is not PSF-matched to the science image. 

315 ``backgroundModel`` : `lsst.afw.math.Chebyshev1Function2D` 

316 Background model that was fit while solving for the PSF-matching kernel 

317 ``psfMatchingKernel`` : `lsst.afw.math.Kernel` 

318 Kernel used to PSF-match the science image to the template. 

319 """ 

320 if self.config.forceCompatibility: 

321 # Compatibility option to maintain old behavior 

322 # This should be removed in the future! 

323 template = template[science.getBBox()] 

324 kernelResult = self.makeKernel.run(science, template, sources, preconvolved=False) 

325 modelParams = kernelResult.backgroundModel.getParameters() 

326 # We must invert the background model if the matching kernel is solved for the science image. 

327 kernelResult.backgroundModel.setParameters([-p for p in modelParams]) 

328 

329 matchedScience = self._convolveExposure(science, kernelResult.psfMatchingKernel, 

330 self.convolutionControl, 

331 psf=template.psf) 

332 

333 difference = _subtractImages(matchedScience, template[science.getBBox()], 

334 backgroundModel=(kernelResult.backgroundModel 

335 if self.config.doSubtractBackground else None)) 

336 

337 # Place back on native photometric scale 

338 difference.maskedImage /= kernelResult.psfMatchingKernel.computeImage( 

339 lsst.afw.image.ImageD(kernelResult.psfMatchingKernel.getDimensions()), False) 

340 correctedExposure = self.finalize(template, science, difference, kernelResult.psfMatchingKernel, 

341 templateMatched=False) 

342 

343 return lsst.pipe.base.Struct(difference=correctedExposure, 

344 matchedTemplate=template, 

345 backgroundModel=kernelResult.backgroundModel, 

346 psfMatchingKernel=kernelResult.psfMatchingKernel,) 

347 

348 def finalize(self, template, science, difference, kernel, 

349 templateMatched=True, 

350 preConvMode=False, 

351 preConvKernel=None, 

352 spatiallyVarying=False): 

353 """Decorrelate the difference image to undo the noise correlations 

354 caused by convolution. 

355 

356 Parameters 

357 ---------- 

358 template : `lsst.afw.image.ExposureF` 

359 Template exposure, warped to match the science exposure. 

360 science : `lsst.afw.image.ExposureF` 

361 Science exposure to subtract from the template. 

362 difference : `lsst.afw.image.ExposureF` 

363 Result of subtracting template and science. 

364 kernel : `lsst.afw.math.Kernel` 

365 An (optionally spatially-varying) PSF matching kernel 

366 templateMatched : `bool`, optional 

367 Was the template PSF-matched to the science image? 

368 preConvMode : `bool`, optional 

369 Was the science image preconvolved with its own PSF 

370 before PSF matching the template? 

371 preConvKernel : `lsst.afw.detection.Psf`, optional 

372 If not `None`, then the science image was pre-convolved with 

373 (the reflection of) this kernel. Must be normalized to sum to 1. 

374 spatiallyVarying : `bool`, optional 

375 Compute the decorrelation kernel spatially varying across the image? 

376 

377 Returns 

378 ------- 

379 correctedExposure : `lsst.afw.image.ExposureF` 

380 The decorrelated image difference. 

381 """ 

382 # Erase existing detection mask planes. 

383 # We don't want the detection mask from the science image 

384 mask = difference.mask 

385 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) 

386 

387 if self.config.doDecorrelation: 

388 self.log.info("Decorrelating image difference.") 

389 correctedExposure = self.decorrelate.run(science, template[science.getBBox()], difference, kernel, 

390 templateMatched=templateMatched, 

391 preConvMode=preConvMode, 

392 preConvKernel=preConvKernel, 

393 spatiallyVarying=spatiallyVarying).correctedExposure 

394 else: 

395 self.log.info("NOT decorrelating image difference.") 

396 correctedExposure = difference 

397 return correctedExposure 

398 

399 @staticmethod 

400 def _validateExposures(template, science): 

401 """Check that the WCS of the two Exposures match, and the template bbox 

402 contains the science bbox. 

403 

404 Parameters 

405 ---------- 

406 template : `lsst.afw.image.ExposureF` 

407 Template exposure, warped to match the science exposure. 

408 science : `lsst.afw.image.ExposureF` 

409 Science exposure to subtract from the template. 

410 

411 Raises 

412 ------ 

413 AssertionError 

414 Raised if the WCS of the template is not equal to the science WCS, 

415 or if the science image is not fully contained in the template 

416 bounding box. 

417 """ 

418 assert template.wcs == science.wcs,\ 

419 "Template and science exposure WCS are not identical." 

420 templateBBox = template.getBBox() 

421 scienceBBox = science.getBBox() 

422 

423 assert templateBBox.contains(scienceBBox),\ 

424 "Template bbox does not contain all of the science image." 

425 

426 @staticmethod 

427 def _convolveExposure(exposure, kernel, convolutionControl, 

428 bbox=None, 

429 psf=None): 

430 """Convolve an exposure with the given kernel. 

431 

432 Parameters 

433 ---------- 

434 exposure : `lsst.afw.Exposure` 

435 exposure to convolve. 

436 kernel : `lsst.afw.math.LinearCombinationKernel` 

437 PSF matching kernel computed in the ``makeKernel`` subtask. 

438 convolutionControl : `lsst.afw.math.ConvolutionControl` 

439 Configuration for convolve algorithm. 

440 bbox : `lsst.geom.Box2I`, optional 

441 Bounding box to trim the convolved exposure to. 

442 psf : `lsst.afw.detection.Psf`, optional 

443 Point spread function (PSF) to set for the convolved exposure. 

444 

445 Returns 

446 ------- 

447 convolvedExp : `lsst.afw.Exposure` 

448 The convolved image. 

449 """ 

450 convolvedExposure = exposure.clone() 

451 if psf is not None: 

452 convolvedExposure.setPsf(psf) 

453 convolvedImage = lsst.afw.image.MaskedImageF(exposure.getBBox()) 

454 lsst.afw.math.convolve(convolvedImage, exposure.maskedImage, kernel, convolutionControl) 

455 convolvedExposure.setMaskedImage(convolvedImage) 

456 if bbox is None: 

457 return convolvedExposure 

458 else: 

459 return convolvedExposure[bbox] 

460 

461 

462def checkTemplateIsSufficient(templateExposure, logger, requiredTemplateFraction=0.): 

463 """Raise NoWorkFound if template coverage < requiredTemplateFraction 

464 

465 Parameters 

466 ---------- 

467 templateExposure : `lsst.afw.image.ExposureF` 

468 The template exposure to check 

469 logger : `lsst.log.Log` 

470 Logger for printing output. 

471 requiredTemplateFraction : `float`, optional 

472 Fraction of pixels of the science image required to have coverage 

473 in the template. 

474 

475 Raises 

476 ------ 

477 lsst.pipe.base.NoWorkFound 

478 Raised if fraction of good pixels, defined as not having NO_DATA 

479 set, is less then the configured requiredTemplateFraction 

480 """ 

481 # Count the number of pixels with the NO_DATA mask bit set 

482 # counting NaN pixels is insufficient because pixels without data are often intepolated over) 

483 pixNoData = np.count_nonzero(templateExposure.mask.array 

484 & templateExposure.mask.getPlaneBitMask('NO_DATA')) 

485 pixGood = templateExposure.getBBox().getArea() - pixNoData 

486 logger.info("template has %d good pixels (%.1f%%)", pixGood, 

487 100*pixGood/templateExposure.getBBox().getArea()) 

488 

489 if pixGood/templateExposure.getBBox().getArea() < requiredTemplateFraction: 

490 message = ("Insufficient Template Coverage. (%.1f%% < %.1f%%) Not attempting subtraction. " 

491 "To force subtraction, set config requiredTemplateFraction=0." % ( 

492 100*pixGood/templateExposure.getBBox().getArea(), 

493 100*requiredTemplateFraction)) 

494 raise lsst.pipe.base.NoWorkFound(message) 

495 

496 

497def _subtractImages(science, template, backgroundModel=None): 

498 """Subtract template from science, propagating relevant metadata. 

499 

500 Parameters 

501 ---------- 

502 science : `lsst.afw.Exposure` 

503 The input science image. 

504 template : `lsst.afw.Exposure` 

505 The template to subtract from the science image. 

506 backgroundModel : `lsst.afw.MaskedImage`, optional 

507 Differential background model 

508 

509 Returns 

510 ------- 

511 difference : `lsst.afw.Exposure` 

512 The subtracted image. 

513 """ 

514 difference = science.clone() 

515 if backgroundModel is not None: 

516 difference.maskedImage -= backgroundModel 

517 difference.maskedImage -= template.maskedImage 

518 return difference