Coverage for python/lsst/ip/diffim/subtractImages.py: 28%

156 statements  

« prev     ^ index     » next       coverage.py v6.4.1, created at 2022-07-09 06:52 -0700

1# This file is part of ip_diffim. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import numpy as np 

23 

24import lsst.afw.image 

25import lsst.afw.math 

26import lsst.geom 

27from lsst.ip.diffim.utils import getPsfFwhm 

28from lsst.meas.algorithms import ScaleVarianceTask 

29import lsst.pex.config 

30import lsst.pipe.base 

31from lsst.pipe.base import connectionTypes 

32from . import MakeKernelTask, DecorrelateALKernelTask 

33 

34__all__ = ["AlardLuptonSubtractConfig", "AlardLuptonSubtractTask"] 

35 

36_dimensions = ("instrument", "visit", "detector") 

37_defaultTemplates = {"coaddName": "deep", "fakesType": ""} 

38 

39 

40class SubtractInputConnections(lsst.pipe.base.PipelineTaskConnections, 

41 dimensions=_dimensions, 

42 defaultTemplates=_defaultTemplates): 

43 template = connectionTypes.Input( 

44 doc="Input warped template to subtract.", 

45 dimensions=("instrument", "visit", "detector"), 

46 storageClass="ExposureF", 

47 name="{fakesType}{coaddName}Diff_templateExp" 

48 ) 

49 science = connectionTypes.Input( 

50 doc="Input science exposure to subtract from.", 

51 dimensions=("instrument", "visit", "detector"), 

52 storageClass="ExposureF", 

53 name="{fakesType}calexp" 

54 ) 

55 sources = connectionTypes.Input( 

56 doc="Sources measured on the science exposure; " 

57 "used to select sources for making the matching kernel.", 

58 dimensions=("instrument", "visit", "detector"), 

59 storageClass="SourceCatalog", 

60 name="{fakesType}src" 

61 ) 

62 

63 

64class SubtractImageOutputConnections(lsst.pipe.base.PipelineTaskConnections, 

65 dimensions=_dimensions, 

66 defaultTemplates=_defaultTemplates): 

67 difference = connectionTypes.Output( 

68 doc="Result of subtracting convolved template from science image.", 

69 dimensions=("instrument", "visit", "detector"), 

70 storageClass="ExposureF", 

71 name="{fakesType}{coaddName}Diff_differenceTempExp", 

72 ) 

73 matchedTemplate = connectionTypes.Output( 

74 doc="Warped and PSF-matched template used to create `subtractedExposure`.", 

75 dimensions=("instrument", "visit", "detector"), 

76 storageClass="ExposureF", 

77 name="{fakesType}{coaddName}Diff_matchedExp", 

78 ) 

79 

80 

81class AlardLuptonSubtractConnections(SubtractInputConnections, SubtractImageOutputConnections): 

82 pass 

83 

84 

85class AlardLuptonSubtractConfig(lsst.pipe.base.PipelineTaskConfig, 

86 pipelineConnections=AlardLuptonSubtractConnections): 

87 mode = lsst.pex.config.ChoiceField( 

88 dtype=str, 

89 default="auto", 

90 allowed={"auto": "Choose which image to convolve at runtime.", 

91 "convolveScience": "Only convolve the science image.", 

92 "convolveTemplate": "Only convolve the template image."}, 

93 doc="Choose which image to convolve at runtime, or require that a specific image is convolved." 

94 ) 

95 makeKernel = lsst.pex.config.ConfigurableField( 

96 target=MakeKernelTask, 

97 doc="Task to construct a matching kernel for convolution.", 

98 ) 

99 doDecorrelation = lsst.pex.config.Field( 

100 dtype=bool, 

101 default=True, 

102 doc="Perform diffim decorrelation to undo pixel correlation due to A&L " 

103 "kernel convolution? If True, also update the diffim PSF." 

104 ) 

105 decorrelate = lsst.pex.config.ConfigurableField( 

106 target=DecorrelateALKernelTask, 

107 doc="Task to decorrelate the image difference.", 

108 ) 

109 requiredTemplateFraction = lsst.pex.config.Field( 

110 dtype=float, 

111 default=0.1, 

112 doc="Abort task if template covers less than this fraction of pixels." 

113 " Setting to 0 will always attempt image subtraction." 

114 ) 

115 doScaleVariance = lsst.pex.config.Field( 

116 dtype=bool, 

117 default=True, 

118 doc="Scale variance of the image difference?" 

119 ) 

120 scaleVariance = lsst.pex.config.ConfigurableField( 

121 target=ScaleVarianceTask, 

122 doc="Subtask to rescale the variance of the template to the statistically expected level." 

123 ) 

124 doSubtractBackground = lsst.pex.config.Field( 

125 doc="Subtract the background fit when solving the kernel?", 

126 dtype=bool, 

127 default=True, 

128 ) 

129 

130 forceCompatibility = lsst.pex.config.Field( 

131 dtype=bool, 

132 default=False, 

133 doc="Set up and run diffim using settings that ensure the results" 

134 "are compatible with the old version in pipe_tasks.", 

135 deprecated="This option is only for backwards compatibility purposes" 

136 " and will be removed after v24.", 

137 ) 

138 

139 def setDefaults(self): 

140 self.makeKernel.kernel.name = "AL" 

141 self.makeKernel.kernel.active.fitForBackground = self.doSubtractBackground 

142 self.makeKernel.kernel.active.spatialKernelOrder = 1 

143 self.makeKernel.kernel.active.spatialBgOrder = 2 

144 

145 def validate(self): 

146 if self.forceCompatibility: 

147 self.mode = "convolveTemplate" 

148 

149 

150class AlardLuptonSubtractTask(lsst.pipe.base.PipelineTask): 

151 """Compute the image difference of a science and template image using 

152 the Alard & Lupton (1998) algorithm. 

153 """ 

154 ConfigClass = AlardLuptonSubtractConfig 

155 _DefaultName = "alardLuptonSubtract" 

156 

157 def __init__(self, **kwargs): 

158 super().__init__(**kwargs) 

159 self.makeSubtask("decorrelate") 

160 self.makeSubtask("makeKernel") 

161 if self.config.doScaleVariance: 

162 self.makeSubtask("scaleVariance") 

163 

164 self.convolutionControl = lsst.afw.math.ConvolutionControl() 

165 # Normalization is an extra, unnecessary, calculation and will result 

166 # in mis-subtraction of the images if there are calibration errors. 

167 self.convolutionControl.setDoNormalize(False) 

168 self.convolutionControl.setDoCopyEdge(True) 

169 

170 def run(self, template, science, sources): 

171 """PSF match, subtract, and decorrelate two images. 

172 

173 Parameters 

174 ---------- 

175 template : `lsst.afw.image.ExposureF` 

176 Template exposure, warped to match the science exposure. 

177 science : `lsst.afw.image.ExposureF` 

178 Science exposure to subtract from the template. 

179 sources : `lsst.afw.table.SourceCatalog` 

180 Identified sources on the science exposure. This catalog is used to 

181 select sources in order to perform the AL PSF matching on stamp 

182 images around them. 

183 

184 Returns 

185 ------- 

186 results : `lsst.pipe.base.Struct` 

187 ``difference`` : `lsst.afw.image.ExposureF` 

188 Result of subtracting template and science. 

189 ``matchedTemplate`` : `lsst.afw.image.ExposureF` 

190 Warped and PSF-matched template exposure. 

191 ``backgroundModel`` : `lsst.afw.math.Chebyshev1Function2D` 

192 Background model that was fit while solving for the PSF-matching kernel 

193 ``psfMatchingKernel`` : `lsst.afw.math.Kernel` 

194 Kernel used to PSF-match the convolved image. 

195 

196 Raises 

197 ------ 

198 RuntimeError 

199 If an unsupported convolution mode is supplied. 

200 lsst.pipe.base.NoWorkFound 

201 Raised if fraction of good pixels, defined as not having NO_DATA 

202 set, is less then the configured requiredTemplateFraction 

203 """ 

204 self._validateExposures(template, science) 

205 checkTemplateIsSufficient(template, self.log, 

206 requiredTemplateFraction=self.config.requiredTemplateFraction) 

207 if self.config.forceCompatibility: 

208 # Compatibility option to maintain old functionality 

209 # This should be removed in the future! 

210 self.log.warning("Running with `config.forceCompatibility=True`") 

211 sources = None 

212 sciencePsfSize = getPsfFwhm(science.psf) 

213 templatePsfSize = getPsfFwhm(template.psf) 

214 self.log.info("Science PSF size: %f", sciencePsfSize) 

215 self.log.info("Template PSF size: %f", templatePsfSize) 

216 if self.config.mode == "auto": 

217 if sciencePsfSize < templatePsfSize: 

218 self.log.info("Template PSF size is greater: convolving science image.") 

219 convolveTemplate = False 

220 else: 

221 self.log.info("Science PSF size is greater: convolving template image.") 

222 convolveTemplate = True 

223 elif self.config.mode == "convolveTemplate": 

224 self.log.info("`convolveTemplate` is set: convolving template image.") 

225 convolveTemplate = True 

226 elif self.config.mode == "convolveScience": 

227 self.log.info("`convolveScience` is set: convolving science image.") 

228 convolveTemplate = False 

229 else: 

230 raise RuntimeError("Cannot handle AlardLuptonSubtract mode: %s", self.config.mode) 

231 

232 if self.config.doScaleVariance and ~self.config.forceCompatibility: 

233 # Scale the variance of the template and science images before 

234 # convolution, subtraction, or decorrelation so that they have the 

235 # correct ratio. 

236 templateVarFactor = self.scaleVariance.run(template.maskedImage) 

237 sciVarFactor = self.scaleVariance.run(science.maskedImage) 

238 self.log.info("Template variance scaling factor: %.2f", templateVarFactor) 

239 self.metadata.add("scaleTemplateVarianceFactor", templateVarFactor) 

240 self.log.info("Science variance scaling factor: %.2f", sciVarFactor) 

241 self.metadata.add("scaleScienceVarianceFactor", sciVarFactor) 

242 

243 kernelSources = self.makeKernel.selectKernelSources(template, science, 

244 candidateList=sources, 

245 preconvolved=False) 

246 if convolveTemplate: 

247 subtractResults = self.runConvolveTemplate(template, science, kernelSources) 

248 else: 

249 subtractResults = self.runConvolveScience(template, science, kernelSources) 

250 

251 if self.config.doScaleVariance and self.config.forceCompatibility: 

252 # The old behavior scaled the variance of the final image difference. 

253 diffimVarFactor = self.scaleVariance.run(subtractResults.difference.maskedImage) 

254 self.log.info("Diffim variance scaling factor: %.2f", diffimVarFactor) 

255 self.metadata.add("scaleDiffimVarianceFactor", diffimVarFactor) 

256 

257 return subtractResults 

258 

259 def runConvolveTemplate(self, template, science, sources): 

260 """Convolve the template image with a PSF-matching kernel and subtract 

261 from the science image. 

262 

263 Parameters 

264 ---------- 

265 template : `lsst.afw.image.ExposureF` 

266 Template exposure, warped to match the science exposure. 

267 science : `lsst.afw.image.ExposureF` 

268 Science exposure to subtract from the template. 

269 sources : `lsst.afw.table.SourceCatalog` 

270 Identified sources on the science exposure. This catalog is used to 

271 select sources in order to perform the AL PSF matching on stamp 

272 images around them. 

273 

274 Returns 

275 ------- 

276 results : `lsst.pipe.base.Struct` 

277 

278 ``difference`` : `lsst.afw.image.ExposureF` 

279 Result of subtracting template and science. 

280 ``matchedTemplate`` : `lsst.afw.image.ExposureF` 

281 Warped and PSF-matched template exposure. 

282 ``backgroundModel`` : `lsst.afw.math.Chebyshev1Function2D` 

283 Background model that was fit while solving for the PSF-matching kernel 

284 ``psfMatchingKernel`` : `lsst.afw.math.Kernel` 

285 Kernel used to PSF-match the template to the science image. 

286 """ 

287 if self.config.forceCompatibility: 

288 # Compatibility option to maintain old behavior 

289 # This should be removed in the future! 

290 template = template[science.getBBox()] 

291 kernelResult = self.makeKernel.run(template, science, sources, preconvolved=False) 

292 

293 matchedTemplate = self._convolveExposure(template, kernelResult.psfMatchingKernel, 

294 self.convolutionControl, 

295 bbox=science.getBBox(), 

296 psf=science.psf, 

297 photoCalib=science.getPhotoCalib()) 

298 difference = _subtractImages(science, matchedTemplate, 

299 backgroundModel=(kernelResult.backgroundModel 

300 if self.config.doSubtractBackground else None)) 

301 correctedExposure = self.finalize(template, science, difference, kernelResult.psfMatchingKernel, 

302 templateMatched=True) 

303 

304 return lsst.pipe.base.Struct(difference=correctedExposure, 

305 matchedTemplate=matchedTemplate, 

306 matchedScience=science, 

307 backgroundModel=kernelResult.backgroundModel, 

308 psfMatchingKernel=kernelResult.psfMatchingKernel) 

309 

310 def runConvolveScience(self, template, science, sources): 

311 """Convolve the science image with a PSF-matching kernel and subtract the template image. 

312 

313 Parameters 

314 ---------- 

315 template : `lsst.afw.image.ExposureF` 

316 Template exposure, warped to match the science exposure. 

317 science : `lsst.afw.image.ExposureF` 

318 Science exposure to subtract from the template. 

319 sources : `lsst.afw.table.SourceCatalog` 

320 Identified sources on the science exposure. This catalog is used to 

321 select sources in order to perform the AL PSF matching on stamp 

322 images around them. 

323 

324 Returns 

325 ------- 

326 results : `lsst.pipe.base.Struct` 

327 

328 ``difference`` : `lsst.afw.image.ExposureF` 

329 Result of subtracting template and science. 

330 ``matchedTemplate`` : `lsst.afw.image.ExposureF` 

331 Warped template exposure. Note that in this case, the template 

332 is not PSF-matched to the science image. 

333 ``backgroundModel`` : `lsst.afw.math.Chebyshev1Function2D` 

334 Background model that was fit while solving for the PSF-matching kernel 

335 ``psfMatchingKernel`` : `lsst.afw.math.Kernel` 

336 Kernel used to PSF-match the science image to the template. 

337 """ 

338 if self.config.forceCompatibility: 

339 # Compatibility option to maintain old behavior 

340 # This should be removed in the future! 

341 template = template[science.getBBox()] 

342 kernelResult = self.makeKernel.run(science, template, sources, preconvolved=False) 

343 modelParams = kernelResult.backgroundModel.getParameters() 

344 # We must invert the background model if the matching kernel is solved for the science image. 

345 kernelResult.backgroundModel.setParameters([-p for p in modelParams]) 

346 

347 kernelImage = lsst.afw.image.ImageD(kernelResult.psfMatchingKernel.getDimensions()) 

348 norm = kernelResult.psfMatchingKernel.computeImage(kernelImage, doNormalize=False) 

349 

350 matchedScience = self._convolveExposure(science, kernelResult.psfMatchingKernel, 

351 self.convolutionControl, 

352 psf=template.psf) 

353 

354 # Place back on native photometric scale 

355 matchedScience.maskedImage /= norm 

356 matchedTemplate = template.clone()[science.getBBox()] 

357 matchedTemplate.maskedImage /= norm 

358 matchedTemplate.setPhotoCalib(science.getPhotoCalib()) 

359 

360 difference = _subtractImages(matchedScience, matchedTemplate, 

361 backgroundModel=(kernelResult.backgroundModel 

362 if self.config.doSubtractBackground else None)) 

363 

364 correctedExposure = self.finalize(template, science, difference, kernelResult.psfMatchingKernel, 

365 templateMatched=False) 

366 

367 return lsst.pipe.base.Struct(difference=correctedExposure, 

368 matchedTemplate=matchedTemplate, 

369 matchedScience=matchedScience, 

370 backgroundModel=kernelResult.backgroundModel, 

371 psfMatchingKernel=kernelResult.psfMatchingKernel,) 

372 

373 def finalize(self, template, science, difference, kernel, 

374 templateMatched=True, 

375 preConvMode=False, 

376 preConvKernel=None, 

377 spatiallyVarying=False): 

378 """Decorrelate the difference image to undo the noise correlations 

379 caused by convolution. 

380 

381 Parameters 

382 ---------- 

383 template : `lsst.afw.image.ExposureF` 

384 Template exposure, warped to match the science exposure. 

385 science : `lsst.afw.image.ExposureF` 

386 Science exposure to subtract from the template. 

387 difference : `lsst.afw.image.ExposureF` 

388 Result of subtracting template and science. 

389 kernel : `lsst.afw.math.Kernel` 

390 An (optionally spatially-varying) PSF matching kernel 

391 templateMatched : `bool`, optional 

392 Was the template PSF-matched to the science image? 

393 preConvMode : `bool`, optional 

394 Was the science image preconvolved with its own PSF 

395 before PSF matching the template? 

396 preConvKernel : `lsst.afw.detection.Psf`, optional 

397 If not `None`, then the science image was pre-convolved with 

398 (the reflection of) this kernel. Must be normalized to sum to 1. 

399 spatiallyVarying : `bool`, optional 

400 Compute the decorrelation kernel spatially varying across the image? 

401 

402 Returns 

403 ------- 

404 correctedExposure : `lsst.afw.image.ExposureF` 

405 The decorrelated image difference. 

406 """ 

407 # Erase existing detection mask planes. 

408 # We don't want the detection mask from the science image 

409 mask = difference.mask 

410 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) 

411 

412 if self.config.doDecorrelation: 

413 self.log.info("Decorrelating image difference.") 

414 correctedExposure = self.decorrelate.run(science, template[science.getBBox()], difference, kernel, 

415 templateMatched=templateMatched, 

416 preConvMode=preConvMode, 

417 preConvKernel=preConvKernel, 

418 spatiallyVarying=spatiallyVarying).correctedExposure 

419 else: 

420 self.log.info("NOT decorrelating image difference.") 

421 correctedExposure = difference 

422 return correctedExposure 

423 

424 @staticmethod 

425 def _validateExposures(template, science): 

426 """Check that the WCS of the two Exposures match, and the template bbox 

427 contains the science bbox. 

428 

429 Parameters 

430 ---------- 

431 template : `lsst.afw.image.ExposureF` 

432 Template exposure, warped to match the science exposure. 

433 science : `lsst.afw.image.ExposureF` 

434 Science exposure to subtract from the template. 

435 

436 Raises 

437 ------ 

438 AssertionError 

439 Raised if the WCS of the template is not equal to the science WCS, 

440 or if the science image is not fully contained in the template 

441 bounding box. 

442 """ 

443 assert template.wcs == science.wcs,\ 

444 "Template and science exposure WCS are not identical." 

445 templateBBox = template.getBBox() 

446 scienceBBox = science.getBBox() 

447 

448 assert templateBBox.contains(scienceBBox),\ 

449 "Template bbox does not contain all of the science image." 

450 

451 @staticmethod 

452 def _convolveExposure(exposure, kernel, convolutionControl, 

453 bbox=None, 

454 psf=None, 

455 photoCalib=None): 

456 """Convolve an exposure with the given kernel. 

457 

458 Parameters 

459 ---------- 

460 exposure : `lsst.afw.Exposure` 

461 exposure to convolve. 

462 kernel : `lsst.afw.math.LinearCombinationKernel` 

463 PSF matching kernel computed in the ``makeKernel`` subtask. 

464 convolutionControl : `lsst.afw.math.ConvolutionControl` 

465 Configuration for convolve algorithm. 

466 bbox : `lsst.geom.Box2I`, optional 

467 Bounding box to trim the convolved exposure to. 

468 psf : `lsst.afw.detection.Psf`, optional 

469 Point spread function (PSF) to set for the convolved exposure. 

470 photoCalib : `lsst.afw.image.PhotoCalib`, optional 

471 Photometric calibration of the convolved exposure. 

472 

473 Returns 

474 ------- 

475 convolvedExp : `lsst.afw.Exposure` 

476 The convolved image. 

477 """ 

478 convolvedExposure = exposure.clone() 

479 if psf is not None: 

480 convolvedExposure.setPsf(psf) 

481 if photoCalib is not None: 

482 convolvedExposure.setPhotoCalib(photoCalib) 

483 convolvedImage = lsst.afw.image.MaskedImageF(exposure.getBBox()) 

484 lsst.afw.math.convolve(convolvedImage, exposure.maskedImage, kernel, convolutionControl) 

485 convolvedExposure.setMaskedImage(convolvedImage) 

486 if bbox is None: 

487 return convolvedExposure 

488 else: 

489 return convolvedExposure[bbox] 

490 

491 

492def checkTemplateIsSufficient(templateExposure, logger, requiredTemplateFraction=0.): 

493 """Raise NoWorkFound if template coverage < requiredTemplateFraction 

494 

495 Parameters 

496 ---------- 

497 templateExposure : `lsst.afw.image.ExposureF` 

498 The template exposure to check 

499 logger : `lsst.log.Log` 

500 Logger for printing output. 

501 requiredTemplateFraction : `float`, optional 

502 Fraction of pixels of the science image required to have coverage 

503 in the template. 

504 

505 Raises 

506 ------ 

507 lsst.pipe.base.NoWorkFound 

508 Raised if fraction of good pixels, defined as not having NO_DATA 

509 set, is less then the configured requiredTemplateFraction 

510 """ 

511 # Count the number of pixels with the NO_DATA mask bit set 

512 # counting NaN pixels is insufficient because pixels without data are often intepolated over) 

513 pixNoData = np.count_nonzero(templateExposure.mask.array 

514 & templateExposure.mask.getPlaneBitMask('NO_DATA')) 

515 pixGood = templateExposure.getBBox().getArea() - pixNoData 

516 logger.info("template has %d good pixels (%.1f%%)", pixGood, 

517 100*pixGood/templateExposure.getBBox().getArea()) 

518 

519 if pixGood/templateExposure.getBBox().getArea() < requiredTemplateFraction: 

520 message = ("Insufficient Template Coverage. (%.1f%% < %.1f%%) Not attempting subtraction. " 

521 "To force subtraction, set config requiredTemplateFraction=0." % ( 

522 100*pixGood/templateExposure.getBBox().getArea(), 

523 100*requiredTemplateFraction)) 

524 raise lsst.pipe.base.NoWorkFound(message) 

525 

526 

527def _subtractImages(science, template, backgroundModel=None): 

528 """Subtract template from science, propagating relevant metadata. 

529 

530 Parameters 

531 ---------- 

532 science : `lsst.afw.Exposure` 

533 The input science image. 

534 template : `lsst.afw.Exposure` 

535 The template to subtract from the science image. 

536 backgroundModel : `lsst.afw.MaskedImage`, optional 

537 Differential background model 

538 

539 Returns 

540 ------- 

541 difference : `lsst.afw.Exposure` 

542 The subtracted image. 

543 """ 

544 difference = science.clone() 

545 if backgroundModel is not None: 

546 difference.maskedImage -= backgroundModel 

547 difference.maskedImage -= template.maskedImage 

548 return difference