Coverage for python/lsst/ip/diffim/subtractImages.py: 24%

296 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-10-20 10:51 +0000

1# This file is part of ip_diffim. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import warnings 

23 

24import numpy as np 

25 

26import lsst.afw.image 

27import lsst.afw.math 

28import lsst.geom 

29from lsst.utils.introspection import find_outside_stacklevel 

30from lsst.ip.diffim.utils import evaluateMeanPsfFwhm, getPsfFwhm 

31from lsst.meas.algorithms import ScaleVarianceTask 

32import lsst.pex.config 

33import lsst.pipe.base 

34from lsst.pex.exceptions import InvalidParameterError 

35from lsst.pipe.base import connectionTypes 

36from . import MakeKernelTask, DecorrelateALKernelTask 

37from lsst.utils.timer import timeMethod 

38 

39__all__ = ["AlardLuptonSubtractConfig", "AlardLuptonSubtractTask", 

40 "AlardLuptonPreconvolveSubtractConfig", "AlardLuptonPreconvolveSubtractTask"] 

41 

42_dimensions = ("instrument", "visit", "detector") 

43_defaultTemplates = {"coaddName": "deep", "fakesType": ""} 

44 

45 

46class SubtractInputConnections(lsst.pipe.base.PipelineTaskConnections, 

47 dimensions=_dimensions, 

48 defaultTemplates=_defaultTemplates): 

49 template = connectionTypes.Input( 

50 doc="Input warped template to subtract.", 

51 dimensions=("instrument", "visit", "detector"), 

52 storageClass="ExposureF", 

53 name="{fakesType}{coaddName}Diff_templateExp" 

54 ) 

55 science = connectionTypes.Input( 

56 doc="Input science exposure to subtract from.", 

57 dimensions=("instrument", "visit", "detector"), 

58 storageClass="ExposureF", 

59 name="{fakesType}calexp" 

60 ) 

61 sources = connectionTypes.Input( 

62 doc="Sources measured on the science exposure; " 

63 "used to select sources for making the matching kernel.", 

64 dimensions=("instrument", "visit", "detector"), 

65 storageClass="SourceCatalog", 

66 name="{fakesType}src" 

67 ) 

68 finalizedPsfApCorrCatalog = connectionTypes.Input( 

69 doc=("Per-visit finalized psf models and aperture correction maps. " 

70 "These catalogs use the detector id for the catalog id, " 

71 "sorted on id for fast lookup."), 

72 dimensions=("instrument", "visit"), 

73 storageClass="ExposureCatalog", 

74 name="finalVisitSummary", 

75 # TODO: remove on DM-39854. 

76 deprecated=( 

77 "Deprecated in favor of visitSummary. Will be removed after v26." 

78 ) 

79 ) 

80 visitSummary = connectionTypes.Input( 

81 doc=("Per-visit catalog with final calibration objects. " 

82 "These catalogs use the detector id for the catalog id, " 

83 "sorted on id for fast lookup."), 

84 dimensions=("instrument", "visit"), 

85 storageClass="ExposureCatalog", 

86 name="finalVisitSummary", 

87 ) 

88 

89 def __init__(self, *, config=None): 

90 super().__init__(config=config) 

91 if not config.doApplyFinalizedPsf: 

92 self.inputs.remove("finalizedPsfApCorrCatalog") 

93 if not config.doApplyExternalCalibrations or config.doApplyFinalizedPsf: 

94 del self.visitSummary 

95 

96 

97class SubtractImageOutputConnections(lsst.pipe.base.PipelineTaskConnections, 

98 dimensions=_dimensions, 

99 defaultTemplates=_defaultTemplates): 

100 difference = connectionTypes.Output( 

101 doc="Result of subtracting convolved template from science image.", 

102 dimensions=("instrument", "visit", "detector"), 

103 storageClass="ExposureF", 

104 name="{fakesType}{coaddName}Diff_differenceTempExp", 

105 ) 

106 matchedTemplate = connectionTypes.Output( 

107 doc="Warped and PSF-matched template used to create `subtractedExposure`.", 

108 dimensions=("instrument", "visit", "detector"), 

109 storageClass="ExposureF", 

110 name="{fakesType}{coaddName}Diff_matchedExp", 

111 ) 

112 

113 

114class SubtractScoreOutputConnections(lsst.pipe.base.PipelineTaskConnections, 

115 dimensions=_dimensions, 

116 defaultTemplates=_defaultTemplates): 

117 scoreExposure = connectionTypes.Output( 

118 doc="The maximum likelihood image, used for the detection of diaSources.", 

119 dimensions=("instrument", "visit", "detector"), 

120 storageClass="ExposureF", 

121 name="{fakesType}{coaddName}Diff_scoreExp", 

122 ) 

123 

124 

125class AlardLuptonSubtractConnections(SubtractInputConnections, SubtractImageOutputConnections): 

126 pass 

127 

128 

129class AlardLuptonSubtractBaseConfig(lsst.pex.config.Config): 

130 makeKernel = lsst.pex.config.ConfigurableField( 

131 target=MakeKernelTask, 

132 doc="Task to construct a matching kernel for convolution.", 

133 ) 

134 doDecorrelation = lsst.pex.config.Field( 

135 dtype=bool, 

136 default=True, 

137 doc="Perform diffim decorrelation to undo pixel correlation due to A&L " 

138 "kernel convolution? If True, also update the diffim PSF." 

139 ) 

140 decorrelate = lsst.pex.config.ConfigurableField( 

141 target=DecorrelateALKernelTask, 

142 doc="Task to decorrelate the image difference.", 

143 ) 

144 requiredTemplateFraction = lsst.pex.config.Field( 

145 dtype=float, 

146 default=0.1, 

147 doc="Abort task if template covers less than this fraction of pixels." 

148 " Setting to 0 will always attempt image subtraction." 

149 ) 

150 doScaleVariance = lsst.pex.config.Field( 

151 dtype=bool, 

152 default=True, 

153 doc="Scale variance of the image difference?" 

154 ) 

155 scaleVariance = lsst.pex.config.ConfigurableField( 

156 target=ScaleVarianceTask, 

157 doc="Subtask to rescale the variance of the template to the statistically expected level." 

158 ) 

159 doSubtractBackground = lsst.pex.config.Field( 

160 doc="Subtract the background fit when solving the kernel?", 

161 dtype=bool, 

162 default=True, 

163 ) 

164 doApplyFinalizedPsf = lsst.pex.config.Field( 

165 doc="Replace science Exposure's psf and aperture correction map" 

166 " with those in finalizedPsfApCorrCatalog.", 

167 dtype=bool, 

168 default=False, 

169 # TODO: remove on DM-39854. 

170 deprecated=( 

171 "Deprecated in favor of doApplyExternalCalibrations. " 

172 "Will be removed after v26." 

173 ) 

174 ) 

175 doApplyExternalCalibrations = lsst.pex.config.Field( 

176 doc=( 

177 "Replace science Exposure's calibration objects with those" 

178 " in visitSummary. Ignored if `doApplyFinalizedPsf is True." 

179 ), 

180 dtype=bool, 

181 default=False, 

182 ) 

183 detectionThreshold = lsst.pex.config.Field( 

184 dtype=float, 

185 default=10, 

186 doc="Minimum signal to noise ratio of detected sources " 

187 "to use for calculating the PSF matching kernel." 

188 ) 

189 badSourceFlags = lsst.pex.config.ListField( 

190 dtype=str, 

191 doc="Flags that, if set, the associated source should not " 

192 "be used to determine the PSF matching kernel.", 

193 default=("sky_source", "slot_Centroid_flag", 

194 "slot_ApFlux_flag", "slot_PsfFlux_flag", ), 

195 ) 

196 badMaskPlanes = lsst.pex.config.ListField( 

197 dtype=str, 

198 default=("NO_DATA", "BAD", "SAT", "EDGE", "FAKE"), 

199 doc="Mask planes to exclude when selecting sources for PSF matching." 

200 ) 

201 preserveTemplateMask = lsst.pex.config.ListField( 

202 dtype=str, 

203 default=("NO_DATA", "BAD", "SAT", "FAKE", "INJECTED", "INJECTED_CORE"), 

204 doc="Mask planes from the template to propagate to the image difference." 

205 ) 

206 

207 def setDefaults(self): 

208 self.makeKernel.kernel.name = "AL" 

209 self.makeKernel.kernel.active.fitForBackground = self.doSubtractBackground 

210 self.makeKernel.kernel.active.spatialKernelOrder = 1 

211 self.makeKernel.kernel.active.spatialBgOrder = 2 

212 

213 

214class AlardLuptonSubtractConfig(AlardLuptonSubtractBaseConfig, lsst.pipe.base.PipelineTaskConfig, 

215 pipelineConnections=AlardLuptonSubtractConnections): 

216 mode = lsst.pex.config.ChoiceField( 

217 dtype=str, 

218 default="convolveTemplate", 

219 allowed={"auto": "Choose which image to convolve at runtime.", 

220 "convolveScience": "Only convolve the science image.", 

221 "convolveTemplate": "Only convolve the template image."}, 

222 doc="Choose which image to convolve at runtime, or require that a specific image is convolved." 

223 ) 

224 

225 

226class AlardLuptonSubtractTask(lsst.pipe.base.PipelineTask): 

227 """Compute the image difference of a science and template image using 

228 the Alard & Lupton (1998) algorithm. 

229 """ 

230 ConfigClass = AlardLuptonSubtractConfig 

231 _DefaultName = "alardLuptonSubtract" 

232 

233 def __init__(self, **kwargs): 

234 super().__init__(**kwargs) 

235 self.makeSubtask("decorrelate") 

236 self.makeSubtask("makeKernel") 

237 if self.config.doScaleVariance: 

238 self.makeSubtask("scaleVariance") 

239 

240 self.convolutionControl = lsst.afw.math.ConvolutionControl() 

241 # Normalization is an extra, unnecessary, calculation and will result 

242 # in mis-subtraction of the images if there are calibration errors. 

243 self.convolutionControl.setDoNormalize(False) 

244 self.convolutionControl.setDoCopyEdge(True) 

245 

246 def _applyExternalCalibrations(self, exposure, visitSummary): 

247 """Replace calibrations (psf, and ApCorrMap) on this exposure with 

248 external ones.". 

249 

250 Parameters 

251 ---------- 

252 exposure : `lsst.afw.image.exposure.Exposure` 

253 Input exposure to adjust calibrations. 

254 visitSummary : `lsst.afw.table.ExposureCatalog` 

255 Exposure catalog with external calibrations to be applied. Catalog 

256 uses the detector id for the catalog id, sorted on id for fast 

257 lookup. 

258 

259 Returns 

260 ------- 

261 exposure : `lsst.afw.image.exposure.Exposure` 

262 Exposure with adjusted calibrations. 

263 """ 

264 detectorId = exposure.info.getDetector().getId() 

265 

266 row = visitSummary.find(detectorId) 

267 if row is None: 

268 self.log.warning("Detector id %s not found in external calibrations catalog; " 

269 "Using original calibrations.", detectorId) 

270 else: 

271 psf = row.getPsf() 

272 apCorrMap = row.getApCorrMap() 

273 if psf is None: 

274 self.log.warning("Detector id %s has None for psf in " 

275 "external calibrations catalog; Using original psf and aperture correction.", 

276 detectorId) 

277 elif apCorrMap is None: 

278 self.log.warning("Detector id %s has None for apCorrMap in " 

279 "external calibrations catalog; Using original psf and aperture correction.", 

280 detectorId) 

281 else: 

282 exposure.setPsf(psf) 

283 exposure.info.setApCorrMap(apCorrMap) 

284 

285 return exposure 

286 

287 @timeMethod 

288 def run(self, template, science, sources, finalizedPsfApCorrCatalog=None, 

289 visitSummary=None): 

290 """PSF match, subtract, and decorrelate two images. 

291 

292 Parameters 

293 ---------- 

294 template : `lsst.afw.image.ExposureF` 

295 Template exposure, warped to match the science exposure. 

296 science : `lsst.afw.image.ExposureF` 

297 Science exposure to subtract from the template. 

298 sources : `lsst.afw.table.SourceCatalog` 

299 Identified sources on the science exposure. This catalog is used to 

300 select sources in order to perform the AL PSF matching on stamp 

301 images around them. 

302 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional 

303 Exposure catalog with finalized psf models and aperture correction 

304 maps to be applied. Catalog uses the detector id for the catalog 

305 id, sorted on id for fast lookup. Deprecated in favor of 

306 ``visitSummary``, and will be removed after v26. 

307 visitSummary : `lsst.afw.table.ExposureCatalog`, optional 

308 Exposure catalog with external calibrations to be applied. Catalog 

309 uses the detector id for the catalog id, sorted on id for fast 

310 lookup. Ignored (for temporary backwards compatibility) if 

311 ``finalizedPsfApCorrCatalog`` is provided. 

312 

313 Returns 

314 ------- 

315 results : `lsst.pipe.base.Struct` 

316 ``difference`` : `lsst.afw.image.ExposureF` 

317 Result of subtracting template and science. 

318 ``matchedTemplate`` : `lsst.afw.image.ExposureF` 

319 Warped and PSF-matched template exposure. 

320 ``backgroundModel`` : `lsst.afw.math.Function2D` 

321 Background model that was fit while solving for the 

322 PSF-matching kernel 

323 ``psfMatchingKernel`` : `lsst.afw.math.Kernel` 

324 Kernel used to PSF-match the convolved image. 

325 

326 Raises 

327 ------ 

328 RuntimeError 

329 If an unsupported convolution mode is supplied. 

330 RuntimeError 

331 If there are too few sources to calculate the PSF matching kernel. 

332 lsst.pipe.base.NoWorkFound 

333 Raised if fraction of good pixels, defined as not having NO_DATA 

334 set, is less then the configured requiredTemplateFraction 

335 """ 

336 

337 if finalizedPsfApCorrCatalog is not None: 

338 warnings.warn( 

339 "The finalizedPsfApCorrCatalog argument is deprecated in favor of the visitSummary " 

340 "argument, and will be removed after v26.", 

341 FutureWarning, 

342 stacklevel=find_outside_stacklevel("lsst.ip.diffim"), 

343 ) 

344 visitSummary = finalizedPsfApCorrCatalog 

345 

346 self._prepareInputs(template, science, visitSummary=visitSummary) 

347 

348 # In the event that getPsfFwhm fails, evaluate the PSF on a grid. 

349 fwhmExposureBuffer = self.config.makeKernel.fwhmExposureBuffer 

350 fwhmExposureGrid = self.config.makeKernel.fwhmExposureGrid 

351 

352 # Calling getPsfFwhm on template.psf fails on some rare occasions when 

353 # the template has no input exposures at the average position of the 

354 # stars. So we try getPsfFwhm first on template, and if that fails we 

355 # evaluate the PSF on a grid specified by fwhmExposure* fields. 

356 # To keep consistent definitions for PSF size on the template and 

357 # science images, we use the same method for both. 

358 try: 

359 templatePsfSize = getPsfFwhm(template.psf) 

360 sciencePsfSize = getPsfFwhm(science.psf) 

361 except InvalidParameterError: 

362 self.log.info("Unable to evaluate PSF at the average position. " 

363 "Evaluting PSF on a grid of points." 

364 ) 

365 templatePsfSize = evaluateMeanPsfFwhm(template, 

366 fwhmExposureBuffer=fwhmExposureBuffer, 

367 fwhmExposureGrid=fwhmExposureGrid 

368 ) 

369 sciencePsfSize = evaluateMeanPsfFwhm(science, 

370 fwhmExposureBuffer=fwhmExposureBuffer, 

371 fwhmExposureGrid=fwhmExposureGrid 

372 ) 

373 self.log.info("Science PSF FWHM: %f pixels", sciencePsfSize) 

374 self.log.info("Template PSF FWHM: %f pixels", templatePsfSize) 

375 self.metadata.add("sciencePsfSize", sciencePsfSize) 

376 self.metadata.add("templatePsfSize", templatePsfSize) 

377 selectSources = self._sourceSelector(sources, science.mask) 

378 

379 if self.config.mode == "auto": 

380 convolveTemplate = _shapeTest(template, 

381 science, 

382 fwhmExposureBuffer=fwhmExposureBuffer, 

383 fwhmExposureGrid=fwhmExposureGrid) 

384 if convolveTemplate: 

385 if sciencePsfSize < templatePsfSize: 

386 self.log.info("Average template PSF size is greater, " 

387 "but science PSF greater in one dimension: convolving template image.") 

388 else: 

389 self.log.info("Science PSF size is greater: convolving template image.") 

390 else: 

391 self.log.info("Template PSF size is greater: convolving science image.") 

392 elif self.config.mode == "convolveTemplate": 

393 self.log.info("`convolveTemplate` is set: convolving template image.") 

394 convolveTemplate = True 

395 elif self.config.mode == "convolveScience": 

396 self.log.info("`convolveScience` is set: convolving science image.") 

397 convolveTemplate = False 

398 else: 

399 raise RuntimeError("Cannot handle AlardLuptonSubtract mode: %s", self.config.mode) 

400 

401 if convolveTemplate: 

402 self.metadata.add("convolvedExposure", "Template") 

403 subtractResults = self.runConvolveTemplate(template, science, selectSources) 

404 else: 

405 self.metadata.add("convolvedExposure", "Science") 

406 subtractResults = self.runConvolveScience(template, science, selectSources) 

407 

408 return subtractResults 

409 

410 def runConvolveTemplate(self, template, science, selectSources): 

411 """Convolve the template image with a PSF-matching kernel and subtract 

412 from the science image. 

413 

414 Parameters 

415 ---------- 

416 template : `lsst.afw.image.ExposureF` 

417 Template exposure, warped to match the science exposure. 

418 science : `lsst.afw.image.ExposureF` 

419 Science exposure to subtract from the template. 

420 selectSources : `lsst.afw.table.SourceCatalog` 

421 Identified sources on the science exposure. This catalog is used to 

422 select sources in order to perform the AL PSF matching on stamp 

423 images around them. 

424 

425 Returns 

426 ------- 

427 results : `lsst.pipe.base.Struct` 

428 

429 ``difference`` : `lsst.afw.image.ExposureF` 

430 Result of subtracting template and science. 

431 ``matchedTemplate`` : `lsst.afw.image.ExposureF` 

432 Warped and PSF-matched template exposure. 

433 ``backgroundModel`` : `lsst.afw.math.Function2D` 

434 Background model that was fit while solving for the PSF-matching kernel 

435 ``psfMatchingKernel`` : `lsst.afw.math.Kernel` 

436 Kernel used to PSF-match the template to the science image. 

437 """ 

438 kernelSources = self.makeKernel.selectKernelSources(template, science, 

439 candidateList=selectSources, 

440 preconvolved=False) 

441 kernelResult = self.makeKernel.run(template, science, kernelSources, 

442 preconvolved=False) 

443 

444 matchedTemplate = self._convolveExposure(template, kernelResult.psfMatchingKernel, 

445 self.convolutionControl, 

446 bbox=science.getBBox(), 

447 psf=science.psf, 

448 photoCalib=science.photoCalib) 

449 

450 difference = _subtractImages(science, matchedTemplate, 

451 backgroundModel=(kernelResult.backgroundModel 

452 if self.config.doSubtractBackground else None)) 

453 correctedExposure = self.finalize(template, science, difference, 

454 kernelResult.psfMatchingKernel, 

455 templateMatched=True) 

456 

457 return lsst.pipe.base.Struct(difference=correctedExposure, 

458 matchedTemplate=matchedTemplate, 

459 matchedScience=science, 

460 backgroundModel=kernelResult.backgroundModel, 

461 psfMatchingKernel=kernelResult.psfMatchingKernel) 

462 

463 def runConvolveScience(self, template, science, selectSources): 

464 """Convolve the science image with a PSF-matching kernel and subtract 

465 the template image. 

466 

467 Parameters 

468 ---------- 

469 template : `lsst.afw.image.ExposureF` 

470 Template exposure, warped to match the science exposure. 

471 science : `lsst.afw.image.ExposureF` 

472 Science exposure to subtract from the template. 

473 selectSources : `lsst.afw.table.SourceCatalog` 

474 Identified sources on the science exposure. This catalog is used to 

475 select sources in order to perform the AL PSF matching on stamp 

476 images around them. 

477 

478 Returns 

479 ------- 

480 results : `lsst.pipe.base.Struct` 

481 

482 ``difference`` : `lsst.afw.image.ExposureF` 

483 Result of subtracting template and science. 

484 ``matchedTemplate`` : `lsst.afw.image.ExposureF` 

485 Warped template exposure. Note that in this case, the template 

486 is not PSF-matched to the science image. 

487 ``backgroundModel`` : `lsst.afw.math.Function2D` 

488 Background model that was fit while solving for the PSF-matching kernel 

489 ``psfMatchingKernel`` : `lsst.afw.math.Kernel` 

490 Kernel used to PSF-match the science image to the template. 

491 """ 

492 bbox = science.getBBox() 

493 kernelSources = self.makeKernel.selectKernelSources(science, template, 

494 candidateList=selectSources, 

495 preconvolved=False) 

496 kernelResult = self.makeKernel.run(science, template, kernelSources, 

497 preconvolved=False) 

498 modelParams = kernelResult.backgroundModel.getParameters() 

499 # We must invert the background model if the matching kernel is solved for the science image. 

500 kernelResult.backgroundModel.setParameters([-p for p in modelParams]) 

501 

502 kernelImage = lsst.afw.image.ImageD(kernelResult.psfMatchingKernel.getDimensions()) 

503 norm = kernelResult.psfMatchingKernel.computeImage(kernelImage, doNormalize=False) 

504 

505 matchedScience = self._convolveExposure(science, kernelResult.psfMatchingKernel, 

506 self.convolutionControl, 

507 psf=template.psf) 

508 

509 # Place back on native photometric scale 

510 matchedScience.maskedImage /= norm 

511 matchedTemplate = template.clone()[bbox] 

512 matchedTemplate.maskedImage /= norm 

513 matchedTemplate.setPhotoCalib(science.photoCalib) 

514 

515 difference = _subtractImages(matchedScience, matchedTemplate, 

516 backgroundModel=(kernelResult.backgroundModel 

517 if self.config.doSubtractBackground else None)) 

518 

519 correctedExposure = self.finalize(template, science, difference, 

520 kernelResult.psfMatchingKernel, 

521 templateMatched=False) 

522 

523 return lsst.pipe.base.Struct(difference=correctedExposure, 

524 matchedTemplate=matchedTemplate, 

525 matchedScience=matchedScience, 

526 backgroundModel=kernelResult.backgroundModel, 

527 psfMatchingKernel=kernelResult.psfMatchingKernel,) 

528 

529 def finalize(self, template, science, difference, kernel, 

530 templateMatched=True, 

531 preConvMode=False, 

532 preConvKernel=None, 

533 spatiallyVarying=False): 

534 """Decorrelate the difference image to undo the noise correlations 

535 caused by convolution. 

536 

537 Parameters 

538 ---------- 

539 template : `lsst.afw.image.ExposureF` 

540 Template exposure, warped to match the science exposure. 

541 science : `lsst.afw.image.ExposureF` 

542 Science exposure to subtract from the template. 

543 difference : `lsst.afw.image.ExposureF` 

544 Result of subtracting template and science. 

545 kernel : `lsst.afw.math.Kernel` 

546 An (optionally spatially-varying) PSF matching kernel 

547 templateMatched : `bool`, optional 

548 Was the template PSF-matched to the science image? 

549 preConvMode : `bool`, optional 

550 Was the science image preconvolved with its own PSF 

551 before PSF matching the template? 

552 preConvKernel : `lsst.afw.detection.Psf`, optional 

553 If not `None`, then the science image was pre-convolved with 

554 (the reflection of) this kernel. Must be normalized to sum to 1. 

555 spatiallyVarying : `bool`, optional 

556 Compute the decorrelation kernel spatially varying across the image? 

557 

558 Returns 

559 ------- 

560 correctedExposure : `lsst.afw.image.ExposureF` 

561 The decorrelated image difference. 

562 """ 

563 # Erase existing detection mask planes. 

564 # We don't want the detection mask from the science image 

565 

566 self.updateMasks(template, science, difference) 

567 

568 if self.config.doDecorrelation: 

569 self.log.info("Decorrelating image difference.") 

570 # We have cleared the template mask plane, so copy the mask plane of 

571 # the image difference so that we can calculate correct statistics 

572 # during decorrelation 

573 correctedExposure = self.decorrelate.run(science, template[science.getBBox()], difference, kernel, 

574 templateMatched=templateMatched, 

575 preConvMode=preConvMode, 

576 preConvKernel=preConvKernel, 

577 spatiallyVarying=spatiallyVarying).correctedExposure 

578 else: 

579 self.log.info("NOT decorrelating image difference.") 

580 correctedExposure = difference 

581 return correctedExposure 

582 

583 def updateMasks(self, template, science, difference): 

584 """Update the mask planes on images for finalizing.""" 

585 

586 bbox = science.getBBox() 

587 mask = difference.mask 

588 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) 

589 

590 if "FAKE" in science.mask.getMaskPlaneDict().keys(): 

591 # propagate the mask plane related to Fake source injection 

592 # NOTE: the fake source injection sets FAKE plane, but it should be INJECTED 

593 # NOTE: This can be removed in DM-40796 

594 

595 self.log.info("Adding injected mask planes") 

596 mask.addMaskPlane("INJECTED") 

597 diffInjectedBitMask = mask.getPlaneBitMask("INJECTED") 

598 

599 mask.addMaskPlane("INJECTED_TEMPLATE") 

600 diffInjTmpltBitMask = mask.getPlaneBitMask("INJECTED_TEMPLATE") 

601 

602 scienceFakeBitMask = science.mask.getPlaneBitMask('FAKE') 

603 tmpltFakeBitMask = template[bbox].mask.getPlaneBitMask('FAKE') 

604 

605 injScienceMaskArray = ((science.mask.array & scienceFakeBitMask) > 0) * diffInjectedBitMask 

606 injTemplateMaskArray = ((template[bbox].mask.array & tmpltFakeBitMask) > 0) * diffInjTmpltBitMask 

607 

608 mask.array |= injScienceMaskArray 

609 mask.array |= injTemplateMaskArray 

610 

611 template[bbox].mask.array[...] = difference.mask.array[...] 

612 

613 @staticmethod 

614 def _validateExposures(template, science): 

615 """Check that the WCS of the two Exposures match, and the template bbox 

616 contains the science bbox. 

617 

618 Parameters 

619 ---------- 

620 template : `lsst.afw.image.ExposureF` 

621 Template exposure, warped to match the science exposure. 

622 science : `lsst.afw.image.ExposureF` 

623 Science exposure to subtract from the template. 

624 

625 Raises 

626 ------ 

627 AssertionError 

628 Raised if the WCS of the template is not equal to the science WCS, 

629 or if the science image is not fully contained in the template 

630 bounding box. 

631 """ 

632 assert template.wcs == science.wcs,\ 

633 "Template and science exposure WCS are not identical." 

634 templateBBox = template.getBBox() 

635 scienceBBox = science.getBBox() 

636 

637 assert templateBBox.contains(scienceBBox),\ 

638 "Template bbox does not contain all of the science image." 

639 

640 def _convolveExposure(self, exposure, kernel, convolutionControl, 

641 bbox=None, 

642 psf=None, 

643 photoCalib=None, 

644 interpolateBadMaskPlanes=False, 

645 ): 

646 """Convolve an exposure with the given kernel. 

647 

648 Parameters 

649 ---------- 

650 exposure : `lsst.afw.Exposure` 

651 exposure to convolve. 

652 kernel : `lsst.afw.math.LinearCombinationKernel` 

653 PSF matching kernel computed in the ``makeKernel`` subtask. 

654 convolutionControl : `lsst.afw.math.ConvolutionControl` 

655 Configuration for convolve algorithm. 

656 bbox : `lsst.geom.Box2I`, optional 

657 Bounding box to trim the convolved exposure to. 

658 psf : `lsst.afw.detection.Psf`, optional 

659 Point spread function (PSF) to set for the convolved exposure. 

660 photoCalib : `lsst.afw.image.PhotoCalib`, optional 

661 Photometric calibration of the convolved exposure. 

662 

663 Returns 

664 ------- 

665 convolvedExp : `lsst.afw.Exposure` 

666 The convolved image. 

667 """ 

668 convolvedExposure = exposure.clone() 

669 if psf is not None: 

670 convolvedExposure.setPsf(psf) 

671 if photoCalib is not None: 

672 convolvedExposure.setPhotoCalib(photoCalib) 

673 if interpolateBadMaskPlanes and self.config.badMaskPlanes is not None: 

674 nInterp = _interpolateImage(convolvedExposure.maskedImage, 

675 self.config.badMaskPlanes) 

676 self.metadata.add("nInterpolated", nInterp) 

677 convolvedImage = lsst.afw.image.MaskedImageF(convolvedExposure.getBBox()) 

678 lsst.afw.math.convolve(convolvedImage, convolvedExposure.maskedImage, kernel, convolutionControl) 

679 convolvedExposure.setMaskedImage(convolvedImage) 

680 if bbox is None: 

681 return convolvedExposure 

682 else: 

683 return convolvedExposure[bbox] 

684 

685 def _sourceSelector(self, sources, mask): 

686 """Select sources from a catalog that meet the selection criteria. 

687 

688 Parameters 

689 ---------- 

690 sources : `lsst.afw.table.SourceCatalog` 

691 Input source catalog to select sources from. 

692 mask : `lsst.afw.image.Mask` 

693 The image mask plane to use to reject sources 

694 based on their location on the ccd. 

695 

696 Returns 

697 ------- 

698 selectSources : `lsst.afw.table.SourceCatalog` 

699 The input source catalog, with flagged and low signal-to-noise 

700 sources removed. 

701 

702 Raises 

703 ------ 

704 RuntimeError 

705 If there are too few sources to compute the PSF matching kernel 

706 remaining after source selection. 

707 """ 

708 flags = np.ones(len(sources), dtype=bool) 

709 for flag in self.config.badSourceFlags: 

710 try: 

711 flags *= ~sources[flag] 

712 except Exception as e: 

713 self.log.warning("Could not apply source flag: %s", e) 

714 sToNFlag = (sources.getPsfInstFlux()/sources.getPsfInstFluxErr()) > self.config.detectionThreshold 

715 flags *= sToNFlag 

716 flags *= self._checkMask(mask, sources, self.config.badMaskPlanes) 

717 selectSources = sources[flags] 

718 self.log.info("%i/%i=%.1f%% of sources selected for PSF matching from the input catalog", 

719 len(selectSources), len(sources), 100*len(selectSources)/len(sources)) 

720 if len(selectSources) < self.config.makeKernel.nStarPerCell: 

721 self.log.error("Too few sources to calculate the PSF matching kernel: " 

722 "%i selected but %i needed for the calculation.", 

723 len(selectSources), self.config.makeKernel.nStarPerCell) 

724 raise RuntimeError("Cannot compute PSF matching kernel: too few sources selected.") 

725 self.metadata.add("nPsfSources", len(selectSources)) 

726 

727 return selectSources.copy(deep=True) 

728 

729 @staticmethod 

730 def _checkMask(mask, sources, badMaskPlanes): 

731 """Exclude sources that are located on masked pixels. 

732 

733 Parameters 

734 ---------- 

735 mask : `lsst.afw.image.Mask` 

736 The image mask plane to use to reject sources 

737 based on the location of their centroid on the ccd. 

738 sources : `lsst.afw.table.SourceCatalog` 

739 The source catalog to evaluate. 

740 badMaskPlanes : `list` of `str` 

741 List of the names of the mask planes to exclude. 

742 

743 Returns 

744 ------- 

745 flags : `numpy.ndarray` of `bool` 

746 Array indicating whether each source in the catalog should be 

747 kept (True) or rejected (False) based on the value of the 

748 mask plane at its location. 

749 """ 

750 setBadMaskPlanes = [ 

751 maskPlane for maskPlane in badMaskPlanes if maskPlane in mask.getMaskPlaneDict() 

752 ] 

753 

754 badPixelMask = mask.getPlaneBitMask(setBadMaskPlanes) 

755 

756 xv = np.rint(sources.getX() - mask.getX0()) 

757 yv = np.rint(sources.getY() - mask.getY0()) 

758 

759 mv = mask.array[yv.astype(int), xv.astype(int)] 

760 flags = np.bitwise_and(mv, badPixelMask) == 0 

761 return flags 

762 

763 def _prepareInputs(self, template, science, visitSummary=None): 

764 """Perform preparatory calculations common to all Alard&Lupton Tasks. 

765 

766 Parameters 

767 ---------- 

768 template : `lsst.afw.image.ExposureF` 

769 Template exposure, warped to match the science exposure. The 

770 variance plane of the template image is modified in place. 

771 science : `lsst.afw.image.ExposureF` 

772 Science exposure to subtract from the template. The variance plane 

773 of the science image is modified in place. 

774 visitSummary : `lsst.afw.table.ExposureCatalog`, optional 

775 Exposure catalog with external calibrations to be applied. Catalog 

776 uses the detector id for the catalog id, sorted on id for fast 

777 lookup. 

778 """ 

779 self._validateExposures(template, science) 

780 if visitSummary is not None: 

781 self._applyExternalCalibrations(science, visitSummary=visitSummary) 

782 checkTemplateIsSufficient(template, self.log, 

783 requiredTemplateFraction=self.config.requiredTemplateFraction) 

784 

785 if self.config.doScaleVariance: 

786 # Scale the variance of the template and science images before 

787 # convolution, subtraction, or decorrelation so that they have the 

788 # correct ratio. 

789 templateVarFactor = self.scaleVariance.run(template.maskedImage) 

790 sciVarFactor = self.scaleVariance.run(science.maskedImage) 

791 self.log.info("Template variance scaling factor: %.2f", templateVarFactor) 

792 self.metadata.add("scaleTemplateVarianceFactor", templateVarFactor) 

793 self.log.info("Science variance scaling factor: %.2f", sciVarFactor) 

794 self.metadata.add("scaleScienceVarianceFactor", sciVarFactor) 

795 self._clearMask(template) 

796 

797 def _clearMask(self, template): 

798 """Clear the mask plane of the template. 

799 

800 Parameters 

801 ---------- 

802 template : `lsst.afw.image.ExposureF` 

803 Template exposure, warped to match the science exposure. 

804 The mask plane will be modified in place. 

805 """ 

806 mask = template.mask 

807 clearMaskPlanes = [maskplane for maskplane in mask.getMaskPlaneDict().keys() 

808 if maskplane not in self.config.preserveTemplateMask] 

809 

810 bitMaskToClear = mask.getPlaneBitMask(clearMaskPlanes) 

811 mask &= ~bitMaskToClear 

812 

813 

814class AlardLuptonPreconvolveSubtractConnections(SubtractInputConnections, 

815 SubtractScoreOutputConnections): 

816 pass 

817 

818 

819class AlardLuptonPreconvolveSubtractConfig(AlardLuptonSubtractBaseConfig, lsst.pipe.base.PipelineTaskConfig, 

820 pipelineConnections=AlardLuptonPreconvolveSubtractConnections): 

821 pass 

822 

823 

824class AlardLuptonPreconvolveSubtractTask(AlardLuptonSubtractTask): 

825 """Subtract a template from a science image, convolving the science image 

826 before computing the kernel, and also convolving the template before 

827 subtraction. 

828 """ 

829 ConfigClass = AlardLuptonPreconvolveSubtractConfig 

830 _DefaultName = "alardLuptonPreconvolveSubtract" 

831 

832 def run(self, template, science, sources, finalizedPsfApCorrCatalog=None, visitSummary=None): 

833 """Preconvolve the science image with its own PSF, 

834 convolve the template image with a PSF-matching kernel and subtract 

835 from the preconvolved science image. 

836 

837 Parameters 

838 ---------- 

839 template : `lsst.afw.image.ExposureF` 

840 The template image, which has previously been warped to the science 

841 image. The template bbox will be padded by a few pixels compared to 

842 the science bbox. 

843 science : `lsst.afw.image.ExposureF` 

844 The science exposure. 

845 sources : `lsst.afw.table.SourceCatalog` 

846 Identified sources on the science exposure. This catalog is used to 

847 select sources in order to perform the AL PSF matching on stamp 

848 images around them. 

849 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional 

850 Exposure catalog with finalized psf models and aperture correction 

851 maps to be applied. Catalog uses the detector id for the catalog 

852 id, sorted on id for fast lookup. Deprecated in favor of 

853 ``visitSummary``, and will be removed after v26. 

854 visitSummary : `lsst.afw.table.ExposureCatalog`, optional 

855 Exposure catalog with complete external calibrations. Catalog uses 

856 the detector id for the catalog id, sorted on id for fast lookup. 

857 Ignored (for temporary backwards compatibility) if 

858 ``finalizedPsfApCorrCatalog`` is provided. 

859 

860 Returns 

861 ------- 

862 results : `lsst.pipe.base.Struct` 

863 ``scoreExposure`` : `lsst.afw.image.ExposureF` 

864 Result of subtracting the convolved template and science 

865 images. Attached PSF is that of the original science image. 

866 ``matchedTemplate`` : `lsst.afw.image.ExposureF` 

867 Warped and PSF-matched template exposure. Attached PSF is that 

868 of the original science image. 

869 ``matchedScience`` : `lsst.afw.image.ExposureF` 

870 The science exposure after convolving with its own PSF. 

871 Attached PSF is that of the original science image. 

872 ``backgroundModel`` : `lsst.afw.math.Function2D` 

873 Background model that was fit while solving for the 

874 PSF-matching kernel 

875 ``psfMatchingKernel`` : `lsst.afw.math.Kernel` 

876 Final kernel used to PSF-match the template to the science 

877 image. 

878 """ 

879 if finalizedPsfApCorrCatalog is not None: 

880 warnings.warn( 

881 "The finalizedPsfApCorrCatalog argument is deprecated in favor of the visitSummary " 

882 "argument, and will be removed after v26.", 

883 FutureWarning, 

884 stacklevel=find_outside_stacklevel("lsst.ip.diffim"), 

885 ) 

886 visitSummary = finalizedPsfApCorrCatalog 

887 

888 self._prepareInputs(template, science, visitSummary=visitSummary) 

889 

890 # TODO: DM-37212 we need to mirror the kernel in order to get correct cross correlation 

891 scienceKernel = science.psf.getKernel() 

892 matchedScience = self._convolveExposure(science, scienceKernel, self.convolutionControl, 

893 interpolateBadMaskPlanes=True) 

894 selectSources = self._sourceSelector(sources, matchedScience.mask) 

895 self.metadata.add("convolvedExposure", "Preconvolution") 

896 

897 subtractResults = self.runPreconvolve(template, science, matchedScience, selectSources, scienceKernel) 

898 

899 return subtractResults 

900 

901 def runPreconvolve(self, template, science, matchedScience, selectSources, preConvKernel): 

902 """Convolve the science image with its own PSF, then convolve the 

903 template with a matching kernel and subtract to form the Score 

904 exposure. 

905 

906 Parameters 

907 ---------- 

908 template : `lsst.afw.image.ExposureF` 

909 Template exposure, warped to match the science exposure. 

910 science : `lsst.afw.image.ExposureF` 

911 Science exposure to subtract from the template. 

912 matchedScience : `lsst.afw.image.ExposureF` 

913 The science exposure, convolved with the reflection of its own PSF. 

914 selectSources : `lsst.afw.table.SourceCatalog` 

915 Identified sources on the science exposure. This catalog is used to 

916 select sources in order to perform the AL PSF matching on stamp 

917 images around them. 

918 preConvKernel : `lsst.afw.math.Kernel` 

919 The reflection of the kernel that was used to preconvolve the 

920 `science` exposure. Must be normalized to sum to 1. 

921 

922 Returns 

923 ------- 

924 results : `lsst.pipe.base.Struct` 

925 

926 ``scoreExposure`` : `lsst.afw.image.ExposureF` 

927 Result of subtracting the convolved template and science 

928 images. Attached PSF is that of the original science image. 

929 ``matchedTemplate`` : `lsst.afw.image.ExposureF` 

930 Warped and PSF-matched template exposure. Attached PSF is that 

931 of the original science image. 

932 ``matchedScience`` : `lsst.afw.image.ExposureF` 

933 The science exposure after convolving with its own PSF. 

934 Attached PSF is that of the original science image. 

935 ``backgroundModel`` : `lsst.afw.math.Function2D` 

936 Background model that was fit while solving for the 

937 PSF-matching kernel 

938 ``psfMatchingKernel`` : `lsst.afw.math.Kernel` 

939 Final kernel used to PSF-match the template to the science 

940 image. 

941 """ 

942 bbox = science.getBBox() 

943 innerBBox = preConvKernel.shrinkBBox(bbox) 

944 

945 kernelSources = self.makeKernel.selectKernelSources(template[innerBBox], matchedScience[innerBBox], 

946 candidateList=selectSources, 

947 preconvolved=True) 

948 kernelResult = self.makeKernel.run(template[innerBBox], matchedScience[innerBBox], kernelSources, 

949 preconvolved=True) 

950 

951 matchedTemplate = self._convolveExposure(template, kernelResult.psfMatchingKernel, 

952 self.convolutionControl, 

953 bbox=bbox, 

954 psf=science.psf, 

955 interpolateBadMaskPlanes=True, 

956 photoCalib=science.photoCalib) 

957 score = _subtractImages(matchedScience, matchedTemplate, 

958 backgroundModel=(kernelResult.backgroundModel 

959 if self.config.doSubtractBackground else None)) 

960 correctedScore = self.finalize(template[bbox], science, score, 

961 kernelResult.psfMatchingKernel, 

962 templateMatched=True, preConvMode=True, 

963 preConvKernel=preConvKernel) 

964 

965 return lsst.pipe.base.Struct(scoreExposure=correctedScore, 

966 matchedTemplate=matchedTemplate, 

967 matchedScience=matchedScience, 

968 backgroundModel=kernelResult.backgroundModel, 

969 psfMatchingKernel=kernelResult.psfMatchingKernel) 

970 

971 

972def checkTemplateIsSufficient(templateExposure, logger, requiredTemplateFraction=0.): 

973 """Raise NoWorkFound if template coverage < requiredTemplateFraction 

974 

975 Parameters 

976 ---------- 

977 templateExposure : `lsst.afw.image.ExposureF` 

978 The template exposure to check 

979 logger : `lsst.log.Log` 

980 Logger for printing output. 

981 requiredTemplateFraction : `float`, optional 

982 Fraction of pixels of the science image required to have coverage 

983 in the template. 

984 

985 Raises 

986 ------ 

987 lsst.pipe.base.NoWorkFound 

988 Raised if fraction of good pixels, defined as not having NO_DATA 

989 set, is less then the configured requiredTemplateFraction 

990 """ 

991 # Count the number of pixels with the NO_DATA mask bit set 

992 # counting NaN pixels is insufficient because pixels without data are often intepolated over) 

993 pixNoData = np.count_nonzero(templateExposure.mask.array 

994 & templateExposure.mask.getPlaneBitMask('NO_DATA')) 

995 pixGood = templateExposure.getBBox().getArea() - pixNoData 

996 logger.info("template has %d good pixels (%.1f%%)", pixGood, 

997 100*pixGood/templateExposure.getBBox().getArea()) 

998 

999 if pixGood/templateExposure.getBBox().getArea() < requiredTemplateFraction: 

1000 message = ("Insufficient Template Coverage. (%.1f%% < %.1f%%) Not attempting subtraction. " 

1001 "To force subtraction, set config requiredTemplateFraction=0." % ( 

1002 100*pixGood/templateExposure.getBBox().getArea(), 

1003 100*requiredTemplateFraction)) 

1004 raise lsst.pipe.base.NoWorkFound(message) 

1005 

1006 

1007def _subtractImages(science, template, backgroundModel=None): 

1008 """Subtract template from science, propagating relevant metadata. 

1009 

1010 Parameters 

1011 ---------- 

1012 science : `lsst.afw.Exposure` 

1013 The input science image. 

1014 template : `lsst.afw.Exposure` 

1015 The template to subtract from the science image. 

1016 backgroundModel : `lsst.afw.MaskedImage`, optional 

1017 Differential background model 

1018 

1019 Returns 

1020 ------- 

1021 difference : `lsst.afw.Exposure` 

1022 The subtracted image. 

1023 """ 

1024 difference = science.clone() 

1025 if backgroundModel is not None: 

1026 difference.maskedImage -= backgroundModel 

1027 difference.maskedImage -= template.maskedImage 

1028 return difference 

1029 

1030 

1031def _shapeTest(exp1, exp2, fwhmExposureBuffer, fwhmExposureGrid): 

1032 """Determine that the PSF of ``exp1`` is not wider than that of ``exp2``. 

1033 

1034 Parameters 

1035 ---------- 

1036 exp1 : `~lsst.afw.image.Exposure` 

1037 Exposure with the reference point spread function (PSF) to evaluate. 

1038 exp2 : `~lsst.afw.image.Exposure` 

1039 Exposure with a candidate point spread function (PSF) to evaluate. 

1040 fwhmExposureBuffer : `float` 

1041 Fractional buffer margin to be left out of all sides of the image 

1042 during the construction of the grid to compute mean PSF FWHM in an 

1043 exposure, if the PSF is not available at its average position. 

1044 fwhmExposureGrid : `int` 

1045 Grid size to compute the mean FWHM in an exposure, if the PSF is not 

1046 available at its average position. 

1047 Returns 

1048 ------- 

1049 result : `bool` 

1050 True if ``exp1`` has a PSF that is not wider than that of ``exp2`` in 

1051 either dimension. 

1052 """ 

1053 try: 

1054 shape1 = getPsfFwhm(exp1.psf, average=False) 

1055 shape2 = getPsfFwhm(exp2.psf, average=False) 

1056 except InvalidParameterError: 

1057 shape1 = evaluateMeanPsfFwhm(exp1, 

1058 fwhmExposureBuffer=fwhmExposureBuffer, 

1059 fwhmExposureGrid=fwhmExposureGrid 

1060 ) 

1061 shape2 = evaluateMeanPsfFwhm(exp2, 

1062 fwhmExposureBuffer=fwhmExposureBuffer, 

1063 fwhmExposureGrid=fwhmExposureGrid 

1064 ) 

1065 return shape1 <= shape2 

1066 

1067 # Results from getPsfFwhm is a tuple of two values, one for each dimension. 

1068 xTest = shape1[0] <= shape2[0] 

1069 yTest = shape1[1] <= shape2[1] 

1070 return xTest | yTest 

1071 

1072 

1073def _interpolateImage(maskedImage, badMaskPlanes, fallbackValue=None): 

1074 """Replace masked image pixels with interpolated values. 

1075 

1076 Parameters 

1077 ---------- 

1078 maskedImage : `lsst.afw.image.MaskedImage` 

1079 Image on which to perform interpolation. 

1080 badMaskPlanes : `list` of `str` 

1081 List of mask planes to interpolate over. 

1082 fallbackValue : `float`, optional 

1083 Value to set when interpolation fails. 

1084 

1085 Returns 

1086 ------- 

1087 result: `float` 

1088 The number of masked pixels that were replaced. 

1089 """ 

1090 imgBadMaskPlanes = [ 

1091 maskPlane for maskPlane in badMaskPlanes if maskPlane in maskedImage.mask.getMaskPlaneDict() 

1092 ] 

1093 

1094 image = maskedImage.image.array 

1095 badPixels = (maskedImage.mask.array & maskedImage.mask.getPlaneBitMask(imgBadMaskPlanes)) > 0 

1096 image[badPixels] = np.nan 

1097 if fallbackValue is None: 

1098 fallbackValue = np.nanmedian(image) 

1099 # For this initial implementation, skip the interpolation and just fill with 

1100 # the median value. 

1101 image[badPixels] = fallbackValue 

1102 return np.sum(badPixels)