Coverage for python/lsst/ip/diffim/subtractImages.py: 25%

280 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-10-15 10:25 +0000

1# This file is part of ip_diffim. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import warnings 

23 

24import numpy as np 

25 

26import lsst.afw.image 

27import lsst.afw.math 

28import lsst.geom 

29from lsst.utils.introspection import find_outside_stacklevel 

30from lsst.ip.diffim.utils import evaluateMeanPsfFwhm, getPsfFwhm 

31from lsst.meas.algorithms import ScaleVarianceTask 

32import lsst.pex.config 

33import lsst.pipe.base 

34from lsst.pex.exceptions import InvalidParameterError 

35from lsst.pipe.base import connectionTypes 

36from . import MakeKernelTask, DecorrelateALKernelTask 

37from lsst.utils.timer import timeMethod 

38 

39__all__ = ["AlardLuptonSubtractConfig", "AlardLuptonSubtractTask", 

40 "AlardLuptonPreconvolveSubtractConfig", "AlardLuptonPreconvolveSubtractTask"] 

41 

42_dimensions = ("instrument", "visit", "detector") 

43_defaultTemplates = {"coaddName": "deep", "fakesType": ""} 

44 

45 

46class SubtractInputConnections(lsst.pipe.base.PipelineTaskConnections, 

47 dimensions=_dimensions, 

48 defaultTemplates=_defaultTemplates): 

49 template = connectionTypes.Input( 

50 doc="Input warped template to subtract.", 

51 dimensions=("instrument", "visit", "detector"), 

52 storageClass="ExposureF", 

53 name="{fakesType}{coaddName}Diff_templateExp" 

54 ) 

55 science = connectionTypes.Input( 

56 doc="Input science exposure to subtract from.", 

57 dimensions=("instrument", "visit", "detector"), 

58 storageClass="ExposureF", 

59 name="{fakesType}calexp" 

60 ) 

61 sources = connectionTypes.Input( 

62 doc="Sources measured on the science exposure; " 

63 "used to select sources for making the matching kernel.", 

64 dimensions=("instrument", "visit", "detector"), 

65 storageClass="SourceCatalog", 

66 name="{fakesType}src" 

67 ) 

68 finalizedPsfApCorrCatalog = connectionTypes.Input( 

69 doc=("Per-visit finalized psf models and aperture correction maps. " 

70 "These catalogs use the detector id for the catalog id, " 

71 "sorted on id for fast lookup."), 

72 dimensions=("instrument", "visit"), 

73 storageClass="ExposureCatalog", 

74 name="finalVisitSummary", 

75 # TODO: remove on DM-39854. 

76 deprecated=( 

77 "Deprecated in favor of visitSummary. Will be removed after v26." 

78 ) 

79 ) 

80 visitSummary = connectionTypes.Input( 

81 doc=("Per-visit catalog with final calibration objects. " 

82 "These catalogs use the detector id for the catalog id, " 

83 "sorted on id for fast lookup."), 

84 dimensions=("instrument", "visit"), 

85 storageClass="ExposureCatalog", 

86 name="finalVisitSummary", 

87 ) 

88 

89 def __init__(self, *, config=None): 

90 super().__init__(config=config) 

91 if not config.doApplyFinalizedPsf: 

92 self.inputs.remove("finalizedPsfApCorrCatalog") 

93 if not config.doApplyExternalCalibrations or config.doApplyFinalizedPsf: 

94 del self.visitSummary 

95 

96 

97class SubtractImageOutputConnections(lsst.pipe.base.PipelineTaskConnections, 

98 dimensions=_dimensions, 

99 defaultTemplates=_defaultTemplates): 

100 difference = connectionTypes.Output( 

101 doc="Result of subtracting convolved template from science image.", 

102 dimensions=("instrument", "visit", "detector"), 

103 storageClass="ExposureF", 

104 name="{fakesType}{coaddName}Diff_differenceTempExp", 

105 ) 

106 matchedTemplate = connectionTypes.Output( 

107 doc="Warped and PSF-matched template used to create `subtractedExposure`.", 

108 dimensions=("instrument", "visit", "detector"), 

109 storageClass="ExposureF", 

110 name="{fakesType}{coaddName}Diff_matchedExp", 

111 ) 

112 

113 

114class SubtractScoreOutputConnections(lsst.pipe.base.PipelineTaskConnections, 

115 dimensions=_dimensions, 

116 defaultTemplates=_defaultTemplates): 

117 scoreExposure = connectionTypes.Output( 

118 doc="The maximum likelihood image, used for the detection of diaSources.", 

119 dimensions=("instrument", "visit", "detector"), 

120 storageClass="ExposureF", 

121 name="{fakesType}{coaddName}Diff_scoreExp", 

122 ) 

123 

124 

125class AlardLuptonSubtractConnections(SubtractInputConnections, SubtractImageOutputConnections): 

126 pass 

127 

128 

129class AlardLuptonSubtractBaseConfig(lsst.pex.config.Config): 

130 makeKernel = lsst.pex.config.ConfigurableField( 

131 target=MakeKernelTask, 

132 doc="Task to construct a matching kernel for convolution.", 

133 ) 

134 doDecorrelation = lsst.pex.config.Field( 

135 dtype=bool, 

136 default=True, 

137 doc="Perform diffim decorrelation to undo pixel correlation due to A&L " 

138 "kernel convolution? If True, also update the diffim PSF." 

139 ) 

140 decorrelate = lsst.pex.config.ConfigurableField( 

141 target=DecorrelateALKernelTask, 

142 doc="Task to decorrelate the image difference.", 

143 ) 

144 requiredTemplateFraction = lsst.pex.config.Field( 

145 dtype=float, 

146 default=0.1, 

147 doc="Abort task if template covers less than this fraction of pixels." 

148 " Setting to 0 will always attempt image subtraction." 

149 ) 

150 doScaleVariance = lsst.pex.config.Field( 

151 dtype=bool, 

152 default=True, 

153 doc="Scale variance of the image difference?" 

154 ) 

155 scaleVariance = lsst.pex.config.ConfigurableField( 

156 target=ScaleVarianceTask, 

157 doc="Subtask to rescale the variance of the template to the statistically expected level." 

158 ) 

159 doSubtractBackground = lsst.pex.config.Field( 

160 doc="Subtract the background fit when solving the kernel?", 

161 dtype=bool, 

162 default=True, 

163 ) 

164 doApplyFinalizedPsf = lsst.pex.config.Field( 

165 doc="Replace science Exposure's psf and aperture correction map" 

166 " with those in finalizedPsfApCorrCatalog.", 

167 dtype=bool, 

168 default=False, 

169 # TODO: remove on DM-39854. 

170 deprecated=( 

171 "Deprecated in favor of doApplyExternalCalibrations. " 

172 "Will be removed after v26." 

173 ) 

174 ) 

175 doApplyExternalCalibrations = lsst.pex.config.Field( 

176 doc=( 

177 "Replace science Exposure's calibration objects with those" 

178 " in visitSummary. Ignored if `doApplyFinalizedPsf is True." 

179 ), 

180 dtype=bool, 

181 default=False, 

182 ) 

183 detectionThreshold = lsst.pex.config.Field( 

184 dtype=float, 

185 default=10, 

186 doc="Minimum signal to noise ratio of detected sources " 

187 "to use for calculating the PSF matching kernel." 

188 ) 

189 badSourceFlags = lsst.pex.config.ListField( 

190 dtype=str, 

191 doc="Flags that, if set, the associated source should not " 

192 "be used to determine the PSF matching kernel.", 

193 default=("sky_source", "slot_Centroid_flag", 

194 "slot_ApFlux_flag", "slot_PsfFlux_flag", ), 

195 ) 

196 badMaskPlanes = lsst.pex.config.ListField( 

197 dtype=str, 

198 default=("NO_DATA", "BAD", "SAT", "EDGE"), 

199 doc="Mask planes to exclude when selecting sources for PSF matching." 

200 ) 

201 preserveTemplateMask = lsst.pex.config.ListField( 

202 dtype=str, 

203 default=("NO_DATA", "BAD", "SAT", "INJECTED", "INJECTED_CORE"), 

204 doc="Mask planes from the template to propagate to the image difference." 

205 ) 

206 

207 def setDefaults(self): 

208 self.makeKernel.kernel.name = "AL" 

209 self.makeKernel.kernel.active.fitForBackground = self.doSubtractBackground 

210 self.makeKernel.kernel.active.spatialKernelOrder = 1 

211 self.makeKernel.kernel.active.spatialBgOrder = 2 

212 

213 

214class AlardLuptonSubtractConfig(AlardLuptonSubtractBaseConfig, lsst.pipe.base.PipelineTaskConfig, 

215 pipelineConnections=AlardLuptonSubtractConnections): 

216 mode = lsst.pex.config.ChoiceField( 

217 dtype=str, 

218 default="convolveTemplate", 

219 allowed={"auto": "Choose which image to convolve at runtime.", 

220 "convolveScience": "Only convolve the science image.", 

221 "convolveTemplate": "Only convolve the template image."}, 

222 doc="Choose which image to convolve at runtime, or require that a specific image is convolved." 

223 ) 

224 

225 

226class AlardLuptonSubtractTask(lsst.pipe.base.PipelineTask): 

227 """Compute the image difference of a science and template image using 

228 the Alard & Lupton (1998) algorithm. 

229 """ 

230 ConfigClass = AlardLuptonSubtractConfig 

231 _DefaultName = "alardLuptonSubtract" 

232 

233 def __init__(self, **kwargs): 

234 super().__init__(**kwargs) 

235 self.makeSubtask("decorrelate") 

236 self.makeSubtask("makeKernel") 

237 if self.config.doScaleVariance: 

238 self.makeSubtask("scaleVariance") 

239 

240 self.convolutionControl = lsst.afw.math.ConvolutionControl() 

241 # Normalization is an extra, unnecessary, calculation and will result 

242 # in mis-subtraction of the images if there are calibration errors. 

243 self.convolutionControl.setDoNormalize(False) 

244 self.convolutionControl.setDoCopyEdge(True) 

245 

246 def _applyExternalCalibrations(self, exposure, visitSummary): 

247 """Replace calibrations (psf, and ApCorrMap) on this exposure with 

248 external ones.". 

249 

250 Parameters 

251 ---------- 

252 exposure : `lsst.afw.image.exposure.Exposure` 

253 Input exposure to adjust calibrations. 

254 visitSummary : `lsst.afw.table.ExposureCatalog` 

255 Exposure catalog with external calibrations to be applied. Catalog 

256 uses the detector id for the catalog id, sorted on id for fast 

257 lookup. 

258 

259 Returns 

260 ------- 

261 exposure : `lsst.afw.image.exposure.Exposure` 

262 Exposure with adjusted calibrations. 

263 """ 

264 detectorId = exposure.info.getDetector().getId() 

265 

266 row = visitSummary.find(detectorId) 

267 if row is None: 

268 self.log.warning("Detector id %s not found in external calibrations catalog; " 

269 "Using original calibrations.", detectorId) 

270 else: 

271 psf = row.getPsf() 

272 apCorrMap = row.getApCorrMap() 

273 if psf is None: 

274 self.log.warning("Detector id %s has None for psf in " 

275 "external calibrations catalog; Using original psf and aperture correction.", 

276 detectorId) 

277 elif apCorrMap is None: 

278 self.log.warning("Detector id %s has None for apCorrMap in " 

279 "external calibrations catalog; Using original psf and aperture correction.", 

280 detectorId) 

281 else: 

282 exposure.setPsf(psf) 

283 exposure.info.setApCorrMap(apCorrMap) 

284 

285 return exposure 

286 

287 @timeMethod 

288 def run(self, template, science, sources, finalizedPsfApCorrCatalog=None, 

289 visitSummary=None): 

290 """PSF match, subtract, and decorrelate two images. 

291 

292 Parameters 

293 ---------- 

294 template : `lsst.afw.image.ExposureF` 

295 Template exposure, warped to match the science exposure. 

296 science : `lsst.afw.image.ExposureF` 

297 Science exposure to subtract from the template. 

298 sources : `lsst.afw.table.SourceCatalog` 

299 Identified sources on the science exposure. This catalog is used to 

300 select sources in order to perform the AL PSF matching on stamp 

301 images around them. 

302 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional 

303 Exposure catalog with finalized psf models and aperture correction 

304 maps to be applied. Catalog uses the detector id for the catalog 

305 id, sorted on id for fast lookup. Deprecated in favor of 

306 ``visitSummary``, and will be removed after v26. 

307 visitSummary : `lsst.afw.table.ExposureCatalog`, optional 

308 Exposure catalog with external calibrations to be applied. Catalog 

309 uses the detector id for the catalog id, sorted on id for fast 

310 lookup. Ignored (for temporary backwards compatibility) if 

311 ``finalizedPsfApCorrCatalog`` is provided. 

312 

313 Returns 

314 ------- 

315 results : `lsst.pipe.base.Struct` 

316 ``difference`` : `lsst.afw.image.ExposureF` 

317 Result of subtracting template and science. 

318 ``matchedTemplate`` : `lsst.afw.image.ExposureF` 

319 Warped and PSF-matched template exposure. 

320 ``backgroundModel`` : `lsst.afw.math.Function2D` 

321 Background model that was fit while solving for the 

322 PSF-matching kernel 

323 ``psfMatchingKernel`` : `lsst.afw.math.Kernel` 

324 Kernel used to PSF-match the convolved image. 

325 

326 Raises 

327 ------ 

328 RuntimeError 

329 If an unsupported convolution mode is supplied. 

330 RuntimeError 

331 If there are too few sources to calculate the PSF matching kernel. 

332 lsst.pipe.base.NoWorkFound 

333 Raised if fraction of good pixels, defined as not having NO_DATA 

334 set, is less then the configured requiredTemplateFraction 

335 """ 

336 

337 if finalizedPsfApCorrCatalog is not None: 

338 warnings.warn( 

339 "The finalizedPsfApCorrCatalog argument is deprecated in favor of the visitSummary " 

340 "argument, and will be removed after v26.", 

341 FutureWarning, 

342 stacklevel=find_outside_stacklevel("lsst.ip.diffim"), 

343 ) 

344 visitSummary = finalizedPsfApCorrCatalog 

345 

346 self._prepareInputs(template, science, visitSummary=visitSummary) 

347 

348 # In the event that getPsfFwhm fails, evaluate the PSF on a grid. 

349 fwhmExposureBuffer = self.config.makeKernel.fwhmExposureBuffer 

350 fwhmExposureGrid = self.config.makeKernel.fwhmExposureGrid 

351 

352 # Calling getPsfFwhm on template.psf fails on some rare occasions when 

353 # the template has no input exposures at the average position of the 

354 # stars. So we try getPsfFwhm first on template, and if that fails we 

355 # evaluate the PSF on a grid specified by fwhmExposure* fields. 

356 # To keep consistent definitions for PSF size on the template and 

357 # science images, we use the same method for both. 

358 try: 

359 templatePsfSize = getPsfFwhm(template.psf) 

360 sciencePsfSize = getPsfFwhm(science.psf) 

361 except InvalidParameterError: 

362 self.log.info("Unable to evaluate PSF at the average position. " 

363 "Evaluting PSF on a grid of points." 

364 ) 

365 templatePsfSize = evaluateMeanPsfFwhm(template, 

366 fwhmExposureBuffer=fwhmExposureBuffer, 

367 fwhmExposureGrid=fwhmExposureGrid 

368 ) 

369 sciencePsfSize = evaluateMeanPsfFwhm(science, 

370 fwhmExposureBuffer=fwhmExposureBuffer, 

371 fwhmExposureGrid=fwhmExposureGrid 

372 ) 

373 self.log.info("Science PSF FWHM: %f pixels", sciencePsfSize) 

374 self.log.info("Template PSF FWHM: %f pixels", templatePsfSize) 

375 self.metadata.add("sciencePsfSize", sciencePsfSize) 

376 self.metadata.add("templatePsfSize", templatePsfSize) 

377 selectSources = self._sourceSelector(sources, science.mask) 

378 

379 if self.config.mode == "auto": 

380 convolveTemplate = _shapeTest(template, 

381 science, 

382 fwhmExposureBuffer=fwhmExposureBuffer, 

383 fwhmExposureGrid=fwhmExposureGrid) 

384 if convolveTemplate: 

385 if sciencePsfSize < templatePsfSize: 

386 self.log.info("Average template PSF size is greater, " 

387 "but science PSF greater in one dimension: convolving template image.") 

388 else: 

389 self.log.info("Science PSF size is greater: convolving template image.") 

390 else: 

391 self.log.info("Template PSF size is greater: convolving science image.") 

392 elif self.config.mode == "convolveTemplate": 

393 self.log.info("`convolveTemplate` is set: convolving template image.") 

394 convolveTemplate = True 

395 elif self.config.mode == "convolveScience": 

396 self.log.info("`convolveScience` is set: convolving science image.") 

397 convolveTemplate = False 

398 else: 

399 raise RuntimeError("Cannot handle AlardLuptonSubtract mode: %s", self.config.mode) 

400 

401 if convolveTemplate: 

402 self.metadata.add("convolvedExposure", "Template") 

403 subtractResults = self.runConvolveTemplate(template, science, selectSources) 

404 else: 

405 self.metadata.add("convolvedExposure", "Science") 

406 subtractResults = self.runConvolveScience(template, science, selectSources) 

407 

408 return subtractResults 

409 

410 def runConvolveTemplate(self, template, science, selectSources): 

411 """Convolve the template image with a PSF-matching kernel and subtract 

412 from the science image. 

413 

414 Parameters 

415 ---------- 

416 template : `lsst.afw.image.ExposureF` 

417 Template exposure, warped to match the science exposure. 

418 science : `lsst.afw.image.ExposureF` 

419 Science exposure to subtract from the template. 

420 selectSources : `lsst.afw.table.SourceCatalog` 

421 Identified sources on the science exposure. This catalog is used to 

422 select sources in order to perform the AL PSF matching on stamp 

423 images around them. 

424 

425 Returns 

426 ------- 

427 results : `lsst.pipe.base.Struct` 

428 

429 ``difference`` : `lsst.afw.image.ExposureF` 

430 Result of subtracting template and science. 

431 ``matchedTemplate`` : `lsst.afw.image.ExposureF` 

432 Warped and PSF-matched template exposure. 

433 ``backgroundModel`` : `lsst.afw.math.Function2D` 

434 Background model that was fit while solving for the PSF-matching kernel 

435 ``psfMatchingKernel`` : `lsst.afw.math.Kernel` 

436 Kernel used to PSF-match the template to the science image. 

437 """ 

438 kernelSources = self.makeKernel.selectKernelSources(template, science, 

439 candidateList=selectSources, 

440 preconvolved=False) 

441 kernelResult = self.makeKernel.run(template, science, kernelSources, 

442 preconvolved=False) 

443 

444 matchedTemplate = self._convolveExposure(template, kernelResult.psfMatchingKernel, 

445 self.convolutionControl, 

446 bbox=science.getBBox(), 

447 psf=science.psf, 

448 photoCalib=science.photoCalib) 

449 

450 difference = _subtractImages(science, matchedTemplate, 

451 backgroundModel=(kernelResult.backgroundModel 

452 if self.config.doSubtractBackground else None)) 

453 correctedExposure = self.finalize(template, science, difference, 

454 kernelResult.psfMatchingKernel, 

455 templateMatched=True) 

456 

457 return lsst.pipe.base.Struct(difference=correctedExposure, 

458 matchedTemplate=matchedTemplate, 

459 matchedScience=science, 

460 backgroundModel=kernelResult.backgroundModel, 

461 psfMatchingKernel=kernelResult.psfMatchingKernel) 

462 

463 def runConvolveScience(self, template, science, selectSources): 

464 """Convolve the science image with a PSF-matching kernel and subtract the template image. 

465 

466 Parameters 

467 ---------- 

468 template : `lsst.afw.image.ExposureF` 

469 Template exposure, warped to match the science exposure. 

470 science : `lsst.afw.image.ExposureF` 

471 Science exposure to subtract from the template. 

472 selectSources : `lsst.afw.table.SourceCatalog` 

473 Identified sources on the science exposure. This catalog is used to 

474 select sources in order to perform the AL PSF matching on stamp 

475 images around them. 

476 

477 Returns 

478 ------- 

479 results : `lsst.pipe.base.Struct` 

480 

481 ``difference`` : `lsst.afw.image.ExposureF` 

482 Result of subtracting template and science. 

483 ``matchedTemplate`` : `lsst.afw.image.ExposureF` 

484 Warped template exposure. Note that in this case, the template 

485 is not PSF-matched to the science image. 

486 ``backgroundModel`` : `lsst.afw.math.Function2D` 

487 Background model that was fit while solving for the PSF-matching kernel 

488 ``psfMatchingKernel`` : `lsst.afw.math.Kernel` 

489 Kernel used to PSF-match the science image to the template. 

490 """ 

491 bbox = science.getBBox() 

492 kernelSources = self.makeKernel.selectKernelSources(science, template, 

493 candidateList=selectSources, 

494 preconvolved=False) 

495 kernelResult = self.makeKernel.run(science, template, kernelSources, 

496 preconvolved=False) 

497 modelParams = kernelResult.backgroundModel.getParameters() 

498 # We must invert the background model if the matching kernel is solved for the science image. 

499 kernelResult.backgroundModel.setParameters([-p for p in modelParams]) 

500 

501 kernelImage = lsst.afw.image.ImageD(kernelResult.psfMatchingKernel.getDimensions()) 

502 norm = kernelResult.psfMatchingKernel.computeImage(kernelImage, doNormalize=False) 

503 

504 matchedScience = self._convolveExposure(science, kernelResult.psfMatchingKernel, 

505 self.convolutionControl, 

506 psf=template.psf) 

507 

508 # Place back on native photometric scale 

509 matchedScience.maskedImage /= norm 

510 matchedTemplate = template.clone()[bbox] 

511 matchedTemplate.maskedImage /= norm 

512 matchedTemplate.setPhotoCalib(science.photoCalib) 

513 

514 difference = _subtractImages(matchedScience, matchedTemplate, 

515 backgroundModel=(kernelResult.backgroundModel 

516 if self.config.doSubtractBackground else None)) 

517 

518 correctedExposure = self.finalize(template, science, difference, 

519 kernelResult.psfMatchingKernel, 

520 templateMatched=False) 

521 

522 return lsst.pipe.base.Struct(difference=correctedExposure, 

523 matchedTemplate=matchedTemplate, 

524 matchedScience=matchedScience, 

525 backgroundModel=kernelResult.backgroundModel, 

526 psfMatchingKernel=kernelResult.psfMatchingKernel,) 

527 

528 def finalize(self, template, science, difference, kernel, 

529 templateMatched=True, 

530 preConvMode=False, 

531 preConvKernel=None, 

532 spatiallyVarying=False): 

533 """Decorrelate the difference image to undo the noise correlations 

534 caused by convolution. 

535 

536 Parameters 

537 ---------- 

538 template : `lsst.afw.image.ExposureF` 

539 Template exposure, warped to match the science exposure. 

540 science : `lsst.afw.image.ExposureF` 

541 Science exposure to subtract from the template. 

542 difference : `lsst.afw.image.ExposureF` 

543 Result of subtracting template and science. 

544 kernel : `lsst.afw.math.Kernel` 

545 An (optionally spatially-varying) PSF matching kernel 

546 templateMatched : `bool`, optional 

547 Was the template PSF-matched to the science image? 

548 preConvMode : `bool`, optional 

549 Was the science image preconvolved with its own PSF 

550 before PSF matching the template? 

551 preConvKernel : `lsst.afw.detection.Psf`, optional 

552 If not `None`, then the science image was pre-convolved with 

553 (the reflection of) this kernel. Must be normalized to sum to 1. 

554 spatiallyVarying : `bool`, optional 

555 Compute the decorrelation kernel spatially varying across the image? 

556 

557 Returns 

558 ------- 

559 correctedExposure : `lsst.afw.image.ExposureF` 

560 The decorrelated image difference. 

561 """ 

562 # Erase existing detection mask planes. 

563 # We don't want the detection mask from the science image 

564 mask = difference.mask 

565 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) 

566 

567 # We have cleared the template mask plane, so copy the mask plane of 

568 # the image difference so that we can calculate correct statistics 

569 # during decorrelation. Do this regardless of whether decorrelation is 

570 # used for consistency. 

571 template[science.getBBox()].mask.array[...] = difference.mask.array[...] 

572 if self.config.doDecorrelation: 

573 self.log.info("Decorrelating image difference.") 

574 # We have cleared the template mask plane, so copy the mask plane of 

575 # the image difference so that we can calculate correct statistics 

576 # during decorrelation 

577 template[science.getBBox()].mask.array[...] = difference.mask.array[...] 

578 correctedExposure = self.decorrelate.run(science, template[science.getBBox()], difference, kernel, 

579 templateMatched=templateMatched, 

580 preConvMode=preConvMode, 

581 preConvKernel=preConvKernel, 

582 spatiallyVarying=spatiallyVarying).correctedExposure 

583 else: 

584 self.log.info("NOT decorrelating image difference.") 

585 correctedExposure = difference 

586 return correctedExposure 

587 

588 @staticmethod 

589 def _validateExposures(template, science): 

590 """Check that the WCS of the two Exposures match, and the template bbox 

591 contains the science bbox. 

592 

593 Parameters 

594 ---------- 

595 template : `lsst.afw.image.ExposureF` 

596 Template exposure, warped to match the science exposure. 

597 science : `lsst.afw.image.ExposureF` 

598 Science exposure to subtract from the template. 

599 

600 Raises 

601 ------ 

602 AssertionError 

603 Raised if the WCS of the template is not equal to the science WCS, 

604 or if the science image is not fully contained in the template 

605 bounding box. 

606 """ 

607 assert template.wcs == science.wcs,\ 

608 "Template and science exposure WCS are not identical." 

609 templateBBox = template.getBBox() 

610 scienceBBox = science.getBBox() 

611 

612 assert templateBBox.contains(scienceBBox),\ 

613 "Template bbox does not contain all of the science image." 

614 

615 def _convolveExposure(self, exposure, kernel, convolutionControl, 

616 bbox=None, 

617 psf=None, 

618 photoCalib=None, 

619 interpolateBadMaskPlanes=False, 

620 ): 

621 """Convolve an exposure with the given kernel. 

622 

623 Parameters 

624 ---------- 

625 exposure : `lsst.afw.Exposure` 

626 exposure to convolve. 

627 kernel : `lsst.afw.math.LinearCombinationKernel` 

628 PSF matching kernel computed in the ``makeKernel`` subtask. 

629 convolutionControl : `lsst.afw.math.ConvolutionControl` 

630 Configuration for convolve algorithm. 

631 bbox : `lsst.geom.Box2I`, optional 

632 Bounding box to trim the convolved exposure to. 

633 psf : `lsst.afw.detection.Psf`, optional 

634 Point spread function (PSF) to set for the convolved exposure. 

635 photoCalib : `lsst.afw.image.PhotoCalib`, optional 

636 Photometric calibration of the convolved exposure. 

637 

638 Returns 

639 ------- 

640 convolvedExp : `lsst.afw.Exposure` 

641 The convolved image. 

642 """ 

643 convolvedExposure = exposure.clone() 

644 if psf is not None: 

645 convolvedExposure.setPsf(psf) 

646 if photoCalib is not None: 

647 convolvedExposure.setPhotoCalib(photoCalib) 

648 if interpolateBadMaskPlanes and self.config.badMaskPlanes is not None: 

649 nInterp = _interpolateImage(convolvedExposure.maskedImage, 

650 self.config.badMaskPlanes) 

651 self.metadata.add("nInterpolated", nInterp) 

652 convolvedImage = lsst.afw.image.MaskedImageF(convolvedExposure.getBBox()) 

653 lsst.afw.math.convolve(convolvedImage, convolvedExposure.maskedImage, kernel, convolutionControl) 

654 convolvedExposure.setMaskedImage(convolvedImage) 

655 if bbox is None: 

656 return convolvedExposure 

657 else: 

658 return convolvedExposure[bbox] 

659 

660 def _sourceSelector(self, sources, mask): 

661 """Select sources from a catalog that meet the selection criteria. 

662 

663 Parameters 

664 ---------- 

665 sources : `lsst.afw.table.SourceCatalog` 

666 Input source catalog to select sources from. 

667 mask : `lsst.afw.image.Mask` 

668 The image mask plane to use to reject sources 

669 based on their location on the ccd. 

670 

671 Returns 

672 ------- 

673 selectSources : `lsst.afw.table.SourceCatalog` 

674 The input source catalog, with flagged and low signal-to-noise 

675 sources removed. 

676 

677 Raises 

678 ------ 

679 RuntimeError 

680 If there are too few sources to compute the PSF matching kernel 

681 remaining after source selection. 

682 """ 

683 flags = np.ones(len(sources), dtype=bool) 

684 for flag in self.config.badSourceFlags: 

685 try: 

686 flags *= ~sources[flag] 

687 except Exception as e: 

688 self.log.warning("Could not apply source flag: %s", e) 

689 sToNFlag = (sources.getPsfInstFlux()/sources.getPsfInstFluxErr()) > self.config.detectionThreshold 

690 flags *= sToNFlag 

691 flags *= self._checkMask(mask, sources, self.config.badMaskPlanes) 

692 selectSources = sources[flags] 

693 self.log.info("%i/%i=%.1f%% of sources selected for PSF matching from the input catalog", 

694 len(selectSources), len(sources), 100*len(selectSources)/len(sources)) 

695 if len(selectSources) < self.config.makeKernel.nStarPerCell: 

696 self.log.error("Too few sources to calculate the PSF matching kernel: " 

697 "%i selected but %i needed for the calculation.", 

698 len(selectSources), self.config.makeKernel.nStarPerCell) 

699 raise RuntimeError("Cannot compute PSF matching kernel: too few sources selected.") 

700 self.metadata.add("nPsfSources", len(selectSources)) 

701 

702 return selectSources.copy(deep=True) 

703 

704 @staticmethod 

705 def _checkMask(mask, sources, badMaskPlanes): 

706 """Exclude sources that are located on masked pixels. 

707 

708 Parameters 

709 ---------- 

710 mask : `lsst.afw.image.Mask` 

711 The image mask plane to use to reject sources 

712 based on the location of their centroid on the ccd. 

713 sources : `lsst.afw.table.SourceCatalog` 

714 The source catalog to evaluate. 

715 badMaskPlanes : `list` of `str` 

716 List of the names of the mask planes to exclude. 

717 

718 Returns 

719 ------- 

720 flags : `numpy.ndarray` of `bool` 

721 Array indicating whether each source in the catalog should be 

722 kept (True) or rejected (False) based on the value of the 

723 mask plane at its location. 

724 """ 

725 badPixelMask = lsst.afw.image.Mask.getPlaneBitMask(badMaskPlanes) 

726 xv = np.rint(sources.getX() - mask.getX0()) 

727 yv = np.rint(sources.getY() - mask.getY0()) 

728 

729 mv = mask.array[yv.astype(int), xv.astype(int)] 

730 flags = np.bitwise_and(mv, badPixelMask) == 0 

731 return flags 

732 

733 def _prepareInputs(self, template, science, visitSummary=None): 

734 """Perform preparatory calculations common to all Alard&Lupton Tasks. 

735 

736 Parameters 

737 ---------- 

738 template : `lsst.afw.image.ExposureF` 

739 Template exposure, warped to match the science exposure. The 

740 variance plane of the template image is modified in place. 

741 science : `lsst.afw.image.ExposureF` 

742 Science exposure to subtract from the template. The variance plane 

743 of the science image is modified in place. 

744 visitSummary : `lsst.afw.table.ExposureCatalog`, optional 

745 Exposure catalog with external calibrations to be applied. Catalog 

746 uses the detector id for the catalog id, sorted on id for fast 

747 lookup. 

748 """ 

749 self._validateExposures(template, science) 

750 if visitSummary is not None: 

751 self._applyExternalCalibrations(science, visitSummary=visitSummary) 

752 checkTemplateIsSufficient(template, self.log, 

753 requiredTemplateFraction=self.config.requiredTemplateFraction) 

754 

755 if self.config.doScaleVariance: 

756 # Scale the variance of the template and science images before 

757 # convolution, subtraction, or decorrelation so that they have the 

758 # correct ratio. 

759 templateVarFactor = self.scaleVariance.run(template.maskedImage) 

760 sciVarFactor = self.scaleVariance.run(science.maskedImage) 

761 self.log.info("Template variance scaling factor: %.2f", templateVarFactor) 

762 self.metadata.add("scaleTemplateVarianceFactor", templateVarFactor) 

763 self.log.info("Science variance scaling factor: %.2f", sciVarFactor) 

764 self.metadata.add("scaleScienceVarianceFactor", sciVarFactor) 

765 self._clearMask(template) 

766 

767 def _clearMask(self, template): 

768 """Clear the mask plane of the template. 

769 

770 Parameters 

771 ---------- 

772 template : `lsst.afw.image.ExposureF` 

773 Template exposure, warped to match the science exposure. 

774 The mask plane will be modified in place. 

775 """ 

776 mask = template.mask 

777 clearMaskPlanes = [maskplane for maskplane in mask.getMaskPlaneDict().keys() 

778 if maskplane not in self.config.preserveTemplateMask] 

779 

780 bitMaskToClear = mask.getPlaneBitMask(clearMaskPlanes) 

781 mask &= ~bitMaskToClear 

782 

783 

784class AlardLuptonPreconvolveSubtractConnections(SubtractInputConnections, 

785 SubtractScoreOutputConnections): 

786 pass 

787 

788 

789class AlardLuptonPreconvolveSubtractConfig(AlardLuptonSubtractBaseConfig, lsst.pipe.base.PipelineTaskConfig, 

790 pipelineConnections=AlardLuptonPreconvolveSubtractConnections): 

791 pass 

792 

793 

794class AlardLuptonPreconvolveSubtractTask(AlardLuptonSubtractTask): 

795 """Subtract a template from a science image, convolving the science image 

796 before computing the kernel, and also convolving the template before 

797 subtraction. 

798 """ 

799 ConfigClass = AlardLuptonPreconvolveSubtractConfig 

800 _DefaultName = "alardLuptonPreconvolveSubtract" 

801 

802 def run(self, template, science, sources, finalizedPsfApCorrCatalog=None, visitSummary=None): 

803 """Preconvolve the science image with its own PSF, 

804 convolve the template image with a PSF-matching kernel and subtract 

805 from the preconvolved science image. 

806 

807 Parameters 

808 ---------- 

809 template : `lsst.afw.image.ExposureF` 

810 The template image, which has previously been warped to the science 

811 image. The template bbox will be padded by a few pixels compared to 

812 the science bbox. 

813 science : `lsst.afw.image.ExposureF` 

814 The science exposure. 

815 sources : `lsst.afw.table.SourceCatalog` 

816 Identified sources on the science exposure. This catalog is used to 

817 select sources in order to perform the AL PSF matching on stamp 

818 images around them. 

819 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional 

820 Exposure catalog with finalized psf models and aperture correction 

821 maps to be applied. Catalog uses the detector id for the catalog 

822 id, sorted on id for fast lookup. Deprecated in favor of 

823 ``visitSummary``, and will be removed after v26. 

824 visitSummary : `lsst.afw.table.ExposureCatalog`, optional 

825 Exposure catalog with complete external calibrations. Catalog uses 

826 the detector id for the catalog id, sorted on id for fast lookup. 

827 Ignored (for temporary backwards compatibility) if 

828 ``finalizedPsfApCorrCatalog`` is provided. 

829 

830 Returns 

831 ------- 

832 results : `lsst.pipe.base.Struct` 

833 ``scoreExposure`` : `lsst.afw.image.ExposureF` 

834 Result of subtracting the convolved template and science 

835 images. Attached PSF is that of the original science image. 

836 ``matchedTemplate`` : `lsst.afw.image.ExposureF` 

837 Warped and PSF-matched template exposure. Attached PSF is that 

838 of the original science image. 

839 ``matchedScience`` : `lsst.afw.image.ExposureF` 

840 The science exposure after convolving with its own PSF. 

841 Attached PSF is that of the original science image. 

842 ``backgroundModel`` : `lsst.afw.math.Function2D` 

843 Background model that was fit while solving for the 

844 PSF-matching kernel 

845 ``psfMatchingKernel`` : `lsst.afw.math.Kernel` 

846 Final kernel used to PSF-match the template to the science 

847 image. 

848 """ 

849 if finalizedPsfApCorrCatalog is not None: 

850 warnings.warn( 

851 "The finalizedPsfApCorrCatalog argument is deprecated in favor of the visitSummary " 

852 "argument, and will be removed after v26.", 

853 FutureWarning, 

854 stacklevel=find_outside_stacklevel("lsst.ip.diffim"), 

855 ) 

856 visitSummary = finalizedPsfApCorrCatalog 

857 

858 self._prepareInputs(template, science, visitSummary=visitSummary) 

859 

860 # TODO: DM-37212 we need to mirror the kernel in order to get correct cross correlation 

861 scienceKernel = science.psf.getKernel() 

862 matchedScience = self._convolveExposure(science, scienceKernel, self.convolutionControl, 

863 interpolateBadMaskPlanes=True) 

864 selectSources = self._sourceSelector(sources, matchedScience.mask) 

865 self.metadata.add("convolvedExposure", "Preconvolution") 

866 

867 subtractResults = self.runPreconvolve(template, science, matchedScience, selectSources, scienceKernel) 

868 

869 return subtractResults 

870 

871 def runPreconvolve(self, template, science, matchedScience, selectSources, preConvKernel): 

872 """Convolve the science image with its own PSF, then convolve the 

873 template with a matching kernel and subtract to form the Score 

874 exposure. 

875 

876 Parameters 

877 ---------- 

878 template : `lsst.afw.image.ExposureF` 

879 Template exposure, warped to match the science exposure. 

880 science : `lsst.afw.image.ExposureF` 

881 Science exposure to subtract from the template. 

882 matchedScience : `lsst.afw.image.ExposureF` 

883 The science exposure, convolved with the reflection of its own PSF. 

884 selectSources : `lsst.afw.table.SourceCatalog` 

885 Identified sources on the science exposure. This catalog is used to 

886 select sources in order to perform the AL PSF matching on stamp 

887 images around them. 

888 preConvKernel : `lsst.afw.math.Kernel` 

889 The reflection of the kernel that was used to preconvolve the 

890 `science` exposure. Must be normalized to sum to 1. 

891 

892 Returns 

893 ------- 

894 results : `lsst.pipe.base.Struct` 

895 

896 ``scoreExposure`` : `lsst.afw.image.ExposureF` 

897 Result of subtracting the convolved template and science 

898 images. Attached PSF is that of the original science image. 

899 ``matchedTemplate`` : `lsst.afw.image.ExposureF` 

900 Warped and PSF-matched template exposure. Attached PSF is that 

901 of the original science image. 

902 ``matchedScience`` : `lsst.afw.image.ExposureF` 

903 The science exposure after convolving with its own PSF. 

904 Attached PSF is that of the original science image. 

905 ``backgroundModel`` : `lsst.afw.math.Function2D` 

906 Background model that was fit while solving for the 

907 PSF-matching kernel 

908 ``psfMatchingKernel`` : `lsst.afw.math.Kernel` 

909 Final kernel used to PSF-match the template to the science 

910 image. 

911 """ 

912 bbox = science.getBBox() 

913 innerBBox = preConvKernel.shrinkBBox(bbox) 

914 

915 kernelSources = self.makeKernel.selectKernelSources(template[innerBBox], matchedScience[innerBBox], 

916 candidateList=selectSources, 

917 preconvolved=True) 

918 kernelResult = self.makeKernel.run(template[innerBBox], matchedScience[innerBBox], kernelSources, 

919 preconvolved=True) 

920 

921 matchedTemplate = self._convolveExposure(template, kernelResult.psfMatchingKernel, 

922 self.convolutionControl, 

923 bbox=bbox, 

924 psf=science.psf, 

925 interpolateBadMaskPlanes=True, 

926 photoCalib=science.photoCalib) 

927 score = _subtractImages(matchedScience, matchedTemplate, 

928 backgroundModel=(kernelResult.backgroundModel 

929 if self.config.doSubtractBackground else None)) 

930 correctedScore = self.finalize(template[bbox], science, score, 

931 kernelResult.psfMatchingKernel, 

932 templateMatched=True, preConvMode=True, 

933 preConvKernel=preConvKernel) 

934 

935 return lsst.pipe.base.Struct(scoreExposure=correctedScore, 

936 matchedTemplate=matchedTemplate, 

937 matchedScience=matchedScience, 

938 backgroundModel=kernelResult.backgroundModel, 

939 psfMatchingKernel=kernelResult.psfMatchingKernel) 

940 

941 

942def checkTemplateIsSufficient(templateExposure, logger, requiredTemplateFraction=0.): 

943 """Raise NoWorkFound if template coverage < requiredTemplateFraction 

944 

945 Parameters 

946 ---------- 

947 templateExposure : `lsst.afw.image.ExposureF` 

948 The template exposure to check 

949 logger : `lsst.log.Log` 

950 Logger for printing output. 

951 requiredTemplateFraction : `float`, optional 

952 Fraction of pixels of the science image required to have coverage 

953 in the template. 

954 

955 Raises 

956 ------ 

957 lsst.pipe.base.NoWorkFound 

958 Raised if fraction of good pixels, defined as not having NO_DATA 

959 set, is less then the configured requiredTemplateFraction 

960 """ 

961 # Count the number of pixels with the NO_DATA mask bit set 

962 # counting NaN pixels is insufficient because pixels without data are often intepolated over) 

963 pixNoData = np.count_nonzero(templateExposure.mask.array 

964 & templateExposure.mask.getPlaneBitMask('NO_DATA')) 

965 pixGood = templateExposure.getBBox().getArea() - pixNoData 

966 logger.info("template has %d good pixels (%.1f%%)", pixGood, 

967 100*pixGood/templateExposure.getBBox().getArea()) 

968 

969 if pixGood/templateExposure.getBBox().getArea() < requiredTemplateFraction: 

970 message = ("Insufficient Template Coverage. (%.1f%% < %.1f%%) Not attempting subtraction. " 

971 "To force subtraction, set config requiredTemplateFraction=0." % ( 

972 100*pixGood/templateExposure.getBBox().getArea(), 

973 100*requiredTemplateFraction)) 

974 raise lsst.pipe.base.NoWorkFound(message) 

975 

976 

977def _subtractImages(science, template, backgroundModel=None): 

978 """Subtract template from science, propagating relevant metadata. 

979 

980 Parameters 

981 ---------- 

982 science : `lsst.afw.Exposure` 

983 The input science image. 

984 template : `lsst.afw.Exposure` 

985 The template to subtract from the science image. 

986 backgroundModel : `lsst.afw.MaskedImage`, optional 

987 Differential background model 

988 

989 Returns 

990 ------- 

991 difference : `lsst.afw.Exposure` 

992 The subtracted image. 

993 """ 

994 difference = science.clone() 

995 if backgroundModel is not None: 

996 difference.maskedImage -= backgroundModel 

997 difference.maskedImage -= template.maskedImage 

998 return difference 

999 

1000 

1001def _shapeTest(exp1, exp2, fwhmExposureBuffer, fwhmExposureGrid): 

1002 """Determine that the PSF of ``exp1`` is not wider than that of ``exp2``. 

1003 

1004 Parameters 

1005 ---------- 

1006 exp1 : `~lsst.afw.image.Exposure` 

1007 Exposure with the reference point spread function (PSF) to evaluate. 

1008 exp2 : `~lsst.afw.image.Exposure` 

1009 Exposure with a candidate point spread function (PSF) to evaluate. 

1010 fwhmExposureBuffer : `float` 

1011 Fractional buffer margin to be left out of all sides of the image 

1012 during the construction of the grid to compute mean PSF FWHM in an 

1013 exposure, if the PSF is not available at its average position. 

1014 fwhmExposureGrid : `int` 

1015 Grid size to compute the mean FWHM in an exposure, if the PSF is not 

1016 available at its average position. 

1017 Returns 

1018 ------- 

1019 result : `bool` 

1020 True if ``exp1`` has a PSF that is not wider than that of ``exp2`` in 

1021 either dimension. 

1022 """ 

1023 try: 

1024 shape1 = getPsfFwhm(exp1.psf, average=False) 

1025 shape2 = getPsfFwhm(exp2.psf, average=False) 

1026 except InvalidParameterError: 

1027 shape1 = evaluateMeanPsfFwhm(exp1, 

1028 fwhmExposureBuffer=fwhmExposureBuffer, 

1029 fwhmExposureGrid=fwhmExposureGrid 

1030 ) 

1031 shape2 = evaluateMeanPsfFwhm(exp2, 

1032 fwhmExposureBuffer=fwhmExposureBuffer, 

1033 fwhmExposureGrid=fwhmExposureGrid 

1034 ) 

1035 return shape1 <= shape2 

1036 

1037 # Results from getPsfFwhm is a tuple of two values, one for each dimension. 

1038 xTest = shape1[0] <= shape2[0] 

1039 yTest = shape1[1] <= shape2[1] 

1040 return xTest | yTest 

1041 

1042 

1043def _interpolateImage(maskedImage, badMaskPlanes, fallbackValue=None): 

1044 """Replace masked image pixels with interpolated values. 

1045 

1046 Parameters 

1047 ---------- 

1048 maskedImage : `lsst.afw.image.MaskedImage` 

1049 Image on which to perform interpolation. 

1050 badMaskPlanes : `list` of `str` 

1051 List of mask planes to interpolate over. 

1052 fallbackValue : `float`, optional 

1053 Value to set when interpolation fails. 

1054 

1055 Returns 

1056 ------- 

1057 result: `float` 

1058 The number of masked pixels that were replaced. 

1059 """ 

1060 image = maskedImage.image.array 

1061 badPixels = (maskedImage.mask.array & maskedImage.mask.getPlaneBitMask(badMaskPlanes)) > 0 

1062 image[badPixels] = np.nan 

1063 if fallbackValue is None: 

1064 fallbackValue = np.nanmedian(image) 

1065 # For this initial implementation, skip the interpolation and just fill with 

1066 # the median value. 

1067 image[badPixels] = fallbackValue 

1068 return np.sum(badPixels)