Coverage for python/lsst/ip/diffim/subtractImages.py: 27%

264 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-08-09 10:32 +0000

1# This file is part of ip_diffim. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import warnings 

23 

24import numpy as np 

25 

26import lsst.afw.image 

27import lsst.afw.math 

28import lsst.geom 

29from lsst.utils.introspection import find_outside_stacklevel 

30from lsst.ip.diffim.utils import evaluateMeanPsfFwhm, getPsfFwhm 

31from lsst.meas.algorithms import ScaleVarianceTask 

32import lsst.pex.config 

33import lsst.pipe.base 

34from lsst.pex.exceptions import InvalidParameterError 

35from lsst.pipe.base import connectionTypes 

36from . import MakeKernelTask, DecorrelateALKernelTask 

37from lsst.utils.timer import timeMethod 

38 

39__all__ = ["AlardLuptonSubtractConfig", "AlardLuptonSubtractTask", 

40 "AlardLuptonPreconvolveSubtractConfig", "AlardLuptonPreconvolveSubtractTask"] 

41 

42_dimensions = ("instrument", "visit", "detector") 

43_defaultTemplates = {"coaddName": "deep", "fakesType": ""} 

44 

45 

46class SubtractInputConnections(lsst.pipe.base.PipelineTaskConnections, 

47 dimensions=_dimensions, 

48 defaultTemplates=_defaultTemplates): 

49 template = connectionTypes.Input( 

50 doc="Input warped template to subtract.", 

51 dimensions=("instrument", "visit", "detector"), 

52 storageClass="ExposureF", 

53 name="{fakesType}{coaddName}Diff_templateExp" 

54 ) 

55 science = connectionTypes.Input( 

56 doc="Input science exposure to subtract from.", 

57 dimensions=("instrument", "visit", "detector"), 

58 storageClass="ExposureF", 

59 name="{fakesType}calexp" 

60 ) 

61 sources = connectionTypes.Input( 

62 doc="Sources measured on the science exposure; " 

63 "used to select sources for making the matching kernel.", 

64 dimensions=("instrument", "visit", "detector"), 

65 storageClass="SourceCatalog", 

66 name="{fakesType}src" 

67 ) 

68 finalizedPsfApCorrCatalog = connectionTypes.Input( 

69 doc=("Per-visit finalized psf models and aperture correction maps. " 

70 "These catalogs use the detector id for the catalog id, " 

71 "sorted on id for fast lookup."), 

72 dimensions=("instrument", "visit"), 

73 storageClass="ExposureCatalog", 

74 name="finalVisitSummary", 

75 # TODO: remove on DM-39854. 

76 deprecated=( 

77 "Deprecated in favor of visitSummary. Will be removed after v26." 

78 ) 

79 ) 

80 visitSummary = connectionTypes.Input( 

81 doc=("Per-visit catalog with final calibration objects. " 

82 "These catalogs use the detector id for the catalog id, " 

83 "sorted on id for fast lookup."), 

84 dimensions=("instrument", "visit"), 

85 storageClass="ExposureCatalog", 

86 name="finalVisitSummary", 

87 ) 

88 

89 def __init__(self, *, config=None): 

90 super().__init__(config=config) 

91 if not config.doApplyFinalizedPsf: 

92 self.inputs.remove("finalizedPsfApCorrCatalog") 

93 if not config.doApplyExternalCalibrations or config.doApplyFinalizedPsf: 

94 del self.visitSummary 

95 

96 

97class SubtractImageOutputConnections(lsst.pipe.base.PipelineTaskConnections, 

98 dimensions=_dimensions, 

99 defaultTemplates=_defaultTemplates): 

100 difference = connectionTypes.Output( 

101 doc="Result of subtracting convolved template from science image.", 

102 dimensions=("instrument", "visit", "detector"), 

103 storageClass="ExposureF", 

104 name="{fakesType}{coaddName}Diff_differenceTempExp", 

105 ) 

106 matchedTemplate = connectionTypes.Output( 

107 doc="Warped and PSF-matched template used to create `subtractedExposure`.", 

108 dimensions=("instrument", "visit", "detector"), 

109 storageClass="ExposureF", 

110 name="{fakesType}{coaddName}Diff_matchedExp", 

111 ) 

112 

113 

114class SubtractScoreOutputConnections(lsst.pipe.base.PipelineTaskConnections, 

115 dimensions=_dimensions, 

116 defaultTemplates=_defaultTemplates): 

117 scoreExposure = connectionTypes.Output( 

118 doc="The maximum likelihood image, used for the detection of diaSources.", 

119 dimensions=("instrument", "visit", "detector"), 

120 storageClass="ExposureF", 

121 name="{fakesType}{coaddName}Diff_scoreExp", 

122 ) 

123 

124 

125class AlardLuptonSubtractConnections(SubtractInputConnections, SubtractImageOutputConnections): 

126 pass 

127 

128 

129class AlardLuptonSubtractBaseConfig(lsst.pex.config.Config): 

130 makeKernel = lsst.pex.config.ConfigurableField( 

131 target=MakeKernelTask, 

132 doc="Task to construct a matching kernel for convolution.", 

133 ) 

134 doDecorrelation = lsst.pex.config.Field( 

135 dtype=bool, 

136 default=True, 

137 doc="Perform diffim decorrelation to undo pixel correlation due to A&L " 

138 "kernel convolution? If True, also update the diffim PSF." 

139 ) 

140 decorrelate = lsst.pex.config.ConfigurableField( 

141 target=DecorrelateALKernelTask, 

142 doc="Task to decorrelate the image difference.", 

143 ) 

144 requiredTemplateFraction = lsst.pex.config.Field( 

145 dtype=float, 

146 default=0.1, 

147 doc="Abort task if template covers less than this fraction of pixels." 

148 " Setting to 0 will always attempt image subtraction." 

149 ) 

150 doScaleVariance = lsst.pex.config.Field( 

151 dtype=bool, 

152 default=True, 

153 doc="Scale variance of the image difference?" 

154 ) 

155 scaleVariance = lsst.pex.config.ConfigurableField( 

156 target=ScaleVarianceTask, 

157 doc="Subtask to rescale the variance of the template to the statistically expected level." 

158 ) 

159 doSubtractBackground = lsst.pex.config.Field( 

160 doc="Subtract the background fit when solving the kernel?", 

161 dtype=bool, 

162 default=True, 

163 ) 

164 doApplyFinalizedPsf = lsst.pex.config.Field( 

165 doc="Replace science Exposure's psf and aperture correction map" 

166 " with those in finalizedPsfApCorrCatalog.", 

167 dtype=bool, 

168 default=False, 

169 # TODO: remove on DM-39854. 

170 deprecated=( 

171 "Deprecated in favor of doApplyExternalCalibrations. " 

172 "Will be removed after v26." 

173 ) 

174 ) 

175 doApplyExternalCalibrations = lsst.pex.config.Field( 

176 doc=( 

177 "Replace science Exposure's calibration objects with those" 

178 " in visitSummary. Ignored if `doApplyFinalizedPsf is True." 

179 ), 

180 dtype=bool, 

181 default=False, 

182 ) 

183 detectionThreshold = lsst.pex.config.Field( 

184 dtype=float, 

185 default=10, 

186 doc="Minimum signal to noise ratio of detected sources " 

187 "to use for calculating the PSF matching kernel." 

188 ) 

189 badSourceFlags = lsst.pex.config.ListField( 

190 dtype=str, 

191 doc="Flags that, if set, the associated source should not " 

192 "be used to determine the PSF matching kernel.", 

193 default=("sky_source", "slot_Centroid_flag", 

194 "slot_ApFlux_flag", "slot_PsfFlux_flag", ), 

195 ) 

196 badMaskPlanes = lsst.pex.config.ListField( 

197 dtype=str, 

198 default=("NO_DATA", "BAD", "SAT", "EDGE"), 

199 doc="Mask planes to exclude when selecting sources for PSF matching." 

200 ) 

201 preserveTemplateMask = lsst.pex.config.ListField( 

202 dtype=str, 

203 default=("NO_DATA", "BAD", "SAT"), 

204 doc="Mask planes from the template to propagate to the image difference." 

205 ) 

206 

207 def setDefaults(self): 

208 self.makeKernel.kernel.name = "AL" 

209 self.makeKernel.kernel.active.fitForBackground = self.doSubtractBackground 

210 self.makeKernel.kernel.active.spatialKernelOrder = 1 

211 self.makeKernel.kernel.active.spatialBgOrder = 2 

212 

213 

214class AlardLuptonSubtractConfig(AlardLuptonSubtractBaseConfig, lsst.pipe.base.PipelineTaskConfig, 

215 pipelineConnections=AlardLuptonSubtractConnections): 

216 mode = lsst.pex.config.ChoiceField( 

217 dtype=str, 

218 default="convolveTemplate", 

219 allowed={"auto": "Choose which image to convolve at runtime.", 

220 "convolveScience": "Only convolve the science image.", 

221 "convolveTemplate": "Only convolve the template image."}, 

222 doc="Choose which image to convolve at runtime, or require that a specific image is convolved." 

223 ) 

224 

225 

226class AlardLuptonSubtractTask(lsst.pipe.base.PipelineTask): 

227 """Compute the image difference of a science and template image using 

228 the Alard & Lupton (1998) algorithm. 

229 """ 

230 ConfigClass = AlardLuptonSubtractConfig 

231 _DefaultName = "alardLuptonSubtract" 

232 

233 def __init__(self, **kwargs): 

234 super().__init__(**kwargs) 

235 self.makeSubtask("decorrelate") 

236 self.makeSubtask("makeKernel") 

237 if self.config.doScaleVariance: 

238 self.makeSubtask("scaleVariance") 

239 

240 self.convolutionControl = lsst.afw.math.ConvolutionControl() 

241 # Normalization is an extra, unnecessary, calculation and will result 

242 # in mis-subtraction of the images if there are calibration errors. 

243 self.convolutionControl.setDoNormalize(False) 

244 self.convolutionControl.setDoCopyEdge(True) 

245 

246 def _applyExternalCalibrations(self, exposure, visitSummary): 

247 """Replace calibrations (psf, and ApCorrMap) on this exposure with 

248 external ones.". 

249 

250 Parameters 

251 ---------- 

252 exposure : `lsst.afw.image.exposure.Exposure` 

253 Input exposure to adjust calibrations. 

254 visitSummary : `lsst.afw.table.ExposureCatalog` 

255 Exposure catalog with external calibrations to be applied. Catalog 

256 uses the detector id for the catalog id, sorted on id for fast 

257 lookup. 

258 

259 Returns 

260 ------- 

261 exposure : `lsst.afw.image.exposure.Exposure` 

262 Exposure with adjusted calibrations. 

263 """ 

264 detectorId = exposure.info.getDetector().getId() 

265 

266 row = visitSummary.find(detectorId) 

267 if row is None: 

268 self.log.warning("Detector id %s not found in external calibrations catalog; " 

269 "Using original calibrations.", detectorId) 

270 else: 

271 psf = row.getPsf() 

272 apCorrMap = row.getApCorrMap() 

273 if psf is None: 

274 self.log.warning("Detector id %s has None for psf in " 

275 "external calibrations catalog; Using original psf and aperture correction.", 

276 detectorId) 

277 elif apCorrMap is None: 

278 self.log.warning("Detector id %s has None for apCorrMap in " 

279 "external calibrations catalog; Using original psf and aperture correction.", 

280 detectorId) 

281 else: 

282 exposure.setPsf(psf) 

283 exposure.info.setApCorrMap(apCorrMap) 

284 

285 return exposure 

286 

287 @timeMethod 

288 def run(self, template, science, sources, finalizedPsfApCorrCatalog=None, 

289 visitSummary=None): 

290 """PSF match, subtract, and decorrelate two images. 

291 

292 Parameters 

293 ---------- 

294 template : `lsst.afw.image.ExposureF` 

295 Template exposure, warped to match the science exposure. 

296 science : `lsst.afw.image.ExposureF` 

297 Science exposure to subtract from the template. 

298 sources : `lsst.afw.table.SourceCatalog` 

299 Identified sources on the science exposure. This catalog is used to 

300 select sources in order to perform the AL PSF matching on stamp 

301 images around them. 

302 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional 

303 Exposure catalog with finalized psf models and aperture correction 

304 maps to be applied. Catalog uses the detector id for the catalog 

305 id, sorted on id for fast lookup. Deprecated in favor of 

306 ``visitSummary``, and will be removed after v26. 

307 visitSummary : `lsst.afw.table.ExposureCatalog`, optional 

308 Exposure catalog with external calibrations to be applied. Catalog 

309 uses the detector id for the catalog id, sorted on id for fast 

310 lookup. Ignored (for temporary backwards compatibility) if 

311 ``finalizedPsfApCorrCatalog`` is provided. 

312 

313 Returns 

314 ------- 

315 results : `lsst.pipe.base.Struct` 

316 ``difference`` : `lsst.afw.image.ExposureF` 

317 Result of subtracting template and science. 

318 ``matchedTemplate`` : `lsst.afw.image.ExposureF` 

319 Warped and PSF-matched template exposure. 

320 ``backgroundModel`` : `lsst.afw.math.Function2D` 

321 Background model that was fit while solving for the 

322 PSF-matching kernel 

323 ``psfMatchingKernel`` : `lsst.afw.math.Kernel` 

324 Kernel used to PSF-match the convolved image. 

325 

326 Raises 

327 ------ 

328 RuntimeError 

329 If an unsupported convolution mode is supplied. 

330 RuntimeError 

331 If there are too few sources to calculate the PSF matching kernel. 

332 lsst.pipe.base.NoWorkFound 

333 Raised if fraction of good pixels, defined as not having NO_DATA 

334 set, is less then the configured requiredTemplateFraction 

335 """ 

336 

337 if finalizedPsfApCorrCatalog is not None: 

338 warnings.warn( 

339 "The finalizedPsfApCorrCatalog argument is deprecated in favor of the visitSummary " 

340 "argument, and will be removed after v26.", 

341 FutureWarning, 

342 stacklevel=find_outside_stacklevel("lsst.ip.diffim"), 

343 ) 

344 visitSummary = finalizedPsfApCorrCatalog 

345 

346 self._prepareInputs(template, science, visitSummary=visitSummary) 

347 

348 # In the event that getPsfFwhm fails, evaluate the PSF on a grid. 

349 fwhmExposureBuffer = self.config.makeKernel.fwhmExposureBuffer 

350 fwhmExposureGrid = self.config.makeKernel.fwhmExposureGrid 

351 

352 # Calling getPsfFwhm on template.psf fails on some rare occasions when 

353 # the template has no input exposures at the average position of the 

354 # stars. So we try getPsfFwhm first on template, and if that fails we 

355 # evaluate the PSF on a grid specified by fwhmExposure* fields. 

356 # To keep consistent definitions for PSF size on the template and 

357 # science images, we use the same method for both. 

358 try: 

359 templatePsfSize = getPsfFwhm(template.psf) 

360 sciencePsfSize = getPsfFwhm(science.psf) 

361 except InvalidParameterError: 

362 self.log.info("Unable to evaluate PSF at the average position. " 

363 "Evaluting PSF on a grid of points." 

364 ) 

365 templatePsfSize = evaluateMeanPsfFwhm(template, 

366 fwhmExposureBuffer=fwhmExposureBuffer, 

367 fwhmExposureGrid=fwhmExposureGrid 

368 ) 

369 sciencePsfSize = evaluateMeanPsfFwhm(science, 

370 fwhmExposureBuffer=fwhmExposureBuffer, 

371 fwhmExposureGrid=fwhmExposureGrid 

372 ) 

373 self.log.info("Science PSF FWHM: %f pixels", sciencePsfSize) 

374 self.log.info("Template PSF FWHM: %f pixels", templatePsfSize) 

375 selectSources = self._sourceSelector(sources, science.mask) 

376 

377 if self.config.mode == "auto": 

378 convolveTemplate = _shapeTest(template, 

379 science, 

380 fwhmExposureBuffer=fwhmExposureBuffer, 

381 fwhmExposureGrid=fwhmExposureGrid) 

382 if convolveTemplate: 

383 if sciencePsfSize < templatePsfSize: 

384 self.log.info("Average template PSF size is greater, " 

385 "but science PSF greater in one dimension: convolving template image.") 

386 else: 

387 self.log.info("Science PSF size is greater: convolving template image.") 

388 else: 

389 self.log.info("Template PSF size is greater: convolving science image.") 

390 elif self.config.mode == "convolveTemplate": 

391 self.log.info("`convolveTemplate` is set: convolving template image.") 

392 convolveTemplate = True 

393 elif self.config.mode == "convolveScience": 

394 self.log.info("`convolveScience` is set: convolving science image.") 

395 convolveTemplate = False 

396 else: 

397 raise RuntimeError("Cannot handle AlardLuptonSubtract mode: %s", self.config.mode) 

398 

399 if convolveTemplate: 

400 subtractResults = self.runConvolveTemplate(template, science, selectSources) 

401 else: 

402 subtractResults = self.runConvolveScience(template, science, selectSources) 

403 

404 return subtractResults 

405 

406 def runConvolveTemplate(self, template, science, selectSources): 

407 """Convolve the template image with a PSF-matching kernel and subtract 

408 from the science image. 

409 

410 Parameters 

411 ---------- 

412 template : `lsst.afw.image.ExposureF` 

413 Template exposure, warped to match the science exposure. 

414 science : `lsst.afw.image.ExposureF` 

415 Science exposure to subtract from the template. 

416 selectSources : `lsst.afw.table.SourceCatalog` 

417 Identified sources on the science exposure. This catalog is used to 

418 select sources in order to perform the AL PSF matching on stamp 

419 images around them. 

420 

421 Returns 

422 ------- 

423 results : `lsst.pipe.base.Struct` 

424 

425 ``difference`` : `lsst.afw.image.ExposureF` 

426 Result of subtracting template and science. 

427 ``matchedTemplate`` : `lsst.afw.image.ExposureF` 

428 Warped and PSF-matched template exposure. 

429 ``backgroundModel`` : `lsst.afw.math.Function2D` 

430 Background model that was fit while solving for the PSF-matching kernel 

431 ``psfMatchingKernel`` : `lsst.afw.math.Kernel` 

432 Kernel used to PSF-match the template to the science image. 

433 """ 

434 kernelSources = self.makeKernel.selectKernelSources(template, science, 

435 candidateList=selectSources, 

436 preconvolved=False) 

437 kernelResult = self.makeKernel.run(template, science, kernelSources, 

438 preconvolved=False) 

439 

440 matchedTemplate = self._convolveExposure(template, kernelResult.psfMatchingKernel, 

441 self.convolutionControl, 

442 bbox=science.getBBox(), 

443 psf=science.psf, 

444 photoCalib=science.photoCalib) 

445 

446 difference = _subtractImages(science, matchedTemplate, 

447 backgroundModel=(kernelResult.backgroundModel 

448 if self.config.doSubtractBackground else None)) 

449 correctedExposure = self.finalize(template, science, difference, 

450 kernelResult.psfMatchingKernel, 

451 templateMatched=True) 

452 

453 return lsst.pipe.base.Struct(difference=correctedExposure, 

454 matchedTemplate=matchedTemplate, 

455 matchedScience=science, 

456 backgroundModel=kernelResult.backgroundModel, 

457 psfMatchingKernel=kernelResult.psfMatchingKernel) 

458 

459 def runConvolveScience(self, template, science, selectSources): 

460 """Convolve the science image with a PSF-matching kernel and subtract the template image. 

461 

462 Parameters 

463 ---------- 

464 template : `lsst.afw.image.ExposureF` 

465 Template exposure, warped to match the science exposure. 

466 science : `lsst.afw.image.ExposureF` 

467 Science exposure to subtract from the template. 

468 selectSources : `lsst.afw.table.SourceCatalog` 

469 Identified sources on the science exposure. This catalog is used to 

470 select sources in order to perform the AL PSF matching on stamp 

471 images around them. 

472 

473 Returns 

474 ------- 

475 results : `lsst.pipe.base.Struct` 

476 

477 ``difference`` : `lsst.afw.image.ExposureF` 

478 Result of subtracting template and science. 

479 ``matchedTemplate`` : `lsst.afw.image.ExposureF` 

480 Warped template exposure. Note that in this case, the template 

481 is not PSF-matched to the science image. 

482 ``backgroundModel`` : `lsst.afw.math.Function2D` 

483 Background model that was fit while solving for the PSF-matching kernel 

484 ``psfMatchingKernel`` : `lsst.afw.math.Kernel` 

485 Kernel used to PSF-match the science image to the template. 

486 """ 

487 bbox = science.getBBox() 

488 kernelSources = self.makeKernel.selectKernelSources(science, template, 

489 candidateList=selectSources, 

490 preconvolved=False) 

491 kernelResult = self.makeKernel.run(science, template, kernelSources, 

492 preconvolved=False) 

493 modelParams = kernelResult.backgroundModel.getParameters() 

494 # We must invert the background model if the matching kernel is solved for the science image. 

495 kernelResult.backgroundModel.setParameters([-p for p in modelParams]) 

496 

497 kernelImage = lsst.afw.image.ImageD(kernelResult.psfMatchingKernel.getDimensions()) 

498 norm = kernelResult.psfMatchingKernel.computeImage(kernelImage, doNormalize=False) 

499 

500 matchedScience = self._convolveExposure(science, kernelResult.psfMatchingKernel, 

501 self.convolutionControl, 

502 psf=template.psf) 

503 

504 # Place back on native photometric scale 

505 matchedScience.maskedImage /= norm 

506 matchedTemplate = template.clone()[bbox] 

507 matchedTemplate.maskedImage /= norm 

508 matchedTemplate.setPhotoCalib(science.photoCalib) 

509 

510 difference = _subtractImages(matchedScience, matchedTemplate, 

511 backgroundModel=(kernelResult.backgroundModel 

512 if self.config.doSubtractBackground else None)) 

513 

514 correctedExposure = self.finalize(template, science, difference, 

515 kernelResult.psfMatchingKernel, 

516 templateMatched=False) 

517 

518 return lsst.pipe.base.Struct(difference=correctedExposure, 

519 matchedTemplate=matchedTemplate, 

520 matchedScience=matchedScience, 

521 backgroundModel=kernelResult.backgroundModel, 

522 psfMatchingKernel=kernelResult.psfMatchingKernel,) 

523 

524 def finalize(self, template, science, difference, kernel, 

525 templateMatched=True, 

526 preConvMode=False, 

527 preConvKernel=None, 

528 spatiallyVarying=False): 

529 """Decorrelate the difference image to undo the noise correlations 

530 caused by convolution. 

531 

532 Parameters 

533 ---------- 

534 template : `lsst.afw.image.ExposureF` 

535 Template exposure, warped to match the science exposure. 

536 science : `lsst.afw.image.ExposureF` 

537 Science exposure to subtract from the template. 

538 difference : `lsst.afw.image.ExposureF` 

539 Result of subtracting template and science. 

540 kernel : `lsst.afw.math.Kernel` 

541 An (optionally spatially-varying) PSF matching kernel 

542 templateMatched : `bool`, optional 

543 Was the template PSF-matched to the science image? 

544 preConvMode : `bool`, optional 

545 Was the science image preconvolved with its own PSF 

546 before PSF matching the template? 

547 preConvKernel : `lsst.afw.detection.Psf`, optional 

548 If not `None`, then the science image was pre-convolved with 

549 (the reflection of) this kernel. Must be normalized to sum to 1. 

550 spatiallyVarying : `bool`, optional 

551 Compute the decorrelation kernel spatially varying across the image? 

552 

553 Returns 

554 ------- 

555 correctedExposure : `lsst.afw.image.ExposureF` 

556 The decorrelated image difference. 

557 """ 

558 # Erase existing detection mask planes. 

559 # We don't want the detection mask from the science image 

560 mask = difference.mask 

561 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) 

562 

563 # We have cleared the template mask plane, so copy the mask plane of 

564 # the image difference so that we can calculate correct statistics 

565 # during decorrelation. Do this regardless of whether decorrelation is 

566 # used for consistency. 

567 template[science.getBBox()].mask.array[...] = difference.mask.array[...] 

568 if self.config.doDecorrelation: 

569 self.log.info("Decorrelating image difference.") 

570 # We have cleared the template mask plane, so copy the mask plane of 

571 # the image difference so that we can calculate correct statistics 

572 # during decorrelation 

573 template[science.getBBox()].mask.array[...] = difference.mask.array[...] 

574 correctedExposure = self.decorrelate.run(science, template[science.getBBox()], difference, kernel, 

575 templateMatched=templateMatched, 

576 preConvMode=preConvMode, 

577 preConvKernel=preConvKernel, 

578 spatiallyVarying=spatiallyVarying).correctedExposure 

579 else: 

580 self.log.info("NOT decorrelating image difference.") 

581 correctedExposure = difference 

582 return correctedExposure 

583 

584 @staticmethod 

585 def _validateExposures(template, science): 

586 """Check that the WCS of the two Exposures match, and the template bbox 

587 contains the science bbox. 

588 

589 Parameters 

590 ---------- 

591 template : `lsst.afw.image.ExposureF` 

592 Template exposure, warped to match the science exposure. 

593 science : `lsst.afw.image.ExposureF` 

594 Science exposure to subtract from the template. 

595 

596 Raises 

597 ------ 

598 AssertionError 

599 Raised if the WCS of the template is not equal to the science WCS, 

600 or if the science image is not fully contained in the template 

601 bounding box. 

602 """ 

603 assert template.wcs == science.wcs,\ 

604 "Template and science exposure WCS are not identical." 

605 templateBBox = template.getBBox() 

606 scienceBBox = science.getBBox() 

607 

608 assert templateBBox.contains(scienceBBox),\ 

609 "Template bbox does not contain all of the science image." 

610 

611 @staticmethod 

612 def _convolveExposure(exposure, kernel, convolutionControl, 

613 bbox=None, 

614 psf=None, 

615 photoCalib=None): 

616 """Convolve an exposure with the given kernel. 

617 

618 Parameters 

619 ---------- 

620 exposure : `lsst.afw.Exposure` 

621 exposure to convolve. 

622 kernel : `lsst.afw.math.LinearCombinationKernel` 

623 PSF matching kernel computed in the ``makeKernel`` subtask. 

624 convolutionControl : `lsst.afw.math.ConvolutionControl` 

625 Configuration for convolve algorithm. 

626 bbox : `lsst.geom.Box2I`, optional 

627 Bounding box to trim the convolved exposure to. 

628 psf : `lsst.afw.detection.Psf`, optional 

629 Point spread function (PSF) to set for the convolved exposure. 

630 photoCalib : `lsst.afw.image.PhotoCalib`, optional 

631 Photometric calibration of the convolved exposure. 

632 

633 Returns 

634 ------- 

635 convolvedExp : `lsst.afw.Exposure` 

636 The convolved image. 

637 """ 

638 convolvedExposure = exposure.clone() 

639 if psf is not None: 

640 convolvedExposure.setPsf(psf) 

641 if photoCalib is not None: 

642 convolvedExposure.setPhotoCalib(photoCalib) 

643 convolvedImage = lsst.afw.image.MaskedImageF(exposure.getBBox()) 

644 lsst.afw.math.convolve(convolvedImage, exposure.maskedImage, kernel, convolutionControl) 

645 convolvedExposure.setMaskedImage(convolvedImage) 

646 if bbox is None: 

647 return convolvedExposure 

648 else: 

649 return convolvedExposure[bbox] 

650 

651 def _sourceSelector(self, sources, mask): 

652 """Select sources from a catalog that meet the selection criteria. 

653 

654 Parameters 

655 ---------- 

656 sources : `lsst.afw.table.SourceCatalog` 

657 Input source catalog to select sources from. 

658 mask : `lsst.afw.image.Mask` 

659 The image mask plane to use to reject sources 

660 based on their location on the ccd. 

661 

662 Returns 

663 ------- 

664 selectSources : `lsst.afw.table.SourceCatalog` 

665 The input source catalog, with flagged and low signal-to-noise 

666 sources removed. 

667 

668 Raises 

669 ------ 

670 RuntimeError 

671 If there are too few sources to compute the PSF matching kernel 

672 remaining after source selection. 

673 """ 

674 flags = np.ones(len(sources), dtype=bool) 

675 for flag in self.config.badSourceFlags: 

676 try: 

677 flags *= ~sources[flag] 

678 except Exception as e: 

679 self.log.warning("Could not apply source flag: %s", e) 

680 sToNFlag = (sources.getPsfInstFlux()/sources.getPsfInstFluxErr()) > self.config.detectionThreshold 

681 flags *= sToNFlag 

682 flags *= self._checkMask(mask, sources, self.config.badMaskPlanes) 

683 selectSources = sources[flags] 

684 self.log.info("%i/%i=%.1f%% of sources selected for PSF matching from the input catalog", 

685 len(selectSources), len(sources), 100*len(selectSources)/len(sources)) 

686 if len(selectSources) < self.config.makeKernel.nStarPerCell: 

687 self.log.error("Too few sources to calculate the PSF matching kernel: " 

688 "%i selected but %i needed for the calculation.", 

689 len(selectSources), self.config.makeKernel.nStarPerCell) 

690 raise RuntimeError("Cannot compute PSF matching kernel: too few sources selected.") 

691 

692 return selectSources.copy(deep=True) 

693 

694 @staticmethod 

695 def _checkMask(mask, sources, badMaskPlanes): 

696 """Exclude sources that are located on masked pixels. 

697 

698 Parameters 

699 ---------- 

700 mask : `lsst.afw.image.Mask` 

701 The image mask plane to use to reject sources 

702 based on the location of their centroid on the ccd. 

703 sources : `lsst.afw.table.SourceCatalog` 

704 The source catalog to evaluate. 

705 badMaskPlanes : `list` of `str` 

706 List of the names of the mask planes to exclude. 

707 

708 Returns 

709 ------- 

710 flags : `numpy.ndarray` of `bool` 

711 Array indicating whether each source in the catalog should be 

712 kept (True) or rejected (False) based on the value of the 

713 mask plane at its location. 

714 """ 

715 badPixelMask = lsst.afw.image.Mask.getPlaneBitMask(badMaskPlanes) 

716 xv = np.rint(sources.getX() - mask.getX0()) 

717 yv = np.rint(sources.getY() - mask.getY0()) 

718 

719 mv = mask.array[yv.astype(int), xv.astype(int)] 

720 flags = np.bitwise_and(mv, badPixelMask) == 0 

721 return flags 

722 

723 def _prepareInputs(self, template, science, visitSummary=None): 

724 """Perform preparatory calculations common to all Alard&Lupton Tasks. 

725 

726 Parameters 

727 ---------- 

728 template : `lsst.afw.image.ExposureF` 

729 Template exposure, warped to match the science exposure. The 

730 variance plane of the template image is modified in place. 

731 science : `lsst.afw.image.ExposureF` 

732 Science exposure to subtract from the template. The variance plane 

733 of the science image is modified in place. 

734 visitSummary : `lsst.afw.table.ExposureCatalog`, optional 

735 Exposure catalog with external calibrations to be applied. Catalog 

736 uses the detector id for the catalog id, sorted on id for fast 

737 lookup. 

738 """ 

739 self._validateExposures(template, science) 

740 if visitSummary is not None: 

741 self._applyExternalCalibrations(science, visitSummary=visitSummary) 

742 checkTemplateIsSufficient(template, self.log, 

743 requiredTemplateFraction=self.config.requiredTemplateFraction) 

744 

745 if self.config.doScaleVariance: 

746 # Scale the variance of the template and science images before 

747 # convolution, subtraction, or decorrelation so that they have the 

748 # correct ratio. 

749 templateVarFactor = self.scaleVariance.run(template.maskedImage) 

750 sciVarFactor = self.scaleVariance.run(science.maskedImage) 

751 self.log.info("Template variance scaling factor: %.2f", templateVarFactor) 

752 self.metadata.add("scaleTemplateVarianceFactor", templateVarFactor) 

753 self.log.info("Science variance scaling factor: %.2f", sciVarFactor) 

754 self.metadata.add("scaleScienceVarianceFactor", sciVarFactor) 

755 self._clearMask(template) 

756 

757 def _clearMask(self, template): 

758 """Clear the mask plane of the template. 

759 

760 Parameters 

761 ---------- 

762 template : `lsst.afw.image.ExposureF` 

763 Template exposure, warped to match the science exposure. 

764 The mask plane will be modified in place. 

765 """ 

766 mask = template.mask 

767 clearMaskPlanes = [maskplane for maskplane in mask.getMaskPlaneDict().keys() 

768 if maskplane not in self.config.preserveTemplateMask] 

769 

770 bitMaskToClear = mask.getPlaneBitMask(clearMaskPlanes) 

771 mask &= ~bitMaskToClear 

772 

773 

774class AlardLuptonPreconvolveSubtractConnections(SubtractInputConnections, 

775 SubtractScoreOutputConnections): 

776 pass 

777 

778 

779class AlardLuptonPreconvolveSubtractConfig(AlardLuptonSubtractBaseConfig, lsst.pipe.base.PipelineTaskConfig, 

780 pipelineConnections=AlardLuptonPreconvolveSubtractConnections): 

781 pass 

782 

783 

784class AlardLuptonPreconvolveSubtractTask(AlardLuptonSubtractTask): 

785 """Subtract a template from a science image, convolving the science image 

786 before computing the kernel, and also convolving the template before 

787 subtraction. 

788 """ 

789 ConfigClass = AlardLuptonPreconvolveSubtractConfig 

790 _DefaultName = "alardLuptonPreconvolveSubtract" 

791 

792 def run(self, template, science, sources, finalizedPsfApCorrCatalog=None, visitSummary=None): 

793 """Preconvolve the science image with its own PSF, 

794 convolve the template image with a PSF-matching kernel and subtract 

795 from the preconvolved science image. 

796 

797 Parameters 

798 ---------- 

799 template : `lsst.afw.image.ExposureF` 

800 The template image, which has previously been warped to the science 

801 image. The template bbox will be padded by a few pixels compared to 

802 the science bbox. 

803 science : `lsst.afw.image.ExposureF` 

804 The science exposure. 

805 sources : `lsst.afw.table.SourceCatalog` 

806 Identified sources on the science exposure. This catalog is used to 

807 select sources in order to perform the AL PSF matching on stamp 

808 images around them. 

809 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional 

810 Exposure catalog with finalized psf models and aperture correction 

811 maps to be applied. Catalog uses the detector id for the catalog 

812 id, sorted on id for fast lookup. Deprecated in favor of 

813 ``visitSummary``, and will be removed after v26. 

814 visitSummary : `lsst.afw.table.ExposureCatalog`, optional 

815 Exposure catalog with complete external calibrations. Catalog uses 

816 the detector id for the catalog id, sorted on id for fast lookup. 

817 Ignored (for temporary backwards compatibility) if 

818 ``finalizedPsfApCorrCatalog`` is provided. 

819 

820 Returns 

821 ------- 

822 results : `lsst.pipe.base.Struct` 

823 ``scoreExposure`` : `lsst.afw.image.ExposureF` 

824 Result of subtracting the convolved template and science 

825 images. Attached PSF is that of the original science image. 

826 ``matchedTemplate`` : `lsst.afw.image.ExposureF` 

827 Warped and PSF-matched template exposure. Attached PSF is that 

828 of the original science image. 

829 ``matchedScience`` : `lsst.afw.image.ExposureF` 

830 The science exposure after convolving with its own PSF. 

831 Attached PSF is that of the original science image. 

832 ``backgroundModel`` : `lsst.afw.math.Function2D` 

833 Background model that was fit while solving for the 

834 PSF-matching kernel 

835 ``psfMatchingKernel`` : `lsst.afw.math.Kernel` 

836 Final kernel used to PSF-match the template to the science 

837 image. 

838 """ 

839 if finalizedPsfApCorrCatalog is not None: 

840 warnings.warn( 

841 "The finalizedPsfApCorrCatalog argument is deprecated in favor of the visitSummary " 

842 "argument, and will be removed after v26.", 

843 FutureWarning, 

844 stacklevel=find_outside_stacklevel("lsst.ip.diffim"), 

845 ) 

846 visitSummary = finalizedPsfApCorrCatalog 

847 

848 self._prepareInputs(template, science, visitSummary=visitSummary) 

849 

850 # TODO: DM-37212 we need to mirror the kernel in order to get correct cross correlation 

851 scienceKernel = science.psf.getKernel() 

852 matchedScience = self._convolveExposure(science, scienceKernel, self.convolutionControl) 

853 selectSources = self._sourceSelector(sources, matchedScience.mask) 

854 

855 subtractResults = self.runPreconvolve(template, science, matchedScience, selectSources, scienceKernel) 

856 

857 return subtractResults 

858 

859 def runPreconvolve(self, template, science, matchedScience, selectSources, preConvKernel): 

860 """Convolve the science image with its own PSF, then convolve the 

861 template with a matching kernel and subtract to form the Score 

862 exposure. 

863 

864 Parameters 

865 ---------- 

866 template : `lsst.afw.image.ExposureF` 

867 Template exposure, warped to match the science exposure. 

868 science : `lsst.afw.image.ExposureF` 

869 Science exposure to subtract from the template. 

870 matchedScience : `lsst.afw.image.ExposureF` 

871 The science exposure, convolved with the reflection of its own PSF. 

872 selectSources : `lsst.afw.table.SourceCatalog` 

873 Identified sources on the science exposure. This catalog is used to 

874 select sources in order to perform the AL PSF matching on stamp 

875 images around them. 

876 preConvKernel : `lsst.afw.math.Kernel` 

877 The reflection of the kernel that was used to preconvolve the 

878 `science` exposure. Must be normalized to sum to 1. 

879 

880 Returns 

881 ------- 

882 results : `lsst.pipe.base.Struct` 

883 

884 ``scoreExposure`` : `lsst.afw.image.ExposureF` 

885 Result of subtracting the convolved template and science 

886 images. Attached PSF is that of the original science image. 

887 ``matchedTemplate`` : `lsst.afw.image.ExposureF` 

888 Warped and PSF-matched template exposure. Attached PSF is that 

889 of the original science image. 

890 ``matchedScience`` : `lsst.afw.image.ExposureF` 

891 The science exposure after convolving with its own PSF. 

892 Attached PSF is that of the original science image. 

893 ``backgroundModel`` : `lsst.afw.math.Function2D` 

894 Background model that was fit while solving for the 

895 PSF-matching kernel 

896 ``psfMatchingKernel`` : `lsst.afw.math.Kernel` 

897 Final kernel used to PSF-match the template to the science 

898 image. 

899 """ 

900 bbox = science.getBBox() 

901 innerBBox = preConvKernel.shrinkBBox(bbox) 

902 

903 kernelSources = self.makeKernel.selectKernelSources(template[innerBBox], matchedScience[innerBBox], 

904 candidateList=selectSources, 

905 preconvolved=True) 

906 kernelResult = self.makeKernel.run(template[innerBBox], matchedScience[innerBBox], kernelSources, 

907 preconvolved=True) 

908 

909 matchedTemplate = self._convolveExposure(template, kernelResult.psfMatchingKernel, 

910 self.convolutionControl, 

911 bbox=bbox, 

912 psf=science.psf, 

913 photoCalib=science.photoCalib) 

914 score = _subtractImages(matchedScience, matchedTemplate, 

915 backgroundModel=(kernelResult.backgroundModel 

916 if self.config.doSubtractBackground else None)) 

917 correctedScore = self.finalize(template[bbox], science, score, 

918 kernelResult.psfMatchingKernel, 

919 templateMatched=True, preConvMode=True, 

920 preConvKernel=preConvKernel) 

921 

922 return lsst.pipe.base.Struct(scoreExposure=correctedScore, 

923 matchedTemplate=matchedTemplate, 

924 matchedScience=matchedScience, 

925 backgroundModel=kernelResult.backgroundModel, 

926 psfMatchingKernel=kernelResult.psfMatchingKernel) 

927 

928 

929def checkTemplateIsSufficient(templateExposure, logger, requiredTemplateFraction=0.): 

930 """Raise NoWorkFound if template coverage < requiredTemplateFraction 

931 

932 Parameters 

933 ---------- 

934 templateExposure : `lsst.afw.image.ExposureF` 

935 The template exposure to check 

936 logger : `lsst.log.Log` 

937 Logger for printing output. 

938 requiredTemplateFraction : `float`, optional 

939 Fraction of pixels of the science image required to have coverage 

940 in the template. 

941 

942 Raises 

943 ------ 

944 lsst.pipe.base.NoWorkFound 

945 Raised if fraction of good pixels, defined as not having NO_DATA 

946 set, is less then the configured requiredTemplateFraction 

947 """ 

948 # Count the number of pixels with the NO_DATA mask bit set 

949 # counting NaN pixels is insufficient because pixels without data are often intepolated over) 

950 pixNoData = np.count_nonzero(templateExposure.mask.array 

951 & templateExposure.mask.getPlaneBitMask('NO_DATA')) 

952 pixGood = templateExposure.getBBox().getArea() - pixNoData 

953 logger.info("template has %d good pixels (%.1f%%)", pixGood, 

954 100*pixGood/templateExposure.getBBox().getArea()) 

955 

956 if pixGood/templateExposure.getBBox().getArea() < requiredTemplateFraction: 

957 message = ("Insufficient Template Coverage. (%.1f%% < %.1f%%) Not attempting subtraction. " 

958 "To force subtraction, set config requiredTemplateFraction=0." % ( 

959 100*pixGood/templateExposure.getBBox().getArea(), 

960 100*requiredTemplateFraction)) 

961 raise lsst.pipe.base.NoWorkFound(message) 

962 

963 

964def _subtractImages(science, template, backgroundModel=None): 

965 """Subtract template from science, propagating relevant metadata. 

966 

967 Parameters 

968 ---------- 

969 science : `lsst.afw.Exposure` 

970 The input science image. 

971 template : `lsst.afw.Exposure` 

972 The template to subtract from the science image. 

973 backgroundModel : `lsst.afw.MaskedImage`, optional 

974 Differential background model 

975 

976 Returns 

977 ------- 

978 difference : `lsst.afw.Exposure` 

979 The subtracted image. 

980 """ 

981 difference = science.clone() 

982 if backgroundModel is not None: 

983 difference.maskedImage -= backgroundModel 

984 difference.maskedImage -= template.maskedImage 

985 return difference 

986 

987 

988def _shapeTest(exp1, exp2, fwhmExposureBuffer, fwhmExposureGrid): 

989 """Determine that the PSF of ``exp1`` is not wider than that of ``exp2``. 

990 

991 Parameters 

992 ---------- 

993 exp1 : `~lsst.afw.image.Exposure` 

994 Exposure with the reference point spread function (PSF) to evaluate. 

995 exp2 : `~lsst.afw.image.Exposure` 

996 Exposure with a candidate point spread function (PSF) to evaluate. 

997 fwhmExposureBuffer : `float` 

998 Fractional buffer margin to be left out of all sides of the image 

999 during the construction of the grid to compute mean PSF FWHM in an 

1000 exposure, if the PSF is not available at its average position. 

1001 fwhmExposureGrid : `int` 

1002 Grid size to compute the mean FWHM in an exposure, if the PSF is not 

1003 available at its average position. 

1004 Returns 

1005 ------- 

1006 result : `bool` 

1007 True if ``exp1`` has a PSF that is not wider than that of ``exp2`` in 

1008 either dimension. 

1009 """ 

1010 try: 

1011 shape1 = getPsfFwhm(exp1.psf, average=False) 

1012 shape2 = getPsfFwhm(exp2.psf, average=False) 

1013 except InvalidParameterError: 

1014 shape1 = evaluateMeanPsfFwhm(exp1, 

1015 fwhmExposureBuffer=fwhmExposureBuffer, 

1016 fwhmExposureGrid=fwhmExposureGrid 

1017 ) 

1018 shape2 = evaluateMeanPsfFwhm(exp2, 

1019 fwhmExposureBuffer=fwhmExposureBuffer, 

1020 fwhmExposureGrid=fwhmExposureGrid 

1021 ) 

1022 return shape1 <= shape2 

1023 

1024 # Results from getPsfFwhm is a tuple of two values, one for each dimension. 

1025 xTest = shape1[0] <= shape2[0] 

1026 yTest = shape1[1] <= shape2[1] 

1027 return xTest | yTest