Coverage for tests/test_subtractTask.py: 6%

698 statements  

« prev     ^ index     » next       coverage.py v7.3.3, created at 2023-12-16 13:38 +0000

1# This file is part of ip_diffim. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import unittest 

23 

24import lsst.afw.math as afwMath 

25import lsst.afw.table as afwTable 

26import lsst.geom 

27import lsst.meas.algorithms as measAlg 

28from lsst.ip.diffim import subtractImages 

29from lsst.pex.config import FieldValidationError 

30from lsst.pipe.base import NoWorkFound 

31import lsst.utils.tests 

32import numpy as np 

33from lsst.ip.diffim.utils import (computeRobustStatistics, computePSFNoiseEquivalentArea, 

34 evaluateMeanPsfFwhm, getPsfFwhm, makeStats, makeTestImage) 

35from lsst.pex.exceptions import InvalidParameterError 

36 

37 

38class CustomCoaddPsf(measAlg.CoaddPsf): 

39 """A custom CoaddPSF that overrides the getAveragePosition method. 

40 """ 

41 def getAveragePosition(self): 

42 return lsst.geom.Point2D(-10000, -10000) 

43 

44 

45class AlardLuptonSubtractTest(lsst.utils.tests.TestCase): 

46 

47 def test_allowed_config_modes(self): 

48 """Verify the allowable modes for convolution. 

49 """ 

50 config = subtractImages.AlardLuptonSubtractTask.ConfigClass() 

51 config.mode = 'auto' 

52 config.mode = 'convolveScience' 

53 config.mode = 'convolveTemplate' 

54 

55 with self.assertRaises(FieldValidationError): 

56 config.mode = 'aotu' 

57 

58 def test_mismatched_template(self): 

59 """Test that an error is raised if the template 

60 does not fully contain the science image. 

61 """ 

62 xSize = 200 

63 ySize = 200 

64 science, sources = makeTestImage(psfSize=2.4, xSize=xSize + 20, ySize=ySize + 20) 

65 template, _ = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize, doApplyCalibration=True) 

66 config = subtractImages.AlardLuptonSubtractTask.ConfigClass() 

67 task = subtractImages.AlardLuptonSubtractTask(config=config) 

68 with self.assertRaises(AssertionError): 

69 task.run(template, science, sources) 

70 

71 def test_incomplete_template_coverage(self): 

72 noiseLevel = 1. 

73 border = 20 

74 xSize = 400 

75 ySize = 400 

76 science, sources = makeTestImage(psfSize=3.0, noiseLevel=noiseLevel, noiseSeed=6, nSrc=50, 

77 xSize=xSize, ySize=ySize) 

78 template, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=7, nSrc=50, 

79 templateBorderSize=border, doApplyCalibration=True, 

80 xSize=xSize, ySize=ySize) 

81 

82 science_height = science.getBBox().getDimensions().getY() 

83 

84 def _run_and_check_coverage(template_coverage, 

85 requiredTemplateFraction=0.1, 

86 minTemplateFractionForExpectedSuccess=0.2): 

87 template_cut = template.clone() 

88 template_height = int(science_height*template_coverage + border) 

89 template_cut.image.array[:, template_height:] = 0 

90 template_cut.mask.array[:, template_height:] = template_cut.mask.getPlaneBitMask('NO_DATA') 

91 config = subtractImages.AlardLuptonSubtractTask.ConfigClass() 

92 config.requiredTemplateFraction = requiredTemplateFraction 

93 config.minTemplateFractionForExpectedSuccess = minTemplateFractionForExpectedSuccess 

94 if template_coverage < requiredTemplateFraction: 

95 doRaise = True 

96 elif template_coverage < minTemplateFractionForExpectedSuccess: 

97 doRaise = True 

98 else: 

99 doRaise = False 

100 task = subtractImages.AlardLuptonSubtractTask(config=config) 

101 if doRaise: 

102 with self.assertRaises(NoWorkFound): 

103 task.run(template_cut, science.clone(), sources.copy(deep=True)) 

104 else: 

105 task.run(template_cut, science.clone(), sources.copy(deep=True)) 

106 _run_and_check_coverage(template_coverage=0.09) 

107 _run_and_check_coverage(template_coverage=0.19) 

108 _run_and_check_coverage(template_coverage=0.7) 

109 

110 def test_clear_template_mask(self): 

111 noiseLevel = 1. 

112 science, sources = makeTestImage(psfSize=3.0, noiseLevel=noiseLevel, noiseSeed=6) 

113 template, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=7, 

114 templateBorderSize=20, doApplyCalibration=True) 

115 diffimEmptyMaskPlanes = ["DETECTED", "DETECTED_NEGATIVE"] 

116 config = subtractImages.AlardLuptonSubtractTask.ConfigClass() 

117 config.doSubtractBackground = False 

118 config.mode = "convolveTemplate" 

119 # Ensure that each each mask plane is set for some pixels 

120 mask = template.mask 

121 x0 = 50 

122 x1 = 75 

123 y0 = 150 

124 y1 = 175 

125 scienceMaskCheck = {} 

126 for maskPlane in mask.getMaskPlaneDict().keys(): 

127 scienceMaskCheck[maskPlane] = np.sum(science.mask.array & mask.getPlaneBitMask(maskPlane) > 0) 

128 mask.array[x0: x1, y0: y1] |= mask.getPlaneBitMask(maskPlane) 

129 self.assertTrue(np.sum(mask.array & mask.getPlaneBitMask(maskPlane) > 0)) 

130 

131 task = subtractImages.AlardLuptonSubtractTask(config=config) 

132 output = task.run(template, science, sources) 

133 # Verify that the template mask has been modified in place 

134 for maskPlane in mask.getMaskPlaneDict().keys(): 

135 if maskPlane in diffimEmptyMaskPlanes: 

136 self.assertTrue(np.sum(mask.array & mask.getPlaneBitMask(maskPlane) == 0)) 

137 elif maskPlane in config.preserveTemplateMask: 

138 self.assertTrue(np.sum(mask.array & mask.getPlaneBitMask(maskPlane) > 0)) 

139 else: 

140 self.assertTrue(np.sum(mask.array & mask.getPlaneBitMask(maskPlane) == 0)) 

141 # Mask planes set in the science image should also be set in the difference 

142 # Except the "DETECTED" planes should have been cleared 

143 diffimMask = output.difference.mask 

144 for maskPlane, scienceSum in scienceMaskCheck.items(): 

145 diffimSum = np.sum(diffimMask.array & mask.getPlaneBitMask(maskPlane) > 0) 

146 if maskPlane in diffimEmptyMaskPlanes: 

147 self.assertEqual(diffimSum, 0) 

148 else: 

149 self.assertTrue(diffimSum >= scienceSum) 

150 

151 def test_equal_images(self): 

152 """Test that running with enough sources produces reasonable output, 

153 with the same size psf in the template and science. 

154 """ 

155 noiseLevel = 1. 

156 science, sources = makeTestImage(psfSize=2.4, noiseLevel=noiseLevel, noiseSeed=6) 

157 template, _ = makeTestImage(psfSize=2.4, noiseLevel=noiseLevel, noiseSeed=7, 

158 templateBorderSize=20, doApplyCalibration=True) 

159 config = subtractImages.AlardLuptonSubtractTask.ConfigClass() 

160 config.doSubtractBackground = False 

161 task = subtractImages.AlardLuptonSubtractTask(config=config) 

162 output = task.run(template, science, sources) 

163 # There shoud be no NaN values in the difference image 

164 self.assertTrue(np.all(np.isfinite(output.difference.image.array))) 

165 # Mean of difference image should be close to zero. 

166 meanError = noiseLevel/np.sqrt(output.difference.image.array.size) 

167 # Make sure to include pixels with the DETECTED mask bit set. 

168 statsCtrl = makeStats(badMaskPlanes=("EDGE", "BAD", "NO_DATA", "DETECTED", "DETECTED_NEGATIVE")) 

169 differenceMean = computeRobustStatistics(output.difference.image, output.difference.mask, statsCtrl) 

170 self.assertFloatsAlmostEqual(differenceMean, 0, atol=5*meanError) 

171 # stddev of difference image should be close to expected value. 

172 differenceStd = computeRobustStatistics(output.difference.image, output.difference.mask, 

173 makeStats(), statistic=afwMath.STDEV) 

174 self.assertFloatsAlmostEqual(differenceStd, np.sqrt(2)*noiseLevel, rtol=0.1) 

175 

176 def test_psf_size(self): 

177 """Test that the image subtract task runs without failing, if 

178 fwhmExposureBuffer and fwhmExposureGrid parameters are set. 

179 """ 

180 noiseLevel = 1. 

181 science, sources = makeTestImage(psfSize=2.4, noiseLevel=noiseLevel, noiseSeed=6) 

182 template, _ = makeTestImage(psfSize=2.4, noiseLevel=noiseLevel, noiseSeed=7, 

183 templateBorderSize=20, doApplyCalibration=True) 

184 

185 schema = afwTable.ExposureTable.makeMinimalSchema() 

186 weightKey = schema.addField("weight", type="D", doc="Coadd weight") 

187 exposureCatalog = afwTable.ExposureCatalog(schema) 

188 kernel = measAlg.DoubleGaussianPsf(7, 7, 2.0).getKernel() 

189 psf = measAlg.KernelPsf(kernel, template.getBBox().getCenter()) 

190 

191 record = exposureCatalog.addNew() 

192 record.setPsf(psf) 

193 record.setWcs(template.wcs) 

194 record.setD(weightKey, 1.0) 

195 record.setBBox(template.getBBox()) 

196 

197 customPsf = CustomCoaddPsf(exposureCatalog, template.wcs) 

198 template.setPsf(customPsf) 

199 

200 # Test that we get an exception if we simply get the FWHM at center. 

201 with self.assertRaises(InvalidParameterError): 

202 getPsfFwhm(template.psf, True) 

203 

204 with self.assertRaises(InvalidParameterError): 

205 getPsfFwhm(template.psf, False) 

206 

207 # Test that evaluateMeanPsfFwhm runs successfully on the template. 

208 evaluateMeanPsfFwhm(template, fwhmExposureBuffer=0.05, fwhmExposureGrid=10) 

209 

210 # Since the PSF is spatially invariant, the FWHM should be the same at 

211 # all points in the science image. 

212 fwhm1 = getPsfFwhm(science.psf, False) 

213 fwhm2 = evaluateMeanPsfFwhm(science, fwhmExposureBuffer=0.05, fwhmExposureGrid=10) 

214 self.assertAlmostEqual(fwhm1[0], fwhm2, places=13) 

215 self.assertAlmostEqual(fwhm1[1], fwhm2, places=13) 

216 

217 self.assertAlmostEqual(evaluateMeanPsfFwhm(science, fwhmExposureBuffer=0.05, 

218 fwhmExposureGrid=10), 

219 getPsfFwhm(science.psf, True), places=7 

220 ) 

221 

222 # Test that the image subtraction task runs successfully. 

223 config = subtractImages.AlardLuptonSubtractTask.ConfigClass() 

224 config.doSubtractBackground = False 

225 task = subtractImages.AlardLuptonSubtractTask(config=config) 

226 

227 # Test that the task runs if we take the mean FWHM on a grid. 

228 with self.assertLogs(level="INFO") as cm: 

229 task.run(template, science, sources) 

230 

231 # Check that evaluateMeanPsfFwhm was called. 

232 # This tests that getPsfFwhm failed raising InvalidParameterError, 

233 # that is caught and handled appropriately. 

234 logMessage = ("INFO:lsst.alardLuptonSubtract:Unable to evaluate PSF at the average position. " 

235 "Evaluting PSF on a grid of points." 

236 ) 

237 self.assertIn(logMessage, cm.output) 

238 

239 def test_auto_convolveTemplate(self): 

240 """Test that auto mode gives the same result as convolveTemplate when 

241 the template psf is the smaller. 

242 """ 

243 noiseLevel = 1. 

244 science, sources = makeTestImage(psfSize=3.0, noiseLevel=noiseLevel, noiseSeed=6) 

245 template, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=7, 

246 templateBorderSize=20, doApplyCalibration=True) 

247 config = subtractImages.AlardLuptonSubtractTask.ConfigClass() 

248 config.doSubtractBackground = False 

249 config.mode = "convolveTemplate" 

250 

251 task = subtractImages.AlardLuptonSubtractTask(config=config) 

252 output = task.run(template.clone(), science.clone(), sources) 

253 

254 config.mode = "auto" 

255 task = subtractImages.AlardLuptonSubtractTask(config=config) 

256 outputAuto = task.run(template, science, sources) 

257 self.assertMaskedImagesEqual(output.difference.maskedImage, outputAuto.difference.maskedImage) 

258 

259 def test_auto_convolveScience(self): 

260 """Test that auto mode gives the same result as convolveScience when 

261 the science psf is the smaller. 

262 """ 

263 noiseLevel = 1. 

264 science, sources = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=6) 

265 template, _ = makeTestImage(psfSize=3.0, noiseLevel=noiseLevel, noiseSeed=7, 

266 templateBorderSize=20, doApplyCalibration=True) 

267 config = subtractImages.AlardLuptonSubtractTask.ConfigClass() 

268 config.doSubtractBackground = False 

269 config.mode = "convolveScience" 

270 

271 task = subtractImages.AlardLuptonSubtractTask(config=config) 

272 output = task.run(template.clone(), science.clone(), sources) 

273 

274 config.mode = "auto" 

275 task = subtractImages.AlardLuptonSubtractTask(config=config) 

276 outputAuto = task.run(template, science, sources) 

277 self.assertMaskedImagesEqual(output.difference.maskedImage, outputAuto.difference.maskedImage) 

278 

279 def test_science_better(self): 

280 """Test that running with enough sources produces reasonable output, 

281 with the science psf being smaller than the template. 

282 """ 

283 statsCtrl = makeStats() 

284 statsCtrlDetect = makeStats(badMaskPlanes=("EDGE", "BAD", "NO_DATA")) 

285 

286 def _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel, templateNoiseLevel): 

287 science, sources = makeTestImage(psfSize=2.0, noiseLevel=scienceNoiseLevel, noiseSeed=6) 

288 template, _ = makeTestImage(psfSize=3.0, noiseLevel=templateNoiseLevel, noiseSeed=7, 

289 templateBorderSize=20, doApplyCalibration=True) 

290 config = subtractImages.AlardLuptonSubtractTask.ConfigClass() 

291 config.doSubtractBackground = False 

292 config.mode = "convolveScience" 

293 task = subtractImages.AlardLuptonSubtractTask(config=config) 

294 output = task.run(template, science, sources) 

295 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"], 1., atol=.05) 

296 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"], 1., atol=.05) 

297 # Mean of difference image should be close to zero. 

298 nGoodPix = np.sum(np.isfinite(output.difference.image.array)) 

299 meanError = (scienceNoiseLevel + templateNoiseLevel)/np.sqrt(nGoodPix) 

300 diffimMean = computeRobustStatistics(output.difference.image, output.difference.mask, 

301 statsCtrlDetect) 

302 

303 self.assertFloatsAlmostEqual(diffimMean, 0, atol=5*meanError) 

304 # stddev of difference image should be close to expected value. 

305 noiseLevel = np.sqrt(scienceNoiseLevel**2 + templateNoiseLevel**2) 

306 varianceMean = computeRobustStatistics(output.difference.variance, output.difference.mask, 

307 statsCtrl) 

308 diffimStd = computeRobustStatistics(output.difference.image, output.difference.mask, 

309 statsCtrl, statistic=afwMath.STDEV) 

310 self.assertFloatsAlmostEqual(varianceMean, noiseLevel**2, rtol=0.1) 

311 self.assertFloatsAlmostEqual(diffimStd, noiseLevel, rtol=0.1) 

312 

313 _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel=1., templateNoiseLevel=1.) 

314 _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel=1., templateNoiseLevel=.1) 

315 _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel=.1, templateNoiseLevel=.1) 

316 

317 def test_template_better(self): 

318 """Test that running with enough sources produces reasonable output, 

319 with the template psf being smaller than the science. 

320 """ 

321 statsCtrl = makeStats() 

322 statsCtrlDetect = makeStats(badMaskPlanes=("EDGE", "BAD", "NO_DATA")) 

323 

324 def _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel, templateNoiseLevel): 

325 science, sources = makeTestImage(psfSize=3.0, noiseLevel=scienceNoiseLevel, noiseSeed=6) 

326 template, _ = makeTestImage(psfSize=2.0, noiseLevel=templateNoiseLevel, noiseSeed=7, 

327 templateBorderSize=20, doApplyCalibration=True) 

328 config = subtractImages.AlardLuptonSubtractTask.ConfigClass() 

329 config.doSubtractBackground = False 

330 task = subtractImages.AlardLuptonSubtractTask(config=config) 

331 output = task.run(template, science, sources) 

332 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"], 1., atol=.05) 

333 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"], 1., atol=.05) 

334 # There should be no NaNs in the image if we convolve the template with a buffer 

335 self.assertTrue(np.all(np.isfinite(output.difference.image.array))) 

336 # Mean of difference image should be close to zero. 

337 meanError = (scienceNoiseLevel + templateNoiseLevel)/np.sqrt(output.difference.image.array.size) 

338 

339 diffimMean = computeRobustStatistics(output.difference.image, output.difference.mask, 

340 statsCtrlDetect) 

341 self.assertFloatsAlmostEqual(diffimMean, 0, atol=5*meanError) 

342 # stddev of difference image should be close to expected value. 

343 noiseLevel = np.sqrt(scienceNoiseLevel**2 + templateNoiseLevel**2) 

344 varianceMean = computeRobustStatistics(output.difference.variance, output.difference.mask, 

345 statsCtrl) 

346 diffimStd = computeRobustStatistics(output.difference.image, output.difference.mask, 

347 statsCtrl, statistic=afwMath.STDEV) 

348 self.assertFloatsAlmostEqual(varianceMean, noiseLevel**2, rtol=0.1) 

349 self.assertFloatsAlmostEqual(diffimStd, noiseLevel, rtol=0.1) 

350 

351 _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel=1., templateNoiseLevel=1.) 

352 _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel=1., templateNoiseLevel=.1) 

353 _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel=.1, templateNoiseLevel=.1) 

354 

355 def test_symmetry(self): 

356 """Test that convolving the science and convolving the template are 

357 symmetric: if the psfs are switched between them, the difference image 

358 should be nearly the same. 

359 """ 

360 noiseLevel = 1. 

361 # Don't include a border for the template, in order to make the results 

362 # comparable when we swap which image is treated as the "science" image. 

363 science, sources = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, 

364 noiseSeed=6, templateBorderSize=0, doApplyCalibration=True) 

365 template, _ = makeTestImage(psfSize=3.0, noiseLevel=noiseLevel, 

366 noiseSeed=7, templateBorderSize=0, doApplyCalibration=True) 

367 config = subtractImages.AlardLuptonSubtractTask.ConfigClass() 

368 config.mode = 'auto' 

369 config.doSubtractBackground = False 

370 task = subtractImages.AlardLuptonSubtractTask(config=config) 

371 

372 # The science image will be modified in place, so use a copy for the second run. 

373 science_better = task.run(template.clone(), science.clone(), sources) 

374 template_better = task.run(science, template, sources) 

375 

376 delta = template_better.difference.clone() 

377 delta.image -= science_better.difference.image 

378 delta.variance -= science_better.difference.variance 

379 delta.mask.array -= science_better.difference.mask.array 

380 

381 statsCtrl = makeStats() 

382 # Mean of delta should be very close to zero. 

383 nGoodPix = np.sum(np.isfinite(delta.image.array)) 

384 meanError = 2*noiseLevel/np.sqrt(nGoodPix) 

385 deltaMean = computeRobustStatistics(delta.image, delta.mask, statsCtrl) 

386 deltaStd = computeRobustStatistics(delta.image, delta.mask, statsCtrl, statistic=afwMath.STDEV) 

387 self.assertFloatsAlmostEqual(deltaMean, 0, atol=5*meanError) 

388 # stddev of difference image should be close to expected value 

389 self.assertFloatsAlmostEqual(deltaStd, 2*np.sqrt(2)*noiseLevel, rtol=.1) 

390 

391 def test_few_sources(self): 

392 """Test with only 1 source, to check that we get a useful error. 

393 """ 

394 xSize = 256 

395 ySize = 256 

396 science, sources = makeTestImage(psfSize=2.4, nSrc=10, xSize=xSize, ySize=ySize) 

397 template, _ = makeTestImage(psfSize=2.0, nSrc=10, xSize=xSize, ySize=ySize, doApplyCalibration=True) 

398 config = subtractImages.AlardLuptonSubtractTask.ConfigClass() 

399 task = subtractImages.AlardLuptonSubtractTask(config=config) 

400 sources = sources[0:1] 

401 with self.assertRaisesRegex(RuntimeError, 

402 "Cannot compute PSF matching kernel: too few sources selected."): 

403 task.run(template, science, sources) 

404 

405 def test_order_equal_images(self): 

406 """Verify that the result is the same regardless of convolution mode 

407 if the images are equivalent. 

408 """ 

409 noiseLevel = .1 

410 seed1 = 6 

411 seed2 = 7 

412 science1, sources1 = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=seed1, 

413 clearEdgeMask=True) 

414 template1, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=seed2, 

415 templateBorderSize=0, doApplyCalibration=True, 

416 clearEdgeMask=True) 

417 config1 = subtractImages.AlardLuptonSubtractTask.ConfigClass() 

418 config1.mode = "convolveTemplate" 

419 config1.doSubtractBackground = False 

420 task1 = subtractImages.AlardLuptonSubtractTask(config=config1) 

421 results_convolveTemplate = task1.run(template1, science1, sources1) 

422 

423 science2, sources2 = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=seed1, 

424 clearEdgeMask=True) 

425 template2, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=seed2, 

426 templateBorderSize=0, doApplyCalibration=True, 

427 clearEdgeMask=True) 

428 config2 = subtractImages.AlardLuptonSubtractTask.ConfigClass() 

429 config2.mode = "convolveScience" 

430 config2.doSubtractBackground = False 

431 task2 = subtractImages.AlardLuptonSubtractTask(config=config2) 

432 results_convolveScience = task2.run(template2, science2, sources2) 

433 bbox = results_convolveTemplate.difference.getBBox().clippedTo( 

434 results_convolveScience.difference.getBBox()) 

435 diff1 = science1.maskedImage.clone()[bbox] 

436 diff1 -= template1.maskedImage[bbox] 

437 diff2 = science2.maskedImage.clone()[bbox] 

438 diff2 -= template2.maskedImage[bbox] 

439 self.assertFloatsAlmostEqual(results_convolveTemplate.difference[bbox].image.array, 

440 diff1.image.array, 

441 atol=noiseLevel*5.) 

442 self.assertFloatsAlmostEqual(results_convolveScience.difference[bbox].image.array, 

443 diff2.image.array, 

444 atol=noiseLevel*5.) 

445 diffErr = noiseLevel*2 

446 self.assertMaskedImagesAlmostEqual(results_convolveTemplate.difference[bbox].maskedImage, 

447 results_convolveScience.difference[bbox].maskedImage, 

448 atol=diffErr*5.) 

449 

450 def test_background_subtraction(self): 

451 """Check that we can recover the background, 

452 and that it is subtracted correctly in the difference image. 

453 """ 

454 noiseLevel = 1. 

455 xSize = 512 

456 ySize = 512 

457 x0 = 123 

458 y0 = 456 

459 template, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=7, 

460 templateBorderSize=20, 

461 xSize=xSize, ySize=ySize, x0=x0, y0=y0, 

462 doApplyCalibration=True) 

463 params = [2.2, 2.1, 2.0, 1.2, 1.1, 1.0] 

464 

465 bbox2D = lsst.geom.Box2D(lsst.geom.Point2D(x0, y0), lsst.geom.Extent2D(xSize, ySize)) 

466 background_model = afwMath.Chebyshev1Function2D(params, bbox2D) 

467 science, sources = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=6, 

468 background=background_model, 

469 xSize=xSize, ySize=ySize, x0=x0, y0=y0) 

470 config = subtractImages.AlardLuptonSubtractTask.ConfigClass() 

471 config.doSubtractBackground = True 

472 

473 config.makeKernel.kernel.name = "AL" 

474 config.makeKernel.kernel.active.fitForBackground = True 

475 config.makeKernel.kernel.active.spatialKernelOrder = 1 

476 config.makeKernel.kernel.active.spatialBgOrder = 2 

477 statsCtrl = makeStats() 

478 

479 def _run_and_check_images(config, statsCtrl, mode): 

480 """Check that the fit background matches the input model. 

481 """ 

482 config.mode = mode 

483 task = subtractImages.AlardLuptonSubtractTask(config=config) 

484 output = task.run(template.clone(), science.clone(), sources) 

485 

486 # We should be fitting the same number of parameters as were in the input model 

487 self.assertEqual(output.backgroundModel.getNParameters(), background_model.getNParameters()) 

488 

489 # The parameters of the background fit should be close to the input model 

490 self.assertFloatsAlmostEqual(np.array(output.backgroundModel.getParameters()), 

491 np.array(params), rtol=0.3) 

492 

493 # stddev of difference image should be close to expected value. 

494 # This will fail if we have mis-subtracted the background. 

495 stdVal = computeRobustStatistics(output.difference.image, output.difference.mask, 

496 statsCtrl, statistic=afwMath.STDEV) 

497 self.assertFloatsAlmostEqual(stdVal, np.sqrt(2)*noiseLevel, rtol=0.1) 

498 

499 _run_and_check_images(config, statsCtrl, "convolveTemplate") 

500 _run_and_check_images(config, statsCtrl, "convolveScience") 

501 

502 def test_scale_variance_convolve_template(self): 

503 """Check variance scaling of the image difference. 

504 """ 

505 scienceNoiseLevel = 4. 

506 templateNoiseLevel = 2. 

507 scaleFactor = 1.345 

508 # Make sure to include pixels with the DETECTED mask bit set. 

509 statsCtrl = makeStats(badMaskPlanes=("EDGE", "BAD", "NO_DATA")) 

510 

511 def _run_and_check_images(science, template, sources, statsCtrl, 

512 doDecorrelation, doScaleVariance, scaleFactor=1.): 

513 """Check that the variance plane matches the expected value for 

514 different configurations of ``doDecorrelation`` and ``doScaleVariance``. 

515 """ 

516 

517 config = subtractImages.AlardLuptonSubtractTask.ConfigClass() 

518 config.doSubtractBackground = False 

519 config.doDecorrelation = doDecorrelation 

520 config.doScaleVariance = doScaleVariance 

521 task = subtractImages.AlardLuptonSubtractTask(config=config) 

522 output = task.run(template.clone(), science.clone(), sources) 

523 if doScaleVariance: 

524 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"], 

525 scaleFactor, atol=0.05) 

526 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"], 

527 scaleFactor, atol=0.05) 

528 

529 scienceNoise = computeRobustStatistics(science.variance, science.mask, statsCtrl) 

530 if doDecorrelation: 

531 templateNoise = computeRobustStatistics(template.variance, template.mask, statsCtrl) 

532 else: 

533 templateNoise = computeRobustStatistics(output.matchedTemplate.variance, 

534 output.matchedTemplate.mask, 

535 statsCtrl) 

536 

537 if doScaleVariance: 

538 templateNoise *= scaleFactor 

539 scienceNoise *= scaleFactor 

540 varMean = computeRobustStatistics(output.difference.variance, output.difference.mask, statsCtrl) 

541 self.assertFloatsAlmostEqual(varMean, scienceNoise + templateNoise, rtol=0.1) 

542 

543 science, sources = makeTestImage(psfSize=3.0, noiseLevel=scienceNoiseLevel, noiseSeed=6) 

544 template, _ = makeTestImage(psfSize=2.0, noiseLevel=templateNoiseLevel, noiseSeed=7, 

545 templateBorderSize=20, doApplyCalibration=True) 

546 # Verify that the variance plane of the difference image is correct 

547 # when the template and science variance planes are correct 

548 _run_and_check_images(science, template, sources, statsCtrl, 

549 doDecorrelation=True, doScaleVariance=True) 

550 _run_and_check_images(science, template, sources, statsCtrl, 

551 doDecorrelation=True, doScaleVariance=False) 

552 _run_and_check_images(science, template, sources, statsCtrl, 

553 doDecorrelation=False, doScaleVariance=True) 

554 _run_and_check_images(science, template, sources, statsCtrl, 

555 doDecorrelation=False, doScaleVariance=False) 

556 

557 # Verify that the variance plane of the difference image is correct 

558 # when the template variance plane is incorrect 

559 template.variance.array /= scaleFactor 

560 science.variance.array /= scaleFactor 

561 _run_and_check_images(science, template, sources, statsCtrl, 

562 doDecorrelation=True, doScaleVariance=True, scaleFactor=scaleFactor) 

563 _run_and_check_images(science, template, sources, statsCtrl, 

564 doDecorrelation=True, doScaleVariance=False, scaleFactor=scaleFactor) 

565 _run_and_check_images(science, template, sources, statsCtrl, 

566 doDecorrelation=False, doScaleVariance=True, scaleFactor=scaleFactor) 

567 _run_and_check_images(science, template, sources, statsCtrl, 

568 doDecorrelation=False, doScaleVariance=False, scaleFactor=scaleFactor) 

569 

570 def test_scale_variance_convolve_science(self): 

571 """Check variance scaling of the image difference. 

572 """ 

573 scienceNoiseLevel = 4. 

574 templateNoiseLevel = 2. 

575 scaleFactor = 1.345 

576 # Make sure to include pixels with the DETECTED mask bit set. 

577 statsCtrl = makeStats(badMaskPlanes=("EDGE", "BAD", "NO_DATA")) 

578 

579 def _run_and_check_images(science, template, sources, statsCtrl, 

580 doDecorrelation, doScaleVariance, scaleFactor=1.): 

581 """Check that the variance plane matches the expected value for 

582 different configurations of ``doDecorrelation`` and ``doScaleVariance``. 

583 """ 

584 

585 config = subtractImages.AlardLuptonSubtractTask.ConfigClass() 

586 config.mode = "convolveScience" 

587 config.doSubtractBackground = False 

588 config.doDecorrelation = doDecorrelation 

589 config.doScaleVariance = doScaleVariance 

590 task = subtractImages.AlardLuptonSubtractTask(config=config) 

591 output = task.run(template.clone(), science.clone(), sources) 

592 if doScaleVariance: 

593 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"], 

594 scaleFactor, atol=0.05) 

595 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"], 

596 scaleFactor, atol=0.05) 

597 

598 templateNoise = computeRobustStatistics(template.variance, template.mask, statsCtrl) 

599 if doDecorrelation: 

600 scienceNoise = computeRobustStatistics(science.variance, science.mask, statsCtrl) 

601 else: 

602 scienceNoise = computeRobustStatistics(output.matchedScience.variance, 

603 output.matchedScience.mask, 

604 statsCtrl) 

605 

606 if doScaleVariance: 

607 templateNoise *= scaleFactor 

608 scienceNoise *= scaleFactor 

609 

610 varMean = computeRobustStatistics(output.difference.variance, output.difference.mask, statsCtrl) 

611 self.assertFloatsAlmostEqual(varMean, scienceNoise + templateNoise, rtol=0.1) 

612 

613 science, sources = makeTestImage(psfSize=2.0, noiseLevel=scienceNoiseLevel, noiseSeed=6) 

614 template, _ = makeTestImage(psfSize=3.0, noiseLevel=templateNoiseLevel, noiseSeed=7, 

615 templateBorderSize=20, doApplyCalibration=True) 

616 # Verify that the variance plane of the difference image is correct 

617 # when the template and science variance planes are correct 

618 _run_and_check_images(science, template, sources, statsCtrl, 

619 doDecorrelation=True, doScaleVariance=True) 

620 _run_and_check_images(science, template, sources, statsCtrl, 

621 doDecorrelation=True, doScaleVariance=False) 

622 _run_and_check_images(science, template, sources, statsCtrl, 

623 doDecorrelation=False, doScaleVariance=True) 

624 _run_and_check_images(science, template, sources, statsCtrl, 

625 doDecorrelation=False, doScaleVariance=False) 

626 

627 # Verify that the variance plane of the difference image is correct 

628 # when the template and science variance planes are incorrect 

629 science.variance.array /= scaleFactor 

630 template.variance.array /= scaleFactor 

631 _run_and_check_images(science, template, sources, statsCtrl, 

632 doDecorrelation=True, doScaleVariance=True, scaleFactor=scaleFactor) 

633 _run_and_check_images(science, template, sources, statsCtrl, 

634 doDecorrelation=True, doScaleVariance=False, scaleFactor=scaleFactor) 

635 _run_and_check_images(science, template, sources, statsCtrl, 

636 doDecorrelation=False, doScaleVariance=True, scaleFactor=scaleFactor) 

637 _run_and_check_images(science, template, sources, statsCtrl, 

638 doDecorrelation=False, doScaleVariance=False, scaleFactor=scaleFactor) 

639 

640 def test_exposure_properties_convolve_template(self): 

641 """Check that all necessary exposure metadata is included 

642 when the template is convolved. 

643 """ 

644 noiseLevel = 1. 

645 seed = 37 

646 rng = np.random.RandomState(seed) 

647 science, sources = makeTestImage(psfSize=3.0, noiseLevel=noiseLevel, noiseSeed=6) 

648 psf = science.psf 

649 psfAvgPos = psf.getAveragePosition() 

650 psfSize = getPsfFwhm(science.psf) 

651 psfImg = psf.computeKernelImage(psfAvgPos) 

652 template, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=7, 

653 templateBorderSize=20, doApplyCalibration=True) 

654 

655 # Generate a random aperture correction map 

656 apCorrMap = lsst.afw.image.ApCorrMap() 

657 for name in ("a", "b", "c"): 

658 apCorrMap.set(name, lsst.afw.math.ChebyshevBoundedField(science.getBBox(), rng.randn(3, 3))) 

659 science.info.setApCorrMap(apCorrMap) 

660 

661 config = subtractImages.AlardLuptonSubtractTask.ConfigClass() 

662 config.mode = "convolveTemplate" 

663 

664 def _run_and_check_images(doDecorrelation): 

665 """Check that the metadata is correct with or without decorrelation. 

666 """ 

667 config.doDecorrelation = doDecorrelation 

668 task = subtractImages.AlardLuptonSubtractTask(config=config) 

669 output = task.run(template.clone(), science.clone(), sources) 

670 psfOut = output.difference.psf 

671 psfAvgPos = psfOut.getAveragePosition() 

672 if doDecorrelation: 

673 # Decorrelation requires recalculating the PSF, 

674 # so it will not be the same as the input 

675 psfOutSize = getPsfFwhm(science.psf) 

676 self.assertFloatsAlmostEqual(psfSize, psfOutSize) 

677 else: 

678 psfOutImg = psfOut.computeKernelImage(psfAvgPos) 

679 self.assertImagesAlmostEqual(psfImg, psfOutImg) 

680 

681 # check PSF, WCS, bbox, filterLabel, photoCalib, aperture correction 

682 self._compare_apCorrMaps(apCorrMap, output.difference.info.getApCorrMap()) 

683 self.assertWcsAlmostEqualOverBBox(science.wcs, output.difference.wcs, science.getBBox()) 

684 self.assertEqual(science.filter, output.difference.filter) 

685 self.assertEqual(science.photoCalib, output.difference.photoCalib) 

686 _run_and_check_images(doDecorrelation=True) 

687 _run_and_check_images(doDecorrelation=False) 

688 

689 def test_exposure_properties_convolve_science(self): 

690 """Check that all necessary exposure metadata is included 

691 when the science image is convolved. 

692 """ 

693 noiseLevel = 1. 

694 seed = 37 

695 rng = np.random.RandomState(seed) 

696 science, sources = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=6) 

697 template, _ = makeTestImage(psfSize=3.0, noiseLevel=noiseLevel, noiseSeed=7, 

698 templateBorderSize=20, doApplyCalibration=True) 

699 psf = template.psf 

700 psfAvgPos = psf.getAveragePosition() 

701 psfSize = getPsfFwhm(template.psf) 

702 psfImg = psf.computeKernelImage(psfAvgPos) 

703 

704 # Generate a random aperture correction map 

705 apCorrMap = lsst.afw.image.ApCorrMap() 

706 for name in ("a", "b", "c"): 

707 apCorrMap.set(name, lsst.afw.math.ChebyshevBoundedField(science.getBBox(), rng.randn(3, 3))) 

708 science.info.setApCorrMap(apCorrMap) 

709 

710 config = subtractImages.AlardLuptonSubtractTask.ConfigClass() 

711 config.mode = "convolveScience" 

712 

713 def _run_and_check_images(doDecorrelation): 

714 """Check that the metadata is correct with or without decorrelation. 

715 """ 

716 config.doDecorrelation = doDecorrelation 

717 task = subtractImages.AlardLuptonSubtractTask(config=config) 

718 output = task.run(template.clone(), science.clone(), sources) 

719 if doDecorrelation: 

720 # Decorrelation requires recalculating the PSF, 

721 # so it will not be the same as the input 

722 psfOutSize = getPsfFwhm(template.psf) 

723 self.assertFloatsAlmostEqual(psfSize, psfOutSize) 

724 else: 

725 psfOut = output.difference.psf 

726 psfAvgPos = psfOut.getAveragePosition() 

727 psfOutImg = psfOut.computeKernelImage(psfAvgPos) 

728 self.assertImagesAlmostEqual(psfImg, psfOutImg) 

729 

730 # check PSF, WCS, bbox, filterLabel, photoCalib, aperture correction 

731 self._compare_apCorrMaps(apCorrMap, output.difference.info.getApCorrMap()) 

732 self.assertWcsAlmostEqualOverBBox(science.wcs, output.difference.wcs, science.getBBox()) 

733 self.assertEqual(science.filter, output.difference.filter) 

734 self.assertEqual(science.photoCalib, output.difference.photoCalib) 

735 

736 _run_and_check_images(doDecorrelation=True) 

737 _run_and_check_images(doDecorrelation=False) 

738 

739 def _compare_apCorrMaps(self, a, b): 

740 """Compare two ApCorrMaps for equality, without assuming that their BoundedFields have the 

741 same addresses (i.e. so we can compare after serialization). 

742 

743 This function is taken from ``ApCorrMapTestCase`` in afw/tests/. 

744 

745 Parameters 

746 ---------- 

747 a, b : `lsst.afw.image.ApCorrMap` 

748 The two aperture correction maps to compare. 

749 """ 

750 self.assertEqual(len(a), len(b)) 

751 for name, value in list(a.items()): 

752 value2 = b.get(name) 

753 self.assertIsNotNone(value2) 

754 self.assertEqual(value.getBBox(), value2.getBBox()) 

755 self.assertFloatsAlmostEqual( 

756 value.getCoefficients(), value2.getCoefficients(), rtol=0.0) 

757 

758 def test_fake_mask_plane_propagation(self): 

759 """Test that we have the mask planes related to fakes in diffim images. 

760 This is testing method called updateMasks 

761 """ 

762 xSize = 200 

763 ySize = 200 

764 science, sources = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize) 

765 science_fake_img, science_fake_sources = makeTestImage( 

766 psfSize=2.4, xSize=xSize, ySize=ySize, seed=7, nSrc=2, noiseLevel=0.25, fluxRange=1 

767 ) 

768 template, _ = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize, doApplyCalibration=True) 

769 tmplt_fake_img, tmplt_fake_sources = makeTestImage( 

770 psfSize=2.4, xSize=xSize, ySize=ySize, seed=9, nSrc=2, noiseLevel=0.25, fluxRange=1 

771 ) 

772 # created fakes and added them to the images 

773 science.image.array += science_fake_img.image.array 

774 template.image.array += tmplt_fake_img.image.array 

775 

776 # TODO: DM-40796 update to INJECTED names when source injection gets refactored 

777 # adding mask planes to both science and template images 

778 science_mask_planes = science.mask.addMaskPlane("FAKE") 

779 template_mask_planes = template.mask.addMaskPlane("FAKE") 

780 

781 for a_science_source in science_fake_sources: 

782 # 3 x 3 masking of the source locations is fine 

783 bbox = lsst.geom.Box2I( 

784 lsst.geom.Point2I(a_science_source.getX(), a_science_source.getY()), lsst.geom.Extent2I(3, 3) 

785 ) 

786 science[bbox].mask.array |= science_mask_planes 

787 

788 for a_template_source in tmplt_fake_sources: 

789 # 3 x 3 masking of the source locations is fine 

790 bbox = lsst.geom.Box2I( 

791 lsst.geom.Point2I(a_template_source.getX(), a_template_source.getY()), 

792 lsst.geom.Extent2I(3, 3) 

793 ) 

794 template[bbox].mask.array |= template_mask_planes 

795 

796 science_fake_masked = (science.mask.array & science.mask.getPlaneBitMask("FAKE")) > 0 

797 template_fake_masked = (template.mask.array & template.mask.getPlaneBitMask("FAKE")) > 0 

798 

799 config = subtractImages.AlardLuptonSubtractTask.ConfigClass() 

800 task = subtractImages.AlardLuptonSubtractTask(config=config) 

801 subtraction = task.run(template, science, sources) 

802 

803 # check subtraction mask plane is set where we set the previous masks 

804 diff_mask = subtraction.difference.mask 

805 

806 # science mask should be now in INJECTED 

807 inj_masked = (diff_mask.array & diff_mask.getPlaneBitMask("INJECTED")) > 0 

808 

809 # template mask should be now in INJECTED_TEMPLATE 

810 injTmplt_masked = (diff_mask.array & diff_mask.getPlaneBitMask("INJECTED_TEMPLATE")) > 0 

811 

812 self.assertEqual(np.sum(inj_masked.astype(int)-science_fake_masked.astype(int)), 0) 

813 self.assertEqual(np.sum(injTmplt_masked.astype(int)-template_fake_masked.astype(int)), 0) 

814 

815 

816class AlardLuptonPreconvolveSubtractTest(lsst.utils.tests.TestCase): 

817 

818 def test_mismatched_template(self): 

819 """Test that an error is raised if the template 

820 does not fully contain the science image. 

821 """ 

822 xSize = 200 

823 ySize = 200 

824 science, sources = makeTestImage(psfSize=2.4, xSize=xSize + 20, ySize=ySize + 20) 

825 template, _ = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize, doApplyCalibration=True) 

826 config = subtractImages.AlardLuptonPreconvolveSubtractTask.ConfigClass() 

827 task = subtractImages.AlardLuptonPreconvolveSubtractTask(config=config) 

828 with self.assertRaises(AssertionError): 

829 task.run(template, science, sources) 

830 

831 def test_equal_images(self): 

832 """Test that running with enough sources produces reasonable output, 

833 with the same size psf in the template and science. 

834 """ 

835 noiseLevel = 1. 

836 xSize = 400 

837 ySize = 400 

838 science, sources = makeTestImage(psfSize=2.4, noiseLevel=noiseLevel, noiseSeed=6, 

839 xSize=xSize, ySize=ySize) 

840 template, _ = makeTestImage(psfSize=2.4, noiseLevel=noiseLevel, noiseSeed=7, 

841 templateBorderSize=20, doApplyCalibration=True, 

842 xSize=xSize, ySize=ySize) 

843 config = subtractImages.AlardLuptonPreconvolveSubtractTask.ConfigClass() 

844 config.doSubtractBackground = False 

845 task = subtractImages.AlardLuptonPreconvolveSubtractTask(config=config) 

846 output = task.run(template, science, sources) 

847 # There shoud be no NaN values in the Score image 

848 self.assertTrue(np.all(np.isfinite(output.scoreExposure.image.array))) 

849 # Mean of Score image should be close to zero. 

850 meanError = noiseLevel/np.sqrt(output.scoreExposure.image.array.size) 

851 # Make sure to include pixels with the DETECTED mask bit set. 

852 statsCtrl = makeStats(badMaskPlanes=("EDGE", "BAD", "NO_DATA")) 

853 scoreMean = computeRobustStatistics(output.scoreExposure.image, 

854 output.scoreExposure.mask, 

855 statsCtrl) 

856 self.assertFloatsAlmostEqual(scoreMean, 0, atol=5*meanError) 

857 nea = computePSFNoiseEquivalentArea(science.psf) 

858 # stddev of Score image should be close to expected value. 

859 scoreStd = computeRobustStatistics(output.scoreExposure.image, output.scoreExposure.mask, 

860 statsCtrl=statsCtrl, statistic=afwMath.STDEV) 

861 self.assertFloatsAlmostEqual(scoreStd, np.sqrt(2)*noiseLevel/np.sqrt(nea), rtol=0.1) 

862 

863 def test_incomplete_template_coverage(self): 

864 noiseLevel = 1. 

865 border = 20 

866 xSize = 400 

867 ySize = 400 

868 science, sources = makeTestImage(psfSize=3.0, noiseLevel=noiseLevel, noiseSeed=6, 

869 xSize=xSize, ySize=ySize) 

870 template, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=7, 

871 templateBorderSize=border, doApplyCalibration=True, 

872 xSize=xSize, ySize=ySize) 

873 

874 science_height = science.getBBox().getDimensions().getY() 

875 

876 def _run_and_check_coverage(template_coverage): 

877 template_cut = template.clone() 

878 template_height = int(science_height*template_coverage + border) 

879 template_cut.image.array[:, template_height:] = 0 

880 template_cut.mask.array[:, template_height:] = template_cut.mask.getPlaneBitMask('NO_DATA') 

881 config = subtractImages.AlardLuptonPreconvolveSubtractTask.ConfigClass() 

882 if template_coverage < config.requiredTemplateFraction: 

883 doRaise = True 

884 elif template_coverage < config.minTemplateFractionForExpectedSuccess: 

885 doRaise = True 

886 else: 

887 doRaise = False 

888 task = subtractImages.AlardLuptonPreconvolveSubtractTask(config=config) 

889 if doRaise: 

890 with self.assertRaises(NoWorkFound): 

891 task.run(template_cut, science.clone(), sources.copy(deep=True)) 

892 else: 

893 task.run(template_cut, science.clone(), sources.copy(deep=True)) 

894 _run_and_check_coverage(template_coverage=0.09) 

895 _run_and_check_coverage(template_coverage=0.19) 

896 _run_and_check_coverage(template_coverage=.7) 

897 

898 def test_clear_template_mask(self): 

899 noiseLevel = 1. 

900 xSize = 400 

901 ySize = 400 

902 science, sources = makeTestImage(psfSize=3.0, noiseLevel=noiseLevel, noiseSeed=6, 

903 xSize=xSize, ySize=ySize) 

904 template, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=7, 

905 templateBorderSize=20, doApplyCalibration=True, 

906 xSize=xSize, ySize=ySize) 

907 diffimEmptyMaskPlanes = ["DETECTED", "DETECTED_NEGATIVE"] 

908 config = subtractImages.AlardLuptonPreconvolveSubtractTask.ConfigClass() 

909 config.doSubtractBackground = False # Ensure that each each mask plane is set for some pixels 

910 mask = template.mask 

911 x0 = 50 

912 x1 = 75 

913 y0 = 150 

914 y1 = 175 

915 scienceMaskCheck = {} 

916 for maskPlane in mask.getMaskPlaneDict().keys(): 

917 scienceMaskCheck[maskPlane] = np.sum(science.mask.array & mask.getPlaneBitMask(maskPlane) > 0) 

918 mask.array[x0: x1, y0: y1] |= mask.getPlaneBitMask(maskPlane) 

919 self.assertTrue(np.sum(mask.array & mask.getPlaneBitMask(maskPlane) > 0)) 

920 

921 task = subtractImages.AlardLuptonPreconvolveSubtractTask(config=config) 

922 output = task.run(template, science, sources) 

923 # Verify that the template mask has been modified in place 

924 for maskPlane in mask.getMaskPlaneDict().keys(): 

925 if maskPlane in diffimEmptyMaskPlanes: 

926 self.assertTrue(np.sum(mask.array & mask.getPlaneBitMask(maskPlane) == 0)) 

927 elif maskPlane in config.preserveTemplateMask: 

928 self.assertTrue(np.sum(mask.array & mask.getPlaneBitMask(maskPlane) > 0)) 

929 else: 

930 self.assertTrue(np.sum(mask.array & mask.getPlaneBitMask(maskPlane) == 0)) 

931 # Mask planes set in the science image should also be set in the difference 

932 # Except the "DETECTED" planes should have been cleared 

933 diffimMask = output.scoreExposure.mask 

934 for maskPlane, scienceSum in scienceMaskCheck.items(): 

935 diffimSum = np.sum(diffimMask.array & mask.getPlaneBitMask(maskPlane) > 0) 

936 if maskPlane in diffimEmptyMaskPlanes: 

937 self.assertEqual(diffimSum, 0) 

938 else: 

939 self.assertTrue(diffimSum >= scienceSum) 

940 

941 def test_agnostic_template_psf(self): 

942 """Test that the Score image is the same whether the template PSF is 

943 larger or smaller than the science image PSF. 

944 """ 

945 noiseLevel = .3 

946 xSize = 400 

947 ySize = 400 

948 science, sources = makeTestImage(psfSize=2.4, noiseLevel=noiseLevel, 

949 noiseSeed=6, templateBorderSize=0, 

950 xSize=xSize, ySize=ySize) 

951 template1, _ = makeTestImage(psfSize=3.0, noiseLevel=noiseLevel, 

952 noiseSeed=7, doApplyCalibration=True, 

953 xSize=xSize, ySize=ySize) 

954 template2, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, 

955 noiseSeed=8, doApplyCalibration=True, 

956 xSize=xSize, ySize=ySize) 

957 config = subtractImages.AlardLuptonPreconvolveSubtractTask.ConfigClass() 

958 config.doSubtractBackground = False 

959 task = subtractImages.AlardLuptonPreconvolveSubtractTask(config=config) 

960 

961 science_better = task.run(template1, science.clone(), sources) 

962 template_better = task.run(template2, science, sources) 

963 bbox = science_better.scoreExposure.getBBox().clippedTo(template_better.scoreExposure.getBBox()) 

964 

965 delta = template_better.scoreExposure[bbox].clone() 

966 delta.image -= science_better.scoreExposure[bbox].image 

967 delta.variance -= science_better.scoreExposure[bbox].variance 

968 delta.mask.array &= science_better.scoreExposure[bbox].mask.array 

969 

970 statsCtrl = makeStats(badMaskPlanes=("EDGE", "BAD", "NO_DATA")) 

971 # Mean of delta should be very close to zero. 

972 nGoodPix = np.sum(np.isfinite(delta.image.array)) 

973 meanError = 2*noiseLevel/np.sqrt(nGoodPix) 

974 deltaMean = computeRobustStatistics(delta.image, delta.mask, statsCtrl) 

975 deltaStd = computeRobustStatistics(delta.image, delta.mask, statsCtrl, 

976 statistic=afwMath.STDEV) 

977 self.assertFloatsAlmostEqual(deltaMean, 0, atol=5*meanError) 

978 nea = computePSFNoiseEquivalentArea(science.psf) 

979 # stddev of Score image should be close to expected value 

980 self.assertFloatsAlmostEqual(deltaStd, np.sqrt(2)*noiseLevel/np.sqrt(nea), rtol=.1) 

981 

982 def test_few_sources(self): 

983 """Test with only 1 source, to check that we get a useful error. 

984 """ 

985 xSize = 256 

986 ySize = 256 

987 science, sources = makeTestImage(psfSize=2.4, nSrc=10, xSize=xSize, ySize=ySize) 

988 template, _ = makeTestImage(psfSize=2.0, nSrc=10, xSize=xSize, ySize=ySize, doApplyCalibration=True) 

989 config = subtractImages.AlardLuptonPreconvolveSubtractTask.ConfigClass() 

990 task = subtractImages.AlardLuptonPreconvolveSubtractTask(config=config) 

991 sources = sources[0:1] 

992 with self.assertRaisesRegex(RuntimeError, 

993 "Cannot compute PSF matching kernel: too few sources selected."): 

994 task.run(template, science, sources) 

995 

996 def test_background_subtraction(self): 

997 """Check that we can recover the background, 

998 and that it is subtracted correctly in the Score image. 

999 """ 

1000 noiseLevel = 1. 

1001 xSize = 512 

1002 ySize = 512 

1003 x0 = 123 

1004 y0 = 456 

1005 template, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=7, 

1006 templateBorderSize=20, 

1007 xSize=xSize, ySize=ySize, x0=x0, y0=y0, 

1008 doApplyCalibration=True) 

1009 params = [2.2, 2.1, 2.0, 1.2, 1.1, 1.0] 

1010 

1011 bbox2D = lsst.geom.Box2D(lsst.geom.Point2D(x0, y0), lsst.geom.Extent2D(xSize, ySize)) 

1012 background_model = afwMath.Chebyshev1Function2D(params, bbox2D) 

1013 science, sources = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=6, 

1014 background=background_model, 

1015 xSize=xSize, ySize=ySize, x0=x0, y0=y0) 

1016 config = subtractImages.AlardLuptonPreconvolveSubtractTask.ConfigClass() 

1017 config.doSubtractBackground = True 

1018 

1019 config.makeKernel.kernel.name = "AL" 

1020 config.makeKernel.kernel.active.fitForBackground = True 

1021 config.makeKernel.kernel.active.spatialKernelOrder = 1 

1022 config.makeKernel.kernel.active.spatialBgOrder = 2 

1023 statsCtrl = makeStats(badMaskPlanes=("EDGE", "BAD", "NO_DATA")) 

1024 

1025 task = subtractImages.AlardLuptonPreconvolveSubtractTask(config=config) 

1026 output = task.run(template.clone(), science.clone(), sources) 

1027 

1028 # We should be fitting the same number of parameters as were in the input model 

1029 self.assertEqual(output.backgroundModel.getNParameters(), background_model.getNParameters()) 

1030 

1031 # The parameters of the background fit should be close to the input model 

1032 self.assertFloatsAlmostEqual(np.array(output.backgroundModel.getParameters()), 

1033 np.array(params), rtol=0.2) 

1034 

1035 # stddev of Score image should be close to expected value. 

1036 # This will fail if we have mis-subtracted the background. 

1037 stdVal = computeRobustStatistics(output.scoreExposure.image, output.scoreExposure.mask, 

1038 statsCtrl, statistic=afwMath.STDEV) 

1039 # get the img psf Noise Equivalent Area value 

1040 nea = computePSFNoiseEquivalentArea(science.psf) 

1041 self.assertFloatsAlmostEqual(stdVal, np.sqrt(2)*noiseLevel/np.sqrt(nea), rtol=0.1) 

1042 

1043 def test_scale_variance(self): 

1044 """Check variance scaling of the Score image. 

1045 """ 

1046 scienceNoiseLevel = 4. 

1047 templateNoiseLevel = 2. 

1048 scaleFactor = 1.345 

1049 xSize = 400 

1050 ySize = 400 

1051 # Make sure to include pixels with the DETECTED mask bit set. 

1052 statsCtrl = makeStats(badMaskPlanes=("EDGE", "BAD", "NO_DATA")) 

1053 

1054 def _run_and_check_images(science, template, sources, statsCtrl, 

1055 doDecorrelation, doScaleVariance, scaleFactor=1.): 

1056 """Check that the variance plane matches the expected value for 

1057 different configurations of ``doDecorrelation`` and ``doScaleVariance``. 

1058 """ 

1059 

1060 config = subtractImages.AlardLuptonPreconvolveSubtractTask.ConfigClass() 

1061 config.doSubtractBackground = False 

1062 config.doDecorrelation = doDecorrelation 

1063 config.doScaleVariance = doScaleVariance 

1064 task = subtractImages.AlardLuptonPreconvolveSubtractTask(config=config) 

1065 output = task.run(template.clone(), science.clone(), sources) 

1066 if doScaleVariance: 

1067 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"], 

1068 scaleFactor, atol=0.05) 

1069 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"], 

1070 scaleFactor, atol=0.05) 

1071 

1072 scienceNoise = computeRobustStatistics(science.variance, science.mask, statsCtrl) 

1073 # get the img psf Noise Equivalent Area value 

1074 nea = computePSFNoiseEquivalentArea(science.psf) 

1075 scienceNoise /= nea 

1076 if doDecorrelation: 

1077 templateNoise = computeRobustStatistics(template.variance, template.mask, statsCtrl) 

1078 templateNoise /= nea 

1079 else: 

1080 # Don't divide by NEA in this case, since the template is convolved 

1081 # and in the same units as the Score exposure. 

1082 templateNoise = computeRobustStatistics(output.matchedTemplate.variance, 

1083 output.matchedTemplate.mask, 

1084 statsCtrl) 

1085 if doScaleVariance: 

1086 templateNoise *= scaleFactor 

1087 scienceNoise *= scaleFactor 

1088 varMean = computeRobustStatistics(output.scoreExposure.variance, 

1089 output.scoreExposure.mask, 

1090 statsCtrl) 

1091 self.assertFloatsAlmostEqual(varMean, scienceNoise + templateNoise, rtol=0.1) 

1092 

1093 science, sources = makeTestImage(psfSize=3.0, noiseLevel=scienceNoiseLevel, noiseSeed=6, 

1094 xSize=xSize, ySize=ySize) 

1095 template, _ = makeTestImage(psfSize=2.0, noiseLevel=templateNoiseLevel, noiseSeed=7, 

1096 templateBorderSize=20, doApplyCalibration=True, 

1097 xSize=xSize, ySize=ySize) 

1098 # Verify that the variance plane of the Score image is correct 

1099 # when the template and science variance planes are correct 

1100 _run_and_check_images(science, template, sources, statsCtrl, 

1101 doDecorrelation=True, doScaleVariance=True) 

1102 _run_and_check_images(science, template, sources, statsCtrl, 

1103 doDecorrelation=True, doScaleVariance=False) 

1104 _run_and_check_images(science, template, sources, statsCtrl, 

1105 doDecorrelation=False, doScaleVariance=True) 

1106 _run_and_check_images(science, template, sources, statsCtrl, 

1107 doDecorrelation=False, doScaleVariance=False) 

1108 

1109 # Verify that the variance plane of the Score image is correct 

1110 # when the template variance plane is incorrect 

1111 template.variance.array /= scaleFactor 

1112 science.variance.array /= scaleFactor 

1113 _run_and_check_images(science, template, sources, statsCtrl, 

1114 doDecorrelation=True, doScaleVariance=True, scaleFactor=scaleFactor) 

1115 _run_and_check_images(science, template, sources, statsCtrl, 

1116 doDecorrelation=True, doScaleVariance=False, scaleFactor=scaleFactor) 

1117 _run_and_check_images(science, template, sources, statsCtrl, 

1118 doDecorrelation=False, doScaleVariance=True, scaleFactor=scaleFactor) 

1119 _run_and_check_images(science, template, sources, statsCtrl, 

1120 doDecorrelation=False, doScaleVariance=False, scaleFactor=scaleFactor) 

1121 

1122 def test_exposure_properties(self): 

1123 """Check that all necessary exposure metadata is included 

1124 with the Score image. 

1125 """ 

1126 noiseLevel = 1. 

1127 xSize = 400 

1128 ySize = 400 

1129 science, sources = makeTestImage(psfSize=3.0, noiseLevel=noiseLevel, noiseSeed=6, 

1130 xSize=xSize, ySize=ySize) 

1131 psf = science.psf 

1132 psfAvgPos = psf.getAveragePosition() 

1133 psfSize = getPsfFwhm(science.psf) 

1134 psfImg = psf.computeKernelImage(psfAvgPos) 

1135 template, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=7, 

1136 templateBorderSize=20, doApplyCalibration=True, 

1137 xSize=xSize, ySize=ySize) 

1138 

1139 config = subtractImages.AlardLuptonPreconvolveSubtractTask.ConfigClass() 

1140 

1141 def _run_and_check_images(doDecorrelation): 

1142 """Check that the metadata is correct with or without decorrelation. 

1143 """ 

1144 config.doDecorrelation = doDecorrelation 

1145 task = subtractImages.AlardLuptonPreconvolveSubtractTask(config=config) 

1146 output = task.run(template.clone(), science.clone(), sources) 

1147 psfOut = output.scoreExposure.psf 

1148 psfAvgPos = psfOut.getAveragePosition() 

1149 if doDecorrelation: 

1150 # Decorrelation requires recalculating the PSF, 

1151 # so it will not be the same as the input 

1152 psfOutSize = getPsfFwhm(science.psf) 

1153 self.assertFloatsAlmostEqual(psfSize, psfOutSize) 

1154 else: 

1155 psfOutImg = psfOut.computeKernelImage(psfAvgPos) 

1156 self.assertImagesAlmostEqual(psfImg, psfOutImg) 

1157 

1158 # check PSF, WCS, bbox, filterLabel, photoCalib 

1159 self.assertWcsAlmostEqualOverBBox(science.wcs, output.scoreExposure.wcs, science.getBBox()) 

1160 self.assertEqual(science.filter, output.scoreExposure.filter) 

1161 self.assertEqual(science.photoCalib, output.scoreExposure.photoCalib) 

1162 _run_and_check_images(doDecorrelation=True) 

1163 _run_and_check_images(doDecorrelation=False) 

1164 

1165 

1166def setup_module(module): 

1167 lsst.utils.tests.init() 

1168 

1169 

1170class MemoryTestCase(lsst.utils.tests.MemoryTestCase): 

1171 pass 

1172 

1173 

1174if __name__ == "__main__": 1174 ↛ 1175line 1174 didn't jump to line 1175, because the condition on line 1174 was never true

1175 lsst.utils.tests.init() 

1176 unittest.main()