Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ip_diffim. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import os 

23import unittest 

24 

25import lsst.utils.tests 

26import lsst.utils 

27import lsst.afw.image as afwImage 

28import lsst.afw.math as afwMath 

29import lsst.geom as geom 

30import lsst.ip.diffim as ipDiffim 

31import lsst.pex.config as pexConfig 

32import lsst.log.utils as logUtils 

33import lsst.afw.table as afwTable 

34 

35logUtils.traceSetAt("ip.diffim", 4) 

36 

37# known input images 

38try: 

39 defDataDir = lsst.utils.getPackageDir('afwdata') 

40except Exception: 

41 defDataDir = None 

42 

43try: 

44 display 

45 defDataDir 

46except NameError: 

47 display = False 

48else: 

49 import lsst.afw.display as afwDisplay 

50 afwDisplay.setDefaultMaskTransparency(75) 

51 

52 

53class DiffimTestCases(lsst.utils.tests.TestCase): 

54 

55 def setUp(self): 

56 schema = afwTable.SourceTable.makeMinimalSchema() 

57 afwTable.Point2DKey.addFields(schema, "Centroid", "input centroid", "pixel") 

58 schema.addField("PsfFlux_instFlux", type=float) 

59 schema.addField("PsfFlux_instFluxErr", type=float) 

60 schema.addField("PsfFlux_flag", type="Flag") 

61 self.table = afwTable.SourceTable.make(schema) 

62 self.table.definePsfFlux("PsfFlux") 

63 self.table.defineCentroid("Centroid") 

64 self.ss = afwTable.SourceCatalog(self.table) 

65 

66 self.config = ipDiffim.ImagePsfMatchTask.ConfigClass() 

67 self.config.kernel.name = "DF" 

68 self.subconfig = self.config.kernel.active 

69 

70 self.ps = pexConfig.makePropertySet(self.subconfig) 

71 self.ps['fitForBackground'] = True # we are testing known background recovery here 

72 self.ps['checkConditionNumber'] = False # just in case 

73 self.ps["useRegularization"] = False 

74 

75 if defDataDir: 

76 defSciencePath = os.path.join(defDataDir, "DC3a-Sim", "sci", "v26-e0", 

77 "v26-e0-c011-a10.sci.fits") 

78 defTemplatePath = os.path.join(defDataDir, "DC3a-Sim", "sci", "v5-e0", 

79 "v5-e0-c011-a10.sci.fits") 

80 

81 scienceExposure = afwImage.ExposureF(defSciencePath) 

82 templateExposure = afwImage.ExposureF(defTemplatePath) 

83 # set XY0 = 0 

84 scienceExposure.setXY0(geom.Point2I(0, 0)) 

85 templateExposure.setXY0(geom.Point2I(0, 0)) 

86 # do the warping first so we don't have any masked pixels in the postage stamps 

87 warper = afwMath.Warper.fromConfig(self.subconfig.warpingConfig) 

88 templateExposure = warper.warpExposure(scienceExposure.getWcs(), templateExposure, 

89 destBBox=scienceExposure.getBBox()) 

90 

91 # Change xy0 

92 # Nice star at position 276, 717 

93 # And should be at index 40, 40 

94 # No masked pixels in this one 

95 self.x02 = 276 

96 self.y02 = 717 

97 size = 40 

98 bbox2 = geom.Box2I(geom.Point2I(self.x02 - size, self.y02 - size), 

99 geom.Point2I(self.x02 + size, self.y02 + size)) 

100 self.scienceImage2 = afwImage.ExposureF(scienceExposure, bbox2, origin=afwImage.LOCAL) 

101 self.templateExposure2 = afwImage.ExposureF(templateExposure, bbox2, origin=afwImage.LOCAL) 

102 

103 def addNoise(self, mi): 

104 img = mi.getImage() 

105 seed = int(afwMath.makeStatistics(mi.getVariance(), afwMath.MEDIAN).getValue()) 

106 rdm = afwMath.Random(afwMath.Random.MT19937, seed) 

107 rdmImage = img.Factory(img.getDimensions()) 

108 afwMath.randomGaussianImage(rdmImage, rdm) 

109 img += rdmImage 

110 return afwMath.makeStatistics(rdmImage, afwMath.MEAN).getValue(afwMath.MEAN) 

111 

112 def verifyDeltaFunctionSolution(self, solution, kSum=1.0, bg=0.0): 

113 # when kSum = 1.0, this agrees to the default precision. when 

114 # kSum != 1.0 I need to go to only 4 digits. 

115 # 

116 # -5.4640810225678728e-06 != 0.0 within 7 places 

117 # 

118 bgSolution = solution.getBackground() 

119 self.assertAlmostEqual(bgSolution, bg, 4) 

120 

121 # again when kSum = 1.0 this agrees. otherwise 

122 # 

123 # 2.7000000605594079 != 2.7000000000000002 within 7 places 

124 # 

125 kSumSolution = solution.getKsum() 

126 self.assertAlmostEqual(kSumSolution, kSum, 5) 

127 

128 kImage = solution.makeKernelImage() 

129 for j in range(kImage.getHeight()): 

130 for i in range(kImage.getWidth()): 

131 

132 if (i == kImage.getWidth() // 2) and (j == kImage.getHeight() // 2): 

133 self.assertAlmostEqual(kImage[i, j, afwImage.LOCAL], kSum, 5) 

134 else: 

135 self.assertAlmostEqual(kImage[i, j, afwImage.LOCAL], 0., 5) 

136 

137 @unittest.skipIf(not defDataDir, "Warning: afwdata is not set up") 

138 def testConstructor(self): 

139 # Original and uninitialized 

140 kc = ipDiffim.KernelCandidateF(self.x02, self.y02, 

141 self.templateExposure2.getMaskedImage(), 

142 self.scienceImage2.getMaskedImage(), 

143 self.ps) 

144 

145 # Kernel not initialized 

146 self.assertEqual(kc.isInitialized(), False) 

147 

148 # But this should be set on construction 

149 try: 

150 kc.getCandidateRating() 

151 except Exception as e: 

152 print(e) 

153 self.fail() 

154 

155 # And these should be filled 

156 try: 

157 kc.getTemplateMaskedImage() 

158 kc.getScienceMaskedImage() 

159 except Exception as e: 

160 print(e) 

161 self.fail() 

162 

163 # And of the right type 

164 self.assertEqual(type(kc.getTemplateMaskedImage()), afwImage.MaskedImageF) 

165 self.assertEqual(type(kc.getScienceMaskedImage()), afwImage.MaskedImageF) 

166 

167 # None of these should work 

168 for kType in (ipDiffim.KernelCandidateF.ORIG, 

169 ipDiffim.KernelCandidateF.PCA, 

170 ipDiffim.KernelCandidateF.RECENT): 

171 for kMethod in (kc.getKernelSolution, 

172 kc.getKernel, 

173 kc.getBackground, 

174 kc.getKsum, 

175 kc.getKernelImage, 

176 kc.getDifferenceImage): 

177 try: 

178 kMethod(kType) 

179 except Exception: 

180 pass 

181 else: 

182 self.fail() 

183 try: 

184 kc.getImage() 

185 except Exception: 

186 pass 

187 else: 

188 self.fail() 

189 

190 @unittest.skipIf(not defDataDir, "Warning: afwdata is not set up") 

191 def testSourceStats(self): 

192 source = self.ss.addNew() 

193 source.setId(1) 

194 source.set(self.table.getCentroidKey().getX(), 276) 

195 source.set(self.table.getCentroidKey().getY(), 717) 

196 source.set("slot_PsfFlux_instFlux", 1.) 

197 

198 kc = ipDiffim.KernelCandidateF(source, 

199 self.templateExposure2.getMaskedImage(), 

200 self.scienceImage2.getMaskedImage(), 

201 self.ps) 

202 kList = ipDiffim.makeKernelBasisList(self.subconfig) 

203 

204 kc.build(kList) 

205 self.assertEqual(kc.isInitialized(), True) 

206 

207 @unittest.skipIf(not defDataDir, "Warning: afwdata is not set up") 

208 def testSourceConstructor(self): 

209 source = self.ss.addNew() 

210 source.setId(1) 

211 source.set(self.table.getCentroidKey().getX(), 276) 

212 source.set(self.table.getCentroidKey().getY(), 717) 

213 source.set("slot_PsfFlux_instFlux", 1.) 

214 

215 kc = ipDiffim.KernelCandidateF(source, 

216 self.templateExposure2.getMaskedImage(), 

217 self.scienceImage2.getMaskedImage(), 

218 self.ps) 

219 

220 # Kernel not initialized 

221 self.assertEqual(kc.isInitialized(), False) 

222 

223 # Check that the source is set 

224 self.assertEqual(kc.getSource(), source) 

225 self.assertEqual(kc.getCandidateRating(), source.getPsfInstFlux()) 

226 

227 # But this should be set on construction 

228 try: 

229 kc.getCandidateRating() 

230 except Exception as e: 

231 print(e) 

232 self.fail() 

233 

234 # And these should be filled 

235 try: 

236 kc.getTemplateMaskedImage() 

237 kc.getScienceMaskedImage() 

238 except Exception as e: 

239 print(e) 

240 self.fail() 

241 

242 # And of the right type 

243 self.assertEqual(type(kc.getTemplateMaskedImage()), afwImage.MaskedImageF) 

244 self.assertEqual(type(kc.getScienceMaskedImage()), afwImage.MaskedImageF) 

245 

246 # None of these should work 

247 for kType in (ipDiffim.KernelCandidateF.ORIG, 

248 ipDiffim.KernelCandidateF.PCA, 

249 ipDiffim.KernelCandidateF.RECENT): 

250 for kMethod in (kc.getKernelSolution, 

251 kc.getKernel, 

252 kc.getBackground, 

253 kc.getKsum, 

254 kc.getKernelImage, 

255 kc.getDifferenceImage): 

256 try: 

257 kMethod(kType) 

258 except Exception: 

259 pass 

260 else: 

261 self.fail() 

262 try: 

263 kc.getImage() 

264 except Exception: 

265 pass 

266 else: 

267 self.fail() 

268 

269 kList = ipDiffim.makeKernelBasisList(self.subconfig) 

270 

271 kc.build(kList) 

272 self.assertEqual(kc.isInitialized(), True) 

273 

274 @unittest.skipIf(not defDataDir, "Warning: afwdata is not set up") 

275 def testDeltaFunctionScaled(self, scaling=2.7, bg=11.3): 

276 sIm = afwImage.MaskedImageF(self.templateExposure2.getMaskedImage(), deep=True) 

277 sIm *= scaling 

278 kc = ipDiffim.KernelCandidateF(self.x02, self.y02, 

279 self.templateExposure2.getMaskedImage(), 

280 sIm, 

281 self.ps) 

282 

283 kList = ipDiffim.makeKernelBasisList(self.subconfig) 

284 kc.build(kList) 

285 self.verifyDeltaFunctionSolution(kc.getKernelSolution(ipDiffim.KernelCandidateF.RECENT), 

286 kSum=scaling) 

287 

288 sIm = afwImage.MaskedImageF(self.templateExposure2.getMaskedImage(), deep=True) 

289 sIm += bg 

290 kc = ipDiffim.KernelCandidateF(self.x02, self.y02, 

291 self.templateExposure2.getMaskedImage(), 

292 sIm, 

293 self.ps) 

294 

295 kList = ipDiffim.makeKernelBasisList(self.subconfig) 

296 kc.build(kList) 

297 self.verifyDeltaFunctionSolution(kc.getKernelSolution(ipDiffim.KernelCandidateF.RECENT), 

298 bg=bg) 

299 

300 @unittest.skipIf(not defDataDir, "Warning: afwdata is not set up") 

301 def testDeltaFunction(self): 

302 # Match an image to itself, with delta-function basis set 

303 # No regularization 

304 kc = ipDiffim.KernelCandidateF(self.x02, self.y02, 

305 self.templateExposure2.getMaskedImage(), 

306 self.templateExposure2.getMaskedImage(), 

307 self.ps) 

308 

309 kList = ipDiffim.makeKernelBasisList(self.subconfig) 

310 

311 kc.build(kList) 

312 self.assertEqual(kc.isInitialized(), True) 

313 

314 # These should work 

315 for kType in (ipDiffim.KernelCandidateF.ORIG, 

316 ipDiffim.KernelCandidateF.RECENT): 

317 for kMethod in (kc.getKernelSolution, 

318 kc.getKernel, 

319 kc.getBackground, 

320 kc.getKsum, 

321 kc.getKernelImage, 

322 kc.getDifferenceImage): 

323 try: 

324 kMethod(kType) 

325 except Exception as e: 

326 print(kMethod, e) 

327 self.fail() 

328 else: 

329 pass 

330 try: 

331 kc.getImage() 

332 except Exception as e: 

333 print(kMethod, e) 

334 self.fail() 

335 else: 

336 pass 

337 

338 # None of these should work 

339 for kType in (ipDiffim.KernelCandidateF.PCA,): 

340 for kMethod in (kc.getKernelSolution, 

341 kc.getKernel, 

342 kc.getBackground, 

343 kc.getKsum, 

344 kc.getKernelImage, 

345 kc.getImage, 

346 kc.getDifferenceImage): 

347 try: 

348 kMethod(kType) 

349 except Exception: 

350 pass 

351 else: 

352 print(kMethod) 

353 self.fail() 

354 

355 self.verifyDeltaFunctionSolution(kc.getKernelSolution(ipDiffim.KernelCandidateF.RECENT)) 

356 

357 @unittest.skipIf(not defDataDir, "Warning: afwdata is not set up") 

358 def testGaussianWithNoise(self): 

359 # Convolve a real image with a gaussian and try and recover 

360 # it. Add noise and perform the same test. 

361 

362 gsize = self.ps["kernelSize"] 

363 gaussFunction = afwMath.GaussianFunction2D(2, 3) 

364 gaussKernel = afwMath.AnalyticKernel(gsize, gsize, gaussFunction) 

365 kImageIn = afwImage.ImageD(geom.Extent2I(gsize, gsize)) 

366 kSumIn = gaussKernel.computeImage(kImageIn, False) 

367 

368 imX, imY = self.templateExposure2.getMaskedImage().getDimensions() 

369 smi = afwImage.MaskedImageF(geom.Extent2I(imX, imY)) 

370 afwMath.convolve(smi, self.templateExposure2.getMaskedImage(), gaussKernel, False) 

371 

372 bbox = gaussKernel.shrinkBBox(smi.getBBox(afwImage.LOCAL)) 

373 

374 tmi2 = afwImage.MaskedImageF(self.templateExposure2.getMaskedImage(), bbox, origin=afwImage.LOCAL) 

375 smi2 = afwImage.MaskedImageF(smi, bbox, origin=afwImage.LOCAL) 

376 

377 kc = ipDiffim.KernelCandidateF(self.x02, self.y02, tmi2, smi2, self.ps) 

378 kList = ipDiffim.makeKernelBasisList(self.subconfig) 

379 kc.build(kList) 

380 self.assertEqual(kc.isInitialized(), True) 

381 kImageOut = kc.getImage() 

382 

383 soln = kc.getKernelSolution(ipDiffim.KernelCandidateF.RECENT) 

384 self.assertAlmostEqual(soln.getKsum(), kSumIn) 

385 # 8.7499380640430563e-06 != 0.0 within 7 places 

386 self.assertAlmostEqual(soln.getBackground(), 0.0, 4) 

387 

388 for j in range(kImageOut.getHeight()): 

389 for i in range(kImageOut.getWidth()): 

390 

391 # in the outskirts of the kernel, the ratio can get screwed because of low S/N 

392 # e.g. 7.45817359824e-09 vs. 1.18062529402e-08 

393 # in the guts of the kernel it should look closer 

394 if kImageIn[i, j, afwImage.LOCAL] > 1e-4: 

395 # sigh, too bad this sort of thing fails.. 

396 # 0.99941584433815966 != 1.0 within 3 places 

397 self.assertAlmostEqual(kImageOut[i, j, afwImage.LOCAL]/kImageIn[i, j, afwImage.LOCAL], 

398 1.0, 2) 

399 

400 # now repeat with noise added; decrease precision of comparison 

401 self.addNoise(smi2) 

402 kc = ipDiffim.KernelCandidateF(self.x02, self.y02, tmi2, smi2, self.ps) 

403 kList = ipDiffim.makeKernelBasisList(self.subconfig) 

404 kc.build(kList) 

405 self.assertEqual(kc.isInitialized(), True) 

406 kImageOut = kc.getImage() 

407 

408 soln = kc.getKernelSolution(ipDiffim.KernelCandidateF.RECENT) 

409 self.assertAlmostEqual(soln.getKsum(), kSumIn, 3) 

410 if not self.ps.get("fitForBackground"): 

411 self.assertEqual(soln.getBackground(), 0.0) 

412 

413 for j in range(kImageOut.getHeight()): 

414 for i in range(kImageOut.getWidth()): 

415 if kImageIn[i, j, afwImage.LOCAL] > 1e-2: 

416 self.assertAlmostEqual(kImageOut[i, j, afwImage.LOCAL], 

417 kImageIn[i, j, afwImage.LOCAL], 2) 

418 

419 def testGaussian(self, imsize=50): 

420 # Convolve a delta function with a known gaussian; try to 

421 # recover using delta-function basis 

422 

423 gsize = self.ps["kernelSize"] 

424 tsize = imsize + gsize 

425 

426 gaussFunction = afwMath.GaussianFunction2D(2, 3) 

427 gaussKernel = afwMath.AnalyticKernel(gsize, gsize, gaussFunction) 

428 kImageIn = afwImage.ImageD(geom.Extent2I(gsize, gsize)) 

429 gaussKernel.computeImage(kImageIn, False) 

430 

431 # template image with a single hot pixel in the exact center 

432 tmi = afwImage.MaskedImageF(geom.Extent2I(tsize, tsize)) 

433 tmi.set(0, 0x0, 1e-4) 

434 cpix = tsize // 2 

435 tmi[cpix, cpix, afwImage.LOCAL] = (1, 0x0, 1) 

436 

437 # science image 

438 smi = afwImage.MaskedImageF(tmi.getDimensions()) 

439 afwMath.convolve(smi, tmi, gaussKernel, False) 

440 

441 # get the actual kernel sum (since the image is not infinite) 

442 gscaling = afwMath.makeStatistics(smi, afwMath.SUM).getValue(afwMath.SUM) 

443 

444 # grab only the non-masked subregion 

445 bbox = gaussKernel.shrinkBBox(smi.getBBox(afwImage.LOCAL)) 

446 

447 tmi2 = afwImage.MaskedImageF(tmi, bbox, origin=afwImage.LOCAL) 

448 smi2 = afwImage.MaskedImageF(smi, bbox, origin=afwImage.LOCAL) 

449 

450 # make sure its a valid subregion! 

451 for j in range(tmi2.getHeight()): 

452 for i in range(tmi2.getWidth()): 

453 self.assertEqual(tmi2.mask[i, j, afwImage.LOCAL], 0) 

454 self.assertEqual(smi2.mask[i, j, afwImage.LOCAL], 0) 

455 

456 kc = ipDiffim.KernelCandidateF(0.0, 0.0, tmi2, smi2, self.ps) 

457 kList = ipDiffim.makeKernelBasisList(self.subconfig) 

458 kc.build(kList) 

459 self.assertEqual(kc.isInitialized(), True) 

460 kImageOut = kc.getImage() 

461 

462 soln = kc.getKernelSolution(ipDiffim.KernelCandidateF.RECENT) 

463 self.assertAlmostEqual(soln.getKsum(), gscaling) 

464 self.assertAlmostEqual(soln.getBackground(), 0.0) 

465 

466 for j in range(kImageOut.getHeight()): 

467 for i in range(kImageOut.getWidth()): 

468 self.assertAlmostEqual(kImageOut[i, j, afwImage.LOCAL]/kImageIn[i, j, afwImage.LOCAL], 

469 1.0, 5) 

470 

471 def testZeroVariance(self, imsize=50): 

472 gsize = self.ps["kernelSize"] 

473 tsize = imsize + gsize 

474 

475 tmi = afwImage.MaskedImageF(geom.Extent2I(tsize, tsize)) 

476 tmi.set(0, 0x0, 1.0) 

477 cpix = tsize // 2 

478 tmi[cpix, cpix, afwImage.LOCAL] = (1, 0x0, 0.0) 

479 smi = afwImage.MaskedImageF(geom.Extent2I(tsize, tsize)) 

480 smi.set(0, 0x0, 1.0) 

481 smi[cpix, cpix, afwImage.LOCAL] = (1, 0x0, 0.0) 

482 

483 kList = ipDiffim.makeKernelBasisList(self.subconfig) 

484 self.ps["constantVarianceWeighting"] = False 

485 kc = ipDiffim.KernelCandidateF(0.0, 0.0, tmi, smi, self.ps) 

486 try: 

487 kc.build(kList) 

488 except Exception: 

489 pass 

490 else: 

491 self.fail() 

492 

493 @unittest.skipIf(not defDataDir, "Warning: afwdata is not set up") 

494 def testConstantWeighting(self): 

495 self.ps["fitForBackground"] = False 

496 self.testGaussian() 

497 self.testGaussianWithNoise() 

498 

499 @unittest.skipIf(not defDataDir, "Warning: afwdata is not set up") 

500 def testNoBackgroundFit(self): 

501 self.ps["constantVarianceWeighting"] = True 

502 self.testGaussian() 

503 

504 def testInsert(self): 

505 mi = afwImage.MaskedImageF(geom.Extent2I(10, 10)) 

506 kc = ipDiffim.makeKernelCandidate(0., 0., mi, mi, self.ps) 

507 kc.setStatus(afwMath.SpatialCellCandidate.GOOD) 

508 

509 sizeCellX = self.ps["sizeCellX"] 

510 sizeCellY = self.ps["sizeCellY"] 

511 kernelCellSet = afwMath.SpatialCellSet(geom.Box2I(geom.Point2I(0, 0), geom.Extent2I(1, 1)), 

512 sizeCellX, sizeCellY) 

513 kernelCellSet.insertCandidate(kc) 

514 nSeen = 0 

515 for cell in kernelCellSet.getCellList(): 

516 for cand in cell.begin(True): 

517 self.assertEqual(cand.getStatus(), afwMath.SpatialCellCandidate.GOOD) 

518 nSeen += 1 

519 self.assertEqual(nSeen, 1) 

520 

521 @unittest.skipIf(not display, "display is None: skipping testDisp") 

522 def testDisp(self): 

523 afwDisplay.Display(frame=1).mtv(self.scienceImage2, 

524 title=self._testMethodName + ": scienceImage2") 

525 afwDisplay.Display(frame=2).mtv(self.templateExposure2, 

526 title=self._testMethodName + ": templateExposure2") 

527 

528 def tearDown(self): 

529 del self.ps 

530 del self.table 

531 del self.ss 

532 if defDataDir: 

533 del self.scienceImage2 

534 del self.templateExposure2 

535 

536##### 

537 

538 

539class TestMemory(lsst.utils.tests.MemoryTestCase): 

540 pass 

541 

542 

543def setup_module(module): 

544 lsst.utils.tests.init() 

545 

546 

547if __name__ == "__main__": 547 ↛ 548line 547 didn't jump to line 548, because the condition on line 547 was never true

548 lsst.utils.tests.init() 

549 unittest.main()