Coverage for tests/test_zogy.py: 16%

182 statements  

« prev     ^ index     » next       coverage.py v7.2.5, created at 2023-05-08 08:40 +0000

1# 

2# LSST Data Management System 

3# Copyright 2016-2017 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21import numpy as np 

22import unittest 

23 

24import lsst.afw.image as afwImage 

25import lsst.afw.math as afwMath 

26import lsst.afw.geom as afwGeom 

27import lsst.daf.base as dafBase 

28import lsst.geom as geom 

29from lsst.ip.diffim.zogy import ZogyTask, ZogyConfig 

30import lsst.meas.algorithms as measAlg 

31import lsst.utils.tests 

32from test_imageDecorrelation import singleGaussian2d 

33 

34try: 

35 type(verbose) 

36except NameError: 

37 verbose = False 

38 

39 

40def setup_module(module): 

41 lsst.utils.tests.init() 

42 

43 

44def makeFakeImages(size=(256, 256), svar=0.04, tvar=0.04, psf1=3.3, psf2=2.2, offset=None, 

45 psf_yvary_factor=0., varSourceChange=1/50., theta1=0., theta2=0., 

46 n_sources=50, seed=66, verbose=False): 

47 """Make two exposures: science and template pair with flux sources and random noise. 

48 In all cases below, index (1) is the science image, and (2) is the template. 

49 

50 Parameters 

51 ---------- 

52 size : `tuple` of `int` 

53 Image pixel size (x,y). Pixel coordinates are set to 

54 (-size[0]//2:size[0]//2, -size[1]//2:size[1]//2) 

55 svar, tvar : `float`, optional 

56 Per pixel variance of the added noise. 

57 psf1, psf2 : `float`, optional 

58 std. dev. of (Gaussian) PSFs for the two images in x,y direction. Default is 

59 [3.3, 3.3] and [2.2, 2.2] for im1 and im2 respectively. 

60 offset : `float`, optional 

61 add a constant (pixel) astrometric offset between the two images. 

62 psf_yvary_factor : `float`, optional 

63 psf_yvary_factor vary the y-width of the PSF across the x-axis of the science image (zero, 

64 the default, means no variation) 

65 varSourceChange : `float`, optional 

66 varSourceChange add this amount of fractional flux to a single source closest to 

67 the center of the science image. 

68 theta1, theta2: `float`, optional 

69 PSF Gaussian rotation angles in degrees. 

70 n_sources : `int`, optional 

71 The number of sources to add to the images. If zero, no sources are 

72 generated just background noise. 

73 seed : `int`, optional 

74 Random number generator seed. 

75 verbose : `bool`, optional 

76 Print some actual values. 

77 

78 Returns 

79 ------- 

80 im1, im2 : `lsst.afw.image.Exposure` 

81 The science and template exposures. 

82 

83 Notes 

84 ----- 

85 If ``n_sources > 0`` and ``varSourceChange > 0.`` exactly one source, 

86 that is closest to the center, will have different fluxes in the two 

87 generated images. The flux on the science image will be higher by 

88 ``varSourceChange`` fraction. 

89 

90 Having sources near the edges really messes up the 

91 fitting (probably because of the convolution). So we make sure no 

92 sources are near the edge. 

93 

94 Also it seems that having the variable source with a large 

95 flux increase also messes up the fitting (seems to lead to 

96 overfitting -- perhaps to the source itself). This might be fixed by 

97 adding more constant sources. 

98 """ 

99 rng = np.random.default_rng(seed) 

100 

101 psf1 = [3.3, 3.3] if psf1 is None else psf1 

102 if not hasattr(psf1, "__len__"): 

103 psf1 = [psf1, psf1] 

104 psf2 = [2.2, 2.2] if psf2 is None else psf2 

105 if not hasattr(psf2, "__len__"): 

106 psf2 = [psf2, psf2] 

107 offset = [0., 0.] if offset is None else offset # astrometric offset (pixels) between the two images 

108 if verbose: 

109 print('Science PSF:', psf1, theta1) 

110 print('Template PSF:', psf2, theta2) 

111 print(np.sqrt(psf1[0]**2 - psf2[0]**2)) 

112 print('Offset:', offset) 

113 

114 xim = np.arange(-size[0]//2, size[0]//2, 1) # Beware that -N//2 != -1*(N//2) for odd numbers 

115 yim = np.arange(-size[1]//2, size[1]//2, 1) 

116 x0im, y0im = np.meshgrid(xim, yim) 

117 

118 im1 = rng.normal(scale=np.sqrt(svar), size=x0im.shape) # variance of science image 

119 im2 = rng.normal(scale=np.sqrt(tvar), size=x0im.shape) # variance of template 

120 

121 if n_sources > 0: 

122 fluxes = rng.uniform(50, 30000, n_sources) 

123 xposns = rng.uniform(xim.min() + 16, xim.max() - 5, n_sources) 

124 yposns = rng.uniform(yim.min() + 16, yim.max() - 5, n_sources) 

125 

126 # Make the source closest to the center of the image the one that increases in flux 

127 ind = np.argmin(xposns**2. + yposns**2.) 

128 

129 # vary the y-width of psf across x-axis of science image (zero means no variation): 

130 psf1_yvary = psf_yvary_factor*(yim.mean() - yposns)/yim.max() 

131 if verbose: 

132 print('PSF y spatial-variation:', psf1_yvary.min(), psf1_yvary.max()) 

133 

134 for i in range(n_sources): 

135 flux = fluxes[i] 

136 tmp = flux*singleGaussian2d(x0im, y0im, xposns[i], yposns[i], psf2[0], psf2[1], theta=theta2) 

137 im2 += tmp 

138 if i == ind: 

139 flux += flux*varSourceChange 

140 tmp = flux*singleGaussian2d(x0im, y0im, xposns[i] + offset[0], yposns[i] + offset[1], 

141 psf1[0], psf1[1] + psf1_yvary[i], theta=theta1) 

142 im1 += tmp 

143 

144 im1_psf = singleGaussian2d(x0im, y0im, 0, 0, psf1[0], psf1[1], theta=theta1) 

145 im2_psf = singleGaussian2d(x0im, y0im, offset[0], offset[1], psf2[0], psf2[1], theta=theta2) 

146 

147 def makeWcs(offset=0): 

148 """ Make a fake Wcs 

149 

150 Parameters 

151 ---------- 

152 offset : float 

153 offset the Wcs by this many pixels. 

154 """ 

155 # taken from $AFW_DIR/tests/testMakeWcs.py 

156 metadata = dafBase.PropertySet() 

157 metadata.set("SIMPLE", "T") 

158 metadata.set("BITPIX", -32) 

159 metadata.set("NAXIS", 2) 

160 metadata.set("NAXIS1", 1024) 

161 metadata.set("NAXIS2", 1153) 

162 metadata.set("RADESYS", 'FK5') 

163 metadata.set("EQUINOX", 2000.) 

164 metadata.setDouble("CRVAL1", 215.604025685476) 

165 metadata.setDouble("CRVAL2", 53.1595451514076) 

166 metadata.setDouble("CRPIX1", 1109.99981456774 + offset) 

167 metadata.setDouble("CRPIX2", 560.018167811613 + offset) 

168 metadata.set("CTYPE1", 'RA---SIN') 

169 metadata.set("CTYPE2", 'DEC--SIN') 

170 metadata.setDouble("CD1_1", 5.10808596133527E-05) 

171 metadata.setDouble("CD1_2", 1.85579539217196E-07) 

172 metadata.setDouble("CD2_2", -5.10281493481982E-05) 

173 metadata.setDouble("CD2_1", -8.27440751733828E-07) 

174 return afwGeom.makeSkyWcs(metadata) 

175 

176 def makeExposure(imgArray, psfArray, imgVariance): 

177 """Convert an image and corresponding PSF into an exposure. 

178 

179 Set the (constant) variance plane equal to ``imgVariance``. 

180 

181 Parameters 

182 ---------- 

183 imgArray : `numpy.ndarray` 

184 2D array containing the image. 

185 psfArray : `numpy.ndarray` 

186 2D array containing the PSF image. 

187 imgVariance : `float` or `numpy.ndarray` 

188 Set the variance plane to this value. If an array, must be broadcastable to ``imgArray.shape``. 

189 

190 Returns 

191 ------- 

192 im1ex : `lsst.afw.image.Exposure` 

193 The new exposure. 

194 """ 

195 # All this code to convert the template image array/psf array into an exposure. 

196 bbox = geom.Box2I(geom.Point2I(0, 0), geom.Point2I(imgArray.shape[1] - 1, imgArray.shape[0] - 1)) 

197 im1ex = afwImage.ExposureD(bbox) 

198 im1ex.image.array[:, :] = imgArray 

199 im1ex.variance.array[:, :] = imgVariance 

200 psfBox = geom.Box2I(geom.Point2I(-12, -12), geom.Point2I(12, 12)) # a 25x25 pixel psf 

201 psf = afwImage.ImageD(psfBox) 

202 psfBox.shift(geom.Extent2I(-(-size[0]//2), -(-size[1]//2))) # -N//2 != -(N//2) for odd numbers 

203 im1_psf_sub = psfArray[psfBox.getMinY():psfBox.getMaxY() + 1, psfBox.getMinX():psfBox.getMaxX() + 1] 

204 psf.getArray()[:, :] = im1_psf_sub 

205 psfK = afwMath.FixedKernel(psf) 

206 psfNew = measAlg.KernelPsf(psfK) 

207 im1ex.setPsf(psfNew) 

208 wcs = makeWcs() 

209 im1ex.setWcs(wcs) 

210 return im1ex 

211 

212 im1ex = makeExposure(im1, im1_psf, svar) # Science image 

213 im2ex = makeExposure(im2, im2_psf, tvar) # Template 

214 

215 return im1ex, im2ex 

216 

217 

218def isPowerOfTwo(x): 

219 """Returns True if x is a power of 2""" 

220 while x > 1: 

221 if x & 1 != 0: 

222 return False 

223 x >>= 1 

224 return True 

225 

226 

227class ZogyTest(lsst.utils.tests.TestCase): 

228 """A test case for the Zogy task. 

229 """ 

230 

231 def setUp(self): 

232 self.psf1_sigma = 3.3 # sigma of psf of science image 

233 self.psf2_sigma = 2.2 # sigma of psf of template image 

234 

235 self.statsControl = afwMath.StatisticsControl() 

236 self.statsControl.setNumSigmaClip(3.) 

237 self.statsControl.setNumIter(3) 

238 self.statsControl.setAndMask(afwImage.Mask 

239 .getPlaneBitMask(["INTRP", "EDGE", "SAT", "CR", 

240 "DETECTED", "BAD", 

241 "NO_DATA", "DETECTED_NEGATIVE"])) 

242 

243 def _computeVarianceMean(self, maskedIm): 

244 statObj = afwMath.makeStatistics(maskedIm.getVariance(), 

245 maskedIm.getMask(), afwMath.MEANCLIP, 

246 self.statsControl) 

247 mn = statObj.getValue(afwMath.MEANCLIP) 

248 return mn 

249 

250 def _computePixelVariance(self, maskedIm): 

251 statObj = afwMath.makeStatistics(maskedIm, afwMath.VARIANCECLIP, 

252 self.statsControl) 

253 var = statObj.getValue(afwMath.VARIANCECLIP) 

254 return var 

255 

256 def _computePixelMean(self, maskedIm): 

257 statObj = afwMath.makeStatistics(maskedIm, afwMath.MEANCLIP, 

258 self.statsControl) 

259 var = statObj.getValue(afwMath.MEANCLIP) 

260 return var 

261 

262 def testFourierTransformConvention(self): 

263 """Test numpy FFT normalization factor convention matches our assumption.""" 

264 D = np.arange(16).reshape(4, 4) 

265 fD = np.real(np.fft.fft2(D)) 

266 self.assertFloatsAlmostEqual( 

267 fD[0, 0], 120., rtol=None, 

268 msg="Numpy FFT does not use expected default normalization" 

269 " convention (1 in forward, 1/Npix in inverse operation).") 

270 

271 def testSplitBorder(self): 

272 """Test outer border box splitting around an inner box""" 

273 config = ZogyConfig() 

274 task = ZogyTask(config=config) 

275 

276 bb = geom.Box2I(geom.Point2I(5, 10), geom.Extent2I(20, 30)) 

277 D = afwImage.ImageI(bb) 

278 innerbox = bb.erodedBy(geom.Extent2I(3, 4)) 

279 D[innerbox] = 1 

280 

281 borderboxes = task.splitBorder(innerbox, bb) 

282 for x in borderboxes: 

283 D[x] += 1 

284 # The splitting should cover all border pixels exactly once 

285 self.assertTrue(np.all(D.array == 1), "Border does not cover all pixels exactly once.") 

286 

287 def testGenerateGrid(self): 

288 """Test that the generated grid covers the whole image""" 

289 config = ZogyConfig() 

290 task = ZogyTask(config=config) 

291 bb = geom.Box2I(geom.Point2I(5, 10), geom.Extent2I(200, 300)) 

292 D = afwImage.ImageI(bb) 

293 grid = task.generateGrid(bb, geom.Extent2I(15, 15), geom.Extent2I(20, 30), powerOfTwo=True) 

294 for x in grid: 

295 h = x.outerBox.getHeight() 

296 w = x.outerBox.getWidth() 

297 self.assertTrue(isPowerOfTwo(h), "Box height is not power of two") 

298 self.assertTrue(isPowerOfTwo(w), "Box width is not power of two") 

299 D[x.innerBox] += 1 

300 self.assertTrue(np.all(D.array == 1), "Grid inner boxes do not cover all pixels exactly once.") 

301 

302 def testWholeImageGrid(self): 

303 """Test that a 1-cell `grid` is actually the whole image""" 

304 config = ZogyConfig() 

305 task = ZogyTask(config=config) 

306 bb = geom.Box2I(geom.Point2I(5, 10), geom.Extent2I(200, 300)) 

307 D = afwImage.ImageI(bb) 

308 grid = task.generateGrid(bb, geom.Extent2I(15, 15), bb.getDimensions()) 

309 self.assertTrue(len(grid) == 1, "Grid length is not 1") 

310 x = grid[0] 

311 D[x.innerBox] += 1 

312 self.assertTrue(np.all(D.array == 1), "Single cell does not cover the original image.") 

313 

314 def testZogyNewImplementation(self): 

315 """DM-25115 implementation test. 

316 

317 Notes 

318 ----- 

319 See diffimTests: tickets/DM-25115_zogy_implementation/DM-25115_zogy_unit_test_development.ipynb 

320 """ 

321 

322 # self.svar = svar # variance of noise in science image 

323 # self.tvar = tvar # variance of noise in template image 

324 

325 # Sourceless case 

326 self.im1ex, self.im2ex \ 

327 = makeFakeImages(size=(256, 256), svar=100., tvar=100., 

328 psf1=self.psf1_sigma, psf2=self.psf2_sigma, 

329 n_sources=0, psf_yvary_factor=0, varSourceChange=0.1, 

330 seed=1, verbose=False) 

331 

332 config = ZogyConfig() 

333 config.scaleByCalibration = False 

334 task = ZogyTask(config=config) 

335 res = task.run(self.im1ex, self.im2ex) 

336 

337 bbox = res.diffExp.getBBox() 

338 subBbox = bbox.erodedBy(lsst.geom.Extent2I(25, 25)) 

339 subExp = res.diffExp[subBbox] 

340 pixvar = self._computePixelVariance(subExp.maskedImage) 

341 varmean = self._computeVarianceMean(subExp.maskedImage) 

342 # Due to 3 sigma clipping, this is not so precise 

343 self.assertFloatsAlmostEqual(pixvar, 200, rtol=0.1, atol=None) 

344 self.assertFloatsAlmostEqual(varmean, 200, rtol=0.05, atol=None) 

345 S = res.scoreExp.image.array / np.sqrt(res.scoreExp.variance.array) 

346 self.assertLess(np.amax(S), 5.) # Source not detected 

347 

348 # ========== 

349 self.im1ex, self.im2ex \ 

350 = makeFakeImages(size=(256, 256), svar=10., tvar=10., 

351 psf1=self.psf1_sigma, psf2=self.psf2_sigma, 

352 n_sources=10, psf_yvary_factor=0, varSourceChange=0.1, 

353 seed=1, verbose=False) 

354 task = ZogyTask(config=config) 

355 res = task.run(self.im1ex, self.im2ex) 

356 S = res.scoreExp.image.array / np.sqrt(res.scoreExp.variance.array) 

357 self.assertGreater(np.amax(S), 5.) # Source detected 

358 

359 

360class MemoryTester(lsst.utils.tests.MemoryTestCase): 

361 pass 

362 

363 

364if __name__ == "__main__": 364 ↛ 365line 364 didn't jump to line 365, because the condition on line 364 was never true

365 lsst.utils.tests.init() 

366 unittest.main()