Coverage for tests/test_zogy.py: 17%
182 statements
« prev ^ index » next coverage.py v6.4.2, created at 2022-08-03 03:19 -0700
« prev ^ index » next coverage.py v6.4.2, created at 2022-08-03 03:19 -0700
1#
2# LSST Data Management System
3# Copyright 2016-2017 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21import numpy as np
22import unittest
24import lsst.afw.image as afwImage
25import lsst.afw.math as afwMath
26import lsst.afw.geom as afwGeom
27import lsst.daf.base as dafBase
28import lsst.geom as geom
29from lsst.ip.diffim.zogy import ZogyTask, ZogyConfig
30import lsst.meas.algorithms as measAlg
31import lsst.utils.tests
32from test_imageDecorrelation import singleGaussian2d
34try:
35 type(verbose)
36except NameError:
37 verbose = False
40def setup_module(module):
41 lsst.utils.tests.init()
44def makeFakeImages(size=(256, 256), svar=0.04, tvar=0.04, psf1=3.3, psf2=2.2, offset=None,
45 psf_yvary_factor=0., varSourceChange=1/50., theta1=0., theta2=0.,
46 n_sources=50, seed=66, verbose=False):
47 """Make two exposures: science and template pair with flux sources and random noise.
48 In all cases below, index (1) is the science image, and (2) is the template.
50 Parameters
51 ----------
52 size : `tuple` of `int`
53 Image pixel size (x,y). Pixel coordinates are set to
54 (-size[0]//2:size[0]//2, -size[1]//2:size[1]//2)
55 svar, tvar : `float`, optional
56 Per pixel variance of the added noise.
57 psf1, psf2 : `float`, optional
58 std. dev. of (Gaussian) PSFs for the two images in x,y direction. Default is
59 [3.3, 3.3] and [2.2, 2.2] for im1 and im2 respectively.
60 offset : `float`, optional
61 add a constant (pixel) astrometric offset between the two images.
62 psf_yvary_factor : `float`, optional
63 psf_yvary_factor vary the y-width of the PSF across the x-axis of the science image (zero,
64 the default, means no variation)
65 varSourceChange : `float`, optional
66 varSourceChange add this amount of fractional flux to a single source closest to
67 the center of the science image.
68 theta1, theta2: `float`, optional
69 PSF Gaussian rotation angles in degrees.
70 n_sources : `int`, optional
71 The number of sources to add to the images. If zero, no sources are
72 generated just background noise.
73 seed : `int`, optional
74 Random number generator seed.
75 verbose : `bool`, optional
76 Print some actual values.
78 Returns
79 -------
80 im1, im2 : `lsst.afw.image.Exposure`
81 The science and template exposures.
83 Notes
84 -----
85 If ``n_sources > 0`` and ``varSourceChange > 0.`` exactly one source,
86 that is closest to the center, will have different fluxes in the two
87 generated images. The flux on the science image will be higher by
88 ``varSourceChange`` fraction.
90 Having sources near the edges really messes up the
91 fitting (probably because of the convolution). So we make sure no
92 sources are near the edge.
94 Also it seems that having the variable source with a large
95 flux increase also messes up the fitting (seems to lead to
96 overfitting -- perhaps to the source itself). This might be fixed by
97 adding more constant sources.
98 """
99 rng = np.random.default_rng(seed)
101 psf1 = [3.3, 3.3] if psf1 is None else psf1
102 if not hasattr(psf1, "__len__"):
103 psf1 = [psf1, psf1]
104 psf2 = [2.2, 2.2] if psf2 is None else psf2
105 if not hasattr(psf2, "__len__"):
106 psf2 = [psf2, psf2]
107 offset = [0., 0.] if offset is None else offset # astrometric offset (pixels) between the two images
108 if verbose:
109 print('Science PSF:', psf1, theta1)
110 print('Template PSF:', psf2, theta2)
111 print(np.sqrt(psf1[0]**2 - psf2[0]**2))
112 print('Offset:', offset)
114 xim = np.arange(-size[0]//2, size[0]//2, 1) # Beware that -N//2 != -1*(N//2) for odd numbers
115 yim = np.arange(-size[1]//2, size[1]//2, 1)
116 x0im, y0im = np.meshgrid(xim, yim)
118 im1 = rng.normal(scale=np.sqrt(svar), size=x0im.shape) # variance of science image
119 im2 = rng.normal(scale=np.sqrt(tvar), size=x0im.shape) # variance of template
121 if n_sources > 0:
122 fluxes = rng.uniform(50, 30000, n_sources)
123 xposns = rng.uniform(xim.min() + 16, xim.max() - 5, n_sources)
124 yposns = rng.uniform(yim.min() + 16, yim.max() - 5, n_sources)
126 # Make the source closest to the center of the image the one that increases in flux
127 ind = np.argmin(xposns**2. + yposns**2.)
129 # vary the y-width of psf across x-axis of science image (zero means no variation):
130 psf1_yvary = psf_yvary_factor*(yim.mean() - yposns)/yim.max()
131 if verbose:
132 print('PSF y spatial-variation:', psf1_yvary.min(), psf1_yvary.max())
134 for i in range(n_sources):
135 flux = fluxes[i]
136 tmp = flux*singleGaussian2d(x0im, y0im, xposns[i], yposns[i], psf2[0], psf2[1], theta=theta2)
137 im2 += tmp
138 if i == ind:
139 flux += flux*varSourceChange
140 tmp = flux*singleGaussian2d(x0im, y0im, xposns[i] + offset[0], yposns[i] + offset[1],
141 psf1[0], psf1[1] + psf1_yvary[i], theta=theta1)
142 im1 += tmp
144 im1_psf = singleGaussian2d(x0im, y0im, 0, 0, psf1[0], psf1[1], theta=theta1)
145 im2_psf = singleGaussian2d(x0im, y0im, offset[0], offset[1], psf2[0], psf2[1], theta=theta2)
147 def makeWcs(offset=0):
148 """ Make a fake Wcs
150 Parameters
151 ----------
152 offset : float
153 offset the Wcs by this many pixels.
154 """
155 # taken from $AFW_DIR/tests/testMakeWcs.py
156 metadata = dafBase.PropertySet()
157 metadata.set("SIMPLE", "T")
158 metadata.set("BITPIX", -32)
159 metadata.set("NAXIS", 2)
160 metadata.set("NAXIS1", 1024)
161 metadata.set("NAXIS2", 1153)
162 metadata.set("RADESYS", 'FK5')
163 metadata.set("EQUINOX", 2000.)
164 metadata.setDouble("CRVAL1", 215.604025685476)
165 metadata.setDouble("CRVAL2", 53.1595451514076)
166 metadata.setDouble("CRPIX1", 1109.99981456774 + offset)
167 metadata.setDouble("CRPIX2", 560.018167811613 + offset)
168 metadata.set("CTYPE1", 'RA---SIN')
169 metadata.set("CTYPE2", 'DEC--SIN')
170 metadata.setDouble("CD1_1", 5.10808596133527E-05)
171 metadata.setDouble("CD1_2", 1.85579539217196E-07)
172 metadata.setDouble("CD2_2", -5.10281493481982E-05)
173 metadata.setDouble("CD2_1", -8.27440751733828E-07)
174 return afwGeom.makeSkyWcs(metadata)
176 def makeExposure(imgArray, psfArray, imgVariance):
177 """Convert an image and corresponding PSF into an exposure.
179 Set the (constant) variance plane equal to ``imgVariance``.
181 Parameters
182 ----------
183 imgArray : `numpy.ndarray`
184 2D array containing the image.
185 psfArray : `numpy.ndarray`
186 2D array containing the PSF image.
187 imgVariance : `float` or `numpy.ndarray`
188 Set the variance plane to this value. If an array, must be broadcastable to ``imgArray.shape``.
190 Returns
191 -------
192 im1ex : `lsst.afw.image.Exposure`
193 The new exposure.
194 """
195 # All this code to convert the template image array/psf array into an exposure.
196 bbox = geom.Box2I(geom.Point2I(0, 0), geom.Point2I(imgArray.shape[1] - 1, imgArray.shape[0] - 1))
197 im1ex = afwImage.ExposureD(bbox)
198 im1ex.image.array[:, :] = imgArray
199 im1ex.variance.array[:, :] = imgVariance
200 psfBox = geom.Box2I(geom.Point2I(-12, -12), geom.Point2I(12, 12)) # a 25x25 pixel psf
201 psf = afwImage.ImageD(psfBox)
202 psfBox.shift(geom.Extent2I(-(-size[0]//2), -(-size[1]//2))) # -N//2 != -(N//2) for odd numbers
203 im1_psf_sub = psfArray[psfBox.getMinY():psfBox.getMaxY() + 1, psfBox.getMinX():psfBox.getMaxX() + 1]
204 psf.getArray()[:, :] = im1_psf_sub
205 psfK = afwMath.FixedKernel(psf)
206 psfNew = measAlg.KernelPsf(psfK)
207 im1ex.setPsf(psfNew)
208 wcs = makeWcs()
209 im1ex.setWcs(wcs)
210 return im1ex
212 im1ex = makeExposure(im1, im1_psf, svar) # Science image
213 im2ex = makeExposure(im2, im2_psf, tvar) # Template
215 return im1ex, im2ex
218def isPowerOfTwo(x):
219 """Returns True if x is a power of 2"""
220 while x > 1:
221 if x & 1 != 0:
222 return False
223 x >>= 1
224 return True
227class ZogyTest(lsst.utils.tests.TestCase):
228 """A test case for the Zogy task.
229 """
231 def setUp(self):
232 self.psf1_sigma = 3.3 # sigma of psf of science image
233 self.psf2_sigma = 2.2 # sigma of psf of template image
235 self.statsControl = afwMath.StatisticsControl()
236 self.statsControl.setNumSigmaClip(3.)
237 self.statsControl.setNumIter(3)
238 self.statsControl.setAndMask(afwImage.Mask
239 .getPlaneBitMask(["INTRP", "EDGE", "SAT", "CR",
240 "DETECTED", "BAD",
241 "NO_DATA", "DETECTED_NEGATIVE"]))
243 def _computeVarianceMean(self, maskedIm):
244 statObj = afwMath.makeStatistics(maskedIm.getVariance(),
245 maskedIm.getMask(), afwMath.MEANCLIP,
246 self.statsControl)
247 mn = statObj.getValue(afwMath.MEANCLIP)
248 return mn
250 def _computePixelVariance(self, maskedIm):
251 statObj = afwMath.makeStatistics(maskedIm, afwMath.VARIANCECLIP,
252 self.statsControl)
253 var = statObj.getValue(afwMath.VARIANCECLIP)
254 return var
256 def _computePixelMean(self, maskedIm):
257 statObj = afwMath.makeStatistics(maskedIm, afwMath.MEANCLIP,
258 self.statsControl)
259 var = statObj.getValue(afwMath.MEANCLIP)
260 return var
262 def testFourierTransformConvention(self):
263 """Test numpy FFT normalization factor convention matches our assumption."""
264 D = np.arange(16).reshape(4, 4)
265 fD = np.real(np.fft.fft2(D))
266 self.assertFloatsAlmostEqual(
267 fD[0, 0], 120., rtol=None,
268 msg="Numpy FFT does not use expected default normalization"
269 " convention (1 in forward, 1/Npix in inverse operation).")
271 def testSplitBorder(self):
272 """Test outer border box splitting around an inner box"""
273 config = ZogyConfig()
274 task = ZogyTask(config=config)
276 bb = geom.Box2I(geom.Point2I(5, 10), geom.Extent2I(20, 30))
277 D = afwImage.ImageI(bb)
278 innerbox = bb.erodedBy(geom.Extent2I(3, 4))
279 D[innerbox] = 1
281 borderboxes = task.splitBorder(innerbox, bb)
282 for x in borderboxes:
283 D[x] += 1
284 # The splitting should cover all border pixels exactly once
285 self.assertTrue(np.all(D.array == 1), "Border does not cover all pixels exactly once.")
287 def testGenerateGrid(self):
288 """Test that the generated grid covers the whole image"""
289 config = ZogyConfig()
290 task = ZogyTask(config=config)
291 bb = geom.Box2I(geom.Point2I(5, 10), geom.Extent2I(200, 300))
292 D = afwImage.ImageI(bb)
293 grid = task.generateGrid(bb, geom.Extent2I(15, 15), geom.Extent2I(20, 30), powerOfTwo=True)
294 for x in grid:
295 h = x.outerBox.getHeight()
296 w = x.outerBox.getWidth()
297 self.assertTrue(isPowerOfTwo(h), "Box height is not power of two")
298 self.assertTrue(isPowerOfTwo(w), "Box width is not power of two")
299 D[x.innerBox] += 1
300 self.assertTrue(np.all(D.array == 1), "Grid inner boxes do not cover all pixels exactly once.")
302 def testWholeImageGrid(self):
303 """Test that a 1-cell `grid` is actually the whole image"""
304 config = ZogyConfig()
305 task = ZogyTask(config=config)
306 bb = geom.Box2I(geom.Point2I(5, 10), geom.Extent2I(200, 300))
307 D = afwImage.ImageI(bb)
308 grid = task.generateGrid(bb, geom.Extent2I(15, 15), bb.getDimensions())
309 self.assertTrue(len(grid) == 1, "Grid length is not 1")
310 x = grid[0]
311 D[x.innerBox] += 1
312 self.assertTrue(np.all(D.array == 1), "Single cell does not cover the original image.")
314 def testZogyNewImplementation(self):
315 """DM-25115 implementation test.
317 Notes
318 -----
319 See diffimTests: tickets/DM-25115_zogy_implementation/DM-25115_zogy_unit_test_development.ipynb
320 """
322 # self.svar = svar # variance of noise in science image
323 # self.tvar = tvar # variance of noise in template image
325 # Sourceless case
326 self.im1ex, self.im2ex \
327 = makeFakeImages(size=(256, 256), svar=100., tvar=100.,
328 psf1=self.psf1_sigma, psf2=self.psf2_sigma,
329 n_sources=0, psf_yvary_factor=0, varSourceChange=0.1,
330 seed=1, verbose=False)
332 config = ZogyConfig()
333 config.scaleByCalibration = False
334 task = ZogyTask(config=config)
335 res = task.run(self.im1ex, self.im2ex)
337 bbox = res.diffExp.getBBox()
338 subBbox = bbox.erodedBy(lsst.geom.Extent2I(25, 25))
339 subExp = res.diffExp[subBbox]
340 pixvar = self._computePixelVariance(subExp.maskedImage)
341 varmean = self._computeVarianceMean(subExp.maskedImage)
342 # Due to 3 sigma clipping, this is not so precise
343 self.assertFloatsAlmostEqual(pixvar, 200, rtol=0.1, atol=None)
344 self.assertFloatsAlmostEqual(varmean, 200, rtol=0.05, atol=None)
345 S = res.scoreExp.image.array / np.sqrt(res.scoreExp.variance.array)
346 self.assertLess(np.amax(S), 5.) # Source not detected
348 # ==========
349 self.im1ex, self.im2ex \
350 = makeFakeImages(size=(256, 256), svar=10., tvar=10.,
351 psf1=self.psf1_sigma, psf2=self.psf2_sigma,
352 n_sources=10, psf_yvary_factor=0, varSourceChange=0.1,
353 seed=1, verbose=False)
354 task = ZogyTask(config=config)
355 res = task.run(self.im1ex, self.im2ex)
356 S = res.scoreExp.image.array / np.sqrt(res.scoreExp.variance.array)
357 self.assertGreater(np.amax(S), 5.) # Source detected
360class MemoryTester(lsst.utils.tests.MemoryTestCase):
361 pass
364if __name__ == "__main__": 364 ↛ 365line 364 didn't jump to line 365, because the condition on line 364 was never true
365 lsst.utils.tests.init()
366 unittest.main()