Coverage for tests/test_imageDecorrelation.py : 16%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2016-2017 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21import unittest
23import numpy as np
25import lsst.utils.tests
26import lsst.afw.image as afwImage
27import lsst.afw.geom as afwGeom
28import lsst.afw.math as afwMath
29import lsst.geom as geom
30import lsst.meas.algorithms as measAlg
31import lsst.daf.base as dafBase
33from lsst.ip.diffim.imageDecorrelation import (DecorrelateALKernelTask,
34 DecorrelateALKernelMapReduceConfig,
35 DecorrelateALKernelSpatialConfig,
36 DecorrelateALKernelSpatialTask)
37from lsst.ip.diffim.imageMapReduce import ImageMapReduceTask
39try:
40 type(verbose)
41except NameError:
42 verbose = False
45def setup_module(module):
46 lsst.utils.tests.init()
49def singleGaussian2d(x, y, xc, yc, sigma_x=1., sigma_y=1., theta=0., ampl=1.):
50 """! Generate a 2-d Gaussian, possibly elongated and rotated, on a grid of pixel
51 coordinates given by x,y.
52 @param x,y each a 1-d numpy.array containing x- and y- coordinates for independent variables,
53 for example `np.arange(-16, 15)`.
54 @param xc,yc each a float giving the centroid of the gaussian
55 @param sigma_x,sigma_y each a float giving the sigma of the gaussian
56 @param theta a float giving the rotation of the gaussian (degrees)
57 @param ampl a float giving the amplitude of the gaussian
58 @return a 2-d numpy.array containing the normalized 2-d Gaussian
60 @Note this can be done in `astropy.modeling` but for now we have it explicitly here.
61 """
62 theta = (theta/180.) * np.pi
63 cos_theta2, sin_theta2 = np.cos(theta)**2., np.sin(theta)**2.
64 sigma_x2, sigma_y2 = sigma_x**2., sigma_y**2.
65 a = cos_theta2/(2.*sigma_x2) + sin_theta2/(2.*sigma_y2)
66 b = -(np.sin(2.*theta))/(4.*sigma_x2) + (np.sin(2.*theta))/(4.*sigma_y2)
67 c = sin_theta2/(2.*sigma_x2) + cos_theta2/(2.*sigma_y2)
68 xxc, yyc = x-xc, y-yc
69 out = np.exp(-(a*(xxc**2.) + 2.*b*xxc*yyc + c*(yyc**2.)))
70 out /= out.sum()
71 return out
74def makeFakeImages(size=(256, 256), svar=0.04, tvar=0.04, psf1=3.3, psf2=2.2, offset=None,
75 psf_yvary_factor=0., varSourceChange=1/50., theta1=0., theta2=0.,
76 n_sources=500, seed=66, verbose=False):
77 """! Make two exposures: a template and a science exposure.
78 Add random sources with randomly-distributed and identical fluxes and a given PSF, then add noise.
79 In all cases below, index (1) is the science image, and (2) is the template.
80 @param size tuple givein image pixel size. Pixel coordinates are set to
81 (-size[0]//2:size[0]//2, -size[1]//2:size[1]//2)
82 @param svar,tar variance of noise to be generated on science/template images. Default is 0.04 for both.
83 @param psf1,psf2 std. dev. of (Gaussian) PSFs for the two images in x,y direction. Default is
84 [3.3, 3.3] and [2.2, 2.2] for im1 and im2 respectively.
85 @param offset add a constant (pixel) astrometric offset between the two images
86 @param psf_yvary_factor vary the y-width of the PSF across the x-axis of the science image (zero,
87 the default, means no variation)
88 @param varSourceChange add this amount of fractional flux to a single source closest to
89 the center of the science image
90 @param n_sources the number of sources to add to the images
91 @param seed the numpy random seed to set prior to image generation
92 @param verbose be verbose
94 @return im1, im2: the science and template afwImage.Exposures
96 @note having sources near the edges really messes up the
97 fitting (probably because of the convolution). So we make sure no
98 sources are near the edge.
99 @note also it seems that having the variable source with a large
100 flux increase also messes up the fitting (seems to lead to
101 overfitting -- perhaps to the source itself). This might be fixed by
102 adding more constant sources.
103 """
104 np.random.seed(seed)
106 psf1 = [3.3, 3.3] if psf1 is None else psf1
107 if not hasattr(psf1, "__len__") and not isinstance(psf1, str):
108 psf1 = [psf1, psf1]
109 psf2 = [2.2, 2.2] if psf2 is None else psf2
110 if not hasattr(psf2, "__len__") and not isinstance(psf2, str):
111 psf2 = [psf2, psf2]
112 offset = [0., 0.] if offset is None else offset # astrometric offset (pixels) between the two images
113 if verbose:
114 print('Science PSF:', psf1, theta1)
115 print('Template PSF:', psf2, theta2)
116 print(np.sqrt(psf1[0]**2 - psf2[0]**2))
117 print('Offset:', offset)
119 xim = np.arange(-size[0]//2, size[0]//2, 1)
120 yim = np.arange(-size[1]//2, size[1]//2, 1)
121 x0im, y0im = np.meshgrid(yim, xim)
122 im1 = np.random.normal(scale=np.sqrt(svar), size=x0im.shape) # variance of science image
123 im2 = np.random.normal(scale=np.sqrt(tvar), size=x0im.shape) # variance of template
125 fluxes = np.random.uniform(50, 30000, n_sources)
126 xposns = np.random.uniform(xim.min()+16, xim.max()-5, n_sources)
127 yposns = np.random.uniform(yim.min()+16, yim.max()-5, n_sources)
129 # Make the source closest to the center of the image the one that increases in flux
130 ind = np.argmin(xposns**2. + yposns**2.)
132 # vary the y-width of psf across x-axis of science image (zero means no variation):
133 psf1_yvary = psf_yvary_factor * (yim.mean() - yposns) / yim.max()
134 if verbose:
135 print('PSF y spatial-variation:', psf1_yvary.min(), psf1_yvary.max())
137 for i in range(n_sources):
138 flux = fluxes[i]
139 tmp = flux * singleGaussian2d(x0im, y0im, xposns[i], yposns[i], psf2[0], psf2[1], theta=theta2)
140 im2 += tmp
141 if i == ind:
142 flux += flux * varSourceChange
143 tmp = flux * singleGaussian2d(x0im, y0im, xposns[i]+offset[0], yposns[i]+offset[1],
144 psf1[0], psf1[1]+psf1_yvary[i], theta=theta1)
145 im1 += tmp
147 im1_psf = singleGaussian2d(x0im, y0im, 0, 0, psf1[0], psf1[1], theta=theta1)
148 im2_psf = singleGaussian2d(x0im, y0im, offset[0], offset[1], psf2[0], psf2[1], theta=theta2)
150 def makeWcs(offset=0):
151 """ Make a fake Wcs
153 Parameters
154 ----------
155 offset : float
156 offset the Wcs by this many pixels.
157 """
158 # taken from $AFW_DIR/tests/testMakeWcs.py
159 metadata = dafBase.PropertySet()
160 metadata.set("SIMPLE", "T")
161 metadata.set("BITPIX", -32)
162 metadata.set("NAXIS", 2)
163 metadata.set("NAXIS1", 1024)
164 metadata.set("NAXIS2", 1153)
165 metadata.set("RADESYS", 'FK5')
166 metadata.set("EQUINOX", 2000.)
167 metadata.setDouble("CRVAL1", 215.604025685476)
168 metadata.setDouble("CRVAL2", 53.1595451514076)
169 metadata.setDouble("CRPIX1", 1109.99981456774 + offset)
170 metadata.setDouble("CRPIX2", 560.018167811613 + offset)
171 metadata.set("CTYPE1", 'RA---SIN')
172 metadata.set("CTYPE2", 'DEC--SIN')
173 metadata.setDouble("CD1_1", 5.10808596133527E-05)
174 metadata.setDouble("CD1_2", 1.85579539217196E-07)
175 metadata.setDouble("CD2_2", -5.10281493481982E-05)
176 metadata.setDouble("CD2_1", -8.27440751733828E-07)
177 return afwGeom.makeSkyWcs(metadata)
179 def makeExposure(imgArray, psfArray, imgVariance):
180 """! Convert an image numpy.array and corresponding PSF numpy.array into an exposure.
182 Add the (constant) variance plane equal to `imgVariance`.
184 @param imgArray 2-d numpy.array containing the image
185 @param psfArray 2-d numpy.array containing the PSF image
186 @param imgVariance variance of input image
187 @return a new exposure containing the image, PSF and desired variance plane
188 """
189 # All this code to convert the template image array/psf array into an exposure.
190 bbox = geom.Box2I(geom.Point2I(0, 0), geom.Point2I(imgArray.shape[1]-1, imgArray.shape[0]-1))
191 im1ex = afwImage.ExposureD(bbox)
192 im1ex.getMaskedImage().getImage().getArray()[:, :] = imgArray
193 im1ex.getMaskedImage().getVariance().getArray()[:, :] = imgVariance
194 psfBox = geom.Box2I(geom.Point2I(-12, -12), geom.Point2I(12, 12)) # a 25x25 pixel psf
195 psf = afwImage.ImageD(psfBox)
196 psfBox.shift(geom.Extent2I(size[0]//2, size[1]//2))
197 im1_psf_sub = psfArray[psfBox.getMinX():psfBox.getMaxX()+1, psfBox.getMinY():psfBox.getMaxY()+1]
198 psf.getArray()[:, :] = im1_psf_sub
199 psfK = afwMath.FixedKernel(psf)
200 psfNew = measAlg.KernelPsf(psfK)
201 im1ex.setPsf(psfNew)
202 wcs = makeWcs()
203 im1ex.setWcs(wcs)
204 return im1ex
206 im1ex = makeExposure(im1, im1_psf, svar) # Science image
207 im2ex = makeExposure(im2, im2_psf, tvar) # Template
209 return im1ex, im2ex
212class DiffimCorrectionTest(lsst.utils.tests.TestCase):
213 """!A test case for the diffim image decorrelation algorithm.
214 """
216 def setUp(self):
217 self.psf1_sigma = 3.3 # sigma of psf of science image
218 self.psf2_sigma = 2.2 # sigma of psf of template image
220 self.statsControl = afwMath.StatisticsControl()
221 self.statsControl.setNumSigmaClip(3.)
222 self.statsControl.setNumIter(3)
223 self.statsControl.setAndMask(afwImage.Mask
224 .getPlaneBitMask(["INTRP", "EDGE", "SAT", "CR",
225 "DETECTED", "BAD",
226 "NO_DATA", "DETECTED_NEGATIVE"]))
228 def _setUpImages(self, svar=0.04, tvar=0.04, varyPsf=0.):
229 """!Generate a fake aligned template and science image.
230 """
232 self.svar = svar # variance of noise in science image
233 self.tvar = tvar # variance of noise in template image
235 self.im1ex, self.im2ex \
236 = makeFakeImages(svar=self.svar, tvar=self.tvar, psf1=self.psf1_sigma, psf2=self.psf2_sigma,
237 n_sources=50, psf_yvary_factor=varyPsf, verbose=False)
239 def _computeVarianceMean(self, maskedIm):
240 statObj = afwMath.makeStatistics(maskedIm.getVariance(),
241 maskedIm.getMask(), afwMath.MEANCLIP,
242 self.statsControl)
243 mn = statObj.getValue(afwMath.MEANCLIP)
244 return mn
246 def _computePixelVariance(self, maskedIm):
247 statObj = afwMath.makeStatistics(maskedIm, afwMath.VARIANCECLIP,
248 self.statsControl)
249 var = statObj.getValue(afwMath.VARIANCECLIP)
250 return var
252 def tearDown(self):
253 del self.im1ex
254 del self.im2ex
256 def _makeAndTestUncorrectedDiffim(self):
257 """Create the (un-decorrelated) diffim, and verify that its variance is too low.
258 """
259 # Create the matching kernel. We used Gaussian PSFs for im1 and im2, so we can compute the "expected"
260 # matching kernel sigma.
261 psf1_sig = self.im1ex.getPsf().computeShape().getDeterminantRadius()
262 psf2_sig = self.im2ex.getPsf().computeShape().getDeterminantRadius()
263 sig_match = np.sqrt((psf1_sig**2. - psf2_sig**2.))
264 # Sanity check - make sure PSFs are correct.
265 self.assertFloatsAlmostEqual(sig_match, np.sqrt((self.psf1_sigma**2. - self.psf2_sigma**2.)),
266 rtol=2e-5)
267 # mKernel = measAlg.SingleGaussianPsf(31, 31, sig_match)
268 x0 = np.arange(-16, 16, 1)
269 y0 = x0.copy()
270 x0im, y0im = np.meshgrid(x0, y0)
271 matchingKernel = singleGaussian2d(x0im, y0im, -1., -1., sigma_x=sig_match, sigma_y=sig_match)
272 kernelImg = afwImage.ImageD(matchingKernel.shape[0], matchingKernel.shape[1])
273 kernelImg.getArray()[:, :] = matchingKernel
274 mKernel = afwMath.FixedKernel(kernelImg)
276 # Create the matched template by convolving the template with the matchingKernel
277 matched_im2ex = self.im2ex.clone()
278 convCntrl = afwMath.ConvolutionControl(False, True, 0)
279 afwMath.convolve(matched_im2ex.getMaskedImage(), self.im2ex.getMaskedImage(), mKernel, convCntrl)
281 # Expected (ideal) variance of difference image
282 expected_var = self.svar + self.tvar
283 if verbose:
284 print('EXPECTED VARIANCE:', expected_var)
286 # Create the diffim (uncorrected)
287 # Uncorrected diffim exposure - variance plane is wrong (too low)
288 tmp_diffExp = self.im1ex.getMaskedImage().clone()
289 tmp_diffExp -= matched_im2ex.getMaskedImage()
290 var = self._computeVarianceMean(tmp_diffExp)
291 self.assertLess(var, expected_var)
293 # Uncorrected diffim exposure - variance is wrong (too low) - same as above but on pixels
294 diffExp = self.im1ex.clone()
295 tmp = diffExp.getMaskedImage()
296 tmp -= matched_im2ex.getMaskedImage()
297 var = self._computePixelVariance(diffExp.getMaskedImage())
298 self.assertLess(var, expected_var)
300 # Uncorrected diffim exposure - variance plane is wrong (too low)
301 mn = self._computeVarianceMean(diffExp.getMaskedImage())
302 self.assertLess(mn, expected_var)
303 if verbose:
304 print('UNCORRECTED VARIANCE:', var, mn)
306 return diffExp, mKernel, expected_var
308 def _runDecorrelationTask(self, diffExp, mKernel):
309 """ Run the decorrelation task on the given diffim with the given matching kernel
310 """
311 task = DecorrelateALKernelTask()
312 decorrResult = task.run(self.im1ex, self.im2ex, diffExp, mKernel)
313 corrected_diffExp = decorrResult.correctedExposure
314 return corrected_diffExp
316 def _testDecorrelation(self, expected_var, corrected_diffExp):
317 """ Check that the variance of the corrected diffim matches the theoretical value.
318 """
319 # Corrected diffim - variance should be close to expected.
320 # We set the tolerance a bit higher here since the simulated images have many bright stars
321 var = self._computePixelVariance(corrected_diffExp.getMaskedImage())
322 self.assertFloatsAlmostEqual(var, expected_var, rtol=0.05)
324 # Check statistics of variance plane in corrected diffim
325 mn = self._computeVarianceMean(corrected_diffExp.getMaskedImage())
326 if verbose:
327 print('CORRECTED VARIANCE:', var, mn)
328 self.assertFloatsAlmostEqual(mn, expected_var, rtol=0.02)
329 self.assertFloatsAlmostEqual(var, mn, rtol=0.05)
330 return var, mn
332 def _testDiffimCorrection(self, svar, tvar):
333 """ Run decorrelation and check the variance of the corrected diffim.
334 """
335 self._setUpImages(svar=svar, tvar=tvar)
336 diffExp, mKernel, expected_var = self._makeAndTestUncorrectedDiffim()
337 corrected_diffExp = self._runDecorrelationTask(diffExp, mKernel)
338 self._testDecorrelation(expected_var, corrected_diffExp)
340 def testDiffimCorrection(self):
341 """Test decorrelated diffim from images with different combinations of variances.
342 """
343 # Same variance
344 self._testDiffimCorrection(svar=0.04, tvar=0.04)
345 # Science image variance is higher than that of the template.
346 self._testDiffimCorrection(svar=0.08, tvar=0.04)
347 # Template variance is higher than that of the science img.
348 self._testDiffimCorrection(svar=0.04, tvar=0.08)
350 def _runDecorrelationTaskMapReduced(self, diffExp, mKernel):
351 """ Run decorrelation using the imageMapReducer.
352 """
353 config = DecorrelateALKernelMapReduceConfig()
354 config.borderSizeX = config.borderSizeY = 3
355 config.reducer.reduceOperation = 'average'
356 task = ImageMapReduceTask(config=config)
357 decorrResult = task.run(diffExp, template=self.im2ex, science=self.im1ex,
358 psfMatchingKernel=mKernel, forceEvenSized=True)
359 corrected_diffExp = decorrResult.exposure
360 return corrected_diffExp
362 def _testDiffimCorrection_mapReduced(self, svar, tvar, varyPsf=0.0):
363 """ Run decorrelation using the imageMapReduce task, and check the variance of
364 the corrected diffim.
365 """
366 self._setUpImages(svar=svar, tvar=tvar, varyPsf=varyPsf)
367 diffExp, mKernel, expected_var = self._makeAndTestUncorrectedDiffim()
368 corrected_diffExp = self._runDecorrelationTaskMapReduced(diffExp, mKernel)
369 self._testDecorrelation(expected_var, corrected_diffExp)
370 # Also compare the diffim generated here vs. the non-ImageMapReduce one
371 corrected_diffExp_OLD = self._runDecorrelationTask(diffExp, mKernel)
372 self.assertMaskedImagesAlmostEqual(corrected_diffExp.getMaskedImage(),
373 corrected_diffExp_OLD.getMaskedImage())
375 def testDiffimCorrection_mapReduced(self):
376 """ Test decorrelated diffim when using the imageMapReduce task.
377 Compare results with those from the original DecorrelateALKernelTask.
378 """
379 # Same variance
380 self._testDiffimCorrection_mapReduced(svar=0.04, tvar=0.04)
381 # Science image variance is higher than that of the template.
382 self._testDiffimCorrection_mapReduced(svar=0.04, tvar=0.08)
383 # Template variance is higher than that of the science img.
384 self._testDiffimCorrection_mapReduced(svar=0.08, tvar=0.04)
386 def _runDecorrelationSpatialTask(self, diffExp, mKernel, spatiallyVarying=False):
387 """ Run decorrelation using the DecorrelateALKernelSpatialTask.
388 """
389 config = DecorrelateALKernelSpatialConfig()
390 task = DecorrelateALKernelSpatialTask(config=config)
391 decorrResult = task.run(scienceExposure=self.im1ex, templateExposure=self.im2ex,
392 subtractedExposure=diffExp, psfMatchingKernel=mKernel,
393 spatiallyVarying=spatiallyVarying)
394 corrected_diffExp = decorrResult.correctedExposure
395 return corrected_diffExp
397 def _testDiffimCorrection_spatialTask(self, svar, tvar, varyPsf=0.0):
398 """Run decorrelation using the DecorrelateALKernelSpatialTask, and
399 check the variance of the corrected diffim. Do it for `spatiallyVarying` both
400 True and False. Also compare the variances between the two `spatiallyVarying`
401 cases.
402 """
403 self._setUpImages(svar=svar, tvar=tvar, varyPsf=varyPsf)
404 diffExp, mKernel, expected_var = self._makeAndTestUncorrectedDiffim()
405 variances = []
406 for spatiallyVarying in [False, True]:
407 corrected_diffExp = self._runDecorrelationSpatialTask(diffExp, mKernel,
408 spatiallyVarying)
409 var, mn = self._testDecorrelation(expected_var, corrected_diffExp)
410 variances.append(var)
411 self.assertFloatsAlmostEqual(variances[0], variances[1], rtol=0.03)
413 def testDiffimCorrection_spatialTask(self):
414 """Test decorrelated diffim when using the DecorrelateALKernelSpatialTask.
415 Compare results with those from the original DecorrelateALKernelTask.
416 """
417 # Same variance
418 self._testDiffimCorrection_spatialTask(svar=0.04, tvar=0.04)
419 # Science image variance is higher than that of the template.
420 self._testDiffimCorrection_spatialTask(svar=0.04, tvar=0.08)
421 # Template variance is higher than that of the science img.
422 self._testDiffimCorrection_spatialTask(svar=0.08, tvar=0.04)
425class MemoryTester(lsst.utils.tests.MemoryTestCase):
426 pass
429if __name__ == "__main__": 429 ↛ 430line 429 didn't jump to line 430, because the condition on line 429 was never true
430 lsst.utils.tests.init()
431 unittest.main()