Coverage for tests/test_stacker.py: 10%
258 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-10 02:46 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-10 02:46 -0800
1# This file is part of afw.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""
23Tests for Stack
25Run with:
26 python test_stacker.py
27or
28 pytest test_stacker.py
29"""
30import unittest
31from functools import reduce
33import numpy as np
35import lsst.geom
36import lsst.afw.image as afwImage
37import lsst.afw.math as afwMath
38import lsst.utils.tests
39import lsst.pex.exceptions as pexEx
40import lsst.afw.display as afwDisplay
42display = False
43afwDisplay.setDefaultMaskTransparency(75)
45######################################
46# main body of code
47######################################
50class StackTestCase(lsst.utils.tests.TestCase):
52 def setUp(self):
53 np.random.seed(1)
54 self.nImg = 10
55 self.nX, self.nY = 64, 64
56 self.values = [1.0, 2.0, 2.0, 3.0, 8.0]
58 def testMean(self):
59 """ Test the statisticsStack() function for a MEAN"""
61 knownMean = 0.0
62 imgList = []
63 for iImg in range(self.nImg):
64 imgList.append(afwImage.ImageF(
65 lsst.geom.Extent2I(self.nX, self.nY), iImg))
66 knownMean += iImg
68 imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN)
69 knownMean /= self.nImg
70 self.assertEqual(imgStack[self.nX//2, self.nY//2, afwImage.LOCAL], knownMean)
72 # Test in-place stacking
73 afwMath.statisticsStack(imgStack, imgList, afwMath.MEAN)
74 self.assertEqual(imgStack[self.nX//2, self.nY//2, afwImage.LOCAL], knownMean)
76 def testStatistics(self):
77 """ Test the statisticsStack() function """
79 imgList = []
80 for val in self.values:
81 imgList.append(afwImage.ImageF(
82 lsst.geom.Extent2I(self.nX, self.nY), val))
84 imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN)
85 mean = reduce(lambda x, y: x+y, self.values)/float(len(self.values))
86 self.assertAlmostEqual(imgStack[self.nX//2, self.nY//2, afwImage.LOCAL], mean)
88 imgStack = afwMath.statisticsStack(imgList, afwMath.MEDIAN)
89 median = sorted(self.values)[len(self.values)//2]
90 self.assertEqual(imgStack[self.nX//2, self.nY//2, afwImage.LOCAL], median)
92 def testWeightedStack(self):
93 """ Test statisticsStack() function when weighting by a variance plane"""
95 sctrl = afwMath.StatisticsControl()
96 sctrl.setWeighted(True)
97 mimgList = []
98 for val in self.values:
99 mimg = afwImage.MaskedImageF(lsst.geom.Extent2I(self.nX, self.nY))
100 mimg.set(val, 0x0, val)
101 mimgList.append(mimg)
102 mimgStack = afwMath.statisticsStack(mimgList, afwMath.MEAN, sctrl)
104 wvalues = [1.0/q for q in self.values]
105 wmean = float(len(self.values)) / reduce(lambda x, y: x + y, wvalues)
106 self.assertAlmostEqual(
107 mimgStack.image[self.nX//2, self.nY//2, afwImage.LOCAL],
108 wmean)
110 # Test in-place stacking
111 afwMath.statisticsStack(mimgStack, mimgList, afwMath.MEAN, sctrl)
112 self.assertAlmostEqual(
113 mimgStack.image[self.nX//2, self.nY//2, afwImage.LOCAL],
114 wmean)
116 def testConstantWeightedStack(self):
117 """ Test statisticsStack() function when weighting by a vector of weights"""
119 sctrl = afwMath.StatisticsControl()
120 imgList = []
121 weights = []
122 for val in self.values:
123 img = afwImage.ImageF(lsst.geom.Extent2I(self.nX, self.nY), val)
124 imgList.append(img)
125 weights.append(val)
126 imgStack = afwMath.statisticsStack(
127 imgList, afwMath.MEAN, sctrl, weights)
129 wsum = reduce(lambda x, y: x + y, self.values)
130 wvalues = [x*x for x in self.values]
131 wmean = reduce(lambda x, y: x + y, wvalues)/float(wsum)
132 self.assertAlmostEqual(imgStack[self.nX//2, self.nY//2, afwImage.LOCAL], wmean)
134 def testRequestMoreThanOneStat(self):
135 """ Make sure we throw an exception if someone requests more than one type of statistics. """
137 sctrl = afwMath.StatisticsControl()
138 imgList = []
139 for val in self.values:
140 img = afwImage.ImageF(lsst.geom.Extent2I(self.nX, self.nY), val)
141 imgList.append(img)
143 def tst():
144 afwMath.statisticsStack(
145 imgList,
146 afwMath.Property(afwMath.MEAN | afwMath.MEANCLIP),
147 sctrl)
149 self.assertRaises(pexEx.InvalidParameterError, tst)
151 def testReturnInputs(self):
152 """ Make sure that a single file put into the stacker is returned unscathed"""
154 imgList = []
156 img = afwImage.MaskedImageF(lsst.geom.Extent2I(10, 20))
157 for y in range(img.getHeight()):
158 simg = img.Factory(
159 img,
160 lsst.geom.Box2I(lsst.geom.Point2I(0, y),
161 lsst.geom.Extent2I(img.getWidth(), 1)),
162 afwImage.LOCAL)
163 simg.set(y)
165 imgList.append(img)
167 imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN)
169 if display:
170 afwDisplay.Display(frame=1).mtv(img, title="input")
171 afwDisplay.Display(frame=2).mtv(imgStack, title="stack")
173 self.assertEqual(img[0, 0, afwImage.LOCAL][0], imgStack[0, 0, afwImage.LOCAL][0])
175 def testStackBadPixels(self):
176 """Check that we properly ignore masked pixels, and set noGoodPixelsMask where there are
177 no good pixels"""
178 mimgVec = []
180 DETECTED = afwImage.Mask.getPlaneBitMask("DETECTED")
181 EDGE = afwImage.Mask.getPlaneBitMask("EDGE")
182 INTRP = afwImage.Mask.getPlaneBitMask("INTRP")
183 SAT = afwImage.Mask.getPlaneBitMask("SAT")
185 sctrl = afwMath.StatisticsControl()
186 sctrl.setNanSafe(False)
187 sctrl.setAndMask(INTRP | SAT)
188 sctrl.setNoGoodPixelsMask(EDGE)
190 # set these pixels to EDGE
191 edgeBBox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0),
192 lsst.geom.Extent2I(20, 20))
193 width, height = 512, 512
194 dim = lsst.geom.Extent2I(width, height)
195 val, maskVal = 10, DETECTED
196 for i in range(4):
197 mimg = afwImage.MaskedImageF(dim)
198 mimg.set(val, maskVal, 1)
199 #
200 # Set part of the image to NaN (with the INTRP bit set)
201 #
202 llc = lsst.geom.Point2I(width//2*(i//2), height//2*(i % 2))
203 bbox = lsst.geom.Box2I(llc, dim//2)
205 smimg = mimg.Factory(mimg, bbox, afwImage.LOCAL)
206 del smimg
207 #
208 # And the bottom corner to SAT
209 #
210 smask = mimg.getMask().Factory(mimg.getMask(), edgeBBox, afwImage.LOCAL)
211 smask |= SAT
212 del smask
214 mimgVec.append(mimg)
216 if display > 1:
217 afwDisplay.Display(frame=i).mtv(mimg, title=str(i))
219 mimgStack = afwMath.statisticsStack(mimgVec, afwMath.MEAN, sctrl)
221 if display:
222 i += 1
223 afwDisplay.Display(frame=i).mtv(mimgStack, title="Stack")
224 i += 1
225 afwDisplay.Display(frame=i).mtv(mimgStack.getVariance(), title="var(Stack)")
226 #
227 # Check the output, ignoring EDGE pixels
228 #
229 sctrl = afwMath.StatisticsControl()
230 sctrl.setAndMask(afwImage.Mask.getPlaneBitMask("EDGE"))
232 stats = afwMath.makeStatistics(
233 mimgStack, afwMath.MIN | afwMath.MAX, sctrl)
234 self.assertEqual(stats.getValue(afwMath.MIN), val)
235 self.assertEqual(stats.getValue(afwMath.MAX), val)
236 #
237 # We have to clear EDGE in the known bad corner to check the mask
238 #
239 smask = mimgStack.mask[edgeBBox, afwImage.LOCAL]
240 self.assertEqual(smask[edgeBBox.getMin(), afwImage.LOCAL], EDGE)
241 smask &= ~EDGE
242 del smask
244 self.assertEqual(
245 afwMath.makeStatistics(mimgStack.getMask(),
246 afwMath.SUM, sctrl).getValue(),
247 maskVal)
249 def testTicket1412(self):
250 """Ticket 1412: ignored mask bits are propegated to output stack."""
252 mimg1 = afwImage.MaskedImageF(lsst.geom.Extent2I(1, 1))
253 mimg1[0, 0, afwImage.LOCAL] = (1, 0x4, 1) # set 0100
254 mimg2 = afwImage.MaskedImageF(lsst.geom.Extent2I(1, 1))
255 mimg2[0, 0, afwImage.LOCAL] = (2, 0x3, 1) # set 0010 and 0001
257 imgList = []
258 imgList.append(mimg1)
259 imgList.append(mimg2)
261 sctrl = afwMath.StatisticsControl()
262 sctrl.setAndMask(0x1) # andmask only 0001
264 # try first with no sctrl (no andmask set), should see 0x0111 for all
265 # output mask pixels
266 imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN)
267 self.assertEqual(imgStack[0, 0, afwImage.LOCAL][1], 0x7)
269 # now try with sctrl (andmask = 0x0001), should see 0x0100 for all
270 # output mask pixels
271 imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN, sctrl)
272 self.assertEqual(imgStack[0, 0, afwImage.LOCAL][1], 0x4)
274 def test2145(self):
275 """The how-to-repeat from #2145"""
276 Size = 5
277 statsCtrl = afwMath.StatisticsControl()
278 statsCtrl.setCalcErrorFromInputVariance(True)
279 maskedImageList = []
280 weightList = []
281 for i in range(3):
282 mi = afwImage.MaskedImageF(Size, Size)
283 imArr, maskArr, varArr = mi.getArrays()
284 imArr[:] = np.random.normal(10, 0.1, (Size, Size))
285 varArr[:] = np.random.normal(10, 0.1, (Size, Size))
286 maskedImageList.append(mi)
287 weightList.append(1.0)
289 stack = afwMath.statisticsStack(
290 maskedImageList, afwMath.MEAN, statsCtrl, weightList)
291 if False:
292 print("image=", stack.getImage().getArray())
293 print("variance=", stack.getVariance().getArray())
294 self.assertNotEqual(np.sum(stack.getVariance().getArray()), 0.0)
296 def testMosaicMode(self):
297 """Test that mosaic mode constructs a variance plane of mean of inputs
298 """
299 SIZE = 5
300 MEAN = 10
301 VAR = 0.1
303 maskedImageList = []
304 weightList = []
305 for i in range(3):
306 mi = afwImage.MaskedImageF(SIZE, SIZE)
307 mi.image.array[:] = np.random.normal(MEAN*i, VAR, (SIZE, SIZE))
308 mi.variance.array[:] = np.random.normal(MEAN*i, VAR, (SIZE, SIZE))
309 maskedImageList.append(mi)
310 # weight each image differently
311 weightList.append(float(i))
313 statsCtrl = afwMath.StatisticsControl()
314 statsCtrl.setCalcErrorMosaicMode(True)
316 # test unweighted mean
317 stack = afwMath.statisticsStack(maskedImageList, afwMath.MEAN, statsCtrl, weightList)
318 self.assertAlmostEqual(np.mean(stack.variance.array), np.mean(stack.image.array), delta=0.1)
320 # test weighted mean
321 statsCtrl.setWeighted(True)
322 stack = afwMath.statisticsStack(maskedImageList, afwMath.MEAN, statsCtrl, weightList)
323 self.assertAlmostEqual(np.mean(stack.variance.array), np.mean(stack.image.array), delta=0.1)
325 def testRejectedMaskPropagation(self):
326 """Test that we can propagate mask bits from rejected pixels, when the amount
327 of rejection crosses a threshold."""
328 rejectedBit = 1 # use this bit to determine whether to reject a pixel
329 propagatedBit = 2 # propagate this bit if a pixel with it set is rejected
330 statsCtrl = afwMath.StatisticsControl()
331 statsCtrl.setMaskPropagationThreshold(propagatedBit, 0.3)
332 statsCtrl.setAndMask(1 << rejectedBit)
333 statsCtrl.setWeighted(True)
334 maskedImageList = []
336 # start with 4 images with no mask bits set
337 partialSum = np.zeros((1, 4), dtype=np.float32)
338 finalImage = np.array([12.0, 12.0, 12.0, 12.0], dtype=np.float32)
339 for i in range(4):
340 mi = afwImage.MaskedImageF(4, 1)
341 imArr, maskArr, varArr = mi.getArrays()
342 imArr[:, :] = np.ones((1, 4), dtype=np.float32)
343 maskedImageList.append(mi)
344 partialSum += imArr
345 # add one more image with all permutations of the first two bits set in
346 # different pixels
347 mi = afwImage.MaskedImageF(4, 1)
348 imArr, maskArr, varArr = mi.getArrays()
349 imArr[0, :] = finalImage
350 maskArr[0, 1] |= (1 << rejectedBit)
351 maskArr[0, 2] |= (1 << propagatedBit)
352 maskArr[0, 3] |= (1 << rejectedBit)
353 maskArr[0, 3] |= (1 << propagatedBit)
354 maskedImageList.append(mi)
356 # these will always be rejected
357 finalImage[1] = 0.0
358 finalImage[3] = 0.0
360 # Uniform weights: we should only see pixel 2 set with propagatedBit, because it's not rejected;
361 # pixel 3 is rejected, but its weight (0.2) below the propagation
362 # threshold (0.3)
363 stack1 = afwMath.statisticsStack(maskedImageList, afwMath.MEAN, statsCtrl, [
364 1.0, 1.0, 1.0, 1.0, 1.0])
365 self.assertEqual(stack1[0, 0, afwImage.LOCAL][1], 0x0)
366 self.assertEqual(stack1[1, 0, afwImage.LOCAL][1], 0x0)
367 self.assertEqual(stack1[2, 0, afwImage.LOCAL][1], 1 << propagatedBit)
368 self.assertEqual(stack1[3, 0, afwImage.LOCAL][1], 0x0)
369 self.assertFloatsAlmostEqual(stack1.getImage().getArray(),
370 (partialSum + finalImage) / np.array([5.0, 4.0, 5.0, 4.0]), rtol=1E-7)
372 # Give the masked image more weight: we should see pixel 2 and pixel 3 set with propagatedBit,
373 # pixel 2 because it's not rejected, and pixel 3 because the weight of the rejection (0.3333)
374 # is above the threshold (0.3)
375 # Note that rejectedBit is never propagated, because we didn't include it in statsCtrl (of course,
376 # normally the bits we'd propagate and the bits we'd reject would be
377 # the same)
378 stack2 = afwMath.statisticsStack(maskedImageList, afwMath.MEAN, statsCtrl, [
379 1.0, 1.0, 1.0, 1.0, 2.0])
380 self.assertEqual(stack2[0, 0, afwImage.LOCAL][1], 0x0)
381 self.assertEqual(stack2[1, 0, afwImage.LOCAL][1], 0x0)
382 self.assertEqual(stack2[2, 0, afwImage.LOCAL][1], 1 << propagatedBit)
383 self.assertEqual(stack2[3, 0, afwImage.LOCAL][1], 1 << propagatedBit)
384 self.assertFloatsAlmostEqual(stack2.getImage().getArray(),
385 (partialSum + 2*finalImage) / np.array([6.0, 4.0, 6.0, 4.0]), rtol=1E-7)
387 def testClipped(self):
388 """Test that we set mask bits when pixels are clipped"""
389 box = lsst.geom.Box2I(lsst.geom.Point2I(12345, 67890), lsst.geom.Extent2I(3, 3))
390 num = 10
391 maskVal = 0xAD
392 value = 0.0
394 images = [afwImage.MaskedImageF(box) for _ in range(num)]
395 statsCtrl = afwMath.StatisticsControl()
396 statsCtrl.setAndMask(maskVal)
397 clipped = 1 << afwImage.Mask().addMaskPlane("CLIPPED")
399 # No clipping: check that vanilla is working
400 for img in images:
401 img.getImage().set(value)
402 img.getMask().set(0)
403 stack = afwMath.statisticsStack(images, afwMath.MEANCLIP, clipped=clipped)
404 self.assertFloatsAlmostEqual(stack.getImage().getArray(), 0.0, atol=0.0)
405 self.assertFloatsAlmostEqual(stack.getMask().getArray(), 0, atol=0.0) # Not floats, but that's OK
407 # Clip a pixel; the CLIPPED bit should be set
408 images[0].getImage()[1, 1, afwImage.LOCAL] = value + 1.0
409 stack = afwMath.statisticsStack(images, afwMath.MEANCLIP, clipped=clipped)
410 self.assertFloatsAlmostEqual(stack.getImage().getArray(), 0.0, atol=0.0)
411 self.assertEqual(stack.mask[1, 1, afwImage.LOCAL], clipped)
413 # Mask a pixel; the CLIPPED bit should be set
414 images[0].getMask()[1, 1, afwImage.LOCAL] = maskVal
415 stack = afwMath.statisticsStack(images, afwMath.MEAN, statsCtrl, clipped=clipped)
416 self.assertFloatsAlmostEqual(stack.getImage().getArray(), 0.0, atol=0.0)
417 self.assertEqual(stack.mask[1, 1, afwImage.LOCAL], clipped)
419 # Excuse that mask; the CLIPPED bit should not be set
420 stack = afwMath.statisticsStack(images, afwMath.MEAN, statsCtrl, clipped=clipped, excuse=maskVal)
421 self.assertFloatsAlmostEqual(stack.getImage().getArray(), 0.0, atol=0.0)
422 self.assertEqual(stack.mask[1, 1, afwImage.LOCAL], 0)
424 # Map that mask value to a different one.
425 rejected = 1 << afwImage.Mask().addMaskPlane("REJECTED")
426 maskMap = [(maskVal, rejected)]
427 images[0].mask[1, 1, afwImage.LOCAL] = 0 # only want to clip, not mask, this one
428 images[1].mask[1, 2, afwImage.LOCAL] = maskVal # only want to mask, not clip, this one
429 stack = afwMath.statisticsStack(images, afwMath.MEANCLIP, statsCtrl, wvector=[], clipped=clipped,
430 maskMap=maskMap)
431 self.assertFloatsAlmostEqual(stack.getImage().getArray(), 0.0, atol=0.0)
432 self.assertEqual(stack.mask[1, 1, afwImage.LOCAL], clipped)
433 self.assertEqual(stack.mask[1, 2, afwImage.LOCAL], rejected)
435#################################################################
436# Test suite boiler plate
437#################################################################
440class TestMemory(lsst.utils.tests.MemoryTestCase):
441 pass
444def setup_module(module):
445 lsst.utils.tests.init()
448if __name__ == "__main__": 448 ↛ 449line 448 didn't jump to line 449, because the condition on line 448 was never true
449 lsst.utils.tests.init()
450 unittest.main()