Coverage for tests/test_stacker.py: 12%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

238 statements  

1# This file is part of afw. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22""" 

23Tests for Stack 

24 

25Run with: 

26 python test_stacker.py 

27or 

28 pytest test_stacker.py 

29""" 

30import unittest 

31from functools import reduce 

32 

33import numpy as np 

34 

35import lsst.geom 

36import lsst.afw.image as afwImage 

37import lsst.afw.math as afwMath 

38import lsst.utils.tests 

39import lsst.pex.exceptions as pexEx 

40import lsst.afw.display as afwDisplay 

41 

42display = False 

43afwDisplay.setDefaultMaskTransparency(75) 

44 

45###################################### 

46# main body of code 

47###################################### 

48 

49 

50class StackTestCase(lsst.utils.tests.TestCase): 

51 

52 def setUp(self): 

53 np.random.seed(1) 

54 self.nImg = 10 

55 self.nX, self.nY = 64, 64 

56 self.values = [1.0, 2.0, 2.0, 3.0, 8.0] 

57 

58 def testMean(self): 

59 """ Test the statisticsStack() function for a MEAN""" 

60 

61 knownMean = 0.0 

62 imgList = [] 

63 for iImg in range(self.nImg): 

64 imgList.append(afwImage.ImageF( 

65 lsst.geom.Extent2I(self.nX, self.nY), iImg)) 

66 knownMean += iImg 

67 

68 imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN) 

69 knownMean /= self.nImg 

70 self.assertEqual(imgStack[self.nX//2, self.nY//2, afwImage.LOCAL], knownMean) 

71 

72 # Test in-place stacking 

73 afwMath.statisticsStack(imgStack, imgList, afwMath.MEAN) 

74 self.assertEqual(imgStack[self.nX//2, self.nY//2, afwImage.LOCAL], knownMean) 

75 

76 def testStatistics(self): 

77 """ Test the statisticsStack() function """ 

78 

79 imgList = [] 

80 for val in self.values: 

81 imgList.append(afwImage.ImageF( 

82 lsst.geom.Extent2I(self.nX, self.nY), val)) 

83 

84 imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN) 

85 mean = reduce(lambda x, y: x+y, self.values)/float(len(self.values)) 

86 self.assertAlmostEqual(imgStack[self.nX//2, self.nY//2, afwImage.LOCAL], mean) 

87 

88 imgStack = afwMath.statisticsStack(imgList, afwMath.MEDIAN) 

89 median = sorted(self.values)[len(self.values)//2] 

90 self.assertEqual(imgStack[self.nX//2, self.nY//2, afwImage.LOCAL], median) 

91 

92 def testWeightedStack(self): 

93 """ Test statisticsStack() function when weighting by a variance plane""" 

94 

95 sctrl = afwMath.StatisticsControl() 

96 sctrl.setWeighted(True) 

97 mimgList = [] 

98 for val in self.values: 

99 mimg = afwImage.MaskedImageF(lsst.geom.Extent2I(self.nX, self.nY)) 

100 mimg.set(val, 0x0, val) 

101 mimgList.append(mimg) 

102 mimgStack = afwMath.statisticsStack(mimgList, afwMath.MEAN, sctrl) 

103 

104 wvalues = [1.0/q for q in self.values] 

105 wmean = float(len(self.values)) / reduce(lambda x, y: x + y, wvalues) 

106 self.assertAlmostEqual( 

107 mimgStack.image[self.nX//2, self.nY//2, afwImage.LOCAL], 

108 wmean) 

109 

110 # Test in-place stacking 

111 afwMath.statisticsStack(mimgStack, mimgList, afwMath.MEAN, sctrl) 

112 self.assertAlmostEqual( 

113 mimgStack.image[self.nX//2, self.nY//2, afwImage.LOCAL], 

114 wmean) 

115 

116 def testConstantWeightedStack(self): 

117 """ Test statisticsStack() function when weighting by a vector of weights""" 

118 

119 sctrl = afwMath.StatisticsControl() 

120 imgList = [] 

121 weights = [] 

122 for val in self.values: 

123 img = afwImage.ImageF(lsst.geom.Extent2I(self.nX, self.nY), val) 

124 imgList.append(img) 

125 weights.append(val) 

126 imgStack = afwMath.statisticsStack( 

127 imgList, afwMath.MEAN, sctrl, weights) 

128 

129 wsum = reduce(lambda x, y: x + y, self.values) 

130 wvalues = [x*x for x in self.values] 

131 wmean = reduce(lambda x, y: x + y, wvalues)/float(wsum) 

132 self.assertAlmostEqual(imgStack[self.nX//2, self.nY//2, afwImage.LOCAL], wmean) 

133 

134 def testRequestMoreThanOneStat(self): 

135 """ Make sure we throw an exception if someone requests more than one type of statistics. """ 

136 

137 sctrl = afwMath.StatisticsControl() 

138 imgList = [] 

139 for val in self.values: 

140 img = afwImage.ImageF(lsst.geom.Extent2I(self.nX, self.nY), val) 

141 imgList.append(img) 

142 

143 def tst(): 

144 afwMath.statisticsStack( 

145 imgList, 

146 afwMath.Property(afwMath.MEAN | afwMath.MEANCLIP), 

147 sctrl) 

148 

149 self.assertRaises(pexEx.InvalidParameterError, tst) 

150 

151 def testReturnInputs(self): 

152 """ Make sure that a single file put into the stacker is returned unscathed""" 

153 

154 imgList = [] 

155 

156 img = afwImage.MaskedImageF(lsst.geom.Extent2I(10, 20)) 

157 for y in range(img.getHeight()): 

158 simg = img.Factory( 

159 img, 

160 lsst.geom.Box2I(lsst.geom.Point2I(0, y), 

161 lsst.geom.Extent2I(img.getWidth(), 1)), 

162 afwImage.LOCAL) 

163 simg.set(y) 

164 

165 imgList.append(img) 

166 

167 imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN) 

168 

169 if display: 

170 afwDisplay.Display(frame=1).mtv(img, title="input") 

171 afwDisplay.Display(frame=2).mtv(imgStack, title="stack") 

172 

173 self.assertEqual(img[0, 0, afwImage.LOCAL][0], imgStack[0, 0, afwImage.LOCAL][0]) 

174 

175 def testStackBadPixels(self): 

176 """Check that we properly ignore masked pixels, and set noGoodPixelsMask where there are 

177 no good pixels""" 

178 mimgVec = [] 

179 

180 DETECTED = afwImage.Mask.getPlaneBitMask("DETECTED") 

181 EDGE = afwImage.Mask.getPlaneBitMask("EDGE") 

182 INTRP = afwImage.Mask.getPlaneBitMask("INTRP") 

183 SAT = afwImage.Mask.getPlaneBitMask("SAT") 

184 

185 sctrl = afwMath.StatisticsControl() 

186 sctrl.setNanSafe(False) 

187 sctrl.setAndMask(INTRP | SAT) 

188 sctrl.setNoGoodPixelsMask(EDGE) 

189 

190 # set these pixels to EDGE 

191 edgeBBox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), 

192 lsst.geom.Extent2I(20, 20)) 

193 width, height = 512, 512 

194 dim = lsst.geom.Extent2I(width, height) 

195 val, maskVal = 10, DETECTED 

196 for i in range(4): 

197 mimg = afwImage.MaskedImageF(dim) 

198 mimg.set(val, maskVal, 1) 

199 # 

200 # Set part of the image to NaN (with the INTRP bit set) 

201 # 

202 llc = lsst.geom.Point2I(width//2*(i//2), height//2*(i % 2)) 

203 bbox = lsst.geom.Box2I(llc, dim//2) 

204 

205 smimg = mimg.Factory(mimg, bbox, afwImage.LOCAL) 

206 del smimg 

207 # 

208 # And the bottom corner to SAT 

209 # 

210 smask = mimg.getMask().Factory(mimg.getMask(), edgeBBox, afwImage.LOCAL) 

211 smask |= SAT 

212 del smask 

213 

214 mimgVec.append(mimg) 

215 

216 if display > 1: 

217 afwDisplay.Display(frame=i).mtv(mimg, title=str(i)) 

218 

219 mimgStack = afwMath.statisticsStack(mimgVec, afwMath.MEAN, sctrl) 

220 

221 if display: 

222 i += 1 

223 afwDisplay.Display(frame=i).mtv(mimgStack, title="Stack") 

224 i += 1 

225 afwDisplay.Display(frame=i).mtv(mimgStack.getVariance(), title="var(Stack)") 

226 # 

227 # Check the output, ignoring EDGE pixels 

228 # 

229 sctrl = afwMath.StatisticsControl() 

230 sctrl.setAndMask(afwImage.Mask.getPlaneBitMask("EDGE")) 

231 

232 stats = afwMath.makeStatistics( 

233 mimgStack, afwMath.MIN | afwMath.MAX, sctrl) 

234 self.assertEqual(stats.getValue(afwMath.MIN), val) 

235 self.assertEqual(stats.getValue(afwMath.MAX), val) 

236 # 

237 # We have to clear EDGE in the known bad corner to check the mask 

238 # 

239 smask = mimgStack.mask[edgeBBox, afwImage.LOCAL] 

240 self.assertEqual(smask[edgeBBox.getMin(), afwImage.LOCAL], EDGE) 

241 smask &= ~EDGE 

242 del smask 

243 

244 self.assertEqual( 

245 afwMath.makeStatistics(mimgStack.getMask(), 

246 afwMath.SUM, sctrl).getValue(), 

247 maskVal) 

248 

249 def testTicket1412(self): 

250 """Ticket 1412: ignored mask bits are propegated to output stack.""" 

251 

252 mimg1 = afwImage.MaskedImageF(lsst.geom.Extent2I(1, 1)) 

253 mimg1[0, 0, afwImage.LOCAL] = (1, 0x4, 1) # set 0100 

254 mimg2 = afwImage.MaskedImageF(lsst.geom.Extent2I(1, 1)) 

255 mimg2[0, 0, afwImage.LOCAL] = (2, 0x3, 1) # set 0010 and 0001 

256 

257 imgList = [] 

258 imgList.append(mimg1) 

259 imgList.append(mimg2) 

260 

261 sctrl = afwMath.StatisticsControl() 

262 sctrl.setAndMask(0x1) # andmask only 0001 

263 

264 # try first with no sctrl (no andmask set), should see 0x0111 for all 

265 # output mask pixels 

266 imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN) 

267 self.assertEqual(imgStack[0, 0, afwImage.LOCAL][1], 0x7) 

268 

269 # now try with sctrl (andmask = 0x0001), should see 0x0100 for all 

270 # output mask pixels 

271 imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN, sctrl) 

272 self.assertEqual(imgStack[0, 0, afwImage.LOCAL][1], 0x4) 

273 

274 def test2145(self): 

275 """The how-to-repeat from #2145""" 

276 Size = 5 

277 statsCtrl = afwMath.StatisticsControl() 

278 statsCtrl.setCalcErrorFromInputVariance(True) 

279 maskedImageList = [] 

280 weightList = [] 

281 for i in range(3): 

282 mi = afwImage.MaskedImageF(Size, Size) 

283 imArr, maskArr, varArr = mi.getArrays() 

284 imArr[:] = np.random.normal(10, 0.1, (Size, Size)) 

285 varArr[:] = np.random.normal(10, 0.1, (Size, Size)) 

286 maskedImageList.append(mi) 

287 weightList.append(1.0) 

288 

289 stack = afwMath.statisticsStack( 

290 maskedImageList, afwMath.MEAN, statsCtrl, weightList) 

291 if False: 

292 print("image=", stack.getImage().getArray()) 

293 print("variance=", stack.getVariance().getArray()) 

294 self.assertNotEqual(np.sum(stack.getVariance().getArray()), 0.0) 

295 

296 def testRejectedMaskPropagation(self): 

297 """Test that we can propagate mask bits from rejected pixels, when the amount 

298 of rejection crosses a threshold.""" 

299 rejectedBit = 1 # use this bit to determine whether to reject a pixel 

300 propagatedBit = 2 # propagate this bit if a pixel with it set is rejected 

301 statsCtrl = afwMath.StatisticsControl() 

302 statsCtrl.setMaskPropagationThreshold(propagatedBit, 0.3) 

303 statsCtrl.setAndMask(1 << rejectedBit) 

304 statsCtrl.setWeighted(True) 

305 maskedImageList = [] 

306 

307 # start with 4 images with no mask bits set 

308 partialSum = np.zeros((1, 4), dtype=np.float32) 

309 finalImage = np.array([12.0, 12.0, 12.0, 12.0], dtype=np.float32) 

310 for i in range(4): 

311 mi = afwImage.MaskedImageF(4, 1) 

312 imArr, maskArr, varArr = mi.getArrays() 

313 imArr[:, :] = np.ones((1, 4), dtype=np.float32) 

314 maskedImageList.append(mi) 

315 partialSum += imArr 

316 # add one more image with all permutations of the first two bits set in 

317 # different pixels 

318 mi = afwImage.MaskedImageF(4, 1) 

319 imArr, maskArr, varArr = mi.getArrays() 

320 imArr[0, :] = finalImage 

321 maskArr[0, 1] |= (1 << rejectedBit) 

322 maskArr[0, 2] |= (1 << propagatedBit) 

323 maskArr[0, 3] |= (1 << rejectedBit) 

324 maskArr[0, 3] |= (1 << propagatedBit) 

325 maskedImageList.append(mi) 

326 

327 # these will always be rejected 

328 finalImage[1] = 0.0 

329 finalImage[3] = 0.0 

330 

331 # Uniform weights: we should only see pixel 2 set with propagatedBit, because it's not rejected; 

332 # pixel 3 is rejected, but its weight (0.2) below the propagation 

333 # threshold (0.3) 

334 stack1 = afwMath.statisticsStack(maskedImageList, afwMath.MEAN, statsCtrl, [ 

335 1.0, 1.0, 1.0, 1.0, 1.0]) 

336 self.assertEqual(stack1[0, 0, afwImage.LOCAL][1], 0x0) 

337 self.assertEqual(stack1[1, 0, afwImage.LOCAL][1], 0x0) 

338 self.assertEqual(stack1[2, 0, afwImage.LOCAL][1], 1 << propagatedBit) 

339 self.assertEqual(stack1[3, 0, afwImage.LOCAL][1], 0x0) 

340 self.assertFloatsAlmostEqual(stack1.getImage().getArray(), 

341 (partialSum + finalImage) / np.array([5.0, 4.0, 5.0, 4.0]), rtol=1E-7) 

342 

343 # Give the masked image more weight: we should see pixel 2 and pixel 3 set with propagatedBit, 

344 # pixel 2 because it's not rejected, and pixel 3 because the weight of the rejection (0.3333) 

345 # is above the threshold (0.3) 

346 # Note that rejectedBit is never propagated, because we didn't include it in statsCtrl (of course, 

347 # normally the bits we'd propagate and the bits we'd reject would be 

348 # the same) 

349 stack2 = afwMath.statisticsStack(maskedImageList, afwMath.MEAN, statsCtrl, [ 

350 1.0, 1.0, 1.0, 1.0, 2.0]) 

351 self.assertEqual(stack2[0, 0, afwImage.LOCAL][1], 0x0) 

352 self.assertEqual(stack2[1, 0, afwImage.LOCAL][1], 0x0) 

353 self.assertEqual(stack2[2, 0, afwImage.LOCAL][1], 1 << propagatedBit) 

354 self.assertEqual(stack2[3, 0, afwImage.LOCAL][1], 1 << propagatedBit) 

355 self.assertFloatsAlmostEqual(stack2.getImage().getArray(), 

356 (partialSum + 2*finalImage) / np.array([6.0, 4.0, 6.0, 4.0]), rtol=1E-7) 

357 

358 def testClipped(self): 

359 """Test that we set mask bits when pixels are clipped""" 

360 box = lsst.geom.Box2I(lsst.geom.Point2I(12345, 67890), lsst.geom.Extent2I(3, 3)) 

361 num = 10 

362 maskVal = 0xAD 

363 value = 0.0 

364 

365 images = [afwImage.MaskedImageF(box) for _ in range(num)] 

366 statsCtrl = afwMath.StatisticsControl() 

367 statsCtrl.setAndMask(maskVal) 

368 clipped = 1 << afwImage.Mask().addMaskPlane("CLIPPED") 

369 

370 # No clipping: check that vanilla is working 

371 for img in images: 

372 img.getImage().set(value) 

373 img.getMask().set(0) 

374 stack = afwMath.statisticsStack(images, afwMath.MEANCLIP, clipped=clipped) 

375 self.assertFloatsAlmostEqual(stack.getImage().getArray(), 0.0, atol=0.0) 

376 self.assertFloatsAlmostEqual(stack.getMask().getArray(), 0, atol=0.0) # Not floats, but that's OK 

377 

378 # Clip a pixel; the CLIPPED bit should be set 

379 images[0].getImage()[1, 1, afwImage.LOCAL] = value + 1.0 

380 stack = afwMath.statisticsStack(images, afwMath.MEANCLIP, clipped=clipped) 

381 self.assertFloatsAlmostEqual(stack.getImage().getArray(), 0.0, atol=0.0) 

382 self.assertEqual(stack.mask[1, 1, afwImage.LOCAL], clipped) 

383 

384 # Mask a pixel; the CLIPPED bit should be set 

385 images[0].getMask()[1, 1, afwImage.LOCAL] = maskVal 

386 stack = afwMath.statisticsStack(images, afwMath.MEAN, statsCtrl, clipped=clipped) 

387 self.assertFloatsAlmostEqual(stack.getImage().getArray(), 0.0, atol=0.0) 

388 self.assertEqual(stack.mask[1, 1, afwImage.LOCAL], clipped) 

389 

390 # Excuse that mask; the CLIPPED bit should not be set 

391 stack = afwMath.statisticsStack(images, afwMath.MEAN, statsCtrl, clipped=clipped, excuse=maskVal) 

392 self.assertFloatsAlmostEqual(stack.getImage().getArray(), 0.0, atol=0.0) 

393 self.assertEqual(stack.mask[1, 1, afwImage.LOCAL], 0) 

394 

395 # Map that mask value to a different one. 

396 rejected = 1 << afwImage.Mask().addMaskPlane("REJECTED") 

397 maskMap = [(maskVal, rejected)] 

398 images[0].mask[1, 1, afwImage.LOCAL] = 0 # only want to clip, not mask, this one 

399 images[1].mask[1, 2, afwImage.LOCAL] = maskVal # only want to mask, not clip, this one 

400 stack = afwMath.statisticsStack(images, afwMath.MEANCLIP, statsCtrl, wvector=[], clipped=clipped, 

401 maskMap=maskMap) 

402 self.assertFloatsAlmostEqual(stack.getImage().getArray(), 0.0, atol=0.0) 

403 self.assertEqual(stack.mask[1, 1, afwImage.LOCAL], clipped) 

404 self.assertEqual(stack.mask[1, 2, afwImage.LOCAL], rejected) 

405 

406################################################################# 

407# Test suite boiler plate 

408################################################################# 

409 

410 

411class TestMemory(lsst.utils.tests.MemoryTestCase): 

412 pass 

413 

414 

415def setup_module(module): 

416 lsst.utils.tests.init() 

417 

418 

419if __name__ == "__main__": 419 ↛ 420line 419 didn't jump to line 420, because the condition on line 419 was never true

420 lsst.utils.tests.init() 

421 unittest.main()