Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of afw. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22""" 

23Tests for MaskedImages 

24 

25Run with: 

26 python test_maskedImageIO.py 

27or 

28 pytest test_maskedImageIO.py 

29""" 

30 

31import contextlib 

32import itertools 

33import os.path 

34import unittest 

35import shutil 

36import tempfile 

37 

38import numpy as np 

39import astropy.io.fits 

40 

41import lsst.utils 

42import lsst.utils.tests 

43import lsst.daf.base as dafBase 

44import lsst.geom 

45import lsst.afw.image as afwImage 

46import lsst.afw.math as afwMath 

47import lsst.afw.display as afwDisplay 

48import lsst.pex.exceptions as pexEx 

49 

50try: 

51 dataDir = lsst.utils.getPackageDir("afwdata") 

52except pexEx.NotFoundError: 

53 dataDir = None 

54 

55try: 

56 type(display) 

57 afwDisplay.setDefaultMaskTransparency(75) 

58except NameError: 

59 display = False 

60 

61 

62class MaskedImageTestCase(lsst.utils.tests.TestCase): 

63 """A test case for MaskedImage""" 

64 

65 def setUp(self): 

66 # Set a (non-standard) initial Mask plane definition 

67 # 

68 # Ideally we'd use the standard dictionary and a non-standard file, but 

69 # a standard file's what we have 

70 # 

71 self.Mask = afwImage.Mask 

72 mask = self.Mask() 

73 

74 # Store the default mask planes for later use 

75 maskPlaneDict = self.Mask().getMaskPlaneDict() 

76 self.defaultMaskPlanes = sorted( 

77 maskPlaneDict, key=maskPlaneDict.__getitem__) 

78 

79 # reset so tests will be deterministic 

80 self.Mask.clearMaskPlaneDict() 

81 for p in ("ZERO", "BAD", "SAT", "INTRP", "CR", "EDGE"): 

82 mask.addMaskPlane(p) 

83 

84 if dataDir is not None: 

85 if False: 

86 self.fileName = os.path.join(dataDir, "Small_MI.fits") 

87 else: 

88 self.fileName = os.path.join( 

89 dataDir, "CFHT", "D4", "cal-53535-i-797722_1.fits") 

90 self.mi = afwImage.MaskedImageF(self.fileName) 

91 

92 def tearDown(self): 

93 if dataDir is not None: 

94 del self.mi 

95 # Reset the mask plane to the default 

96 self.Mask.clearMaskPlaneDict() 

97 for p in self.defaultMaskPlanes: 

98 self.Mask.addMaskPlane(p) 

99 

100 @unittest.skipIf(dataDir is None, "afwdata not setup") 

101 def testFitsRead(self): 

102 """Check if we read MaskedImages""" 

103 

104 image = self.mi.getImage() 

105 mask = self.mi.getMask() 

106 

107 if display: 

108 afwDisplay.Display(frame=0).mtv(self.mi, title="Image") 

109 

110 self.assertEqual(image[32, 1, afwImage.LOCAL], 3728) 

111 self.assertEqual(mask[0, 0, afwImage.LOCAL], 2) # == BAD 

112 

113 @unittest.skipIf(dataDir is None, "afwdata not setup") 

114 def testFitsReadImage(self): 

115 """Check if we can read a single-HDU image as a MaskedImage, setting the mask and variance 

116 planes to zero.""" 

117 filename = os.path.join(dataDir, "data", "small_img.fits") 

118 image = afwImage.ImageF(filename) 

119 maskedImage = afwImage.MaskedImageF(filename) 

120 exposure = afwImage.ExposureF(filename) 

121 self.assertEqual(image[0, 0, afwImage.LOCAL], maskedImage.image[0, 0, afwImage.LOCAL]) 

122 self.assertEqual( 

123 image[0, 0, afwImage.LOCAL], exposure.getMaskedImage().image[0, 0, afwImage.LOCAL]) 

124 self.assertTrue(np.all(maskedImage.getMask().getArray() == 0)) 

125 self.assertTrue( 

126 np.all(exposure.getMaskedImage().getMask().getArray() == 0)) 

127 self.assertTrue(np.all(maskedImage.getVariance().getArray() == 0.0)) 

128 self.assertTrue( 

129 np.all(exposure.getMaskedImage().getVariance().getArray() == 0.0)) 

130 

131 @unittest.skipIf(dataDir is None, "afwdata not setup") 

132 def testFitsReadConform(self): 

133 """Check if we read MaskedImages and make them replace Mask's plane dictionary""" 

134 

135 metadata, bbox, conformMasks = None, lsst.geom.Box2I(), True 

136 self.mi = afwImage.MaskedImageF( 

137 self.fileName, metadata, bbox, afwImage.LOCAL, conformMasks) 

138 

139 image = self.mi.getImage() 

140 mask = self.mi.getMask() 

141 

142 self.assertEqual(image[32, 1, afwImage.LOCAL], 3728) 

143 # i.e. not shifted 1 place to the right 

144 self.assertEqual(mask[0, 0, afwImage.LOCAL], 1) 

145 

146 self.assertEqual(mask.getMaskPlane("CR"), 3, 

147 "Plane CR has value specified in FITS file") 

148 

149 @unittest.skipIf(dataDir is None, "afwdata not setup") 

150 def testFitsReadNoConform2(self): 

151 """Check that reading a mask doesn't invalidate the plane dictionary""" 

152 

153 testMask = afwImage.Mask(self.fileName, hdu=2) 

154 

155 mask = self.mi.getMask() 

156 mask |= testMask 

157 

158 @unittest.skipIf(dataDir is None, "afwdata not setup") 

159 def testFitsReadConform2(self): 

160 """Check that conforming a mask invalidates the plane dictionary""" 

161 

162 hdu, metadata, bbox, conformMasks = 2, None, lsst.geom.Box2I(), True 

163 testMask = afwImage.Mask(self.fileName, 

164 hdu, metadata, bbox, afwImage.LOCAL, conformMasks) 

165 

166 mask = self.mi.getMask() 

167 

168 def tst(mask=mask): 

169 mask |= testMask 

170 

171 self.assertRaises(pexEx.RuntimeError, tst) 

172 

173 def testTicket617(self): 

174 """Test reading an F64 image and converting it to a MaskedImage""" 

175 im = afwImage.ImageD(lsst.geom.Extent2I(100, 100)) 

176 im.set(666) 

177 afwImage.MaskedImageD(im) 

178 

179 def testReadWriteXY0(self): 

180 """Test that we read and write (X0, Y0) correctly""" 

181 im = afwImage.MaskedImageF(lsst.geom.Extent2I(10, 20)) 

182 

183 x0, y0 = 1, 2 

184 im.setXY0(x0, y0) 

185 with lsst.utils.tests.getTempFilePath(".fits") as tmpFile: 

186 im.writeFits(tmpFile) 

187 

188 im2 = im.Factory(tmpFile) 

189 self.assertEqual(im2.getX0(), x0) 

190 self.assertEqual(im2.getY0(), y0) 

191 

192 self.assertEqual(im2.getImage().getX0(), x0) 

193 self.assertEqual(im2.getImage().getY0(), y0) 

194 

195 self.assertEqual(im2.getMask().getX0(), x0) 

196 self.assertEqual(im2.getMask().getY0(), y0) 

197 

198 self.assertEqual(im2.getVariance().getX0(), x0) 

199 self.assertEqual(im2.getVariance().getY0(), y0) 

200 

201 @unittest.skipIf(dataDir is None, "afwdata not setup") 

202 def testReadFitsWithOptions(self): 

203 xy0Offset = lsst.geom.Extent2I(7, 5) 

204 bbox = lsst.geom.Box2I(lsst.geom.Point2I(10, 11), lsst.geom.Extent2I(31, 22)) 

205 

206 with lsst.utils.tests.getTempFilePath(".fits") as filepath: 

207 # write a temporary version of the image with non-zero XY0 

208 imagePath = os.path.join(dataDir, "data", "med.fits") 

209 maskedImage = afwImage.MaskedImageD(imagePath) 

210 maskedImage.setXY0(lsst.geom.Point2I(xy0Offset)) 

211 maskedImage.writeFits(filepath) 

212 

213 for ImageClass, imageOrigin in itertools.product( 

214 (afwImage.MaskedImageF, afwImage.MaskedImageD), 

215 (None, "LOCAL", "PARENT"), 

216 ): 

217 with self.subTest(ImageClass=ImageClass, imageOrigin=imageOrigin): 

218 fullImage = ImageClass(filepath) 

219 options = dafBase.PropertySet() 

220 options.set("llcX", bbox.getMinX()) 

221 options.set("llcY", bbox.getMinY()) 

222 options.set("width", bbox.getWidth()) 

223 options.set("height", bbox.getHeight()) 

224 if imageOrigin is not None: 

225 options.set("imageOrigin", imageOrigin) 

226 image1 = ImageClass.readFitsWithOptions(filepath, options) 

227 readBBoxParent = lsst.geom.Box2I(bbox) 

228 if imageOrigin == "LOCAL": 

229 readBBoxParent.shift(xy0Offset) 

230 self.assertMaskedImagesEqual(image1, ImageClass(fullImage, readBBoxParent)) 

231 

232 for name in ("llcY", "width", "height"): 

233 badOptions = options.deepCopy() 

234 badOptions.remove(name) 

235 with self.assertRaises(pexEx.NotFoundError): 

236 ImageClass.readFitsWithOptions(filepath, badOptions) 

237 

238 badOptions = options.deepCopy() 

239 badOptions.set("imageOrigin", "INVALID") 

240 with self.assertRaises(RuntimeError): 

241 ImageClass.readFitsWithOptions(filepath, badOptions) 

242 

243 

244@contextlib.contextmanager 

245def tmpFits(*hdus): 

246 # Given a list of numpy arrays, create a temporary FITS file that 

247 # contains them as consecutive HDUs. Yield it, then remove it. 

248 hdus = [astropy.io.fits.PrimaryHDU(hdus[0])] + \ 

249 [astropy.io.fits.ImageHDU(hdu) for hdu in hdus[1:]] 

250 hdulist = astropy.io.fits.HDUList(hdus) 

251 tempdir = tempfile.mkdtemp() 

252 try: 

253 filename = os.path.join(tempdir, 'test.fits') 

254 hdulist.writeto(filename) 

255 yield filename 

256 finally: 

257 shutil.rmtree(tempdir) 

258 

259 

260class MultiExtensionTestCase: 

261 """Base class for testing that we correctly read multi-extension FITS files. 

262 

263 MEF files may be read to either MaskedImage or Exposure objects. We apply 

264 the same set of tests to each by subclassing and defining _constructImage 

265 and _checkImage. 

266 """ 

267 # When persisting a MaskedImage (or derivative, e.g. Exposure) to FITS, we impose a data 

268 # model which the combination of the limits of the FITS structure and the desire to maintain 

269 # backwards compatibility make it hard to express. We attempt to make this as safe as 

270 # possible by handling the following situations and logging appropriate warnings: 

271 # 

272 # Note that Exposures always set needAllHdus to False. 

273 # 

274 # 1. If needAllHdus is true: 

275 # 1.1 If the user has specified a non-default HDU, we throw. 

276 # 1.2 If the user has not specified an HDU (or has specified one equal to the default): 

277 # 1.2.1 If any of the image, mask or variance is unreadable (eg because they don't 

278 # exist, or they have the wrong data type), we throw. 

279 # 1.2.2 Otherwise, we return the MaskedImage with image/mask/variance set as 

280 # expected. 

281 # 2. If needAllHdus is false: 

282 # 2.1 If the user has specified a non-default HDU: 

283 # 2.1.1 If the user specified HDU is unreadable, we throw. 

284 # 2.1.2 Otherwise, we return the contents of that HDU as the image and default 

285 # (=empty) mask & variance. 

286 # 2.2 If the user has not specified an HDU, or has specified one equal to the default: 

287 # 2.2.1 If the default HDU is unreadable, we throw. 

288 # 2.2.2 Otherwise, we attempt to read both mask and variance from the FITS file, 

289 # and return them together with the image. If one or both are unreadable, 

290 # we fall back to an empty default for the missing data and return the 

291 # remainder.. 

292 # 

293 # See also the discussion at DM-2599. 

294 

295 def _checkMaskedImage(self, mim, width, height, val1, val2, val3): 

296 # Check that the input image has dimensions width & height and that the image, mask and 

297 # variance have mean val1, val2 & val3 respectively. 

298 self.assertEqual(mim.getWidth(), width) 

299 self.assertEqual(mim.getHeight(), width) 

300 self.assertEqual( 

301 afwMath.makeStatistics(mim.getImage(), afwMath.MEAN).getValue(), 

302 val1) 

303 s = afwMath.makeStatistics(mim.getMask(), afwMath.SUM | afwMath.NPOINT) 

304 self.assertEqual(float(s.getValue(afwMath.SUM)) / s.getValue(afwMath.NPOINT), 

305 val2) 

306 self.assertEqual( 

307 afwMath.makeStatistics(mim.getVariance(), afwMath.MEAN).getValue(), 

308 val3) 

309 

310 def testUnreadableExtensionAsImage(self): 

311 # Test for case 2.1.1 above. 

312 with tmpFits(None, np.array([[1]]), np.array([[2]], dtype=np.int16), None) as fitsfile: 

313 self.assertRaises(Exception, self._constructImage, fitsfile, 3) 

314 

315 def testReadableExtensionAsImage(self): 

316 # Test for case 2.1.2 above. 

317 with tmpFits(None, np.array([[1]]), np.array([[2]], dtype=np.int16), 

318 np.array([[3]])) as fitsfile: 

319 self._checkImage(self._constructImage(fitsfile, 3), 1, 1, 3, 0, 0) 

320 

321 def testUnreadbleDefaultAsImage(self): 

322 # Test for case 2.2.1 above. 

323 with tmpFits(None, None, np.array([[2]], dtype=np.int16), np.array([[3]])) as fitsfile: 

324 self.assertRaises(Exception, self._constructImage, fitsfile) 

325 

326 def testUnreadbleOptionalExtensions(self): 

327 # Test for case 2.2.2 above. 

328 # Unreadable mask. 

329 with tmpFits(None, np.array([[1]]), None, np.array([[3]])) as fitsfile: 

330 self._checkImage(self._constructImage(fitsfile), 1, 1, 1, 0, 3) 

331 # Unreadable variance. 

332 with tmpFits(None, np.array([[1]]), np.array([[2]], dtype=np.int16), None) as fitsfile: 

333 self._checkImage(self._constructImage(fitsfile, needAllHdus=False), 

334 1, 1, 1, 2, 0) 

335 

336 

337class MaskedMultiExtensionTestCase(MultiExtensionTestCase, lsst.utils.tests.TestCase): 

338 """Derived version of MultiExtensionTestCase for MaskedImages.""" 

339 

340 def _constructImage(self, filename, hdu=None, needAllHdus=False): 

341 # Construct an instance of MaskedImageF by loading from filename. If hdu 

342 # is specified, load that HDU specifically. Pass through needAllHdus 

343 # to the MaskedImageF constructor. This function exists only to stub 

344 # default arguments into the constructor for parameters which we are 

345 # not exercising in this test. 

346 if hdu: 

347 filename = f"{filename}[{hdu}]" 

348 return afwImage.MaskedImageF(filename, None, lsst.geom.Box2I(), afwImage.PARENT, False, needAllHdus) 

349 

350 def _checkImage(self, *args, **kwargs): 

351 self._checkMaskedImage(*args, **kwargs) 

352 

353 def testNeedAllHdus(self): 

354 # Tests for cases 1.1 & 1.2.2 above. 

355 # We'll regard it as ok for the user to specify any of: 

356 # * No HDU; 

357 # * The "zeroeth" (primary) HDU; 

358 # * The first (first extension) HDU. 

359 # Any others should raise when needAllHdus is true 

360 with tmpFits(None, np.array([[1]]), np.array([[2]], dtype=np.int16), 

361 np.array([[3]])) as fitsfile: 

362 # No HDU specified -> ok. 

363 self._checkImage(self._constructImage(fitsfile, needAllHdus=True), 

364 1, 1, 1, 2, 3) 

365 # First HDU -> ok. 

366 self._checkImage( 

367 self._constructImage(fitsfile, 0, needAllHdus=True), 

368 1, 1, 1, 2, 3) 

369 # First HDU -> ok. 

370 self._checkImage( 

371 self._constructImage(fitsfile, 1, needAllHdus=True), 

372 1, 1, 1, 2, 3) 

373 # Second HDU -> raises. 

374 self.assertRaises(Exception, self._constructImage, 

375 fitsfile, 2, needAllHdus=True) 

376 

377 def testUnreadableImage(self): 

378 # Test for case 1.2.1 above. 

379 with tmpFits(None, None, np.array([[2]], dtype=np.int16), np.array([[3]])) as fitsfile: 

380 self.assertRaises(Exception, self._constructImage, 

381 fitsfile, None, needAllHdus=True) 

382 

383 def testUnreadableMask(self): 

384 # Test for case 1.2.1 above. 

385 with tmpFits(None, np.array([[1]]), None, np.array([[3]])) as fitsfile: 

386 self.assertRaises(Exception, self._constructImage, 

387 fitsfile, None, needAllHdus=True) 

388 

389 def testUnreadableVariance(self): 

390 # Test for case 1.2.1 above. 

391 with tmpFits(None, np.array([[1]]), np.array([[2]], dtype=np.int16), None) as fitsfile: 

392 self.assertRaises(Exception, self._constructImage, 

393 fitsfile, None, needAllHdus=True) 

394 

395 

396class ExposureMultiExtensionTestCase(MultiExtensionTestCase, lsst.utils.tests.TestCase): 

397 """Derived version of MultiExtensionTestCase for Exposures.""" 

398 

399 def _constructImage(self, filename, hdu=None, needAllHdus=False): 

400 # Construct an instance of ExposureF by loading from filename. If hdu 

401 # is specified, load that HDU specifically. needAllHdus exists for API 

402 # compatibility, but should always be False. This function exists only 

403 # to stub default arguments into the constructor for parameters which 

404 # we are not exercising in this test. 

405 if hdu: 

406 filename = f"{filename}[{hdu}]" 

407 if needAllHdus: 

408 raise ValueError("Cannot needAllHdus with Exposure") 

409 return afwImage.ExposureF(filename) 

410 

411 def _checkImage(self, im, width, height, val1, val2, val3): 

412 self._checkMaskedImage(im.getMaskedImage(), width, 

413 height, val1, val2, val3) 

414 

415 

416class TestMemory(lsst.utils.tests.MemoryTestCase): 

417 pass 

418 

419 

420def setup_module(module): 

421 lsst.utils.tests.init() 

422 

423 

424if __name__ == "__main__": 424 ↛ 425line 424 didn't jump to line 425, because the condition on line 424 was never true

425 lsst.utils.tests.init() 

426 unittest.main()