Coverage for tests/test_fitsCompression.py: 14%

366 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-03-30 02:46 -0700

1# 

2# LSST Data Management System 

3# Copyright 2017 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23import os 

24import unittest 

25import itertools 

26 

27import numpy as np 

28import astropy.io.fits 

29 

30import lsst.utils 

31import lsst.daf.base 

32import lsst.geom 

33import lsst.afw.geom 

34import lsst.afw.image 

35import lsst.afw.fits 

36import lsst.utils.tests 

37from lsst.afw.image import LOCAL 

38from lsst.afw.fits import ImageScalingOptions, ImageCompressionOptions 

39 

40 

41def checkAstropy(image, filename, hduNum=0): 

42 """Check that astropy can read our file 

43 

44 We don't insist on equality for low BITPIX (8, 16) when the original 

45 type is double-precision: astropy will (quite reasonably) read that 

46 into a single-precision image and apply bscale,bzero to that so it's 

47 going to be subject to roundoff. 

48 

49 Parameters 

50 ---------- 

51 image : `lsst.afw.image.Image` 

52 Image read by our own code. 

53 filename : `str` 

54 Filename of FITS file to read with astropy. 

55 hduNum : `int` 

56 HDU number of interest. 

57 """ 

58 print("Astropy currently doesn't read our compressed images perfectly.") 

59 return 

60 

61 def parseVersion(version): 

62 return tuple(int(vv) for vv in np.array(version.split("."))) 

63 

64 if parseVersion(astropy.__version__) <= parseVersion("2.0.1"): 

65 # astropy 2.0.1 and earlier have problems: 

66 # * Doesn't support GZIP_2: https://github.com/astropy/astropy/pull/6486 

67 # * Uses the wrong array type: https://github.com/astropy/astropy/pull/6492 

68 print(f"Refusing to check with astropy version {astropy.__version__} due to astropy bugs") 

69 return 

70 hdu = astropy.io.fits.open(filename)[hduNum] 

71 if hdu.header["BITPIX"] in (8, 16) and isinstance(image, lsst.afw.image.ImageD): 

72 return 

73 dtype = image.getArray().dtype 

74 theirs = hdu.data.astype(dtype) 

75 # Allow for minor differences due to arithmetic: +/- 1 in the last place 

76 np.testing.assert_array_max_ulp(theirs, image.getArray()) 

77 

78 

79class ImageScalingTestCase(lsst.utils.tests.TestCase): 

80 """Tests of image scaling 

81 

82 The pattern here is to create an image, write it out with a 

83 specific scaling algorithm, read it back in and test that everything 

84 is as we expect. We do this for each scaling algorithm in its own 

85 test, and within that test iterate over various parameters (input 

86 image type, BITPIX, etc.). The image we create has a few features 

87 (low, high and masked pixels) that we check. 

88 """ 

89 def setUp(self): 

90 self.bbox = lsst.geom.Box2I(lsst.geom.Point2I(123, 456), lsst.geom.Extent2I(7, 8)) 

91 self.base = 456 # Base value for pixels 

92 self.highValue = 789 # Value for high pixel 

93 self.lowValue = 123 # Value for low pixel 

94 self.maskedValue = 12345 # Value for masked pixel (to throw off statistics) 

95 self.highPixel = lsst.geom.Point2I(1, 1) # Location of high pixel 

96 self.lowPixel = lsst.geom.Point2I(2, 2) # Location of low pixel 

97 self.maskedPixel = lsst.geom.Point2I(3, 3) # Location of masked pixel 

98 self.badMask = "BAD" # Mask plane to set for masked pixel 

99 self.stdev = 5.0 # Noise stdev to add to image 

100 

101 def makeImage(self, ImageClass, scaling, addNoise=True): 

102 """Make an image for testing 

103 

104 We create an image, persist and unpersist it, returning 

105 some data to the caller. 

106 

107 Parameters 

108 ---------- 

109 ImageClass : `type`, an `lsst.afw.image.Image` class 

110 Class of image to create. 

111 scaling : `lsst.afw.fits.ImageScalingOptions` 

112 Scaling to apply during persistence. 

113 addNoise : `bool` 

114 Add noise to image? 

115 

116 Returns 

117 ------- 

118 image : `lsst.afw.image.Image` (ImageClass) 

119 Created image. 

120 unpersisted : `lsst.afw.image.Image` (ImageClass) 

121 Unpersisted image. 

122 bscale, bzero : `float` 

123 FITS scale factor and zero used. 

124 minValue, maxValue : `float` 

125 Minimum and maximum value given the nominated scaling. 

126 """ 

127 image = ImageClass(self.bbox) 

128 mask = lsst.afw.image.Mask(self.bbox) 

129 mask.addMaskPlane(self.badMask) 

130 bad = mask.getPlaneBitMask(self.badMask) 

131 image.set(self.base) 

132 image[self.highPixel, LOCAL] = self.highValue 

133 image[self.lowPixel, LOCAL] = self.lowValue 

134 image[self.maskedPixel, LOCAL] = self.maskedValue 

135 mask[self.maskedPixel, LOCAL] = bad 

136 

137 rng = np.random.RandomState(12345) 

138 dtype = image.getArray().dtype 

139 if addNoise: 

140 image.getArray()[:] += rng.normal(0.0, self.stdev, image.getArray().shape).astype(dtype) 

141 

142 with lsst.utils.tests.getTempFilePath(".fits") as filename: 

143 with lsst.afw.fits.Fits(filename, "w") as fits: 

144 options = lsst.afw.fits.ImageWriteOptions(scaling) 

145 header = lsst.daf.base.PropertyList() 

146 image.writeFits(fits, options, header, mask) 

147 unpersisted = ImageClass(filename) 

148 self.assertEqual(image.getBBox(), unpersisted.getBBox()) 

149 

150 header = lsst.afw.fits.readMetadata(filename) 

151 bscale = header.getScalar("BSCALE") 

152 bzero = header.getScalar("BZERO") 

153 

154 if scaling.algorithm != ImageScalingOptions.NONE: 

155 self.assertEqual(header.getScalar("BITPIX"), scaling.bitpix) 

156 

157 if scaling.bitpix == 8: # unsigned, says FITS 

158 maxValue = bscale*(2**scaling.bitpix - 1) + bzero 

159 minValue = bzero 

160 else: 

161 maxValue = bscale*(2**(scaling.bitpix - 1) - 1) + bzero 

162 if scaling.bitpix == 32: 

163 # cfitsio pads 10 values, and so do we 

164 minValue = -bscale*(2**(scaling.bitpix - 1) - 10) + bzero 

165 else: 

166 minValue = -bscale*(2**(scaling.bitpix - 1)) + bzero 

167 

168 # Convert scalars to the appropriate type 

169 maxValue = np.array(maxValue, dtype=image.getArray().dtype) 

170 minValue = np.array(minValue, dtype=image.getArray().dtype) 

171 

172 checkAstropy(unpersisted, filename) 

173 

174 return image, unpersisted, bscale, bzero, minValue, maxValue 

175 

176 def checkPixel(self, unpersisted, original, xy, expected, rtol=None, atol=None): 

177 """Check one of the special pixels 

178 

179 After checking, we set this pixel to the original value so 

180 it's then easy to compare the entire image. 

181 

182 Parameters 

183 ---------- 

184 unpersisted : `lsst.afw.image.Image` 

185 Unpersisted image. 

186 original : `lsst.afw.image.Image` 

187 Original image. 

188 xy : `tuple` of two `int`s 

189 Position of pixel to check. 

190 expected : scalar 

191 Expected value of pixel. 

192 rtol, atol : `float` or `None` 

193 Relative/absolute tolerance for comparison. 

194 """ 

195 if np.isnan(expected): 

196 self.assertTrue(np.isnan(unpersisted[xy, LOCAL])) 

197 else: 

198 self.assertFloatsAlmostEqual(unpersisted[xy, LOCAL], expected, rtol=rtol, atol=atol) 

199 unpersisted[xy, LOCAL] = original[xy, LOCAL] # for ease of comparison of the whole image 

200 

201 def checkSpecialPixels(self, original, unpersisted, maxValue, minValue, rtol=None, atol=None): 

202 """Check the special pixels 

203 

204 Parameters 

205 ---------- 

206 original : `lsst.afw.image.Image` 

207 Original image. 

208 unpersisted : `lsst.afw.image.Image` 

209 Unpersisted image. 

210 minValue, maxValue : `float` 

211 Minimum and maximum value given the nominated scaling. 

212 rtol, atol : `float` or `None` 

213 Relative/absolute tolerance for comparison. 

214 """ 

215 highValue = original[self.highPixel, LOCAL] 

216 lowValue = original[self.lowPixel, LOCAL] 

217 maskedValue = original[self.maskedPixel, LOCAL] 

218 

219 expectHigh = min(highValue, maxValue) 

220 expectLow = max(lowValue, minValue) 

221 expectMasked = min(maskedValue, maxValue) 

222 

223 if unpersisted.getArray().dtype in (np.float32, np.float64): 

224 if highValue >= maxValue: 

225 expectHigh = np.nan 

226 if maskedValue >= maxValue: 

227 expectMasked = np.nan 

228 

229 self.checkPixel(unpersisted, original, self.highPixel, expectHigh, rtol=rtol, atol=atol) 

230 self.checkPixel(unpersisted, original, self.lowPixel, expectLow, rtol=rtol, atol=atol) 

231 self.checkPixel(unpersisted, original, self.maskedPixel, expectMasked, rtol=rtol, atol=atol) 

232 

233 def checkRange(self, ImageClass, bitpix): 

234 """Check that the RANGE scaling works 

235 

236 Parameters 

237 ---------- 

238 ImageClass : `type`, an `lsst.afw.image.Image` class 

239 Class of image to create. 

240 bitpix : `int` 

241 Bits per pixel for FITS image. 

242 """ 

243 scaling = ImageScalingOptions(ImageScalingOptions.RANGE, bitpix, [u"BAD"], fuzz=False) 

244 original, unpersisted, bscale, bzero, minValue, maxValue = self.makeImage(ImageClass, scaling, False) 

245 

246 numValues = 2**bitpix - 1 

247 numValues -= 2 # Padding on either end 

248 if bitpix == 32: 

249 numValues -= 10 

250 bscaleExpect = (self.highValue - self.lowValue)/numValues 

251 self.assertFloatsAlmostEqual(bscale, bscaleExpect, atol=1.0e-6) # F32 resolution 

252 

253 rtol = 1.0/2**(bitpix - 1) 

254 self.checkSpecialPixels(original, unpersisted, maxValue, minValue, atol=bscale) 

255 self.assertImagesAlmostEqual(original, unpersisted, rtol=rtol) 

256 

257 def checkStdev(self, ImageClass, bitpix, algorithm, quantizeLevel, quantizePad): 

258 """Check that one of the STDEV scaling algorithms work 

259 

260 Parameters 

261 ---------- 

262 ImageClass : `type`, an `lsst.afw.image.Image` class 

263 Class of image to create. 

264 bitpix : `int` 

265 Bits per pixel for FITS image. 

266 algorithm : `lsst.afw.fits.ImageScalingOptions.ScalingAlgorithm` 

267 Scaling algorithm to apply (one of the STDEV_*). 

268 quantizeLevel : `float` 

269 Quantization level. 

270 quantizePad : `float` 

271 Quantization padding. 

272 """ 

273 scaling = lsst.afw.fits.ImageScalingOptions(algorithm, bitpix, [u"BAD"], fuzz=False, 

274 quantizeLevel=quantizeLevel, quantizePad=quantizePad) 

275 

276 makeImageResults = self.makeImage(ImageClass, scaling) 

277 original, unpersisted, bscale, bzero, minValue, maxValue = makeImageResults 

278 

279 self.assertFloatsAlmostEqual(bscale, self.stdev/quantizeLevel, rtol=3.0/quantizeLevel) 

280 self.checkSpecialPixels(original, unpersisted, maxValue, minValue, atol=bscale) 

281 self.assertImagesAlmostEqual(original, unpersisted, atol=bscale) 

282 

283 def testRange(self): 

284 """Test that the RANGE scaling works on floating-point inputs 

285 

286 We deliberately don't include BITPIX=64 because int64 provides 

287 a larger dynamic range than 'double BSCALE' can handle. 

288 """ 

289 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD) 

290 bitpixList = (8, 16, 32) 

291 for cls, bitpix in itertools.product(classList, bitpixList): 

292 self.checkRange(cls, bitpix) 

293 

294 def testStdev(self): 

295 """Test that the STDEV scalings work on floating-point inputs 

296 

297 We deliberately don't include BITPIX=64 because int64 provides 

298 a larger dynamic range than 'double BSCALE' can handle. 

299 

300 We deliberately don't include BITPIX=8 because that provides 

301 only a tiny dynamic range where everything goes out of range easily. 

302 """ 

303 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD) 

304 bitpixList = (16, 32) 

305 algorithmList = (ImageScalingOptions.STDEV_POSITIVE, ImageScalingOptions.STDEV_NEGATIVE, 

306 ImageScalingOptions.STDEV_BOTH) 

307 quantizeLevelList = (2.0, 10.0, 100.0) 

308 quantizePadList = (5.0, 10.0, 100.0) 

309 for values in itertools.product(classList, bitpixList, algorithmList, 

310 quantizeLevelList, quantizePadList): 

311 self.checkStdev(*values) 

312 

313 def testRangeFailures(self): 

314 """Test that the RANGE scaling fails on integer inputs""" 

315 classList = (lsst.afw.image.ImageU, lsst.afw.image.ImageI, lsst.afw.image.ImageL) 

316 bitpixList = (8, 16, 32) 

317 for cls, bitpix in itertools.product(classList, bitpixList): 

318 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError): 

319 self.checkRange(cls, bitpix) 

320 

321 def testStdevFailures(self): 

322 """Test that the STDEV scalings fail on integer inputs""" 

323 classList = (lsst.afw.image.ImageU, lsst.afw.image.ImageI, lsst.afw.image.ImageL) 

324 bitpixList = (16, 32) 

325 algorithmList = (ImageScalingOptions.STDEV_POSITIVE, ImageScalingOptions.STDEV_NEGATIVE, 

326 ImageScalingOptions.STDEV_BOTH) 

327 for cls, bitpix, algorithm in itertools.product(classList, bitpixList, algorithmList): 

328 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError): 

329 self.checkStdev(cls, bitpix, algorithm, 10.0, 10.0) 

330 

331 def checkNone(self, ImageClass, bitpix): 

332 """Check that the NONE scaling algorithm works 

333 

334 Parameters 

335 ---------- 

336 ImageClass : `type`, an `lsst.afw.image.Image` class 

337 Class of image to create. 

338 bitpix : `int` 

339 Bits per pixel for FITS image. 

340 """ 

341 scaling = ImageScalingOptions(ImageScalingOptions.NONE, bitpix, [u"BAD"], fuzz=False) 

342 original, unpersisted, bscale, bzero, minValue, maxValue = self.makeImage(ImageClass, scaling) 

343 self.assertFloatsAlmostEqual(bscale, 1.0, atol=0.0) 

344 self.assertFloatsAlmostEqual(bzero, 0.0, atol=0.0) 

345 self.assertImagesAlmostEqual(original, unpersisted, atol=0.0) 

346 

347 def testNone(self): 

348 """Test that the NONE scaling works on floating-point inputs""" 

349 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD) 

350 bitpixList = (8, 16, 32) 

351 for cls, bitpix in itertools.product(classList, bitpixList): 

352 self.checkNone(cls, bitpix) 

353 

354 def checkManual(self, ImageClass, bitpix): 

355 """Check that the MANUAL scaling algorithm works 

356 

357 Parameters 

358 ---------- 

359 ImageClass : `type`, an `lsst.afw.image.Image` class 

360 Class of image to create. 

361 bitpix : `int` 

362 Bits per pixel for FITS image. 

363 """ 

364 bscaleSet = 1.2345 

365 bzeroSet = self.base 

366 scaling = ImageScalingOptions(ImageScalingOptions.MANUAL, bitpix, [u"BAD"], bscale=bscaleSet, 

367 bzero=bzeroSet, fuzz=False) 

368 original, unpersisted, bscale, bzero, minValue, maxValue = self.makeImage(ImageClass, scaling) 

369 self.assertFloatsAlmostEqual(bscale, bscaleSet, atol=0.0) 

370 self.assertFloatsAlmostEqual(bzero, bzeroSet, atol=0.0) 

371 self.assertImagesAlmostEqual(original, unpersisted, atol=bscale) 

372 

373 def testManual(self): 

374 """Test that the MANUAL scaling works on floating-point inputs""" 

375 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD) 

376 bitpixList = (16, 32) 

377 for cls, bitpix in itertools.product(classList, bitpixList): 

378 self.checkNone(cls, bitpix) 

379 

380 

381class ImageCompressionTestCase(lsst.utils.tests.TestCase): 

382 """Tests of image compression 

383 

384 We test compression both with and without loss (quantisation/scaling). 

385 

386 The pattern here is to create an image, write it out with a 

387 specific compression algorithm, read it back in and test that everything 

388 is as we expect. We do this for each compression algorithm in its own 

389 test, and within that test iterate over various parameters (input 

390 image type, BITPIX, etc.). 

391 

392 We print the (inverse) compression ratio for interest. Note that 

393 these should not be considered to be representative of the 

394 compression that will be achieved on scientific data, since the 

395 images created here have different qualities than scientific data 

396 that will affect the compression ratio (e.g., size, noise properties). 

397 """ 

398 def setUp(self): 

399 self.bbox = lsst.geom.Box2I(lsst.geom.Point2I(123, 456), lsst.geom.Extent2I(7, 8)) 

400 self.background = 12345.6789 # Background value 

401 self.noise = 67.89 # Noise (stdev) 

402 self.maskPlanes = ["FOO", "BAR"] # Mask planes to add 

403 self.extension = "." + self.__class__.__name__ + ".fits" # extension name for temp files 

404 

405 def readWriteImage(self, ImageClass, image, filename, options, *args): 

406 """Read the image after it has been written 

407 

408 This implementation does the persistence using methods on the 

409 ImageClass. 

410 

411 Parameters 

412 ---------- 

413 ImageClass : `type`, an `lsst.afw.image.Image` class 

414 Class of image to create. 

415 image : `lsst.afw.image.Image` 

416 Image to compress. 

417 filename : `str` 

418 Filename to which to write. 

419 options : `lsst.afw.fits.ImageWriteOptions` 

420 Options for writing. 

421 """ 

422 image.writeFits(filename, options, *args) 

423 return ImageClass(filename) 

424 

425 def makeImage(self, ImageClass): 

426 """Create an image 

427 

428 Parameters 

429 ---------- 

430 ImageClass : `type`, an `lsst.afw.image.Image` class 

431 Class of image to create. 

432 

433 Returns 

434 ------- 

435 image : `ImageClass` 

436 The created image. 

437 """ 

438 image = ImageClass(self.bbox) 

439 rng = np.random.RandomState(12345) 

440 dtype = image.getArray().dtype 

441 noise = rng.normal(0.0, self.noise, image.getArray().shape).astype(dtype) 

442 image.getArray()[:] = np.array(self.background, dtype=dtype) + noise 

443 return image 

444 

445 def makeMask(self): 

446 """Create a mask 

447 

448 Note that we generate a random distribution of mask pixel values, 

449 which is very different from the usual distribution in science images. 

450 

451 Returns 

452 ------- 

453 mask : `lsst.afw.image.Mask` 

454 The created mask. 

455 """ 

456 mask = lsst.afw.image.Mask(self.bbox) 

457 rng = np.random.RandomState(12345) 

458 dtype = mask.getArray().dtype 

459 mask.getArray()[:] = rng.randint(0, 2**(dtype.itemsize*8 - 1), mask.getArray().shape, dtype=dtype) 

460 for plane in self.maskPlanes: 

461 mask.addMaskPlane(plane) 

462 return mask 

463 

464 def checkCompressedImage(self, ImageClass, image, compression, scaling=None, atol=0.0): 

465 """Check that compression works on an image 

466 

467 Parameters 

468 ---------- 

469 ImageClass : `type`, an `lsst.afw.image.Image` class 

470 Class of image. 

471 image : `lsst.afw.image.Image` 

472 Image to compress. 

473 compression : `lsst.afw.fits.ImageCompressionOptions` 

474 Compression parameters. 

475 scaling : `lsst.afw.fits.ImageScalingOptions` or `None` 

476 Scaling parameters for lossy compression (optional). 

477 atol : `float` 

478 Absolute tolerance for comparing unpersisted image. 

479 

480 Returns 

481 ------- 

482 unpersisted : `ImageClass` 

483 The unpersisted image. 

484 """ 

485 with lsst.utils.tests.getTempFilePath(self.extension) as filename: 

486 if scaling: 

487 options = lsst.afw.fits.ImageWriteOptions(compression, scaling) 

488 else: 

489 options = lsst.afw.fits.ImageWriteOptions(compression) 

490 unpersisted = self.readWriteImage(ImageClass, image, filename, options) 

491 

492 fileSize = os.stat(filename).st_size 

493 fitsBlockSize = 2880 # All sizes in FITS are a multiple of this 

494 numBlocks = 1 + np.ceil(self.bbox.getArea()*image.getArray().dtype.itemsize/fitsBlockSize) 

495 uncompressedSize = fitsBlockSize*numBlocks 

496 print(ImageClass, compression.algorithm, fileSize, uncompressedSize, fileSize/uncompressedSize) 

497 

498 self.assertEqual(image.getBBox(), unpersisted.getBBox()) 

499 self.assertImagesAlmostEqual(unpersisted, image, atol=atol) 

500 

501 checkAstropy(unpersisted, filename, 1) 

502 

503 return unpersisted 

504 

505 def testLosslessFloat(self): 

506 """Test lossless compression of floating-point image""" 

507 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD) 

508 algorithmList = ("GZIP", "GZIP_SHUFFLE") # Lossless float compression requires GZIP 

509 for cls, algorithm in itertools.product(classList, algorithmList): 

510 image = self.makeImage(cls) 

511 compression = ImageCompressionOptions(lsst.afw.fits.compressionAlgorithmFromString(algorithm)) 

512 self.checkCompressedImage(cls, image, compression, atol=0.0) 

513 

514 def testLosslessInt(self): 

515 """Test lossless compression of integer image 

516 

517 We deliberately don't test `lsst.afw.image.ImageL` because 

518 compression of LONGLONG images is unsupported by cfitsio. 

519 """ 

520 classList = (lsst.afw.image.ImageU, lsst.afw.image.ImageI) 

521 algorithmList = ("GZIP", "GZIP_SHUFFLE", "RICE") 

522 for cls, algorithm in itertools.product(classList, algorithmList): 

523 compression = ImageCompressionOptions(lsst.afw.fits.compressionAlgorithmFromString(algorithm)) 

524 image = self.makeImage(cls) 

525 self.checkCompressedImage(cls, image, compression, atol=0.0) 

526 

527 def testLongLong(self): 

528 """Test graceful failure when compressing ImageL 

529 

530 We deliberately don't test `lsst.afw.image.ImageL` because 

531 compression of LONGLONG images is unsupported by cfitsio. 

532 """ 

533 algorithmList = ("GZIP", "GZIP_SHUFFLE", "RICE") 

534 for algorithm in algorithmList: 

535 compression = ImageCompressionOptions(lsst.afw.fits.compressionAlgorithmFromString(algorithm)) 

536 cls = lsst.afw.image.ImageL 

537 image = self.makeImage(cls) 

538 with self.assertRaises(lsst.afw.fits.FitsError): 

539 self.checkCompressedImage(cls, image, compression) 

540 

541 def testMask(self): 

542 """Test compression of mask 

543 

544 We deliberately don't test PLIO compression (which is designed for 

545 masks) because our default mask type (32) has too much dynamic range 

546 for PLIO (limit of 24 bits). 

547 """ 

548 for algorithm in ("GZIP", "GZIP_SHUFFLE", "RICE"): 

549 compression = ImageCompressionOptions(lsst.afw.fits.compressionAlgorithmFromString(algorithm)) 

550 mask = self.makeMask() 

551 unpersisted = self.checkCompressedImage(lsst.afw.image.Mask, mask, compression, atol=0.0) 

552 for mp in mask.getMaskPlaneDict(): 

553 self.assertIn(mp, unpersisted.getMaskPlaneDict()) 

554 unpersisted.getPlaneBitMask(mp) 

555 

556 def testLossyFloatCfitsio(self): 

557 """Test lossy compresion of floating-point images with cfitsio 

558 

559 cfitsio does the compression, controlled through the 'quantizeLevel' 

560 parameter. Note that cfitsio doesn't have access to our masks when 

561 it does its statistics. 

562 """ 

563 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD) 

564 algorithmList = ("GZIP", "GZIP_SHUFFLE", "RICE") 

565 quantizeList = (4.0, 10.0) 

566 for cls, algorithm, quantizeLevel in itertools.product(classList, algorithmList, quantizeList): 

567 compression = ImageCompressionOptions(lsst.afw.fits.compressionAlgorithmFromString(algorithm), 

568 quantizeLevel=quantizeLevel) 

569 image = self.makeImage(cls) 

570 self.checkCompressedImage(cls, image, compression, atol=self.noise/quantizeLevel) 

571 

572 def testLossyFloatOurs(self): 

573 """Test lossy compression of floating-point images ourselves 

574 

575 We do lossy compression by scaling first. We have full control over 

576 the scaling (multiple scaling algorithms), and we have access to our 

577 own masks when we do statistics. 

578 """ 

579 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD) 

580 algorithmList = ("GZIP", "GZIP_SHUFFLE", "RICE") 

581 bitpixList = (16, 32) 

582 quantizeList = (4.0, 10.0) 

583 for cls, algorithm, bitpix, quantize in itertools.product(classList, algorithmList, bitpixList, 

584 quantizeList): 

585 compression = ImageCompressionOptions(lsst.afw.fits.compressionAlgorithmFromString(algorithm), 

586 quantizeLevel=0.0) 

587 scaling = ImageScalingOptions(ImageScalingOptions.STDEV_BOTH, bitpix, quantizeLevel=quantize, 

588 fuzz=True) 

589 image = self.makeImage(cls) 

590 self.checkCompressedImage(cls, image, compression, scaling, atol=self.noise/quantize) 

591 

592 def readWriteMaskedImage(self, image, filename, imageOptions, maskOptions, varianceOptions): 

593 """Read the MaskedImage after it has been written 

594 

595 This implementation does the persistence using methods on the 

596 MaskedImage class. 

597 

598 Parameters 

599 ---------- 

600 image : `lsst.afw.image.Image` 

601 Image to compress. 

602 filename : `str` 

603 Filename to which to write. 

604 imageOptions, maskOptions, varianceOptions : `lsst.afw.fits.ImageWriteOptions` 

605 Options for writing the image, mask and variance planes. 

606 """ 

607 image.writeFits(filename, imageOptions, maskOptions, varianceOptions) 

608 if hasattr(image, "getMaskedImage"): 

609 image = image.getMaskedImage() 

610 return lsst.afw.image.MaskedImageF(filename) 

611 

612 def checkCompressedMaskedImage(self, image, imageOptions, maskOptions, varianceOptions, atol=0.0): 

613 """Check that compression works on a MaskedImage 

614 

615 Parameters 

616 ---------- 

617 image : `lsst.afw.image.MaskedImage` or `lsst.afw.image.Exposure` 

618 MaskedImage or exposure to compress. 

619 imageOptions, maskOptions, varianceOptions : `lsst.afw.fits.ImageWriteOptions` 

620 Parameters for writing (compression and scaling) the image, mask 

621 and variance planes. 

622 atol : `float` 

623 Absolute tolerance for comparing unpersisted image. 

624 """ 

625 with lsst.utils.tests.getTempFilePath(self.extension) as filename: 

626 self.readWriteMaskedImage(image, filename, imageOptions, maskOptions, varianceOptions) 

627 unpersisted = type(image)(filename) 

628 if hasattr(image, "getMaskedImage"): 

629 image = image.getMaskedImage() 

630 unpersisted = unpersisted.getMaskedImage() 

631 self.assertEqual(image.getBBox(), unpersisted.getBBox()) 

632 self.assertImagesAlmostEqual(unpersisted.getImage(), image.getImage(), atol=atol) 

633 self.assertImagesAlmostEqual(unpersisted.getMask(), image.getMask(), atol=atol) 

634 self.assertImagesAlmostEqual(unpersisted.getVariance(), image.getVariance(), atol=atol) 

635 

636 for mp in image.getMask().getMaskPlaneDict(): 

637 self.assertIn(mp, unpersisted.getMask().getMaskPlaneDict()) 

638 unpersisted.getMask().getPlaneBitMask(mp) 

639 

640 def checkMaskedImage(self, imageOptions, maskOptions, varianceOptions, atol=0.0): 

641 """Check that we can compress a MaskedImage and Exposure 

642 

643 Parameters 

644 ---------- 

645 imageOptions, maskOptions, varianceOptions : `lsst.afw.fits.ImageWriteOptions` 

646 Parameters for writing (compression and scaling) the image, mask 

647 and variance planes. 

648 atol : `float` 

649 Absolute tolerance for comparing unpersisted image. 

650 """ 

651 image = lsst.afw.image.makeMaskedImage(self.makeImage(lsst.afw.image.ImageF), 

652 self.makeMask(), self.makeImage(lsst.afw.image.ImageF)) 

653 self.checkCompressedMaskedImage(image, imageOptions, maskOptions, varianceOptions, atol=atol) 

654 exp = lsst.afw.image.makeExposure(image) 

655 self.checkCompressedMaskedImage(exp, imageOptions, maskOptions, varianceOptions, atol=atol) 

656 

657 def testMaskedImage(self): 

658 """Test compression of MaskedImage 

659 

660 We test lossless, lossy cfitsio and lossy LSST compression. 

661 """ 

662 # Lossless 

663 lossless = lsst.afw.fits.ImageCompressionOptions(ImageCompressionOptions.GZIP_SHUFFLE) 

664 options = lsst.afw.fits.ImageWriteOptions(lossless) 

665 self.checkMaskedImage(options, options, options, atol=0.0) 

666 

667 # Lossy cfitsio compression 

668 quantize = 4.0 

669 cfitsio = lsst.afw.fits.ImageCompressionOptions(ImageCompressionOptions.GZIP_SHUFFLE, True, quantize) 

670 imageOptions = lsst.afw.fits.ImageWriteOptions(cfitsio) 

671 maskOptions = lsst.afw.fits.ImageWriteOptions(lossless) 

672 self.checkMaskedImage(imageOptions, maskOptions, imageOptions, atol=self.noise/quantize) 

673 

674 # Lossy our compression 

675 quantize = 10.0 

676 compression = lsst.afw.fits.ImageCompressionOptions(ImageCompressionOptions.RICE, True, 0.0) 

677 scaling = lsst.afw.fits.ImageScalingOptions(ImageScalingOptions.STDEV_BOTH, 32, 

678 quantizeLevel=quantize) 

679 imageOptions = lsst.afw.fits.ImageWriteOptions(compression, scaling) 

680 maskOptions = lsst.afw.fits.ImageWriteOptions(compression) 

681 self.checkMaskedImage(imageOptions, maskOptions, imageOptions, atol=self.noise/quantize) 

682 

683 def testQuantization(self): 

684 """Test that our quantization produces the same values as cfitsio 

685 

686 Our quantization is more configurable (e.g., choice of scaling algorithm, 

687 specifying mask planes) and extensible (logarithmic, asinh scalings) 

688 than cfitsio's. However, cfitsio uses its own fuzz ("subtractive dithering") 

689 when reading the data, so if we don't want to add random values twice, 

690 we need to be sure that we're using the same random values. To check that, 

691 we write one image with our scaling+compression, and one with cfitsio's 

692 compression using exactly the BSCALE and dither seed we used for our own. 

693 That way, the two codes will quantize independently, and we can compare 

694 the results. 

695 """ 

696 bscaleSet = 1.0 

697 bzeroSet = self.background - 10*self.noise 

698 algorithm = ImageCompressionOptions.GZIP 

699 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD) 

700 tilesList = ((4, 5), (0, 0), (0, 5), (4, 0), (0, 1)) 

701 for cls, tiles in itertools.product(classList, tilesList): 

702 tiles = np.array(tiles, dtype=np.int64) 

703 compression = ImageCompressionOptions(algorithm, tiles, -bscaleSet) 

704 original = self.makeImage(cls) 

705 with lsst.utils.tests.getTempFilePath(self.extension) as filename: 

706 with lsst.afw.fits.Fits(filename, "w") as fits: 

707 options = lsst.afw.fits.ImageWriteOptions(compression) 

708 original.writeFits(fits, options) 

709 cfitsio = cls(filename) 

710 header = lsst.afw.fits.readMetadata(filename, 1) 

711 seed = header.getScalar("ZDITHER0") 

712 self.assertEqual(header.getScalar("BSCALE"), bscaleSet) 

713 

714 compression = ImageCompressionOptions(algorithm, tiles, 0.0) 

715 scaling = ImageScalingOptions(ImageScalingOptions.MANUAL, 32, [u"BAD"], bscale=bscaleSet, 

716 bzero=bzeroSet, fuzz=True, seed=seed) 

717 unpersisted = self.checkCompressedImage(cls, original, compression, scaling, atol=bscaleSet) 

718 oursDiff = unpersisted.getArray() - original.getArray() 

719 cfitsioDiff = cfitsio.getArray() - original.getArray() 

720 self.assertImagesAlmostEqual(oursDiff, cfitsioDiff, atol=0.0) 

721 

722 

723def optionsToPropertySet(options): 

724 """Convert the ImageWriteOptions to a PropertySet 

725 

726 This allows us to pass the options into the persistence framework 

727 as the "additionalData". 

728 """ 

729 ps = lsst.daf.base.PropertySet() 

730 ps.set("compression.algorithm", lsst.afw.fits.compressionAlgorithmToString(options.compression.algorithm)) 

731 ps.set("compression.columns", options.compression.tiles[0]) 

732 ps.set("compression.rows", options.compression.tiles[1]) 

733 ps.set("compression.quantizeLevel", options.compression.quantizeLevel) 

734 

735 ps.set("scaling.algorithm", lsst.afw.fits.scalingAlgorithmToString(options.scaling.algorithm)) 

736 ps.set("scaling.bitpix", options.scaling.bitpix) 

737 ps.setString("scaling.maskPlanes", options.scaling.maskPlanes) 

738 ps.set("scaling.fuzz", options.scaling.fuzz) 

739 ps.set("scaling.seed", options.scaling.seed) 

740 ps.set("scaling.quantizeLevel", options.scaling.quantizeLevel) 

741 ps.set("scaling.quantizePad", options.scaling.quantizePad) 

742 ps.set("scaling.bscale", options.scaling.bscale) 

743 ps.set("scaling.bzero", options.scaling.bzero) 

744 return ps 

745 

746 

747def persistUnpersist(ImageClass, image, filename, additionalData): 

748 """Use read/writeFitsWithOptions to persist and unpersist an image 

749 

750 Parameters 

751 ---------- 

752 ImageClass : `type`, an `lsst.afw.image.Image` class 

753 Class of image. 

754 image : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage` 

755 Image to compress. 

756 filename : `str` 

757 Filename to write. 

758 additionalData : `lsst.daf.base.PropertySet` 

759 Additional data for persistence framework. 

760 

761 Returns 

762 ------- 

763 unpersisted : `ImageClass` 

764 The unpersisted image. 

765 """ 

766 additionalData.set("visit", 12345) 

767 additionalData.set("ccd", 67) 

768 

769 image.writeFitsWithOptions(filename, additionalData) 

770 return ImageClass.readFits(filename) 

771 

772 

773class PersistenceTestCase(ImageCompressionTestCase): 

774 """Test compression using the persistence framework 

775 

776 We override the I/O methods to use the persistence framework. 

777 """ 

778 def testQuantization(self): 

779 """Not appropriate --- disable""" 

780 pass 

781 

782 def readWriteImage(self, ImageClass, image, filename, options): 

783 """Read the image after it has been written 

784 

785 This implementation uses the persistence framework. 

786 

787 Parameters 

788 ---------- 

789 ImageClass : `type`, an `lsst.afw.image.Image` class 

790 Class of image to create. 

791 image : `lsst.afw.image.Image` 

792 Image to compress. 

793 filename : `str` 

794 Filename to which to write. 

795 options : `lsst.afw.fits.ImageWriteOptions` 

796 Options for writing. 

797 """ 

798 additionalData = lsst.daf.base.PropertySet() 

799 additionalData.set("image", optionsToPropertySet(options)) 

800 return persistUnpersist(ImageClass, image, filename, additionalData) 

801 

802 def readWriteMaskedImage(self, image, filename, imageOptions, maskOptions, varianceOptions): 

803 """Read the MaskedImage after it has been written 

804 

805 This implementation uses the persistence framework. 

806 

807 Parameters 

808 ---------- 

809 image : `lsst.afw.image.MaskedImage` 

810 Image to compress. 

811 filename : `str` 

812 Filename to which to write. 

813 imageOptions, maskOptions, varianceOptions : `lsst.afw.fits.ImageWriteOptions` 

814 Options for writing the image, mask and variance planes. 

815 """ 

816 additionalData = lsst.daf.base.PropertySet() 

817 additionalData.set("image", optionsToPropertySet(imageOptions)) 

818 additionalData.set("mask", optionsToPropertySet(maskOptions)) 

819 additionalData.set("variance", optionsToPropertySet(varianceOptions)) 

820 return persistUnpersist(lsst.afw.image.MaskedImageF, image, filename, additionalData) 

821 

822 

823class EmptyExposureTestCase(lsst.utils.tests.TestCase): 

824 """Test that an empty image can be written 

825 

826 We sometimes use an empty lsst.afw.image.Exposure as a vehicle for 

827 persisting other things, e.g., Wcs, Calib. cfitsio compression will 

828 choke on an empty image, so make sure we're dealing with that. 

829 """ 

830 def checkEmptyExposure(self, algorithm): 

831 """Check that we can persist an empty Exposure 

832 

833 Parameters 

834 ---------- 

835 algorithm : `lsst.afw.fits.ImageCompressionOptions.CompressionAlgorithm` 

836 Compression algorithm to try. 

837 """ 

838 exp = lsst.afw.image.ExposureF(0, 0) 

839 degrees = lsst.geom.degrees 

840 cdMatrix = np.array([[1.0e-4, 0.0], [0.0, 1.0e-4]], dtype=float) 

841 exp.setWcs(lsst.afw.geom.makeSkyWcs(crval=lsst.geom.SpherePoint(0*degrees, 0*degrees), 

842 crpix=lsst.geom.Point2D(0.0, 0.0), 

843 cdMatrix=cdMatrix)) 

844 imageOptions = lsst.afw.fits.ImageWriteOptions(ImageCompressionOptions(algorithm)) 

845 maskOptions = lsst.afw.fits.ImageWriteOptions(exp.getMaskedImage().getMask()) 

846 varianceOptions = lsst.afw.fits.ImageWriteOptions(ImageCompressionOptions(algorithm)) 

847 with lsst.utils.tests.getTempFilePath(".fits") as filename: 

848 exp.writeFits(filename, imageOptions, maskOptions, varianceOptions) 

849 unpersisted = type(exp)(filename) 

850 self.assertEqual(unpersisted.getMaskedImage().getDimensions(), lsst.geom.Extent2I(0, 0)) 

851 self.assertEqual(unpersisted.getWcs(), exp.getWcs()) 

852 

853 def testEmptyExposure(self): 

854 """Persist an empty Exposure with compression""" 

855 algorithmList = ("GZIP", "GZIP_SHUFFLE", "RICE") 

856 for algorithm in algorithmList: 

857 self.checkEmptyExposure(lsst.afw.fits.compressionAlgorithmFromString(algorithm)) 

858 

859 

860class TestMemory(lsst.utils.tests.MemoryTestCase): 

861 pass 

862 

863 

864def setup_module(module): 

865 lsst.utils.tests.init() 

866 

867 

868if __name__ == "__main__": 868 ↛ 869line 868 didn't jump to line 869, because the condition on line 868 was never true

869 import sys 

870 setup_module(sys.modules[__name__]) 

871 unittest.main()