Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2017 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23import os 

24import unittest 

25import itertools 

26 

27import numpy as np 

28import astropy.io.fits 

29 

30import lsst.utils 

31import lsst.daf.base 

32import lsst.daf.persistence 

33import lsst.geom 

34import lsst.afw.geom 

35import lsst.afw.image 

36import lsst.afw.fits 

37import lsst.utils.tests 

38from lsst.afw.image import LOCAL 

39from lsst.afw.fits import ImageScalingOptions, ImageCompressionOptions 

40 

41 

42def checkAstropy(image, filename, hduNum=0): 

43 """Check that astropy can read our file 

44 

45 We don't insist on equality for low BITPIX (8, 16) when the original 

46 type is double-precision: astropy will (quite reasonably) read that 

47 into a single-precision image and apply bscale,bzero to that so it's 

48 going to be subject to roundoff. 

49 

50 Parameters 

51 ---------- 

52 image : `lsst.afw.image.Image` 

53 Image read by our own code. 

54 filename : `str` 

55 Filename of FITS file to read with astropy. 

56 hduNum : `int` 

57 HDU number of interest. 

58 """ 

59 print("Astropy currently doesn't read our compressed images perfectly.") 

60 return 

61 

62 def parseVersion(version): 

63 return tuple(int(vv) for vv in np.array(version.split("."))) 

64 

65 if parseVersion(astropy.__version__) <= parseVersion("2.0.1"): 

66 # astropy 2.0.1 and earlier have problems: 

67 # * Doesn't support GZIP_2: https://github.com/astropy/astropy/pull/6486 

68 # * Uses the wrong array type: https://github.com/astropy/astropy/pull/6492 

69 print(f"Refusing to check with astropy version {astropy.__version__} due to astropy bugs") 

70 return 

71 hdu = astropy.io.fits.open(filename)[hduNum] 

72 if hdu.header["BITPIX"] in (8, 16) and isinstance(image, lsst.afw.image.ImageD): 

73 return 

74 dtype = image.getArray().dtype 

75 theirs = hdu.data.astype(dtype) 

76 # Allow for minor differences due to arithmetic: +/- 1 in the last place 

77 np.testing.assert_array_max_ulp(theirs, image.getArray()) 

78 

79 

80class ImageScalingTestCase(lsst.utils.tests.TestCase): 

81 """Tests of image scaling 

82 

83 The pattern here is to create an image, write it out with a 

84 specific scaling algorithm, read it back in and test that everything 

85 is as we expect. We do this for each scaling algorithm in its own 

86 test, and within that test iterate over various parameters (input 

87 image type, BITPIX, etc.). The image we create has a few features 

88 (low, high and masked pixels) that we check. 

89 """ 

90 def setUp(self): 

91 self.bbox = lsst.geom.Box2I(lsst.geom.Point2I(123, 456), lsst.geom.Extent2I(7, 8)) 

92 self.base = 456 # Base value for pixels 

93 self.highValue = 789 # Value for high pixel 

94 self.lowValue = 123 # Value for low pixel 

95 self.maskedValue = 12345 # Value for masked pixel (to throw off statistics) 

96 self.highPixel = lsst.geom.Point2I(1, 1) # Location of high pixel 

97 self.lowPixel = lsst.geom.Point2I(2, 2) # Location of low pixel 

98 self.maskedPixel = lsst.geom.Point2I(3, 3) # Location of masked pixel 

99 self.badMask = "BAD" # Mask plane to set for masked pixel 

100 self.stdev = 5.0 # Noise stdev to add to image 

101 

102 def makeImage(self, ImageClass, scaling, addNoise=True): 

103 """Make an image for testing 

104 

105 We create an image, persist and unpersist it, returning 

106 some data to the caller. 

107 

108 Parameters 

109 ---------- 

110 ImageClass : `type`, an `lsst.afw.image.Image` class 

111 Class of image to create. 

112 scaling : `lsst.afw.fits.ImageScalingOptions` 

113 Scaling to apply during persistence. 

114 addNoise : `bool` 

115 Add noise to image? 

116 

117 Returns 

118 ------- 

119 image : `lsst.afw.image.Image` (ImageClass) 

120 Created image. 

121 unpersisted : `lsst.afw.image.Image` (ImageClass) 

122 Unpersisted image. 

123 bscale, bzero : `float` 

124 FITS scale factor and zero used. 

125 minValue, maxValue : `float` 

126 Minimum and maximum value given the nominated scaling. 

127 """ 

128 image = ImageClass(self.bbox) 

129 mask = lsst.afw.image.Mask(self.bbox) 

130 mask.addMaskPlane(self.badMask) 

131 bad = mask.getPlaneBitMask(self.badMask) 

132 image.set(self.base) 

133 image[self.highPixel, LOCAL] = self.highValue 

134 image[self.lowPixel, LOCAL] = self.lowValue 

135 image[self.maskedPixel, LOCAL] = self.maskedValue 

136 mask[self.maskedPixel, LOCAL] = bad 

137 

138 rng = np.random.RandomState(12345) 

139 dtype = image.getArray().dtype 

140 if addNoise: 

141 image.getArray()[:] += rng.normal(0.0, self.stdev, image.getArray().shape).astype(dtype) 

142 

143 with lsst.utils.tests.getTempFilePath(".fits") as filename: 

144 with lsst.afw.fits.Fits(filename, "w") as fits: 

145 options = lsst.afw.fits.ImageWriteOptions(scaling) 

146 header = lsst.daf.base.PropertyList() 

147 image.writeFits(fits, options, header, mask) 

148 unpersisted = ImageClass(filename) 

149 self.assertEqual(image.getBBox(), unpersisted.getBBox()) 

150 

151 header = lsst.afw.fits.readMetadata(filename) 

152 bscale = header.getScalar("BSCALE") 

153 bzero = header.getScalar("BZERO") 

154 

155 if scaling.algorithm != ImageScalingOptions.NONE: 

156 self.assertEqual(header.getScalar("BITPIX"), scaling.bitpix) 

157 

158 if scaling.bitpix == 8: # unsigned, says FITS 

159 maxValue = bscale*(2**scaling.bitpix - 1) + bzero 

160 minValue = bzero 

161 else: 

162 maxValue = bscale*(2**(scaling.bitpix - 1) - 1) + bzero 

163 if scaling.bitpix == 32: 

164 # cfitsio pads 10 values, and so do we 

165 minValue = -bscale*(2**(scaling.bitpix - 1) - 10) + bzero 

166 else: 

167 minValue = -bscale*(2**(scaling.bitpix - 1)) + bzero 

168 

169 # Convert scalars to the appropriate type 

170 maxValue = np.array(maxValue, dtype=image.getArray().dtype) 

171 minValue = np.array(minValue, dtype=image.getArray().dtype) 

172 

173 checkAstropy(unpersisted, filename) 

174 

175 return image, unpersisted, bscale, bzero, minValue, maxValue 

176 

177 def checkPixel(self, unpersisted, original, xy, expected, rtol=None, atol=None): 

178 """Check one of the special pixels 

179 

180 After checking, we set this pixel to the original value so 

181 it's then easy to compare the entire image. 

182 

183 Parameters 

184 ---------- 

185 unpersisted : `lsst.afw.image.Image` 

186 Unpersisted image. 

187 original : `lsst.afw.image.Image` 

188 Original image. 

189 xy : `tuple` of two `int`s 

190 Position of pixel to check. 

191 expected : scalar 

192 Expected value of pixel. 

193 rtol, atol : `float` or `None` 

194 Relative/absolute tolerance for comparison. 

195 """ 

196 if np.isnan(expected): 

197 self.assertTrue(np.isnan(unpersisted[xy, LOCAL])) 

198 else: 

199 self.assertFloatsAlmostEqual(unpersisted[xy, LOCAL], expected, rtol=rtol, atol=atol) 

200 unpersisted[xy, LOCAL] = original[xy, LOCAL] # for ease of comparison of the whole image 

201 

202 def checkSpecialPixels(self, original, unpersisted, maxValue, minValue, rtol=None, atol=None): 

203 """Check the special pixels 

204 

205 Parameters 

206 ---------- 

207 original : `lsst.afw.image.Image` 

208 Original image. 

209 unpersisted : `lsst.afw.image.Image` 

210 Unpersisted image. 

211 minValue, maxValue : `float` 

212 Minimum and maximum value given the nominated scaling. 

213 rtol, atol : `float` or `None` 

214 Relative/absolute tolerance for comparison. 

215 """ 

216 highValue = original[self.highPixel, LOCAL] 

217 lowValue = original[self.lowPixel, LOCAL] 

218 maskedValue = original[self.maskedPixel, LOCAL] 

219 

220 expectHigh = min(highValue, maxValue) 

221 expectLow = max(lowValue, minValue) 

222 expectMasked = min(maskedValue, maxValue) 

223 

224 if unpersisted.getArray().dtype in (np.float32, np.float64): 

225 if highValue >= maxValue: 

226 expectHigh = np.nan 

227 if maskedValue >= maxValue: 

228 expectMasked = np.nan 

229 

230 self.checkPixel(unpersisted, original, self.highPixel, expectHigh, rtol=rtol, atol=atol) 

231 self.checkPixel(unpersisted, original, self.lowPixel, expectLow, rtol=rtol, atol=atol) 

232 self.checkPixel(unpersisted, original, self.maskedPixel, expectMasked, rtol=rtol, atol=atol) 

233 

234 def checkRange(self, ImageClass, bitpix): 

235 """Check that the RANGE scaling works 

236 

237 Parameters 

238 ---------- 

239 ImageClass : `type`, an `lsst.afw.image.Image` class 

240 Class of image to create. 

241 bitpix : `int` 

242 Bits per pixel for FITS image. 

243 """ 

244 scaling = ImageScalingOptions(ImageScalingOptions.RANGE, bitpix, [u"BAD"], fuzz=False) 

245 original, unpersisted, bscale, bzero, minValue, maxValue = self.makeImage(ImageClass, scaling, False) 

246 

247 numValues = 2**bitpix - 1 

248 numValues -= 2 # Padding on either end 

249 if bitpix == 32: 

250 numValues -= 10 

251 bscaleExpect = (self.highValue - self.lowValue)/numValues 

252 self.assertFloatsAlmostEqual(bscale, bscaleExpect, atol=1.0e-6) # F32 resolution 

253 

254 rtol = 1.0/2**(bitpix - 1) 

255 self.checkSpecialPixels(original, unpersisted, maxValue, minValue, atol=bscale) 

256 self.assertImagesAlmostEqual(original, unpersisted, rtol=rtol) 

257 

258 def checkStdev(self, ImageClass, bitpix, algorithm, quantizeLevel, quantizePad): 

259 """Check that one of the STDEV scaling algorithms work 

260 

261 Parameters 

262 ---------- 

263 ImageClass : `type`, an `lsst.afw.image.Image` class 

264 Class of image to create. 

265 bitpix : `int` 

266 Bits per pixel for FITS image. 

267 algorithm : `lsst.afw.fits.ImageScalingOptions.ScalingAlgorithm` 

268 Scaling algorithm to apply (one of the STDEV_*). 

269 quantizeLevel : `float` 

270 Quantization level. 

271 quantizePad : `float` 

272 Quantization padding. 

273 """ 

274 scaling = lsst.afw.fits.ImageScalingOptions(algorithm, bitpix, [u"BAD"], fuzz=False, 

275 quantizeLevel=quantizeLevel, quantizePad=quantizePad) 

276 

277 makeImageResults = self.makeImage(ImageClass, scaling) 

278 original, unpersisted, bscale, bzero, minValue, maxValue = makeImageResults 

279 

280 self.assertFloatsAlmostEqual(bscale, self.stdev/quantizeLevel, rtol=3.0/quantizeLevel) 

281 self.checkSpecialPixels(original, unpersisted, maxValue, minValue, atol=bscale) 

282 self.assertImagesAlmostEqual(original, unpersisted, atol=bscale) 

283 

284 def testRange(self): 

285 """Test that the RANGE scaling works on floating-point inputs 

286 

287 We deliberately don't include BITPIX=64 because int64 provides 

288 a larger dynamic range than 'double BSCALE' can handle. 

289 """ 

290 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD) 

291 bitpixList = (8, 16, 32) 

292 for cls, bitpix in itertools.product(classList, bitpixList): 

293 self.checkRange(cls, bitpix) 

294 

295 def testStdev(self): 

296 """Test that the STDEV scalings work on floating-point inputs 

297 

298 We deliberately don't include BITPIX=64 because int64 provides 

299 a larger dynamic range than 'double BSCALE' can handle. 

300 

301 We deliberately don't include BITPIX=8 because that provides 

302 only a tiny dynamic range where everything goes out of range easily. 

303 """ 

304 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD) 

305 bitpixList = (16, 32) 

306 algorithmList = (ImageScalingOptions.STDEV_POSITIVE, ImageScalingOptions.STDEV_NEGATIVE, 

307 ImageScalingOptions.STDEV_BOTH) 

308 quantizeLevelList = (2.0, 10.0, 100.0) 

309 quantizePadList = (5.0, 10.0, 100.0) 

310 for values in itertools.product(classList, bitpixList, algorithmList, 

311 quantizeLevelList, quantizePadList): 

312 self.checkStdev(*values) 

313 

314 def testRangeFailures(self): 

315 """Test that the RANGE scaling fails on integer inputs""" 

316 classList = (lsst.afw.image.ImageU, lsst.afw.image.ImageI, lsst.afw.image.ImageL) 

317 bitpixList = (8, 16, 32) 

318 for cls, bitpix in itertools.product(classList, bitpixList): 

319 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError): 

320 self.checkRange(cls, bitpix) 

321 

322 def testStdevFailures(self): 

323 """Test that the STDEV scalings fail on integer inputs""" 

324 classList = (lsst.afw.image.ImageU, lsst.afw.image.ImageI, lsst.afw.image.ImageL) 

325 bitpixList = (16, 32) 

326 algorithmList = (ImageScalingOptions.STDEV_POSITIVE, ImageScalingOptions.STDEV_NEGATIVE, 

327 ImageScalingOptions.STDEV_BOTH) 

328 for cls, bitpix, algorithm in itertools.product(classList, bitpixList, algorithmList): 

329 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError): 

330 self.checkStdev(cls, bitpix, algorithm, 10.0, 10.0) 

331 

332 def checkNone(self, ImageClass, bitpix): 

333 """Check that the NONE scaling algorithm works 

334 

335 Parameters 

336 ---------- 

337 ImageClass : `type`, an `lsst.afw.image.Image` class 

338 Class of image to create. 

339 bitpix : `int` 

340 Bits per pixel for FITS image. 

341 """ 

342 scaling = ImageScalingOptions(ImageScalingOptions.NONE, bitpix, [u"BAD"], fuzz=False) 

343 original, unpersisted, bscale, bzero, minValue, maxValue = self.makeImage(ImageClass, scaling) 

344 self.assertFloatsAlmostEqual(bscale, 1.0, atol=0.0) 

345 self.assertFloatsAlmostEqual(bzero, 0.0, atol=0.0) 

346 self.assertImagesAlmostEqual(original, unpersisted, atol=0.0) 

347 

348 def testNone(self): 

349 """Test that the NONE scaling works on floating-point inputs""" 

350 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD) 

351 bitpixList = (8, 16, 32) 

352 for cls, bitpix in itertools.product(classList, bitpixList): 

353 self.checkNone(cls, bitpix) 

354 

355 def checkManual(self, ImageClass, bitpix): 

356 """Check that the MANUAL scaling algorithm works 

357 

358 Parameters 

359 ---------- 

360 ImageClass : `type`, an `lsst.afw.image.Image` class 

361 Class of image to create. 

362 bitpix : `int` 

363 Bits per pixel for FITS image. 

364 """ 

365 bscaleSet = 1.2345 

366 bzeroSet = self.base 

367 scaling = ImageScalingOptions(ImageScalingOptions.MANUAL, bitpix, [u"BAD"], bscale=bscaleSet, 

368 bzero=bzeroSet, fuzz=False) 

369 original, unpersisted, bscale, bzero, minValue, maxValue = self.makeImage(ImageClass, scaling) 

370 self.assertFloatsAlmostEqual(bscale, bscaleSet, atol=0.0) 

371 self.assertFloatsAlmostEqual(bzero, bzeroSet, atol=0.0) 

372 self.assertImagesAlmostEqual(original, unpersisted, atol=bscale) 

373 

374 def testManual(self): 

375 """Test that the MANUAL scaling works on floating-point inputs""" 

376 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD) 

377 bitpixList = (16, 32) 

378 for cls, bitpix in itertools.product(classList, bitpixList): 

379 self.checkNone(cls, bitpix) 

380 

381 

382class ImageCompressionTestCase(lsst.utils.tests.TestCase): 

383 """Tests of image compression 

384 

385 We test compression both with and without loss (quantisation/scaling). 

386 

387 The pattern here is to create an image, write it out with a 

388 specific compression algorithm, read it back in and test that everything 

389 is as we expect. We do this for each compression algorithm in its own 

390 test, and within that test iterate over various parameters (input 

391 image type, BITPIX, etc.). 

392 

393 We print the (inverse) compression ratio for interest. Note that 

394 these should not be considered to be representative of the 

395 compression that will be achieved on scientific data, since the 

396 images created here have different qualities than scientific data 

397 that will affect the compression ratio (e.g., size, noise properties). 

398 """ 

399 def setUp(self): 

400 self.bbox = lsst.geom.Box2I(lsst.geom.Point2I(123, 456), lsst.geom.Extent2I(7, 8)) 

401 self.background = 12345.6789 # Background value 

402 self.noise = 67.89 # Noise (stdev) 

403 self.maskPlanes = ["FOO", "BAR"] # Mask planes to add 

404 self.extension = "." + self.__class__.__name__ + ".fits" # extension name for temp files 

405 

406 def readWriteImage(self, ImageClass, image, filename, options, *args): 

407 """Read the image after it has been written 

408 

409 This implementation does the persistence using methods on the 

410 ImageClass. 

411 

412 Parameters 

413 ---------- 

414 ImageClass : `type`, an `lsst.afw.image.Image` class 

415 Class of image to create. 

416 image : `lsst.afw.image.Image` 

417 Image to compress. 

418 filename : `str` 

419 Filename to which to write. 

420 options : `lsst.afw.fits.ImageWriteOptions` 

421 Options for writing. 

422 """ 

423 image.writeFits(filename, options, *args) 

424 return ImageClass(filename) 

425 

426 def makeImage(self, ImageClass): 

427 """Create an image 

428 

429 Parameters 

430 ---------- 

431 ImageClass : `type`, an `lsst.afw.image.Image` class 

432 Class of image to create. 

433 

434 Returns 

435 ------- 

436 image : `ImageClass` 

437 The created image. 

438 """ 

439 image = ImageClass(self.bbox) 

440 rng = np.random.RandomState(12345) 

441 dtype = image.getArray().dtype 

442 noise = rng.normal(0.0, self.noise, image.getArray().shape).astype(dtype) 

443 image.getArray()[:] = np.array(self.background, dtype=dtype) + noise 

444 return image 

445 

446 def makeMask(self): 

447 """Create a mask 

448 

449 Note that we generate a random distribution of mask pixel values, 

450 which is very different from the usual distribution in science images. 

451 

452 Returns 

453 ------- 

454 mask : `lsst.afw.image.Mask` 

455 The created mask. 

456 """ 

457 mask = lsst.afw.image.Mask(self.bbox) 

458 rng = np.random.RandomState(12345) 

459 dtype = mask.getArray().dtype 

460 mask.getArray()[:] = rng.randint(0, 2**(dtype.itemsize*8 - 1), mask.getArray().shape, dtype=dtype) 

461 for plane in self.maskPlanes: 

462 mask.addMaskPlane(plane) 

463 return mask 

464 

465 def checkCompressedImage(self, ImageClass, image, compression, scaling=None, atol=0.0): 

466 """Check that compression works on an image 

467 

468 Parameters 

469 ---------- 

470 ImageClass : `type`, an `lsst.afw.image.Image` class 

471 Class of image. 

472 image : `lsst.afw.image.Image` 

473 Image to compress. 

474 compression : `lsst.afw.fits.ImageCompressionOptions` 

475 Compression parameters. 

476 scaling : `lsst.afw.fits.ImageScalingOptions` or `None` 

477 Scaling parameters for lossy compression (optional). 

478 atol : `float` 

479 Absolute tolerance for comparing unpersisted image. 

480 

481 Returns 

482 ------- 

483 unpersisted : `ImageClass` 

484 The unpersisted image. 

485 """ 

486 with lsst.utils.tests.getTempFilePath(self.extension) as filename: 

487 if scaling: 

488 options = lsst.afw.fits.ImageWriteOptions(compression, scaling) 

489 else: 

490 options = lsst.afw.fits.ImageWriteOptions(compression) 

491 unpersisted = self.readWriteImage(ImageClass, image, filename, options) 

492 

493 fileSize = os.stat(filename).st_size 

494 fitsBlockSize = 2880 # All sizes in FITS are a multiple of this 

495 numBlocks = 1 + np.ceil(self.bbox.getArea()*image.getArray().dtype.itemsize/fitsBlockSize) 

496 uncompressedSize = fitsBlockSize*numBlocks 

497 print(ImageClass, compression.algorithm, fileSize, uncompressedSize, fileSize/uncompressedSize) 

498 

499 self.assertEqual(image.getBBox(), unpersisted.getBBox()) 

500 self.assertImagesAlmostEqual(unpersisted, image, atol=atol) 

501 

502 checkAstropy(unpersisted, filename, 1) 

503 

504 return unpersisted 

505 

506 def testLosslessFloat(self): 

507 """Test lossless compression of floating-point image""" 

508 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD) 

509 algorithmList = ("GZIP", "GZIP_SHUFFLE") # Lossless float compression requires GZIP 

510 for cls, algorithm in itertools.product(classList, algorithmList): 

511 image = self.makeImage(cls) 

512 compression = ImageCompressionOptions(lsst.afw.fits.compressionAlgorithmFromString(algorithm)) 

513 self.checkCompressedImage(cls, image, compression, atol=0.0) 

514 

515 def testLosslessInt(self): 

516 """Test lossless compression of integer image 

517 

518 We deliberately don't test `lsst.afw.image.ImageL` because 

519 compression of LONGLONG images is unsupported by cfitsio. 

520 """ 

521 classList = (lsst.afw.image.ImageU, lsst.afw.image.ImageI) 

522 algorithmList = ("GZIP", "GZIP_SHUFFLE", "RICE") 

523 for cls, algorithm in itertools.product(classList, algorithmList): 

524 compression = ImageCompressionOptions(lsst.afw.fits.compressionAlgorithmFromString(algorithm)) 

525 image = self.makeImage(cls) 

526 self.checkCompressedImage(cls, image, compression, atol=0.0) 

527 

528 def testLongLong(self): 

529 """Test graceful failure when compressing ImageL 

530 

531 We deliberately don't test `lsst.afw.image.ImageL` because 

532 compression of LONGLONG images is unsupported by cfitsio. 

533 """ 

534 algorithmList = ("GZIP", "GZIP_SHUFFLE", "RICE") 

535 for algorithm in algorithmList: 

536 compression = ImageCompressionOptions(lsst.afw.fits.compressionAlgorithmFromString(algorithm)) 

537 cls = lsst.afw.image.ImageL 

538 image = self.makeImage(cls) 

539 with self.assertRaises(lsst.afw.fits.FitsError): 

540 self.checkCompressedImage(cls, image, compression) 

541 

542 def testMask(self): 

543 """Test compression of mask 

544 

545 We deliberately don't test PLIO compression (which is designed for 

546 masks) because our default mask type (32) has too much dynamic range 

547 for PLIO (limit of 24 bits). 

548 """ 

549 for algorithm in ("GZIP", "GZIP_SHUFFLE", "RICE"): 

550 compression = ImageCompressionOptions(lsst.afw.fits.compressionAlgorithmFromString(algorithm)) 

551 mask = self.makeMask() 

552 unpersisted = self.checkCompressedImage(lsst.afw.image.Mask, mask, compression, atol=0.0) 

553 for mp in mask.getMaskPlaneDict(): 

554 self.assertIn(mp, unpersisted.getMaskPlaneDict()) 

555 unpersisted.getPlaneBitMask(mp) 

556 

557 def testLossyFloatCfitsio(self): 

558 """Test lossy compresion of floating-point images with cfitsio 

559 

560 cfitsio does the compression, controlled through the 'quantizeLevel' 

561 parameter. Note that cfitsio doesn't have access to our masks when 

562 it does its statistics. 

563 """ 

564 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD) 

565 algorithmList = ("GZIP", "GZIP_SHUFFLE", "RICE") 

566 quantizeList = (4.0, 10.0) 

567 for cls, algorithm, quantizeLevel in itertools.product(classList, algorithmList, quantizeList): 

568 compression = ImageCompressionOptions(lsst.afw.fits.compressionAlgorithmFromString(algorithm), 

569 quantizeLevel=quantizeLevel) 

570 image = self.makeImage(cls) 

571 self.checkCompressedImage(cls, image, compression, atol=self.noise/quantizeLevel) 

572 

573 def testLossyFloatOurs(self): 

574 """Test lossy compression of floating-point images ourselves 

575 

576 We do lossy compression by scaling first. We have full control over 

577 the scaling (multiple scaling algorithms), and we have access to our 

578 own masks when we do statistics. 

579 """ 

580 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD) 

581 algorithmList = ("GZIP", "GZIP_SHUFFLE", "RICE") 

582 bitpixList = (16, 32) 

583 quantizeList = (4.0, 10.0) 

584 for cls, algorithm, bitpix, quantize in itertools.product(classList, algorithmList, bitpixList, 

585 quantizeList): 

586 compression = ImageCompressionOptions(lsst.afw.fits.compressionAlgorithmFromString(algorithm), 

587 quantizeLevel=0.0) 

588 scaling = ImageScalingOptions(ImageScalingOptions.STDEV_BOTH, bitpix, quantizeLevel=quantize, 

589 fuzz=True) 

590 image = self.makeImage(cls) 

591 self.checkCompressedImage(cls, image, compression, scaling, atol=self.noise/quantize) 

592 

593 def readWriteMaskedImage(self, image, filename, imageOptions, maskOptions, varianceOptions): 

594 """Read the MaskedImage after it has been written 

595 

596 This implementation does the persistence using methods on the 

597 MaskedImage class. 

598 

599 Parameters 

600 ---------- 

601 image : `lsst.afw.image.Image` 

602 Image to compress. 

603 filename : `str` 

604 Filename to which to write. 

605 imageOptions, maskOptions, varianceOptions : `lsst.afw.fits.ImageWriteOptions` 

606 Options for writing the image, mask and variance planes. 

607 """ 

608 image.writeFits(filename, imageOptions, maskOptions, varianceOptions) 

609 if hasattr(image, "getMaskedImage"): 

610 image = image.getMaskedImage() 

611 return lsst.afw.image.MaskedImageF(filename) 

612 

613 def checkCompressedMaskedImage(self, image, imageOptions, maskOptions, varianceOptions, atol=0.0): 

614 """Check that compression works on a MaskedImage 

615 

616 Parameters 

617 ---------- 

618 image : `lsst.afw.image.MaskedImage` or `lsst.afw.image.Exposure` 

619 MaskedImage or exposure to compress. 

620 imageOptions, maskOptions, varianceOptions : `lsst.afw.fits.ImageWriteOptions` 

621 Parameters for writing (compression and scaling) the image, mask 

622 and variance planes. 

623 atol : `float` 

624 Absolute tolerance for comparing unpersisted image. 

625 """ 

626 with lsst.utils.tests.getTempFilePath(self.extension) as filename: 

627 self.readWriteMaskedImage(image, filename, imageOptions, maskOptions, varianceOptions) 

628 unpersisted = type(image)(filename) 

629 if hasattr(image, "getMaskedImage"): 

630 image = image.getMaskedImage() 

631 unpersisted = unpersisted.getMaskedImage() 

632 self.assertEqual(image.getBBox(), unpersisted.getBBox()) 

633 self.assertImagesAlmostEqual(unpersisted.getImage(), image.getImage(), atol=atol) 

634 self.assertImagesAlmostEqual(unpersisted.getMask(), image.getMask(), atol=atol) 

635 self.assertImagesAlmostEqual(unpersisted.getVariance(), image.getVariance(), atol=atol) 

636 

637 for mp in image.getMask().getMaskPlaneDict(): 

638 self.assertIn(mp, unpersisted.getMask().getMaskPlaneDict()) 

639 unpersisted.getMask().getPlaneBitMask(mp) 

640 

641 def checkMaskedImage(self, imageOptions, maskOptions, varianceOptions, atol=0.0): 

642 """Check that we can compress a MaskedImage and Exposure 

643 

644 Parameters 

645 ---------- 

646 imageOptions, maskOptions, varianceOptions : `lsst.afw.fits.ImageWriteOptions` 

647 Parameters for writing (compression and scaling) the image, mask 

648 and variance planes. 

649 atol : `float` 

650 Absolute tolerance for comparing unpersisted image. 

651 """ 

652 image = lsst.afw.image.makeMaskedImage(self.makeImage(lsst.afw.image.ImageF), 

653 self.makeMask(), self.makeImage(lsst.afw.image.ImageF)) 

654 self.checkCompressedMaskedImage(image, imageOptions, maskOptions, varianceOptions, atol=atol) 

655 exp = lsst.afw.image.makeExposure(image) 

656 self.checkCompressedMaskedImage(exp, imageOptions, maskOptions, varianceOptions, atol=atol) 

657 

658 def testMaskedImage(self): 

659 """Test compression of MaskedImage 

660 

661 We test lossless, lossy cfitsio and lossy LSST compression. 

662 """ 

663 # Lossless 

664 lossless = lsst.afw.fits.ImageCompressionOptions(ImageCompressionOptions.GZIP_SHUFFLE) 

665 options = lsst.afw.fits.ImageWriteOptions(lossless) 

666 self.checkMaskedImage(options, options, options, atol=0.0) 

667 

668 # Lossy cfitsio compression 

669 quantize = 4.0 

670 cfitsio = lsst.afw.fits.ImageCompressionOptions(ImageCompressionOptions.GZIP_SHUFFLE, True, quantize) 

671 imageOptions = lsst.afw.fits.ImageWriteOptions(cfitsio) 

672 maskOptions = lsst.afw.fits.ImageWriteOptions(lossless) 

673 self.checkMaskedImage(imageOptions, maskOptions, imageOptions, atol=self.noise/quantize) 

674 

675 # Lossy our compression 

676 quantize = 10.0 

677 compression = lsst.afw.fits.ImageCompressionOptions(ImageCompressionOptions.RICE, True, 0.0) 

678 scaling = lsst.afw.fits.ImageScalingOptions(ImageScalingOptions.STDEV_BOTH, 32, 

679 quantizeLevel=quantize) 

680 imageOptions = lsst.afw.fits.ImageWriteOptions(compression, scaling) 

681 maskOptions = lsst.afw.fits.ImageWriteOptions(compression) 

682 self.checkMaskedImage(imageOptions, maskOptions, imageOptions, atol=self.noise/quantize) 

683 

684 def testQuantization(self): 

685 """Test that our quantization produces the same values as cfitsio 

686 

687 Our quantization is more configurable (e.g., choice of scaling algorithm, 

688 specifying mask planes) and extensible (logarithmic, asinh scalings) 

689 than cfitsio's. However, cfitsio uses its own fuzz ("subtractive dithering") 

690 when reading the data, so if we don't want to add random values twice, 

691 we need to be sure that we're using the same random values. To check that, 

692 we write one image with our scaling+compression, and one with cfitsio's 

693 compression using exactly the BSCALE and dither seed we used for our own. 

694 That way, the two codes will quantize independently, and we can compare 

695 the results. 

696 """ 

697 bscaleSet = 1.0 

698 bzeroSet = self.background - 10*self.noise 

699 algorithm = ImageCompressionOptions.GZIP 

700 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD) 

701 tilesList = ((4, 5), (0, 0), (0, 5), (4, 0), (0, 1)) 

702 for cls, tiles in itertools.product(classList, tilesList): 

703 tiles = np.array(tiles, dtype=np.int64) 

704 compression = ImageCompressionOptions(algorithm, tiles, -bscaleSet) 

705 original = self.makeImage(cls) 

706 with lsst.utils.tests.getTempFilePath(self.extension) as filename: 

707 with lsst.afw.fits.Fits(filename, "w") as fits: 

708 options = lsst.afw.fits.ImageWriteOptions(compression) 

709 original.writeFits(fits, options) 

710 cfitsio = cls(filename) 

711 header = lsst.afw.fits.readMetadata(filename, 1) 

712 seed = header.getScalar("ZDITHER0") 

713 self.assertEqual(header.getScalar("BSCALE"), bscaleSet) 

714 

715 compression = ImageCompressionOptions(algorithm, tiles, 0.0) 

716 scaling = ImageScalingOptions(ImageScalingOptions.MANUAL, 32, [u"BAD"], bscale=bscaleSet, 

717 bzero=bzeroSet, fuzz=True, seed=seed) 

718 unpersisted = self.checkCompressedImage(cls, original, compression, scaling, atol=bscaleSet) 

719 oursDiff = unpersisted.getArray() - original.getArray() 

720 cfitsioDiff = cfitsio.getArray() - original.getArray() 

721 self.assertImagesAlmostEqual(oursDiff, cfitsioDiff, atol=0.0) 

722 

723 

724def optionsToPropertySet(options): 

725 """Convert the ImageWriteOptions to a PropertySet 

726 

727 This allows us to pass the options into the persistence framework 

728 as the "additionalData". 

729 """ 

730 ps = lsst.daf.base.PropertySet() 

731 ps.set("compression.algorithm", lsst.afw.fits.compressionAlgorithmToString(options.compression.algorithm)) 

732 ps.set("compression.columns", options.compression.tiles[0]) 

733 ps.set("compression.rows", options.compression.tiles[1]) 

734 ps.set("compression.quantizeLevel", options.compression.quantizeLevel) 

735 

736 ps.set("scaling.algorithm", lsst.afw.fits.scalingAlgorithmToString(options.scaling.algorithm)) 

737 ps.set("scaling.bitpix", options.scaling.bitpix) 

738 ps.setString("scaling.maskPlanes", options.scaling.maskPlanes) 

739 ps.set("scaling.fuzz", options.scaling.fuzz) 

740 ps.set("scaling.seed", options.scaling.seed) 

741 ps.set("scaling.quantizeLevel", options.scaling.quantizeLevel) 

742 ps.set("scaling.quantizePad", options.scaling.quantizePad) 

743 ps.set("scaling.bscale", options.scaling.bscale) 

744 ps.set("scaling.bzero", options.scaling.bzero) 

745 return ps 

746 

747 

748def persistUnpersist(ImageClass, image, filename, additionalData): 

749 """Use read/writeFitsWithOptions to persist and unpersist an image 

750 

751 Parameters 

752 ---------- 

753 ImageClass : `type`, an `lsst.afw.image.Image` class 

754 Class of image. 

755 image : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage` 

756 Image to compress. 

757 filename : `str` 

758 Filename to write. 

759 additionalData : `lsst.daf.base.PropertySet` 

760 Additional data for persistence framework. 

761 

762 Returns 

763 ------- 

764 unpersisted : `ImageClass` 

765 The unpersisted image. 

766 """ 

767 additionalData.set("visit", 12345) 

768 additionalData.set("ccd", 67) 

769 

770 image.writeFitsWithOptions(filename, additionalData) 

771 return ImageClass.readFits(filename) 

772 

773 

774class PersistenceTestCase(ImageCompressionTestCase): 

775 """Test compression using the persistence framework 

776 

777 We override the I/O methods to use the persistence framework. 

778 """ 

779 def testQuantization(self): 

780 """Not appropriate --- disable""" 

781 pass 

782 

783 def readWriteImage(self, ImageClass, image, filename, options): 

784 """Read the image after it has been written 

785 

786 This implementation uses the persistence framework. 

787 

788 Parameters 

789 ---------- 

790 ImageClass : `type`, an `lsst.afw.image.Image` class 

791 Class of image to create. 

792 image : `lsst.afw.image.Image` 

793 Image to compress. 

794 filename : `str` 

795 Filename to which to write. 

796 options : `lsst.afw.fits.ImageWriteOptions` 

797 Options for writing. 

798 """ 

799 additionalData = lsst.daf.base.PropertySet() 

800 additionalData.set("image", optionsToPropertySet(options)) 

801 return persistUnpersist(ImageClass, image, filename, additionalData) 

802 

803 def readWriteMaskedImage(self, image, filename, imageOptions, maskOptions, varianceOptions): 

804 """Read the MaskedImage after it has been written 

805 

806 This implementation uses the persistence framework. 

807 

808 Parameters 

809 ---------- 

810 image : `lsst.afw.image.MaskedImage` 

811 Image to compress. 

812 filename : `str` 

813 Filename to which to write. 

814 imageOptions, maskOptions, varianceOptions : `lsst.afw.fits.ImageWriteOptions` 

815 Options for writing the image, mask and variance planes. 

816 """ 

817 additionalData = lsst.daf.base.PropertySet() 

818 additionalData.set("image", optionsToPropertySet(imageOptions)) 

819 additionalData.set("mask", optionsToPropertySet(maskOptions)) 

820 additionalData.set("variance", optionsToPropertySet(varianceOptions)) 

821 return persistUnpersist(lsst.afw.image.MaskedImageF, image, filename, additionalData) 

822 

823 

824class EmptyExposureTestCase(lsst.utils.tests.TestCase): 

825 """Test that an empty image can be written 

826 

827 We sometimes use an empty lsst.afw.image.Exposure as a vehicle for 

828 persisting other things, e.g., Wcs, Calib. cfitsio compression will 

829 choke on an empty image, so make sure we're dealing with that. 

830 """ 

831 def checkEmptyExposure(self, algorithm): 

832 """Check that we can persist an empty Exposure 

833 

834 Parameters 

835 ---------- 

836 algorithm : `lsst.afw.fits.ImageCompressionOptions.CompressionAlgorithm` 

837 Compression algorithm to try. 

838 """ 

839 exp = lsst.afw.image.ExposureF(0, 0) 

840 degrees = lsst.geom.degrees 

841 cdMatrix = np.array([[1.0e-4, 0.0], [0.0, 1.0e-4]], dtype=float) 

842 exp.setWcs(lsst.afw.geom.makeSkyWcs(crval=lsst.geom.SpherePoint(0*degrees, 0*degrees), 

843 crpix=lsst.geom.Point2D(0.0, 0.0), 

844 cdMatrix=cdMatrix)) 

845 imageOptions = lsst.afw.fits.ImageWriteOptions(ImageCompressionOptions(algorithm)) 

846 maskOptions = lsst.afw.fits.ImageWriteOptions(exp.getMaskedImage().getMask()) 

847 varianceOptions = lsst.afw.fits.ImageWriteOptions(ImageCompressionOptions(algorithm)) 

848 with lsst.utils.tests.getTempFilePath(".fits") as filename: 

849 exp.writeFits(filename, imageOptions, maskOptions, varianceOptions) 

850 unpersisted = type(exp)(filename) 

851 self.assertEqual(unpersisted.getMaskedImage().getDimensions(), lsst.geom.Extent2I(0, 0)) 

852 self.assertEqual(unpersisted.getWcs(), exp.getWcs()) 

853 

854 def testEmptyExposure(self): 

855 """Persist an empty Exposure with compression""" 

856 algorithmList = ("GZIP", "GZIP_SHUFFLE", "RICE") 

857 for algorithm in algorithmList: 

858 self.checkEmptyExposure(lsst.afw.fits.compressionAlgorithmFromString(algorithm)) 

859 

860 

861class TestMemory(lsst.utils.tests.MemoryTestCase): 

862 pass 

863 

864 

865def setup_module(module): 

866 lsst.utils.tests.init() 

867 

868 

869if __name__ == "__main__": 869 ↛ 870line 869 didn't jump to line 870, because the condition on line 869 was never true

870 import sys 

871 setup_module(sys.modules[__name__]) 

872 unittest.main()