Coverage for tests/test_fitsCompression.py: 14%
374 statements
« prev ^ index » next coverage.py v7.1.0, created at 2023-02-05 17:50 -0800
« prev ^ index » next coverage.py v7.1.0, created at 2023-02-05 17:50 -0800
1#
2# LSST Data Management System
3# Copyright 2017 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
23import os
24import unittest
25import itertools
27import numpy as np
28import astropy.io.fits
30import lsst.utils
31import lsst.daf.base
32import lsst.daf.persistence
33import lsst.geom
34import lsst.afw.geom
35import lsst.afw.image
36import lsst.afw.fits
37import lsst.utils.tests
38from lsst.afw.image import LOCAL
39from lsst.afw.fits import ImageScalingOptions, ImageCompressionOptions
42def checkAstropy(image, filename, hduNum=0):
43 """Check that astropy can read our file
45 We don't insist on equality for low BITPIX (8, 16) when the original
46 type is double-precision: astropy will (quite reasonably) read that
47 into a single-precision image and apply bscale,bzero to that so it's
48 going to be subject to roundoff.
50 Parameters
51 ----------
52 image : `lsst.afw.image.Image`
53 Image read by our own code.
54 filename : `str`
55 Filename of FITS file to read with astropy.
56 hduNum : `int`
57 HDU number of interest.
58 """
59 print("Astropy currently doesn't read our compressed images perfectly.")
60 return
62 def parseVersion(version):
63 return tuple(int(vv) for vv in np.array(version.split(".")))
65 if parseVersion(astropy.__version__) <= parseVersion("2.0.1"):
66 # astropy 2.0.1 and earlier have problems:
67 # * Doesn't support GZIP_2: https://github.com/astropy/astropy/pull/6486
68 # * Uses the wrong array type: https://github.com/astropy/astropy/pull/6492
69 print(f"Refusing to check with astropy version {astropy.__version__} due to astropy bugs")
70 return
71 hdu = astropy.io.fits.open(filename)[hduNum]
72 if hdu.header["BITPIX"] in (8, 16) and isinstance(image, lsst.afw.image.ImageD):
73 return
74 dtype = image.getArray().dtype
75 theirs = hdu.data.astype(dtype)
76 # Allow for minor differences due to arithmetic: +/- 1 in the last place
77 np.testing.assert_array_max_ulp(theirs, image.getArray())
80class ImageScalingTestCase(lsst.utils.tests.TestCase):
81 """Tests of image scaling
83 The pattern here is to create an image, write it out with a
84 specific scaling algorithm, read it back in and test that everything
85 is as we expect. We do this for each scaling algorithm in its own
86 test, and within that test iterate over various parameters (input
87 image type, BITPIX, etc.). The image we create has a few features
88 (low, high and masked pixels) that we check.
89 """
90 def setUp(self):
91 self.bbox = lsst.geom.Box2I(lsst.geom.Point2I(123, 456), lsst.geom.Extent2I(7, 8))
92 self.base = 456 # Base value for pixels
93 self.highValue = 789 # Value for high pixel
94 self.lowValue = 123 # Value for low pixel
95 self.maskedValue = 12345 # Value for masked pixel (to throw off statistics)
96 self.highPixel = lsst.geom.Point2I(1, 1) # Location of high pixel
97 self.lowPixel = lsst.geom.Point2I(2, 2) # Location of low pixel
98 self.maskedPixel = lsst.geom.Point2I(3, 3) # Location of masked pixel
99 self.badMask = "BAD" # Mask plane to set for masked pixel
100 self.stdev = 5.0 # Noise stdev to add to image
102 def makeImage(self, ImageClass, scaling, addNoise=True):
103 """Make an image for testing
105 We create an image, persist and unpersist it, returning
106 some data to the caller.
108 Parameters
109 ----------
110 ImageClass : `type`, an `lsst.afw.image.Image` class
111 Class of image to create.
112 scaling : `lsst.afw.fits.ImageScalingOptions`
113 Scaling to apply during persistence.
114 addNoise : `bool`
115 Add noise to image?
117 Returns
118 -------
119 image : `lsst.afw.image.Image` (ImageClass)
120 Created image.
121 unpersisted : `lsst.afw.image.Image` (ImageClass)
122 Unpersisted image.
123 bscale, bzero : `float`
124 FITS scale factor and zero used.
125 minValue, maxValue : `float`
126 Minimum and maximum value given the nominated scaling.
127 """
128 image = ImageClass(self.bbox)
129 mask = lsst.afw.image.Mask(self.bbox)
130 mask.addMaskPlane(self.badMask)
131 bad = mask.getPlaneBitMask(self.badMask)
132 image.set(self.base)
133 image[self.highPixel, LOCAL] = self.highValue
134 image[self.lowPixel, LOCAL] = self.lowValue
135 image[self.maskedPixel, LOCAL] = self.maskedValue
136 mask[self.maskedPixel, LOCAL] = bad
138 rng = np.random.RandomState(12345)
139 dtype = image.getArray().dtype
140 if addNoise:
141 image.getArray()[:] += rng.normal(0.0, self.stdev, image.getArray().shape).astype(dtype)
143 with lsst.utils.tests.getTempFilePath(".fits") as filename:
144 with lsst.afw.fits.Fits(filename, "w") as fits:
145 options = lsst.afw.fits.ImageWriteOptions(scaling)
146 header = lsst.daf.base.PropertyList()
147 image.writeFits(fits, options, header, mask)
148 unpersisted = ImageClass(filename)
149 self.assertEqual(image.getBBox(), unpersisted.getBBox())
151 header = lsst.afw.fits.readMetadata(filename)
152 bscale = header.getScalar("BSCALE")
153 bzero = header.getScalar("BZERO")
155 if scaling.algorithm != ImageScalingOptions.NONE:
156 self.assertEqual(header.getScalar("BITPIX"), scaling.bitpix)
158 if scaling.bitpix == 8: # unsigned, says FITS
159 maxValue = bscale*(2**scaling.bitpix - 1) + bzero
160 minValue = bzero
161 else:
162 maxValue = bscale*(2**(scaling.bitpix - 1) - 1) + bzero
163 if scaling.bitpix == 32:
164 # cfitsio pads 10 values, and so do we
165 minValue = -bscale*(2**(scaling.bitpix - 1) - 10) + bzero
166 else:
167 minValue = -bscale*(2**(scaling.bitpix - 1)) + bzero
169 # Convert scalars to the appropriate type
170 maxValue = np.array(maxValue, dtype=image.getArray().dtype)
171 minValue = np.array(minValue, dtype=image.getArray().dtype)
173 checkAstropy(unpersisted, filename)
175 return image, unpersisted, bscale, bzero, minValue, maxValue
177 def checkPixel(self, unpersisted, original, xy, expected, rtol=None, atol=None):
178 """Check one of the special pixels
180 After checking, we set this pixel to the original value so
181 it's then easy to compare the entire image.
183 Parameters
184 ----------
185 unpersisted : `lsst.afw.image.Image`
186 Unpersisted image.
187 original : `lsst.afw.image.Image`
188 Original image.
189 xy : `tuple` of two `int`s
190 Position of pixel to check.
191 expected : scalar
192 Expected value of pixel.
193 rtol, atol : `float` or `None`
194 Relative/absolute tolerance for comparison.
195 """
196 if np.isnan(expected):
197 self.assertTrue(np.isnan(unpersisted[xy, LOCAL]))
198 else:
199 self.assertFloatsAlmostEqual(unpersisted[xy, LOCAL], expected, rtol=rtol, atol=atol)
200 unpersisted[xy, LOCAL] = original[xy, LOCAL] # for ease of comparison of the whole image
202 def checkSpecialPixels(self, original, unpersisted, maxValue, minValue, rtol=None, atol=None):
203 """Check the special pixels
205 Parameters
206 ----------
207 original : `lsst.afw.image.Image`
208 Original image.
209 unpersisted : `lsst.afw.image.Image`
210 Unpersisted image.
211 minValue, maxValue : `float`
212 Minimum and maximum value given the nominated scaling.
213 rtol, atol : `float` or `None`
214 Relative/absolute tolerance for comparison.
215 """
216 highValue = original[self.highPixel, LOCAL]
217 lowValue = original[self.lowPixel, LOCAL]
218 maskedValue = original[self.maskedPixel, LOCAL]
220 expectHigh = min(highValue, maxValue)
221 expectLow = max(lowValue, minValue)
222 expectMasked = min(maskedValue, maxValue)
224 if unpersisted.getArray().dtype in (np.float32, np.float64):
225 if highValue >= maxValue:
226 expectHigh = np.nan
227 if maskedValue >= maxValue:
228 expectMasked = np.nan
230 self.checkPixel(unpersisted, original, self.highPixel, expectHigh, rtol=rtol, atol=atol)
231 self.checkPixel(unpersisted, original, self.lowPixel, expectLow, rtol=rtol, atol=atol)
232 self.checkPixel(unpersisted, original, self.maskedPixel, expectMasked, rtol=rtol, atol=atol)
234 def checkRange(self, ImageClass, bitpix):
235 """Check that the RANGE scaling works
237 Parameters
238 ----------
239 ImageClass : `type`, an `lsst.afw.image.Image` class
240 Class of image to create.
241 bitpix : `int`
242 Bits per pixel for FITS image.
243 """
244 scaling = ImageScalingOptions(ImageScalingOptions.RANGE, bitpix, [u"BAD"], fuzz=False)
245 original, unpersisted, bscale, bzero, minValue, maxValue = self.makeImage(ImageClass, scaling, False)
247 numValues = 2**bitpix - 1
248 numValues -= 2 # Padding on either end
249 if bitpix == 32:
250 numValues -= 10
251 bscaleExpect = (self.highValue - self.lowValue)/numValues
252 self.assertFloatsAlmostEqual(bscale, bscaleExpect, atol=1.0e-6) # F32 resolution
254 rtol = 1.0/2**(bitpix - 1)
255 self.checkSpecialPixels(original, unpersisted, maxValue, minValue, atol=bscale)
256 self.assertImagesAlmostEqual(original, unpersisted, rtol=rtol)
258 def checkStdev(self, ImageClass, bitpix, algorithm, quantizeLevel, quantizePad):
259 """Check that one of the STDEV scaling algorithms work
261 Parameters
262 ----------
263 ImageClass : `type`, an `lsst.afw.image.Image` class
264 Class of image to create.
265 bitpix : `int`
266 Bits per pixel for FITS image.
267 algorithm : `lsst.afw.fits.ImageScalingOptions.ScalingAlgorithm`
268 Scaling algorithm to apply (one of the STDEV_*).
269 quantizeLevel : `float`
270 Quantization level.
271 quantizePad : `float`
272 Quantization padding.
273 """
274 scaling = lsst.afw.fits.ImageScalingOptions(algorithm, bitpix, [u"BAD"], fuzz=False,
275 quantizeLevel=quantizeLevel, quantizePad=quantizePad)
277 makeImageResults = self.makeImage(ImageClass, scaling)
278 original, unpersisted, bscale, bzero, minValue, maxValue = makeImageResults
280 self.assertFloatsAlmostEqual(bscale, self.stdev/quantizeLevel, rtol=3.0/quantizeLevel)
281 self.checkSpecialPixels(original, unpersisted, maxValue, minValue, atol=bscale)
282 self.assertImagesAlmostEqual(original, unpersisted, atol=bscale)
284 def testRange(self):
285 """Test that the RANGE scaling works on floating-point inputs
287 We deliberately don't include BITPIX=64 because int64 provides
288 a larger dynamic range than 'double BSCALE' can handle.
289 """
290 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD)
291 bitpixList = (8, 16, 32)
292 for cls, bitpix in itertools.product(classList, bitpixList):
293 self.checkRange(cls, bitpix)
295 def testStdev(self):
296 """Test that the STDEV scalings work on floating-point inputs
298 We deliberately don't include BITPIX=64 because int64 provides
299 a larger dynamic range than 'double BSCALE' can handle.
301 We deliberately don't include BITPIX=8 because that provides
302 only a tiny dynamic range where everything goes out of range easily.
303 """
304 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD)
305 bitpixList = (16, 32)
306 algorithmList = (ImageScalingOptions.STDEV_POSITIVE, ImageScalingOptions.STDEV_NEGATIVE,
307 ImageScalingOptions.STDEV_BOTH)
308 quantizeLevelList = (2.0, 10.0, 100.0)
309 quantizePadList = (5.0, 10.0, 100.0)
310 for values in itertools.product(classList, bitpixList, algorithmList,
311 quantizeLevelList, quantizePadList):
312 self.checkStdev(*values)
314 def testRangeFailures(self):
315 """Test that the RANGE scaling fails on integer inputs"""
316 classList = (lsst.afw.image.ImageU, lsst.afw.image.ImageI, lsst.afw.image.ImageL)
317 bitpixList = (8, 16, 32)
318 for cls, bitpix in itertools.product(classList, bitpixList):
319 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError):
320 self.checkRange(cls, bitpix)
322 def testStdevFailures(self):
323 """Test that the STDEV scalings fail on integer inputs"""
324 classList = (lsst.afw.image.ImageU, lsst.afw.image.ImageI, lsst.afw.image.ImageL)
325 bitpixList = (16, 32)
326 algorithmList = (ImageScalingOptions.STDEV_POSITIVE, ImageScalingOptions.STDEV_NEGATIVE,
327 ImageScalingOptions.STDEV_BOTH)
328 for cls, bitpix, algorithm in itertools.product(classList, bitpixList, algorithmList):
329 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError):
330 self.checkStdev(cls, bitpix, algorithm, 10.0, 10.0)
332 def checkNone(self, ImageClass, bitpix):
333 """Check that the NONE scaling algorithm works
335 Parameters
336 ----------
337 ImageClass : `type`, an `lsst.afw.image.Image` class
338 Class of image to create.
339 bitpix : `int`
340 Bits per pixel for FITS image.
341 """
342 scaling = ImageScalingOptions(ImageScalingOptions.NONE, bitpix, [u"BAD"], fuzz=False)
343 original, unpersisted, bscale, bzero, minValue, maxValue = self.makeImage(ImageClass, scaling)
344 self.assertFloatsAlmostEqual(bscale, 1.0, atol=0.0)
345 self.assertFloatsAlmostEqual(bzero, 0.0, atol=0.0)
346 self.assertImagesAlmostEqual(original, unpersisted, atol=0.0)
348 def testNone(self):
349 """Test that the NONE scaling works on floating-point inputs"""
350 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD)
351 bitpixList = (8, 16, 32)
352 for cls, bitpix in itertools.product(classList, bitpixList):
353 self.checkNone(cls, bitpix)
355 def checkManual(self, ImageClass, bitpix):
356 """Check that the MANUAL scaling algorithm works
358 Parameters
359 ----------
360 ImageClass : `type`, an `lsst.afw.image.Image` class
361 Class of image to create.
362 bitpix : `int`
363 Bits per pixel for FITS image.
364 """
365 bscaleSet = 1.2345
366 bzeroSet = self.base
367 scaling = ImageScalingOptions(ImageScalingOptions.MANUAL, bitpix, [u"BAD"], bscale=bscaleSet,
368 bzero=bzeroSet, fuzz=False)
369 original, unpersisted, bscale, bzero, minValue, maxValue = self.makeImage(ImageClass, scaling)
370 self.assertFloatsAlmostEqual(bscale, bscaleSet, atol=0.0)
371 self.assertFloatsAlmostEqual(bzero, bzeroSet, atol=0.0)
372 self.assertImagesAlmostEqual(original, unpersisted, atol=bscale)
374 def testManual(self):
375 """Test that the MANUAL scaling works on floating-point inputs"""
376 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD)
377 bitpixList = (16, 32)
378 for cls, bitpix in itertools.product(classList, bitpixList):
379 self.checkNone(cls, bitpix)
382class ImageCompressionTestCase(lsst.utils.tests.TestCase):
383 """Tests of image compression
385 We test compression both with and without loss (quantisation/scaling).
387 The pattern here is to create an image, write it out with a
388 specific compression algorithm, read it back in and test that everything
389 is as we expect. We do this for each compression algorithm in its own
390 test, and within that test iterate over various parameters (input
391 image type, BITPIX, etc.).
393 We print the (inverse) compression ratio for interest. Note that
394 these should not be considered to be representative of the
395 compression that will be achieved on scientific data, since the
396 images created here have different qualities than scientific data
397 that will affect the compression ratio (e.g., size, noise properties).
398 """
399 def setUp(self):
400 self.bbox = lsst.geom.Box2I(lsst.geom.Point2I(123, 456), lsst.geom.Extent2I(7, 8))
401 self.background = 12345.6789 # Background value
402 self.noise = 67.89 # Noise (stdev)
403 self.maskPlanes = ["FOO", "BAR"] # Mask planes to add
404 self.extension = "." + self.__class__.__name__ + ".fits" # extension name for temp files
406 def readWriteImage(self, ImageClass, image, filename, options, *args):
407 """Read the image after it has been written
409 This implementation does the persistence using methods on the
410 ImageClass.
412 Parameters
413 ----------
414 ImageClass : `type`, an `lsst.afw.image.Image` class
415 Class of image to create.
416 image : `lsst.afw.image.Image`
417 Image to compress.
418 filename : `str`
419 Filename to which to write.
420 options : `lsst.afw.fits.ImageWriteOptions`
421 Options for writing.
422 """
423 image.writeFits(filename, options, *args)
424 return ImageClass(filename)
426 def makeImage(self, ImageClass):
427 """Create an image
429 Parameters
430 ----------
431 ImageClass : `type`, an `lsst.afw.image.Image` class
432 Class of image to create.
434 Returns
435 -------
436 image : `ImageClass`
437 The created image.
438 """
439 image = ImageClass(self.bbox)
440 rng = np.random.RandomState(12345)
441 dtype = image.getArray().dtype
442 noise = rng.normal(0.0, self.noise, image.getArray().shape).astype(dtype)
443 image.getArray()[:] = np.array(self.background, dtype=dtype) + noise
444 return image
446 def makeMask(self):
447 """Create a mask
449 Note that we generate a random distribution of mask pixel values,
450 which is very different from the usual distribution in science images.
452 Returns
453 -------
454 mask : `lsst.afw.image.Mask`
455 The created mask.
456 """
457 mask = lsst.afw.image.Mask(self.bbox)
458 rng = np.random.RandomState(12345)
459 dtype = mask.getArray().dtype
460 mask.getArray()[:] = rng.randint(0, 2**(dtype.itemsize*8 - 1), mask.getArray().shape, dtype=dtype)
461 for plane in self.maskPlanes:
462 mask.addMaskPlane(plane)
463 return mask
465 def checkCompressedImage(self, ImageClass, image, compression, scaling=None, atol=0.0):
466 """Check that compression works on an image
468 Parameters
469 ----------
470 ImageClass : `type`, an `lsst.afw.image.Image` class
471 Class of image.
472 image : `lsst.afw.image.Image`
473 Image to compress.
474 compression : `lsst.afw.fits.ImageCompressionOptions`
475 Compression parameters.
476 scaling : `lsst.afw.fits.ImageScalingOptions` or `None`
477 Scaling parameters for lossy compression (optional).
478 atol : `float`
479 Absolute tolerance for comparing unpersisted image.
481 Returns
482 -------
483 unpersisted : `ImageClass`
484 The unpersisted image.
485 """
486 with lsst.utils.tests.getTempFilePath(self.extension) as filename:
487 if scaling:
488 options = lsst.afw.fits.ImageWriteOptions(compression, scaling)
489 else:
490 options = lsst.afw.fits.ImageWriteOptions(compression)
491 unpersisted = self.readWriteImage(ImageClass, image, filename, options)
493 fileSize = os.stat(filename).st_size
494 fitsBlockSize = 2880 # All sizes in FITS are a multiple of this
495 numBlocks = 1 + np.ceil(self.bbox.getArea()*image.getArray().dtype.itemsize/fitsBlockSize)
496 uncompressedSize = fitsBlockSize*numBlocks
497 print(ImageClass, compression.algorithm, fileSize, uncompressedSize, fileSize/uncompressedSize)
499 self.assertEqual(image.getBBox(), unpersisted.getBBox())
500 self.assertImagesAlmostEqual(unpersisted, image, atol=atol)
502 checkAstropy(unpersisted, filename, 1)
504 return unpersisted
506 def testLosslessFloat(self):
507 """Test lossless compression of floating-point image"""
508 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD)
509 algorithmList = ("GZIP", "GZIP_SHUFFLE") # Lossless float compression requires GZIP
510 for cls, algorithm in itertools.product(classList, algorithmList):
511 image = self.makeImage(cls)
512 compression = ImageCompressionOptions(lsst.afw.fits.compressionAlgorithmFromString(algorithm))
513 self.checkCompressedImage(cls, image, compression, atol=0.0)
515 def testLosslessInt(self):
516 """Test lossless compression of integer image
518 We deliberately don't test `lsst.afw.image.ImageL` because
519 compression of LONGLONG images is unsupported by cfitsio.
520 """
521 classList = (lsst.afw.image.ImageU, lsst.afw.image.ImageI)
522 algorithmList = ("GZIP", "GZIP_SHUFFLE", "RICE")
523 for cls, algorithm in itertools.product(classList, algorithmList):
524 compression = ImageCompressionOptions(lsst.afw.fits.compressionAlgorithmFromString(algorithm))
525 image = self.makeImage(cls)
526 self.checkCompressedImage(cls, image, compression, atol=0.0)
528 def testLongLong(self):
529 """Test graceful failure when compressing ImageL
531 We deliberately don't test `lsst.afw.image.ImageL` because
532 compression of LONGLONG images is unsupported by cfitsio.
533 """
534 algorithmList = ("GZIP", "GZIP_SHUFFLE", "RICE")
535 for algorithm in algorithmList:
536 compression = ImageCompressionOptions(lsst.afw.fits.compressionAlgorithmFromString(algorithm))
537 cls = lsst.afw.image.ImageL
538 image = self.makeImage(cls)
539 with self.assertRaises(lsst.afw.fits.FitsError):
540 self.checkCompressedImage(cls, image, compression)
542 def testMask(self):
543 """Test compression of mask
545 We deliberately don't test PLIO compression (which is designed for
546 masks) because our default mask type (32) has too much dynamic range
547 for PLIO (limit of 24 bits).
548 """
549 for algorithm in ("GZIP", "GZIP_SHUFFLE", "RICE"):
550 compression = ImageCompressionOptions(lsst.afw.fits.compressionAlgorithmFromString(algorithm))
551 mask = self.makeMask()
552 unpersisted = self.checkCompressedImage(lsst.afw.image.Mask, mask, compression, atol=0.0)
553 for mp in mask.getMaskPlaneDict():
554 self.assertIn(mp, unpersisted.getMaskPlaneDict())
555 unpersisted.getPlaneBitMask(mp)
557 def testLossyFloatCfitsio(self):
558 """Test lossy compresion of floating-point images with cfitsio
560 cfitsio does the compression, controlled through the 'quantizeLevel'
561 parameter. Note that cfitsio doesn't have access to our masks when
562 it does its statistics.
563 """
564 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD)
565 algorithmList = ("GZIP", "GZIP_SHUFFLE", "RICE")
566 quantizeList = (4.0, 10.0)
567 for cls, algorithm, quantizeLevel in itertools.product(classList, algorithmList, quantizeList):
568 compression = ImageCompressionOptions(lsst.afw.fits.compressionAlgorithmFromString(algorithm),
569 quantizeLevel=quantizeLevel)
570 image = self.makeImage(cls)
571 self.checkCompressedImage(cls, image, compression, atol=self.noise/quantizeLevel)
573 def testLossyFloatOurs(self):
574 """Test lossy compression of floating-point images ourselves
576 We do lossy compression by scaling first. We have full control over
577 the scaling (multiple scaling algorithms), and we have access to our
578 own masks when we do statistics.
579 """
580 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD)
581 algorithmList = ("GZIP", "GZIP_SHUFFLE", "RICE")
582 bitpixList = (16, 32)
583 quantizeList = (4.0, 10.0)
584 for cls, algorithm, bitpix, quantize in itertools.product(classList, algorithmList, bitpixList,
585 quantizeList):
586 compression = ImageCompressionOptions(lsst.afw.fits.compressionAlgorithmFromString(algorithm),
587 quantizeLevel=0.0)
588 scaling = ImageScalingOptions(ImageScalingOptions.STDEV_BOTH, bitpix, quantizeLevel=quantize,
589 fuzz=True)
590 image = self.makeImage(cls)
591 self.checkCompressedImage(cls, image, compression, scaling, atol=self.noise/quantize)
593 def readWriteMaskedImage(self, image, filename, imageOptions, maskOptions, varianceOptions):
594 """Read the MaskedImage after it has been written
596 This implementation does the persistence using methods on the
597 MaskedImage class.
599 Parameters
600 ----------
601 image : `lsst.afw.image.Image`
602 Image to compress.
603 filename : `str`
604 Filename to which to write.
605 imageOptions, maskOptions, varianceOptions : `lsst.afw.fits.ImageWriteOptions`
606 Options for writing the image, mask and variance planes.
607 """
608 image.writeFits(filename, imageOptions, maskOptions, varianceOptions)
609 if hasattr(image, "getMaskedImage"):
610 image = image.getMaskedImage()
611 return lsst.afw.image.MaskedImageF(filename)
613 def checkCompressedMaskedImage(self, image, imageOptions, maskOptions, varianceOptions, atol=0.0):
614 """Check that compression works on a MaskedImage
616 Parameters
617 ----------
618 image : `lsst.afw.image.MaskedImage` or `lsst.afw.image.Exposure`
619 MaskedImage or exposure to compress.
620 imageOptions, maskOptions, varianceOptions : `lsst.afw.fits.ImageWriteOptions`
621 Parameters for writing (compression and scaling) the image, mask
622 and variance planes.
623 atol : `float`
624 Absolute tolerance for comparing unpersisted image.
625 """
626 with lsst.utils.tests.getTempFilePath(self.extension) as filename:
627 self.readWriteMaskedImage(image, filename, imageOptions, maskOptions, varianceOptions)
628 unpersisted = type(image)(filename)
629 if hasattr(image, "getMaskedImage"):
630 image = image.getMaskedImage()
631 unpersisted = unpersisted.getMaskedImage()
632 self.assertEqual(image.getBBox(), unpersisted.getBBox())
633 self.assertImagesAlmostEqual(unpersisted.getImage(), image.getImage(), atol=atol)
634 self.assertImagesAlmostEqual(unpersisted.getMask(), image.getMask(), atol=atol)
635 self.assertImagesAlmostEqual(unpersisted.getVariance(), image.getVariance(), atol=atol)
637 for mp in image.getMask().getMaskPlaneDict():
638 self.assertIn(mp, unpersisted.getMask().getMaskPlaneDict())
639 unpersisted.getMask().getPlaneBitMask(mp)
641 def checkMaskedImage(self, imageOptions, maskOptions, varianceOptions, atol=0.0):
642 """Check that we can compress a MaskedImage and Exposure
644 Parameters
645 ----------
646 imageOptions, maskOptions, varianceOptions : `lsst.afw.fits.ImageWriteOptions`
647 Parameters for writing (compression and scaling) the image, mask
648 and variance planes.
649 atol : `float`
650 Absolute tolerance for comparing unpersisted image.
651 """
652 image = lsst.afw.image.makeMaskedImage(self.makeImage(lsst.afw.image.ImageF),
653 self.makeMask(), self.makeImage(lsst.afw.image.ImageF))
654 self.checkCompressedMaskedImage(image, imageOptions, maskOptions, varianceOptions, atol=atol)
655 exp = lsst.afw.image.makeExposure(image)
656 self.checkCompressedMaskedImage(exp, imageOptions, maskOptions, varianceOptions, atol=atol)
658 def testMaskedImage(self):
659 """Test compression of MaskedImage
661 We test lossless, lossy cfitsio and lossy LSST compression.
662 """
663 # Lossless
664 lossless = lsst.afw.fits.ImageCompressionOptions(ImageCompressionOptions.GZIP_SHUFFLE)
665 options = lsst.afw.fits.ImageWriteOptions(lossless)
666 self.checkMaskedImage(options, options, options, atol=0.0)
668 # Lossy cfitsio compression
669 quantize = 4.0
670 cfitsio = lsst.afw.fits.ImageCompressionOptions(ImageCompressionOptions.GZIP_SHUFFLE, True, quantize)
671 imageOptions = lsst.afw.fits.ImageWriteOptions(cfitsio)
672 maskOptions = lsst.afw.fits.ImageWriteOptions(lossless)
673 self.checkMaskedImage(imageOptions, maskOptions, imageOptions, atol=self.noise/quantize)
675 # Lossy our compression
676 quantize = 10.0
677 compression = lsst.afw.fits.ImageCompressionOptions(ImageCompressionOptions.RICE, True, 0.0)
678 scaling = lsst.afw.fits.ImageScalingOptions(ImageScalingOptions.STDEV_BOTH, 32,
679 quantizeLevel=quantize)
680 imageOptions = lsst.afw.fits.ImageWriteOptions(compression, scaling)
681 maskOptions = lsst.afw.fits.ImageWriteOptions(compression)
682 self.checkMaskedImage(imageOptions, maskOptions, imageOptions, atol=self.noise/quantize)
684 def testQuantization(self):
685 """Test that our quantization produces the same values as cfitsio
687 Our quantization is more configurable (e.g., choice of scaling algorithm,
688 specifying mask planes) and extensible (logarithmic, asinh scalings)
689 than cfitsio's. However, cfitsio uses its own fuzz ("subtractive dithering")
690 when reading the data, so if we don't want to add random values twice,
691 we need to be sure that we're using the same random values. To check that,
692 we write one image with our scaling+compression, and one with cfitsio's
693 compression using exactly the BSCALE and dither seed we used for our own.
694 That way, the two codes will quantize independently, and we can compare
695 the results.
696 """
697 bscaleSet = 1.0
698 bzeroSet = self.background - 10*self.noise
699 algorithm = ImageCompressionOptions.GZIP
700 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD)
701 tilesList = ((4, 5), (0, 0), (0, 5), (4, 0), (0, 1))
702 for cls, tiles in itertools.product(classList, tilesList):
703 tiles = np.array(tiles, dtype=np.int64)
704 compression = ImageCompressionOptions(algorithm, tiles, -bscaleSet)
705 original = self.makeImage(cls)
706 with lsst.utils.tests.getTempFilePath(self.extension) as filename:
707 with lsst.afw.fits.Fits(filename, "w") as fits:
708 options = lsst.afw.fits.ImageWriteOptions(compression)
709 original.writeFits(fits, options)
710 cfitsio = cls(filename)
711 header = lsst.afw.fits.readMetadata(filename, 1)
712 seed = header.getScalar("ZDITHER0")
713 self.assertEqual(header.getScalar("BSCALE"), bscaleSet)
715 compression = ImageCompressionOptions(algorithm, tiles, 0.0)
716 scaling = ImageScalingOptions(ImageScalingOptions.MANUAL, 32, [u"BAD"], bscale=bscaleSet,
717 bzero=bzeroSet, fuzz=True, seed=seed)
718 unpersisted = self.checkCompressedImage(cls, original, compression, scaling, atol=bscaleSet)
719 oursDiff = unpersisted.getArray() - original.getArray()
720 cfitsioDiff = cfitsio.getArray() - original.getArray()
721 self.assertImagesAlmostEqual(oursDiff, cfitsioDiff, atol=0.0)
724def optionsToPropertySet(options):
725 """Convert the ImageWriteOptions to a PropertySet
727 This allows us to pass the options into the persistence framework
728 as the "additionalData".
729 """
730 ps = lsst.daf.base.PropertySet()
731 ps.set("compression.algorithm", lsst.afw.fits.compressionAlgorithmToString(options.compression.algorithm))
732 ps.set("compression.columns", options.compression.tiles[0])
733 ps.set("compression.rows", options.compression.tiles[1])
734 ps.set("compression.quantizeLevel", options.compression.quantizeLevel)
736 ps.set("scaling.algorithm", lsst.afw.fits.scalingAlgorithmToString(options.scaling.algorithm))
737 ps.set("scaling.bitpix", options.scaling.bitpix)
738 ps.setString("scaling.maskPlanes", options.scaling.maskPlanes)
739 ps.set("scaling.fuzz", options.scaling.fuzz)
740 ps.set("scaling.seed", options.scaling.seed)
741 ps.set("scaling.quantizeLevel", options.scaling.quantizeLevel)
742 ps.set("scaling.quantizePad", options.scaling.quantizePad)
743 ps.set("scaling.bscale", options.scaling.bscale)
744 ps.set("scaling.bzero", options.scaling.bzero)
745 return ps
748def persistUnpersist(ImageClass, image, filename, additionalData):
749 """Use read/writeFitsWithOptions to persist and unpersist an image
751 Parameters
752 ----------
753 ImageClass : `type`, an `lsst.afw.image.Image` class
754 Class of image.
755 image : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage`
756 Image to compress.
757 filename : `str`
758 Filename to write.
759 additionalData : `lsst.daf.base.PropertySet`
760 Additional data for persistence framework.
762 Returns
763 -------
764 unpersisted : `ImageClass`
765 The unpersisted image.
766 """
767 additionalData.set("visit", 12345)
768 additionalData.set("ccd", 67)
770 image.writeFitsWithOptions(filename, additionalData)
771 return ImageClass.readFits(filename)
774class PersistenceTestCase(ImageCompressionTestCase):
775 """Test compression using the persistence framework
777 We override the I/O methods to use the persistence framework.
778 """
779 def testQuantization(self):
780 """Not appropriate --- disable"""
781 pass
783 def readWriteImage(self, ImageClass, image, filename, options):
784 """Read the image after it has been written
786 This implementation uses the persistence framework.
788 Parameters
789 ----------
790 ImageClass : `type`, an `lsst.afw.image.Image` class
791 Class of image to create.
792 image : `lsst.afw.image.Image`
793 Image to compress.
794 filename : `str`
795 Filename to which to write.
796 options : `lsst.afw.fits.ImageWriteOptions`
797 Options for writing.
798 """
799 additionalData = lsst.daf.base.PropertySet()
800 additionalData.set("image", optionsToPropertySet(options))
801 return persistUnpersist(ImageClass, image, filename, additionalData)
803 def readWriteMaskedImage(self, image, filename, imageOptions, maskOptions, varianceOptions):
804 """Read the MaskedImage after it has been written
806 This implementation uses the persistence framework.
808 Parameters
809 ----------
810 image : `lsst.afw.image.MaskedImage`
811 Image to compress.
812 filename : `str`
813 Filename to which to write.
814 imageOptions, maskOptions, varianceOptions : `lsst.afw.fits.ImageWriteOptions`
815 Options for writing the image, mask and variance planes.
816 """
817 additionalData = lsst.daf.base.PropertySet()
818 additionalData.set("image", optionsToPropertySet(imageOptions))
819 additionalData.set("mask", optionsToPropertySet(maskOptions))
820 additionalData.set("variance", optionsToPropertySet(varianceOptions))
821 return persistUnpersist(lsst.afw.image.MaskedImageF, image, filename, additionalData)
824class EmptyExposureTestCase(lsst.utils.tests.TestCase):
825 """Test that an empty image can be written
827 We sometimes use an empty lsst.afw.image.Exposure as a vehicle for
828 persisting other things, e.g., Wcs, Calib. cfitsio compression will
829 choke on an empty image, so make sure we're dealing with that.
830 """
831 def checkEmptyExposure(self, algorithm):
832 """Check that we can persist an empty Exposure
834 Parameters
835 ----------
836 algorithm : `lsst.afw.fits.ImageCompressionOptions.CompressionAlgorithm`
837 Compression algorithm to try.
838 """
839 exp = lsst.afw.image.ExposureF(0, 0)
840 degrees = lsst.geom.degrees
841 cdMatrix = np.array([[1.0e-4, 0.0], [0.0, 1.0e-4]], dtype=float)
842 exp.setWcs(lsst.afw.geom.makeSkyWcs(crval=lsst.geom.SpherePoint(0*degrees, 0*degrees),
843 crpix=lsst.geom.Point2D(0.0, 0.0),
844 cdMatrix=cdMatrix))
845 imageOptions = lsst.afw.fits.ImageWriteOptions(ImageCompressionOptions(algorithm))
846 maskOptions = lsst.afw.fits.ImageWriteOptions(exp.getMaskedImage().getMask())
847 varianceOptions = lsst.afw.fits.ImageWriteOptions(ImageCompressionOptions(algorithm))
848 with lsst.utils.tests.getTempFilePath(".fits") as filename:
849 exp.writeFits(filename, imageOptions, maskOptions, varianceOptions)
850 unpersisted = type(exp)(filename)
851 self.assertEqual(unpersisted.getMaskedImage().getDimensions(), lsst.geom.Extent2I(0, 0))
852 self.assertEqual(unpersisted.getWcs(), exp.getWcs())
854 def testEmptyExposure(self):
855 """Persist an empty Exposure with compression"""
856 algorithmList = ("GZIP", "GZIP_SHUFFLE", "RICE")
857 for algorithm in algorithmList:
858 self.checkEmptyExposure(lsst.afw.fits.compressionAlgorithmFromString(algorithm))
861class TestMemory(lsst.utils.tests.MemoryTestCase):
862 pass
865def setup_module(module):
866 lsst.utils.tests.init()
869if __name__ == "__main__": 869 ↛ 870line 869 didn't jump to line 870, because the condition on line 869 was never true
870 import sys
871 setup_module(sys.modules[__name__])
872 unittest.main()