Coverage for tests/test_fitsCompression.py: 16%
366 statements
« prev ^ index » next coverage.py v6.4, created at 2022-06-02 03:42 -0700
« prev ^ index » next coverage.py v6.4, created at 2022-06-02 03:42 -0700
1#
2# LSST Data Management System
3# Copyright 2017 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
23import os
24import unittest
25import itertools
27import numpy as np
28import astropy.io.fits
30import lsst.utils
31import lsst.daf.base
32import lsst.geom
33import lsst.afw.geom
34import lsst.afw.image
35import lsst.afw.fits
36import lsst.utils.tests
37from lsst.afw.image import LOCAL
38from lsst.afw.fits import ImageScalingOptions, ImageCompressionOptions
41def checkAstropy(image, filename, hduNum=0):
42 """Check that astropy can read our file
44 We don't insist on equality for low BITPIX (8, 16) when the original
45 type is double-precision: astropy will (quite reasonably) read that
46 into a single-precision image and apply bscale,bzero to that so it's
47 going to be subject to roundoff.
49 Parameters
50 ----------
51 image : `lsst.afw.image.Image`
52 Image read by our own code.
53 filename : `str`
54 Filename of FITS file to read with astropy.
55 hduNum : `int`
56 HDU number of interest.
57 """
58 print("Astropy currently doesn't read our compressed images perfectly.")
59 return
61 def parseVersion(version):
62 return tuple(int(vv) for vv in np.array(version.split(".")))
64 if parseVersion(astropy.__version__) <= parseVersion("2.0.1"):
65 # astropy 2.0.1 and earlier have problems:
66 # * Doesn't support GZIP_2: https://github.com/astropy/astropy/pull/6486
67 # * Uses the wrong array type: https://github.com/astropy/astropy/pull/6492
68 print(f"Refusing to check with astropy version {astropy.__version__} due to astropy bugs")
69 return
70 hdu = astropy.io.fits.open(filename)[hduNum]
71 if hdu.header["BITPIX"] in (8, 16) and isinstance(image, lsst.afw.image.ImageD):
72 return
73 dtype = image.getArray().dtype
74 theirs = hdu.data.astype(dtype)
75 # Allow for minor differences due to arithmetic: +/- 1 in the last place
76 np.testing.assert_array_max_ulp(theirs, image.getArray())
79class ImageScalingTestCase(lsst.utils.tests.TestCase):
80 """Tests of image scaling
82 The pattern here is to create an image, write it out with a
83 specific scaling algorithm, read it back in and test that everything
84 is as we expect. We do this for each scaling algorithm in its own
85 test, and within that test iterate over various parameters (input
86 image type, BITPIX, etc.). The image we create has a few features
87 (low, high and masked pixels) that we check.
88 """
89 def setUp(self):
90 self.bbox = lsst.geom.Box2I(lsst.geom.Point2I(123, 456), lsst.geom.Extent2I(7, 8))
91 self.base = 456 # Base value for pixels
92 self.highValue = 789 # Value for high pixel
93 self.lowValue = 123 # Value for low pixel
94 self.maskedValue = 12345 # Value for masked pixel (to throw off statistics)
95 self.highPixel = lsst.geom.Point2I(1, 1) # Location of high pixel
96 self.lowPixel = lsst.geom.Point2I(2, 2) # Location of low pixel
97 self.maskedPixel = lsst.geom.Point2I(3, 3) # Location of masked pixel
98 self.badMask = "BAD" # Mask plane to set for masked pixel
99 self.stdev = 5.0 # Noise stdev to add to image
101 def makeImage(self, ImageClass, scaling, addNoise=True):
102 """Make an image for testing
104 We create an image, persist and unpersist it, returning
105 some data to the caller.
107 Parameters
108 ----------
109 ImageClass : `type`, an `lsst.afw.image.Image` class
110 Class of image to create.
111 scaling : `lsst.afw.fits.ImageScalingOptions`
112 Scaling to apply during persistence.
113 addNoise : `bool`
114 Add noise to image?
116 Returns
117 -------
118 image : `lsst.afw.image.Image` (ImageClass)
119 Created image.
120 unpersisted : `lsst.afw.image.Image` (ImageClass)
121 Unpersisted image.
122 bscale, bzero : `float`
123 FITS scale factor and zero used.
124 minValue, maxValue : `float`
125 Minimum and maximum value given the nominated scaling.
126 """
127 image = ImageClass(self.bbox)
128 mask = lsst.afw.image.Mask(self.bbox)
129 mask.addMaskPlane(self.badMask)
130 bad = mask.getPlaneBitMask(self.badMask)
131 image.set(self.base)
132 image[self.highPixel, LOCAL] = self.highValue
133 image[self.lowPixel, LOCAL] = self.lowValue
134 image[self.maskedPixel, LOCAL] = self.maskedValue
135 mask[self.maskedPixel, LOCAL] = bad
137 rng = np.random.RandomState(12345)
138 dtype = image.getArray().dtype
139 if addNoise:
140 image.getArray()[:] += rng.normal(0.0, self.stdev, image.getArray().shape).astype(dtype)
142 with lsst.utils.tests.getTempFilePath(".fits") as filename:
143 with lsst.afw.fits.Fits(filename, "w") as fits:
144 options = lsst.afw.fits.ImageWriteOptions(scaling)
145 header = lsst.daf.base.PropertyList()
146 image.writeFits(fits, options, header, mask)
147 unpersisted = ImageClass(filename)
148 self.assertEqual(image.getBBox(), unpersisted.getBBox())
150 header = lsst.afw.fits.readMetadata(filename)
151 bscale = header.getScalar("BSCALE")
152 bzero = header.getScalar("BZERO")
154 if scaling.algorithm != ImageScalingOptions.NONE:
155 self.assertEqual(header.getScalar("BITPIX"), scaling.bitpix)
157 if scaling.bitpix == 8: # unsigned, says FITS
158 maxValue = bscale*(2**scaling.bitpix - 1) + bzero
159 minValue = bzero
160 else:
161 maxValue = bscale*(2**(scaling.bitpix - 1) - 1) + bzero
162 if scaling.bitpix == 32:
163 # cfitsio pads 10 values, and so do we
164 minValue = -bscale*(2**(scaling.bitpix - 1) - 10) + bzero
165 else:
166 minValue = -bscale*(2**(scaling.bitpix - 1)) + bzero
168 # Convert scalars to the appropriate type
169 maxValue = np.array(maxValue, dtype=image.getArray().dtype)
170 minValue = np.array(minValue, dtype=image.getArray().dtype)
172 checkAstropy(unpersisted, filename)
174 return image, unpersisted, bscale, bzero, minValue, maxValue
176 def checkPixel(self, unpersisted, original, xy, expected, rtol=None, atol=None):
177 """Check one of the special pixels
179 After checking, we set this pixel to the original value so
180 it's then easy to compare the entire image.
182 Parameters
183 ----------
184 unpersisted : `lsst.afw.image.Image`
185 Unpersisted image.
186 original : `lsst.afw.image.Image`
187 Original image.
188 xy : `tuple` of two `int`s
189 Position of pixel to check.
190 expected : scalar
191 Expected value of pixel.
192 rtol, atol : `float` or `None`
193 Relative/absolute tolerance for comparison.
194 """
195 if np.isnan(expected):
196 self.assertTrue(np.isnan(unpersisted[xy, LOCAL]))
197 else:
198 self.assertFloatsAlmostEqual(unpersisted[xy, LOCAL], expected, rtol=rtol, atol=atol)
199 unpersisted[xy, LOCAL] = original[xy, LOCAL] # for ease of comparison of the whole image
201 def checkSpecialPixels(self, original, unpersisted, maxValue, minValue, rtol=None, atol=None):
202 """Check the special pixels
204 Parameters
205 ----------
206 original : `lsst.afw.image.Image`
207 Original image.
208 unpersisted : `lsst.afw.image.Image`
209 Unpersisted image.
210 minValue, maxValue : `float`
211 Minimum and maximum value given the nominated scaling.
212 rtol, atol : `float` or `None`
213 Relative/absolute tolerance for comparison.
214 """
215 highValue = original[self.highPixel, LOCAL]
216 lowValue = original[self.lowPixel, LOCAL]
217 maskedValue = original[self.maskedPixel, LOCAL]
219 expectHigh = min(highValue, maxValue)
220 expectLow = max(lowValue, minValue)
221 expectMasked = min(maskedValue, maxValue)
223 if unpersisted.getArray().dtype in (np.float32, np.float64):
224 if highValue >= maxValue:
225 expectHigh = np.nan
226 if maskedValue >= maxValue:
227 expectMasked = np.nan
229 self.checkPixel(unpersisted, original, self.highPixel, expectHigh, rtol=rtol, atol=atol)
230 self.checkPixel(unpersisted, original, self.lowPixel, expectLow, rtol=rtol, atol=atol)
231 self.checkPixel(unpersisted, original, self.maskedPixel, expectMasked, rtol=rtol, atol=atol)
233 def checkRange(self, ImageClass, bitpix):
234 """Check that the RANGE scaling works
236 Parameters
237 ----------
238 ImageClass : `type`, an `lsst.afw.image.Image` class
239 Class of image to create.
240 bitpix : `int`
241 Bits per pixel for FITS image.
242 """
243 scaling = ImageScalingOptions(ImageScalingOptions.RANGE, bitpix, [u"BAD"], fuzz=False)
244 original, unpersisted, bscale, bzero, minValue, maxValue = self.makeImage(ImageClass, scaling, False)
246 numValues = 2**bitpix - 1
247 numValues -= 2 # Padding on either end
248 if bitpix == 32:
249 numValues -= 10
250 bscaleExpect = (self.highValue - self.lowValue)/numValues
251 self.assertFloatsAlmostEqual(bscale, bscaleExpect, atol=1.0e-6) # F32 resolution
253 rtol = 1.0/2**(bitpix - 1)
254 self.checkSpecialPixels(original, unpersisted, maxValue, minValue, atol=bscale)
255 self.assertImagesAlmostEqual(original, unpersisted, rtol=rtol)
257 def checkStdev(self, ImageClass, bitpix, algorithm, quantizeLevel, quantizePad):
258 """Check that one of the STDEV scaling algorithms work
260 Parameters
261 ----------
262 ImageClass : `type`, an `lsst.afw.image.Image` class
263 Class of image to create.
264 bitpix : `int`
265 Bits per pixel for FITS image.
266 algorithm : `lsst.afw.fits.ImageScalingOptions.ScalingAlgorithm`
267 Scaling algorithm to apply (one of the STDEV_*).
268 quantizeLevel : `float`
269 Quantization level.
270 quantizePad : `float`
271 Quantization padding.
272 """
273 scaling = lsst.afw.fits.ImageScalingOptions(algorithm, bitpix, [u"BAD"], fuzz=False,
274 quantizeLevel=quantizeLevel, quantizePad=quantizePad)
276 makeImageResults = self.makeImage(ImageClass, scaling)
277 original, unpersisted, bscale, bzero, minValue, maxValue = makeImageResults
279 self.assertFloatsAlmostEqual(bscale, self.stdev/quantizeLevel, rtol=3.0/quantizeLevel)
280 self.checkSpecialPixels(original, unpersisted, maxValue, minValue, atol=bscale)
281 self.assertImagesAlmostEqual(original, unpersisted, atol=bscale)
283 def testRange(self):
284 """Test that the RANGE scaling works on floating-point inputs
286 We deliberately don't include BITPIX=64 because int64 provides
287 a larger dynamic range than 'double BSCALE' can handle.
288 """
289 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD)
290 bitpixList = (8, 16, 32)
291 for cls, bitpix in itertools.product(classList, bitpixList):
292 self.checkRange(cls, bitpix)
294 def testStdev(self):
295 """Test that the STDEV scalings work on floating-point inputs
297 We deliberately don't include BITPIX=64 because int64 provides
298 a larger dynamic range than 'double BSCALE' can handle.
300 We deliberately don't include BITPIX=8 because that provides
301 only a tiny dynamic range where everything goes out of range easily.
302 """
303 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD)
304 bitpixList = (16, 32)
305 algorithmList = (ImageScalingOptions.STDEV_POSITIVE, ImageScalingOptions.STDEV_NEGATIVE,
306 ImageScalingOptions.STDEV_BOTH)
307 quantizeLevelList = (2.0, 10.0, 100.0)
308 quantizePadList = (5.0, 10.0, 100.0)
309 for values in itertools.product(classList, bitpixList, algorithmList,
310 quantizeLevelList, quantizePadList):
311 self.checkStdev(*values)
313 def testRangeFailures(self):
314 """Test that the RANGE scaling fails on integer inputs"""
315 classList = (lsst.afw.image.ImageU, lsst.afw.image.ImageI, lsst.afw.image.ImageL)
316 bitpixList = (8, 16, 32)
317 for cls, bitpix in itertools.product(classList, bitpixList):
318 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError):
319 self.checkRange(cls, bitpix)
321 def testStdevFailures(self):
322 """Test that the STDEV scalings fail on integer inputs"""
323 classList = (lsst.afw.image.ImageU, lsst.afw.image.ImageI, lsst.afw.image.ImageL)
324 bitpixList = (16, 32)
325 algorithmList = (ImageScalingOptions.STDEV_POSITIVE, ImageScalingOptions.STDEV_NEGATIVE,
326 ImageScalingOptions.STDEV_BOTH)
327 for cls, bitpix, algorithm in itertools.product(classList, bitpixList, algorithmList):
328 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError):
329 self.checkStdev(cls, bitpix, algorithm, 10.0, 10.0)
331 def checkNone(self, ImageClass, bitpix):
332 """Check that the NONE scaling algorithm works
334 Parameters
335 ----------
336 ImageClass : `type`, an `lsst.afw.image.Image` class
337 Class of image to create.
338 bitpix : `int`
339 Bits per pixel for FITS image.
340 """
341 scaling = ImageScalingOptions(ImageScalingOptions.NONE, bitpix, [u"BAD"], fuzz=False)
342 original, unpersisted, bscale, bzero, minValue, maxValue = self.makeImage(ImageClass, scaling)
343 self.assertFloatsAlmostEqual(bscale, 1.0, atol=0.0)
344 self.assertFloatsAlmostEqual(bzero, 0.0, atol=0.0)
345 self.assertImagesAlmostEqual(original, unpersisted, atol=0.0)
347 def testNone(self):
348 """Test that the NONE scaling works on floating-point inputs"""
349 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD)
350 bitpixList = (8, 16, 32)
351 for cls, bitpix in itertools.product(classList, bitpixList):
352 self.checkNone(cls, bitpix)
354 def checkManual(self, ImageClass, bitpix):
355 """Check that the MANUAL scaling algorithm works
357 Parameters
358 ----------
359 ImageClass : `type`, an `lsst.afw.image.Image` class
360 Class of image to create.
361 bitpix : `int`
362 Bits per pixel for FITS image.
363 """
364 bscaleSet = 1.2345
365 bzeroSet = self.base
366 scaling = ImageScalingOptions(ImageScalingOptions.MANUAL, bitpix, [u"BAD"], bscale=bscaleSet,
367 bzero=bzeroSet, fuzz=False)
368 original, unpersisted, bscale, bzero, minValue, maxValue = self.makeImage(ImageClass, scaling)
369 self.assertFloatsAlmostEqual(bscale, bscaleSet, atol=0.0)
370 self.assertFloatsAlmostEqual(bzero, bzeroSet, atol=0.0)
371 self.assertImagesAlmostEqual(original, unpersisted, atol=bscale)
373 def testManual(self):
374 """Test that the MANUAL scaling works on floating-point inputs"""
375 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD)
376 bitpixList = (16, 32)
377 for cls, bitpix in itertools.product(classList, bitpixList):
378 self.checkNone(cls, bitpix)
381class ImageCompressionTestCase(lsst.utils.tests.TestCase):
382 """Tests of image compression
384 We test compression both with and without loss (quantisation/scaling).
386 The pattern here is to create an image, write it out with a
387 specific compression algorithm, read it back in and test that everything
388 is as we expect. We do this for each compression algorithm in its own
389 test, and within that test iterate over various parameters (input
390 image type, BITPIX, etc.).
392 We print the (inverse) compression ratio for interest. Note that
393 these should not be considered to be representative of the
394 compression that will be achieved on scientific data, since the
395 images created here have different qualities than scientific data
396 that will affect the compression ratio (e.g., size, noise properties).
397 """
398 def setUp(self):
399 self.bbox = lsst.geom.Box2I(lsst.geom.Point2I(123, 456), lsst.geom.Extent2I(7, 8))
400 self.background = 12345.6789 # Background value
401 self.noise = 67.89 # Noise (stdev)
402 self.maskPlanes = ["FOO", "BAR"] # Mask planes to add
403 self.extension = "." + self.__class__.__name__ + ".fits" # extension name for temp files
405 def readWriteImage(self, ImageClass, image, filename, options, *args):
406 """Read the image after it has been written
408 This implementation does the persistence using methods on the
409 ImageClass.
411 Parameters
412 ----------
413 ImageClass : `type`, an `lsst.afw.image.Image` class
414 Class of image to create.
415 image : `lsst.afw.image.Image`
416 Image to compress.
417 filename : `str`
418 Filename to which to write.
419 options : `lsst.afw.fits.ImageWriteOptions`
420 Options for writing.
421 """
422 image.writeFits(filename, options, *args)
423 return ImageClass(filename)
425 def makeImage(self, ImageClass):
426 """Create an image
428 Parameters
429 ----------
430 ImageClass : `type`, an `lsst.afw.image.Image` class
431 Class of image to create.
433 Returns
434 -------
435 image : `ImageClass`
436 The created image.
437 """
438 image = ImageClass(self.bbox)
439 rng = np.random.RandomState(12345)
440 dtype = image.getArray().dtype
441 noise = rng.normal(0.0, self.noise, image.getArray().shape).astype(dtype)
442 image.getArray()[:] = np.array(self.background, dtype=dtype) + noise
443 return image
445 def makeMask(self):
446 """Create a mask
448 Note that we generate a random distribution of mask pixel values,
449 which is very different from the usual distribution in science images.
451 Returns
452 -------
453 mask : `lsst.afw.image.Mask`
454 The created mask.
455 """
456 mask = lsst.afw.image.Mask(self.bbox)
457 rng = np.random.RandomState(12345)
458 dtype = mask.getArray().dtype
459 mask.getArray()[:] = rng.randint(0, 2**(dtype.itemsize*8 - 1), mask.getArray().shape, dtype=dtype)
460 for plane in self.maskPlanes:
461 mask.addMaskPlane(plane)
462 return mask
464 def checkCompressedImage(self, ImageClass, image, compression, scaling=None, atol=0.0):
465 """Check that compression works on an image
467 Parameters
468 ----------
469 ImageClass : `type`, an `lsst.afw.image.Image` class
470 Class of image.
471 image : `lsst.afw.image.Image`
472 Image to compress.
473 compression : `lsst.afw.fits.ImageCompressionOptions`
474 Compression parameters.
475 scaling : `lsst.afw.fits.ImageScalingOptions` or `None`
476 Scaling parameters for lossy compression (optional).
477 atol : `float`
478 Absolute tolerance for comparing unpersisted image.
480 Returns
481 -------
482 unpersisted : `ImageClass`
483 The unpersisted image.
484 """
485 with lsst.utils.tests.getTempFilePath(self.extension) as filename:
486 if scaling:
487 options = lsst.afw.fits.ImageWriteOptions(compression, scaling)
488 else:
489 options = lsst.afw.fits.ImageWriteOptions(compression)
490 unpersisted = self.readWriteImage(ImageClass, image, filename, options)
492 fileSize = os.stat(filename).st_size
493 fitsBlockSize = 2880 # All sizes in FITS are a multiple of this
494 numBlocks = 1 + np.ceil(self.bbox.getArea()*image.getArray().dtype.itemsize/fitsBlockSize)
495 uncompressedSize = fitsBlockSize*numBlocks
496 print(ImageClass, compression.algorithm, fileSize, uncompressedSize, fileSize/uncompressedSize)
498 self.assertEqual(image.getBBox(), unpersisted.getBBox())
499 self.assertImagesAlmostEqual(unpersisted, image, atol=atol)
501 checkAstropy(unpersisted, filename, 1)
503 return unpersisted
505 def testLosslessFloat(self):
506 """Test lossless compression of floating-point image"""
507 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD)
508 algorithmList = ("GZIP", "GZIP_SHUFFLE") # Lossless float compression requires GZIP
509 for cls, algorithm in itertools.product(classList, algorithmList):
510 image = self.makeImage(cls)
511 compression = ImageCompressionOptions(lsst.afw.fits.compressionAlgorithmFromString(algorithm))
512 self.checkCompressedImage(cls, image, compression, atol=0.0)
514 def testLosslessInt(self):
515 """Test lossless compression of integer image
517 We deliberately don't test `lsst.afw.image.ImageL` because
518 compression of LONGLONG images is unsupported by cfitsio.
519 """
520 classList = (lsst.afw.image.ImageU, lsst.afw.image.ImageI)
521 algorithmList = ("GZIP", "GZIP_SHUFFLE", "RICE")
522 for cls, algorithm in itertools.product(classList, algorithmList):
523 compression = ImageCompressionOptions(lsst.afw.fits.compressionAlgorithmFromString(algorithm))
524 image = self.makeImage(cls)
525 self.checkCompressedImage(cls, image, compression, atol=0.0)
527 def testLongLong(self):
528 """Test graceful failure when compressing ImageL
530 We deliberately don't test `lsst.afw.image.ImageL` because
531 compression of LONGLONG images is unsupported by cfitsio.
532 """
533 algorithmList = ("GZIP", "GZIP_SHUFFLE", "RICE")
534 for algorithm in algorithmList:
535 compression = ImageCompressionOptions(lsst.afw.fits.compressionAlgorithmFromString(algorithm))
536 cls = lsst.afw.image.ImageL
537 image = self.makeImage(cls)
538 with self.assertRaises(lsst.afw.fits.FitsError):
539 self.checkCompressedImage(cls, image, compression)
541 def testMask(self):
542 """Test compression of mask
544 We deliberately don't test PLIO compression (which is designed for
545 masks) because our default mask type (32) has too much dynamic range
546 for PLIO (limit of 24 bits).
547 """
548 for algorithm in ("GZIP", "GZIP_SHUFFLE", "RICE"):
549 compression = ImageCompressionOptions(lsst.afw.fits.compressionAlgorithmFromString(algorithm))
550 mask = self.makeMask()
551 unpersisted = self.checkCompressedImage(lsst.afw.image.Mask, mask, compression, atol=0.0)
552 for mp in mask.getMaskPlaneDict():
553 self.assertIn(mp, unpersisted.getMaskPlaneDict())
554 unpersisted.getPlaneBitMask(mp)
556 def testLossyFloatCfitsio(self):
557 """Test lossy compresion of floating-point images with cfitsio
559 cfitsio does the compression, controlled through the 'quantizeLevel'
560 parameter. Note that cfitsio doesn't have access to our masks when
561 it does its statistics.
562 """
563 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD)
564 algorithmList = ("GZIP", "GZIP_SHUFFLE", "RICE")
565 quantizeList = (4.0, 10.0)
566 for cls, algorithm, quantizeLevel in itertools.product(classList, algorithmList, quantizeList):
567 compression = ImageCompressionOptions(lsst.afw.fits.compressionAlgorithmFromString(algorithm),
568 quantizeLevel=quantizeLevel)
569 image = self.makeImage(cls)
570 self.checkCompressedImage(cls, image, compression, atol=self.noise/quantizeLevel)
572 def testLossyFloatOurs(self):
573 """Test lossy compression of floating-point images ourselves
575 We do lossy compression by scaling first. We have full control over
576 the scaling (multiple scaling algorithms), and we have access to our
577 own masks when we do statistics.
578 """
579 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD)
580 algorithmList = ("GZIP", "GZIP_SHUFFLE", "RICE")
581 bitpixList = (16, 32)
582 quantizeList = (4.0, 10.0)
583 for cls, algorithm, bitpix, quantize in itertools.product(classList, algorithmList, bitpixList,
584 quantizeList):
585 compression = ImageCompressionOptions(lsst.afw.fits.compressionAlgorithmFromString(algorithm),
586 quantizeLevel=0.0)
587 scaling = ImageScalingOptions(ImageScalingOptions.STDEV_BOTH, bitpix, quantizeLevel=quantize,
588 fuzz=True)
589 image = self.makeImage(cls)
590 self.checkCompressedImage(cls, image, compression, scaling, atol=self.noise/quantize)
592 def readWriteMaskedImage(self, image, filename, imageOptions, maskOptions, varianceOptions):
593 """Read the MaskedImage after it has been written
595 This implementation does the persistence using methods on the
596 MaskedImage class.
598 Parameters
599 ----------
600 image : `lsst.afw.image.Image`
601 Image to compress.
602 filename : `str`
603 Filename to which to write.
604 imageOptions, maskOptions, varianceOptions : `lsst.afw.fits.ImageWriteOptions`
605 Options for writing the image, mask and variance planes.
606 """
607 image.writeFits(filename, imageOptions, maskOptions, varianceOptions)
608 if hasattr(image, "getMaskedImage"):
609 image = image.getMaskedImage()
610 return lsst.afw.image.MaskedImageF(filename)
612 def checkCompressedMaskedImage(self, image, imageOptions, maskOptions, varianceOptions, atol=0.0):
613 """Check that compression works on a MaskedImage
615 Parameters
616 ----------
617 image : `lsst.afw.image.MaskedImage` or `lsst.afw.image.Exposure`
618 MaskedImage or exposure to compress.
619 imageOptions, maskOptions, varianceOptions : `lsst.afw.fits.ImageWriteOptions`
620 Parameters for writing (compression and scaling) the image, mask
621 and variance planes.
622 atol : `float`
623 Absolute tolerance for comparing unpersisted image.
624 """
625 with lsst.utils.tests.getTempFilePath(self.extension) as filename:
626 self.readWriteMaskedImage(image, filename, imageOptions, maskOptions, varianceOptions)
627 unpersisted = type(image)(filename)
628 if hasattr(image, "getMaskedImage"):
629 image = image.getMaskedImage()
630 unpersisted = unpersisted.getMaskedImage()
631 self.assertEqual(image.getBBox(), unpersisted.getBBox())
632 self.assertImagesAlmostEqual(unpersisted.getImage(), image.getImage(), atol=atol)
633 self.assertImagesAlmostEqual(unpersisted.getMask(), image.getMask(), atol=atol)
634 self.assertImagesAlmostEqual(unpersisted.getVariance(), image.getVariance(), atol=atol)
636 for mp in image.getMask().getMaskPlaneDict():
637 self.assertIn(mp, unpersisted.getMask().getMaskPlaneDict())
638 unpersisted.getMask().getPlaneBitMask(mp)
640 def checkMaskedImage(self, imageOptions, maskOptions, varianceOptions, atol=0.0):
641 """Check that we can compress a MaskedImage and Exposure
643 Parameters
644 ----------
645 imageOptions, maskOptions, varianceOptions : `lsst.afw.fits.ImageWriteOptions`
646 Parameters for writing (compression and scaling) the image, mask
647 and variance planes.
648 atol : `float`
649 Absolute tolerance for comparing unpersisted image.
650 """
651 image = lsst.afw.image.makeMaskedImage(self.makeImage(lsst.afw.image.ImageF),
652 self.makeMask(), self.makeImage(lsst.afw.image.ImageF))
653 self.checkCompressedMaskedImage(image, imageOptions, maskOptions, varianceOptions, atol=atol)
654 exp = lsst.afw.image.makeExposure(image)
655 self.checkCompressedMaskedImage(exp, imageOptions, maskOptions, varianceOptions, atol=atol)
657 def testMaskedImage(self):
658 """Test compression of MaskedImage
660 We test lossless, lossy cfitsio and lossy LSST compression.
661 """
662 # Lossless
663 lossless = lsst.afw.fits.ImageCompressionOptions(ImageCompressionOptions.GZIP_SHUFFLE)
664 options = lsst.afw.fits.ImageWriteOptions(lossless)
665 self.checkMaskedImage(options, options, options, atol=0.0)
667 # Lossy cfitsio compression
668 quantize = 4.0
669 cfitsio = lsst.afw.fits.ImageCompressionOptions(ImageCompressionOptions.GZIP_SHUFFLE, True, quantize)
670 imageOptions = lsst.afw.fits.ImageWriteOptions(cfitsio)
671 maskOptions = lsst.afw.fits.ImageWriteOptions(lossless)
672 self.checkMaskedImage(imageOptions, maskOptions, imageOptions, atol=self.noise/quantize)
674 # Lossy our compression
675 quantize = 10.0
676 compression = lsst.afw.fits.ImageCompressionOptions(ImageCompressionOptions.RICE, True, 0.0)
677 scaling = lsst.afw.fits.ImageScalingOptions(ImageScalingOptions.STDEV_BOTH, 32,
678 quantizeLevel=quantize)
679 imageOptions = lsst.afw.fits.ImageWriteOptions(compression, scaling)
680 maskOptions = lsst.afw.fits.ImageWriteOptions(compression)
681 self.checkMaskedImage(imageOptions, maskOptions, imageOptions, atol=self.noise/quantize)
683 def testQuantization(self):
684 """Test that our quantization produces the same values as cfitsio
686 Our quantization is more configurable (e.g., choice of scaling algorithm,
687 specifying mask planes) and extensible (logarithmic, asinh scalings)
688 than cfitsio's. However, cfitsio uses its own fuzz ("subtractive dithering")
689 when reading the data, so if we don't want to add random values twice,
690 we need to be sure that we're using the same random values. To check that,
691 we write one image with our scaling+compression, and one with cfitsio's
692 compression using exactly the BSCALE and dither seed we used for our own.
693 That way, the two codes will quantize independently, and we can compare
694 the results.
695 """
696 bscaleSet = 1.0
697 bzeroSet = self.background - 10*self.noise
698 algorithm = ImageCompressionOptions.GZIP
699 classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD)
700 tilesList = ((4, 5), (0, 0), (0, 5), (4, 0), (0, 1))
701 for cls, tiles in itertools.product(classList, tilesList):
702 tiles = np.array(tiles, dtype=np.int64)
703 compression = ImageCompressionOptions(algorithm, tiles, -bscaleSet)
704 original = self.makeImage(cls)
705 with lsst.utils.tests.getTempFilePath(self.extension) as filename:
706 with lsst.afw.fits.Fits(filename, "w") as fits:
707 options = lsst.afw.fits.ImageWriteOptions(compression)
708 original.writeFits(fits, options)
709 cfitsio = cls(filename)
710 header = lsst.afw.fits.readMetadata(filename, 1)
711 seed = header.getScalar("ZDITHER0")
712 self.assertEqual(header.getScalar("BSCALE"), bscaleSet)
714 compression = ImageCompressionOptions(algorithm, tiles, 0.0)
715 scaling = ImageScalingOptions(ImageScalingOptions.MANUAL, 32, [u"BAD"], bscale=bscaleSet,
716 bzero=bzeroSet, fuzz=True, seed=seed)
717 unpersisted = self.checkCompressedImage(cls, original, compression, scaling, atol=bscaleSet)
718 oursDiff = unpersisted.getArray() - original.getArray()
719 cfitsioDiff = cfitsio.getArray() - original.getArray()
720 self.assertImagesAlmostEqual(oursDiff, cfitsioDiff, atol=0.0)
723def optionsToPropertySet(options):
724 """Convert the ImageWriteOptions to a PropertySet
726 This allows us to pass the options into the persistence framework
727 as the "additionalData".
728 """
729 ps = lsst.daf.base.PropertySet()
730 ps.set("compression.algorithm", lsst.afw.fits.compressionAlgorithmToString(options.compression.algorithm))
731 ps.set("compression.columns", options.compression.tiles[0])
732 ps.set("compression.rows", options.compression.tiles[1])
733 ps.set("compression.quantizeLevel", options.compression.quantizeLevel)
735 ps.set("scaling.algorithm", lsst.afw.fits.scalingAlgorithmToString(options.scaling.algorithm))
736 ps.set("scaling.bitpix", options.scaling.bitpix)
737 ps.setString("scaling.maskPlanes", options.scaling.maskPlanes)
738 ps.set("scaling.fuzz", options.scaling.fuzz)
739 ps.set("scaling.seed", options.scaling.seed)
740 ps.set("scaling.quantizeLevel", options.scaling.quantizeLevel)
741 ps.set("scaling.quantizePad", options.scaling.quantizePad)
742 ps.set("scaling.bscale", options.scaling.bscale)
743 ps.set("scaling.bzero", options.scaling.bzero)
744 return ps
747def persistUnpersist(ImageClass, image, filename, additionalData):
748 """Use read/writeFitsWithOptions to persist and unpersist an image
750 Parameters
751 ----------
752 ImageClass : `type`, an `lsst.afw.image.Image` class
753 Class of image.
754 image : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage`
755 Image to compress.
756 filename : `str`
757 Filename to write.
758 additionalData : `lsst.daf.base.PropertySet`
759 Additional data for persistence framework.
761 Returns
762 -------
763 unpersisted : `ImageClass`
764 The unpersisted image.
765 """
766 additionalData.set("visit", 12345)
767 additionalData.set("ccd", 67)
769 image.writeFitsWithOptions(filename, additionalData)
770 return ImageClass.readFits(filename)
773class PersistenceTestCase(ImageCompressionTestCase):
774 """Test compression using the persistence framework
776 We override the I/O methods to use the persistence framework.
777 """
778 def testQuantization(self):
779 """Not appropriate --- disable"""
780 pass
782 def readWriteImage(self, ImageClass, image, filename, options):
783 """Read the image after it has been written
785 This implementation uses the persistence framework.
787 Parameters
788 ----------
789 ImageClass : `type`, an `lsst.afw.image.Image` class
790 Class of image to create.
791 image : `lsst.afw.image.Image`
792 Image to compress.
793 filename : `str`
794 Filename to which to write.
795 options : `lsst.afw.fits.ImageWriteOptions`
796 Options for writing.
797 """
798 additionalData = lsst.daf.base.PropertySet()
799 additionalData.set("image", optionsToPropertySet(options))
800 return persistUnpersist(ImageClass, image, filename, additionalData)
802 def readWriteMaskedImage(self, image, filename, imageOptions, maskOptions, varianceOptions):
803 """Read the MaskedImage after it has been written
805 This implementation uses the persistence framework.
807 Parameters
808 ----------
809 image : `lsst.afw.image.MaskedImage`
810 Image to compress.
811 filename : `str`
812 Filename to which to write.
813 imageOptions, maskOptions, varianceOptions : `lsst.afw.fits.ImageWriteOptions`
814 Options for writing the image, mask and variance planes.
815 """
816 additionalData = lsst.daf.base.PropertySet()
817 additionalData.set("image", optionsToPropertySet(imageOptions))
818 additionalData.set("mask", optionsToPropertySet(maskOptions))
819 additionalData.set("variance", optionsToPropertySet(varianceOptions))
820 return persistUnpersist(lsst.afw.image.MaskedImageF, image, filename, additionalData)
823class EmptyExposureTestCase(lsst.utils.tests.TestCase):
824 """Test that an empty image can be written
826 We sometimes use an empty lsst.afw.image.Exposure as a vehicle for
827 persisting other things, e.g., Wcs, Calib. cfitsio compression will
828 choke on an empty image, so make sure we're dealing with that.
829 """
830 def checkEmptyExposure(self, algorithm):
831 """Check that we can persist an empty Exposure
833 Parameters
834 ----------
835 algorithm : `lsst.afw.fits.ImageCompressionOptions.CompressionAlgorithm`
836 Compression algorithm to try.
837 """
838 exp = lsst.afw.image.ExposureF(0, 0)
839 degrees = lsst.geom.degrees
840 cdMatrix = np.array([[1.0e-4, 0.0], [0.0, 1.0e-4]], dtype=float)
841 exp.setWcs(lsst.afw.geom.makeSkyWcs(crval=lsst.geom.SpherePoint(0*degrees, 0*degrees),
842 crpix=lsst.geom.Point2D(0.0, 0.0),
843 cdMatrix=cdMatrix))
844 imageOptions = lsst.afw.fits.ImageWriteOptions(ImageCompressionOptions(algorithm))
845 maskOptions = lsst.afw.fits.ImageWriteOptions(exp.getMaskedImage().getMask())
846 varianceOptions = lsst.afw.fits.ImageWriteOptions(ImageCompressionOptions(algorithm))
847 with lsst.utils.tests.getTempFilePath(".fits") as filename:
848 exp.writeFits(filename, imageOptions, maskOptions, varianceOptions)
849 unpersisted = type(exp)(filename)
850 self.assertEqual(unpersisted.getMaskedImage().getDimensions(), lsst.geom.Extent2I(0, 0))
851 self.assertEqual(unpersisted.getWcs(), exp.getWcs())
853 def testEmptyExposure(self):
854 """Persist an empty Exposure with compression"""
855 algorithmList = ("GZIP", "GZIP_SHUFFLE", "RICE")
856 for algorithm in algorithmList:
857 self.checkEmptyExposure(lsst.afw.fits.compressionAlgorithmFromString(algorithm))
860class TestMemory(lsst.utils.tests.MemoryTestCase):
861 pass
864def setup_module(module):
865 lsst.utils.tests.init()
868if __name__ == "__main__": 868 ↛ 869line 868 didn't jump to line 869, because the condition on line 868 was never true
869 import sys
870 setup_module(sys.modules[__name__])
871 unittest.main()