Coverage for tests/test_butlerFits.py: 17%
223 statements
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-29 17:03 +0000
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-29 17:03 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24import os
25import shutil
26import tempfile
27import unittest
28from typing import TYPE_CHECKING
30import astropy.table
31import lsst.afw.cameraGeom.testUtils # for test asserts injected into TestCase
32import lsst.afw.image
33import lsst.pex.config
34import lsst.utils.tests
35from lsst.afw.fits import readMetadata
36from lsst.afw.image import LOCAL, ExposureFitsReader
37from lsst.afw.math import flipImage
38from lsst.daf.base import PropertyList, PropertySet
39from lsst.daf.butler import Config, DatasetType, StorageClassFactory
40from lsst.daf.butler.tests import addDatasetType, makeTestCollection, makeTestRepo
41from lsst.geom import Box2I, Extent2I, Point2I
42from lsst.obs.base.exposureAssembler import ExposureAssembler
43from lsst.obs.base.tests import make_ramp_exposure_trimmed, make_ramp_exposure_untrimmed
45if TYPE_CHECKING: 45 ↛ 46line 45 didn't jump to line 46, because the condition on line 45 was never true
46 from lsst.daf.butler import DatasetRef
48TESTDIR = os.path.dirname(__file__)
50BUTLER_CONFIG = """
51storageClasses:
52 ExposureCompositeF:
53 inheritsFrom: ExposureF
54datastore:
55 # Want to check disassembly so can't use InMemory
56 cls: lsst.daf.butler.datastores.fileDatastore.FileDatastore
57 formatters:
58 ExposureCompositeF: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
59 lossless:
60 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
61 parameters:
62 recipe: lossless
63 uncompressed:
64 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
65 parameters:
66 recipe: noCompression
67 lossy:
68 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
69 parameters:
70 recipe: lossyBasic
71 composites:
72 disassembled:
73 ExposureCompositeF: True
74"""
76# Components present in the test file
77COMPONENTS = {
78 "wcs",
79 "image",
80 "mask",
81 "coaddInputs",
82 "psf",
83 "visitInfo",
84 "variance",
85 "metadata",
86 "photoCalib",
87 "filter",
88 "validPolygon",
89 "transmissionCurve",
90 "detector",
91 "apCorrMap",
92 "summaryStats",
93 "id",
94}
95READ_COMPONENTS = {
96 "bbox",
97 "xy0",
98 "dimensions",
99}
102class ButlerFitsTests(lsst.utils.tests.TestCase):
103 """Tests for butler interaction with FITS files."""
105 @classmethod
106 def setUpClass(cls):
107 """Create a new butler once only."""
108 cls.storageClassFactory = StorageClassFactory()
110 cls.root = tempfile.mkdtemp(dir=TESTDIR)
112 dataIds = {
113 "instrument": ["DummyCam"],
114 "physical_filter": ["d-r"],
115 "visit": [42, 43, 44],
116 }
118 # Ensure that we test in a directory that will include some
119 # metacharacters
120 subdir = "sub?#dir"
121 butlerRoot = os.path.join(cls.root, subdir)
123 cls.creatorButler = makeTestRepo(butlerRoot, dataIds, config=Config.fromYaml(BUTLER_CONFIG))
125 # Create dataset types used by the tests
126 for datasetTypeName, storageClassName in (
127 ("calexp", "ExposureF"),
128 ("unknown", "ExposureCompositeF"),
129 ("testCatalog", "SourceCatalog"),
130 ("lossless", "ExposureF"),
131 ("uncompressed", "ExposureF"),
132 ("lossy", "ExposureF"),
133 ):
134 storageClass = cls.storageClassFactory.getStorageClass(storageClassName)
135 addDatasetType(cls.creatorButler, datasetTypeName, set(dataIds), storageClass)
137 # And some dataset types that have no dimensions for easy testing
138 for datasetTypeName, storageClassName in (
139 ("ps", "PropertySet"),
140 ("pl", "PropertyList"),
141 ("int_exp_trimmed", "ExposureI"),
142 ("int_exp_untrimmed", "ExposureI"),
143 ):
144 storageClass = cls.storageClassFactory.getStorageClass(storageClassName)
145 addDatasetType(cls.creatorButler, datasetTypeName, {}, storageClass)
147 @classmethod
148 def tearDownClass(cls):
149 if cls.root is not None:
150 shutil.rmtree(cls.root, ignore_errors=True)
152 def setUp(self):
153 self.butler = makeTestCollection(self.creatorButler)
155 def makeExampleCatalog(self) -> lsst.afw.table.SourceCatalog:
156 catalogPath = os.path.join(TESTDIR, "data", "source_catalog.fits")
157 return lsst.afw.table.SourceCatalog.readFits(catalogPath)
159 def assertCatalogEqual(
160 self, inputCatalog: lsst.afw.table.SourceCatalog, outputCatalog: lsst.afw.table.SourceCatalog
161 ) -> None:
162 self.assertIsInstance(outputCatalog, lsst.afw.table.SourceCatalog)
163 inputTable = inputCatalog.getTable()
164 inputRecord = inputCatalog[0]
165 outputTable = outputCatalog.getTable()
166 outputRecord = outputCatalog[0]
167 self.assertEqual(inputRecord.getPsfInstFlux(), outputRecord.getPsfInstFlux())
168 self.assertEqual(inputRecord.getPsfFluxFlag(), outputRecord.getPsfFluxFlag())
169 self.assertEqual(
170 inputTable.getSchema().getAliasMap().get("slot_Centroid"),
171 outputTable.getSchema().getAliasMap().get("slot_Centroid"),
172 )
173 self.assertEqual(inputRecord.getCentroid(), outputRecord.getCentroid())
174 self.assertFloatsAlmostEqual(
175 inputRecord.getCentroidErr()[0, 0], outputRecord.getCentroidErr()[0, 0], rtol=1e-6
176 )
177 self.assertFloatsAlmostEqual(
178 inputRecord.getCentroidErr()[1, 1], outputRecord.getCentroidErr()[1, 1], rtol=1e-6
179 )
180 self.assertEqual(
181 inputTable.getSchema().getAliasMap().get("slot_Shape"),
182 outputTable.getSchema().getAliasMap().get("slot_Shape"),
183 )
184 self.assertFloatsAlmostEqual(
185 inputRecord.getShapeErr()[0, 0], outputRecord.getShapeErr()[0, 0], rtol=1e-6
186 )
187 self.assertFloatsAlmostEqual(
188 inputRecord.getShapeErr()[1, 1], outputRecord.getShapeErr()[1, 1], rtol=1e-6
189 )
190 self.assertFloatsAlmostEqual(
191 inputRecord.getShapeErr()[2, 2], outputRecord.getShapeErr()[2, 2], rtol=1e-6
192 )
194 def runFundamentalTypeTest(self, datasetTypeName, entity):
195 """Put and get the supplied entity and compare."""
196 ref = self.butler.put(entity, datasetTypeName)
197 butler_ps = self.butler.get(ref)
198 self.assertEqual(butler_ps, entity)
200 # Break the contact by ensuring that we are writing YAML
201 uri = self.butler.getURI(ref)
202 self.assertEqual(uri.getExtension(), ".yaml", f"Check extension of {uri}")
204 def testFundamentalTypes(self) -> None:
205 """Ensure that some fundamental stack types round trip."""
206 ps = PropertySet()
207 ps["a.b"] = 5
208 ps["c.d.e"] = "string"
209 self.runFundamentalTypeTest("ps", ps)
211 pl = PropertyList()
212 pl["A"] = 1
213 pl.setComment("A", "An int comment")
214 pl["B"] = "string"
215 pl.setComment("B", "A string comment")
216 self.runFundamentalTypeTest("pl", pl)
218 def testFitsCatalog(self) -> None:
219 """Test reading of a FITS catalog."""
220 catalog = self.makeExampleCatalog()
221 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"}
222 ref = self.butler.put(catalog, "testCatalog", dataId)
223 stored = self.butler.get(ref)
224 self.assertCatalogEqual(catalog, stored)
226 # Override the storage class.
227 astropy_table = self.butler.get(ref, storageClass="AstropyTable")
228 self.assertIsInstance(astropy_table, astropy.table.Table)
229 self.assertEqual(len(astropy_table), len(stored))
231 def testExposureCompositePutGetConcrete(self) -> None:
232 """Test composite with no disassembly."""
233 ref = self.runExposureCompositePutGetTest("calexp")
235 uri = self.butler.getURI(ref)
236 self.assertTrue(uri.exists(), f"Checking URI {uri} existence")
238 def testExposureCompositePutGetVirtual(self) -> None:
239 """Testing composite disassembly."""
240 ref = self.runExposureCompositePutGetTest("unknown")
242 primary, components = self.butler.getURIs(ref)
243 self.assertIsNone(primary)
244 self.assertEqual(set(components), COMPONENTS)
245 for compName, uri in components.items():
246 self.assertTrue(uri.exists(), f"Checking URI {uri} existence for component {compName}")
248 def runExposureCompositePutGetTest(self, datasetTypeName: str) -> DatasetRef:
249 example = os.path.join(TESTDIR, "data", "calexp.fits")
250 exposure = lsst.afw.image.ExposureF(example)
252 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"}
253 ref = self.butler.put(exposure, datasetTypeName, dataId)
255 # Get the full thing
256 composite = self.butler.get(datasetTypeName, dataId)
258 # There is no assert for Exposure so just look at maskedImage
259 self.assertMaskedImagesEqual(composite.maskedImage, exposure.maskedImage)
261 # Helper for extracting components
262 assembler = ExposureAssembler(ref.datasetType.storageClass)
264 # Check all possible components that can be read
265 allComponents = set()
266 allComponents.update(COMPONENTS, READ_COMPONENTS)
268 # Get each component from butler independently
269 for compName in allComponents:
270 compTypeName = DatasetType.nameWithComponent(datasetTypeName, compName)
271 component = self.butler.get(compTypeName, dataId)
273 reference = assembler.getComponent(exposure, compName)
275 self.assertIsInstance(component, type(reference), f"Checking type of component {compName}")
277 if compName in ("image", "variance"):
278 self.assertImagesEqual(component, reference)
279 elif compName == "mask":
280 self.assertMasksEqual(component, reference)
281 elif compName == "wcs":
282 self.assertWcsAlmostEqualOverBBox(component, reference, exposure.getBBox())
283 elif compName == "coaddInputs":
284 self.assertEqual(
285 len(component.visits), len(reference.visits), f"cf visits {component.visits}"
286 )
287 self.assertEqual(len(component.ccds), len(reference.ccds), f"cf CCDs {component.ccds}")
288 elif compName == "psf":
289 # Equality for PSF does not work
290 pass
291 elif compName == "filter":
292 self.assertEqual(component, reference)
293 elif compName == "id":
294 self.assertEqual(component, reference)
295 elif compName == "visitInfo":
296 self.assertEqual(component, reference, "VisitInfo comparison")
297 elif compName == "metadata":
298 # The component metadata has extra fields in it so cannot
299 # compare directly.
300 for k, v in reference.items():
301 self.assertEqual(component[k], v)
302 elif compName == "photoCalib":
303 # This example has a
304 # "spatially constant with mean: inf error: nan" entry
305 # which does not compare directly.
306 self.assertEqual(str(component), str(reference))
307 self.assertIn("spatially constant with mean: 1.99409", str(component), "Checking photoCalib")
308 elif compName in ("bbox", "xy0", "dimensions", "validPolygon"):
309 self.assertEqual(component, reference)
310 elif compName == "apCorrMap":
311 self.assertEqual(set(component.keys()), set(reference.keys()))
312 elif compName == "transmissionCurve":
313 self.assertEqual(component.getThroughputAtBounds(), reference.getThroughputAtBounds())
314 elif compName == "detector":
315 c_amps = {a.getName() for a in component.getAmplifiers()}
316 r_amps = {a.getName() for a in reference.getAmplifiers()}
317 self.assertEqual(c_amps, r_amps)
318 elif compName == "summaryStats":
319 self.assertEqual(component.psfSigma, reference.psfSigma)
320 else:
321 raise RuntimeError(f"Unexpected component '{compName}' encountered in test")
323 # Full Exposure with parameters
324 inBBox = Box2I(minimum=Point2I(3, 3), maximum=Point2I(21, 16))
325 parameters = {"bbox": inBBox, "origin": LOCAL}
326 subset = self.butler.get(datasetTypeName, dataId, parameters=parameters)
327 outBBox = subset.getBBox()
328 self.assertEqual(inBBox, outBBox)
329 self.assertImagesEqual(subset.getImage(), exposure.subset(inBBox, origin=LOCAL).getImage())
331 return ref
333 def putFits(self, exposure, datasetTypeName, visit):
334 """Put different datasetTypes and return information."""
335 dataId = {"visit": visit, "instrument": "DummyCam", "physical_filter": "d-r"}
336 refC = self.butler.put(exposure, datasetTypeName, dataId)
337 uriC = self.butler.getURI(refC)
338 stat = os.stat(uriC.ospath)
339 size = stat.st_size
340 # We can't use butler's Exposure storage class metadata component,
341 # because that intentionally strips keywords that science code
342 # shouldn't ever read (to at least weaken our assumptions that we write
343 # to FITS). Instead use a lower-level method on the URI for this test.
344 meta = readMetadata(uriC.ospath)
345 return meta, size
347 def testCompression(self):
348 """Test that we can write compressed and uncompressed FITS."""
349 example = os.path.join(TESTDIR, "data", "small.fits")
350 exposure = lsst.afw.image.ExposureF(example)
352 # Write a lossless compressed
353 metaC, sizeC = self.putFits(exposure, "lossless", 42)
354 self.assertEqual(metaC["TTYPE1"], "COMPRESSED_DATA")
355 self.assertEqual(metaC["ZCMPTYPE"], "GZIP_2")
357 # Write an uncompressed FITS file
358 metaN, sizeN = self.putFits(exposure, "uncompressed", 43)
359 self.assertNotIn("ZCMPTYPE", metaN)
361 # Write an uncompressed FITS file
362 metaL, sizeL = self.putFits(exposure, "lossy", 44)
363 self.assertEqual(metaL["TTYPE1"], "COMPRESSED_DATA")
364 self.assertEqual(metaL["ZCMPTYPE"], "RICE_1")
366 self.assertNotEqual(sizeC, sizeN)
367 # Data file is so small that Lossy and Compressed are dominated
368 # by the extra compression tables
369 self.assertEqual(sizeL, sizeC)
371 def testExposureFormatterAmpParameter(self):
372 """Test the FitsExposureFormatter implementation of the Exposure
373 StorageClass's 'amp' and 'detector' parameters.
374 """
375 # Our example exposure file has a realistic detector (looks like HSC),
376 # but the image itself doesn't match it. So we just load the detector,
377 # and use it to make our own more useful images and put them.
378 detector = ExposureFitsReader(os.path.join(TESTDIR, "data", "calexp.fits")).readDetector()
379 trimmed_full = make_ramp_exposure_trimmed(detector)
380 untrimmed_full = make_ramp_exposure_untrimmed(detector)
381 trimmed_ref = self.butler.put(trimmed_full, "int_exp_trimmed")
382 untrimmed_ref = self.butler.put(untrimmed_full, "int_exp_untrimmed")
383 for n, amp in enumerate(detector):
384 # Try to read each amp as it is on disk, with a variety of
385 # parameters that should all do the same thing.
386 for amp_parameter in [amp, amp.getName(), n]:
387 for parameters in [{"amp": amp_parameter}, {"amp": amp_parameter, "detector": detector}]:
388 with self.subTest(parameters=parameters):
389 test_trimmed = self.butler.get(trimmed_ref, parameters=parameters)
390 test_untrimmed = self.butler.get(untrimmed_ref, parameters=parameters)
391 self.assertImagesEqual(test_trimmed.image, trimmed_full[amp.getBBox()].image)
392 self.assertImagesEqual(test_untrimmed.image, untrimmed_full[amp.getRawBBox()].image)
393 self.assertEqual(len(test_trimmed.getDetector()), 1)
394 self.assertEqual(len(test_untrimmed.getDetector()), 1)
395 self.assertAmplifiersEqual(test_trimmed.getDetector()[0], amp)
396 self.assertAmplifiersEqual(test_untrimmed.getDetector()[0], amp)
397 # Try to read various transformed versions of the original amp,
398 # to make sure flips and offsets are applied correctly.
399 # First flip X only.
400 amp_t1 = amp.rebuild().transform(outFlipX=True).finish()
401 test_t1_trimmed = self.butler.get(trimmed_ref, parameters={"amp": amp_t1})
402 self.assertImagesEqual(
403 test_t1_trimmed.image, flipImage(trimmed_full[amp.getBBox()].image, flipLR=True, flipTB=False)
404 )
405 self.assertAmplifiersEqual(test_t1_trimmed.getDetector()[0], amp_t1)
406 test_t1_untrimmed = self.butler.get(untrimmed_ref, parameters={"amp": amp_t1})
407 self.assertImagesEqual(
408 test_t1_untrimmed.image,
409 flipImage(untrimmed_full[amp.getRawBBox()].image, flipLR=True, flipTB=False),
410 )
411 self.assertAmplifiersEqual(test_t1_trimmed.getDetector()[0], amp_t1)
412 # Flip Y only.
413 amp_t2 = amp.rebuild().transform(outFlipY=True).finish()
414 test_t2_trimmed = self.butler.get(trimmed_ref, parameters={"amp": amp_t2})
415 self.assertImagesEqual(
416 test_t2_trimmed.image, flipImage(trimmed_full[amp.getBBox()].image, flipLR=False, flipTB=True)
417 )
418 self.assertAmplifiersEqual(test_t2_trimmed.getDetector()[0], amp_t2)
419 test_t2_untrimmed = self.butler.get(untrimmed_ref, parameters={"amp": amp_t2})
420 self.assertImagesEqual(
421 test_t2_untrimmed.image,
422 flipImage(untrimmed_full[amp.getRawBBox()].image, flipLR=False, flipTB=True),
423 )
424 self.assertAmplifiersEqual(test_t2_trimmed.getDetector()[0], amp_t2)
425 # Add an XY offset only.
426 amp_t3 = amp.rebuild().transform(outOffset=Extent2I(5, 4)).finish()
427 test_t3_trimmed = self.butler.get(trimmed_ref, parameters={"amp": amp_t3})
428 self.assertImagesEqual(test_t3_trimmed.image, trimmed_full[amp.getBBox()].image)
429 self.assertAmplifiersEqual(test_t3_trimmed.getDetector()[0], amp_t3)
430 test_t3_untrimmed = self.butler.get(untrimmed_ref, parameters={"amp": amp_t3})
431 self.assertImagesEqual(test_t3_untrimmed.image, untrimmed_full[amp.getRawBBox()].image)
432 self.assertAmplifiersEqual(test_t3_trimmed.getDetector()[0], amp_t3)
435if __name__ == "__main__": 435 ↛ 436line 435 didn't jump to line 436, because the condition on line 435 was never true
436 unittest.main()