Coverage for tests/test_butlerFits.py: 16%
223 statements
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-19 04:51 -0700
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-19 04:51 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24import os
25import shutil
26import tempfile
27import unittest
28from typing import TYPE_CHECKING
30import astropy.table
31import lsst.afw.cameraGeom.testUtils # for test asserts injected into TestCase
32import lsst.afw.image
33import lsst.pex.config
34import lsst.utils.tests
35from lsst.afw.fits import readMetadata
36from lsst.afw.image import LOCAL, ExposureFitsReader
37from lsst.afw.math import flipImage
38from lsst.daf.base import PropertyList, PropertySet
39from lsst.daf.butler import Config, DatasetType, StorageClassFactory
40from lsst.daf.butler.tests import DatasetTestHelper, addDatasetType, makeTestCollection, makeTestRepo
41from lsst.geom import Box2I, Extent2I, Point2I
42from lsst.obs.base.exposureAssembler import ExposureAssembler
43from lsst.obs.base.tests import make_ramp_exposure_trimmed, make_ramp_exposure_untrimmed
45if TYPE_CHECKING: 45 ↛ 46line 45 didn't jump to line 46, because the condition on line 45 was never true
46 from lsst.daf.butler import DatasetRef
48TESTDIR = os.path.dirname(__file__)
50BUTLER_CONFIG = """
51storageClasses:
52 ExposureCompositeF:
53 inheritsFrom: ExposureF
54datastore:
55 # Want to check disassembly so can't use InMemory
56 cls: lsst.daf.butler.datastores.fileDatastore.FileDatastore
57 formatters:
58 ExposureCompositeF: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
59 lossless:
60 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
61 parameters:
62 recipe: lossless
63 uncompressed:
64 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
65 parameters:
66 recipe: noCompression
67 lossy:
68 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
69 parameters:
70 recipe: lossyBasic
71 composites:
72 disassembled:
73 ExposureCompositeF: True
74"""
76# Components present in the test file
77COMPONENTS = {
78 "wcs",
79 "image",
80 "mask",
81 "coaddInputs",
82 "psf",
83 "visitInfo",
84 "variance",
85 "metadata",
86 "photoCalib",
87 "filter",
88 "validPolygon",
89 "transmissionCurve",
90 "detector",
91 "apCorrMap",
92 "summaryStats",
93 "id",
94}
95READ_COMPONENTS = {
96 "bbox",
97 "xy0",
98 "dimensions",
99}
102class ButlerFitsTests(DatasetTestHelper, lsst.utils.tests.TestCase):
103 @classmethod
104 def setUpClass(cls):
105 """Create a new butler once only."""
107 cls.storageClassFactory = StorageClassFactory()
109 cls.root = tempfile.mkdtemp(dir=TESTDIR)
111 dataIds = {
112 "instrument": ["DummyCam"],
113 "physical_filter": ["d-r"],
114 "visit": [42, 43, 44],
115 }
117 # Ensure that we test in a directory that will include some
118 # metacharacters
119 subdir = "sub?#dir"
120 butlerRoot = os.path.join(cls.root, subdir)
122 cls.creatorButler = makeTestRepo(butlerRoot, dataIds, config=Config.fromYaml(BUTLER_CONFIG))
124 # Create dataset types used by the tests
125 for datasetTypeName, storageClassName in (
126 ("calexp", "ExposureF"),
127 ("unknown", "ExposureCompositeF"),
128 ("testCatalog", "SourceCatalog"),
129 ("lossless", "ExposureF"),
130 ("uncompressed", "ExposureF"),
131 ("lossy", "ExposureF"),
132 ):
133 storageClass = cls.storageClassFactory.getStorageClass(storageClassName)
134 addDatasetType(cls.creatorButler, datasetTypeName, set(dataIds), storageClass)
136 # And some dataset types that have no dimensions for easy testing
137 for datasetTypeName, storageClassName in (
138 ("ps", "PropertySet"),
139 ("pl", "PropertyList"),
140 ("int_exp_trimmed", "ExposureI"),
141 ("int_exp_untrimmed", "ExposureI"),
142 ):
143 storageClass = cls.storageClassFactory.getStorageClass(storageClassName)
144 addDatasetType(cls.creatorButler, datasetTypeName, {}, storageClass)
146 @classmethod
147 def tearDownClass(cls):
148 if cls.root is not None:
149 shutil.rmtree(cls.root, ignore_errors=True)
151 def setUp(self):
152 self.butler = makeTestCollection(self.creatorButler)
154 def makeExampleCatalog(self) -> lsst.afw.table.SourceCatalog:
155 catalogPath = os.path.join(TESTDIR, "data", "source_catalog.fits")
156 return lsst.afw.table.SourceCatalog.readFits(catalogPath)
158 def assertCatalogEqual(
159 self, inputCatalog: lsst.afw.table.SourceCatalog, outputCatalog: lsst.afw.table.SourceCatalog
160 ) -> None:
161 self.assertIsInstance(outputCatalog, lsst.afw.table.SourceCatalog)
162 inputTable = inputCatalog.getTable()
163 inputRecord = inputCatalog[0]
164 outputTable = outputCatalog.getTable()
165 outputRecord = outputCatalog[0]
166 self.assertEqual(inputRecord.getPsfInstFlux(), outputRecord.getPsfInstFlux())
167 self.assertEqual(inputRecord.getPsfFluxFlag(), outputRecord.getPsfFluxFlag())
168 self.assertEqual(
169 inputTable.getSchema().getAliasMap().get("slot_Centroid"),
170 outputTable.getSchema().getAliasMap().get("slot_Centroid"),
171 )
172 self.assertEqual(inputRecord.getCentroid(), outputRecord.getCentroid())
173 self.assertFloatsAlmostEqual(
174 inputRecord.getCentroidErr()[0, 0], outputRecord.getCentroidErr()[0, 0], rtol=1e-6
175 )
176 self.assertFloatsAlmostEqual(
177 inputRecord.getCentroidErr()[1, 1], outputRecord.getCentroidErr()[1, 1], rtol=1e-6
178 )
179 self.assertEqual(
180 inputTable.getSchema().getAliasMap().get("slot_Shape"),
181 outputTable.getSchema().getAliasMap().get("slot_Shape"),
182 )
183 self.assertFloatsAlmostEqual(
184 inputRecord.getShapeErr()[0, 0], outputRecord.getShapeErr()[0, 0], rtol=1e-6
185 )
186 self.assertFloatsAlmostEqual(
187 inputRecord.getShapeErr()[1, 1], outputRecord.getShapeErr()[1, 1], rtol=1e-6
188 )
189 self.assertFloatsAlmostEqual(
190 inputRecord.getShapeErr()[2, 2], outputRecord.getShapeErr()[2, 2], rtol=1e-6
191 )
193 def runFundamentalTypeTest(self, datasetTypeName, entity):
194 """Put and get the supplied entity and compare."""
195 ref = self.butler.put(entity, datasetTypeName)
196 butler_ps = self.butler.get(ref)
197 self.assertEqual(butler_ps, entity)
199 # Break the contact by ensuring that we are writing YAML
200 uri = self.butler.getURI(ref)
201 self.assertEqual(uri.getExtension(), ".yaml", f"Check extension of {uri}")
203 def testFundamentalTypes(self) -> None:
204 """Ensure that some fundamental stack types round trip."""
205 ps = PropertySet()
206 ps["a.b"] = 5
207 ps["c.d.e"] = "string"
208 self.runFundamentalTypeTest("ps", ps)
210 pl = PropertyList()
211 pl["A"] = 1
212 pl.setComment("A", "An int comment")
213 pl["B"] = "string"
214 pl.setComment("B", "A string comment")
215 self.runFundamentalTypeTest("pl", pl)
217 def testFitsCatalog(self) -> None:
218 """Test reading of a FITS catalog"""
219 catalog = self.makeExampleCatalog()
220 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"}
221 ref = self.butler.put(catalog, "testCatalog", dataId)
222 stored = self.butler.get(ref)
223 self.assertCatalogEqual(catalog, stored)
225 # Override the storage class.
226 astropy_table = self.butler.get(ref, storageClass="AstropyTable")
227 self.assertIsInstance(astropy_table, astropy.table.Table)
228 self.assertEqual(len(astropy_table), len(stored))
230 def testExposureCompositePutGetConcrete(self) -> None:
231 """Test composite with no disassembly"""
232 ref = self.runExposureCompositePutGetTest("calexp")
234 uri = self.butler.getURI(ref)
235 self.assertTrue(uri.exists(), f"Checking URI {uri} existence")
237 def testExposureCompositePutGetVirtual(self) -> None:
238 """Testing composite disassembly"""
239 ref = self.runExposureCompositePutGetTest("unknown")
241 primary, components = self.butler.getURIs(ref)
242 self.assertIsNone(primary)
243 self.assertEqual(set(components), COMPONENTS)
244 for compName, uri in components.items():
245 self.assertTrue(uri.exists(), f"Checking URI {uri} existence for component {compName}")
247 def runExposureCompositePutGetTest(self, datasetTypeName: str) -> DatasetRef:
248 example = os.path.join(TESTDIR, "data", "calexp.fits")
249 exposure = lsst.afw.image.ExposureF(example)
251 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"}
252 ref = self.butler.put(exposure, datasetTypeName, dataId)
254 # Get the full thing
255 composite = self.butler.get(datasetTypeName, dataId)
257 # There is no assert for Exposure so just look at maskedImage
258 self.assertMaskedImagesEqual(composite.maskedImage, exposure.maskedImage)
260 # Helper for extracting components
261 assembler = ExposureAssembler(ref.datasetType.storageClass)
263 # Check all possible components that can be read
264 allComponents = set()
265 allComponents.update(COMPONENTS, READ_COMPONENTS)
267 # Get each component from butler independently
268 for compName in allComponents:
269 compTypeName = DatasetType.nameWithComponent(datasetTypeName, compName)
270 component = self.butler.get(compTypeName, dataId)
272 reference = assembler.getComponent(exposure, compName)
274 self.assertIsInstance(component, type(reference), f"Checking type of component {compName}")
276 if compName in ("image", "variance"):
277 self.assertImagesEqual(component, reference)
278 elif compName == "mask":
279 self.assertMasksEqual(component, reference)
280 elif compName == "wcs":
281 self.assertWcsAlmostEqualOverBBox(component, reference, exposure.getBBox())
282 elif compName == "coaddInputs":
283 self.assertEqual(
284 len(component.visits), len(reference.visits), f"cf visits {component.visits}"
285 )
286 self.assertEqual(len(component.ccds), len(reference.ccds), f"cf CCDs {component.ccds}")
287 elif compName == "psf":
288 # Equality for PSF does not work
289 pass
290 elif compName == "filter":
291 self.assertEqual(component, reference)
292 elif compName == "id":
293 self.assertEqual(component, reference)
294 elif compName == "visitInfo":
295 self.assertEqual(component, reference, "VisitInfo comparison")
296 elif compName == "metadata":
297 # The component metadata has extra fields in it so cannot
298 # compare directly.
299 for k, v in reference.items():
300 self.assertEqual(component[k], v)
301 elif compName == "photoCalib":
302 # This example has a
303 # "spatially constant with mean: inf error: nan" entry
304 # which does not compare directly.
305 self.assertEqual(str(component), str(reference))
306 self.assertIn("spatially constant with mean: 1.99409", str(component), "Checking photoCalib")
307 elif compName in ("bbox", "xy0", "dimensions", "validPolygon"):
308 self.assertEqual(component, reference)
309 elif compName == "apCorrMap":
310 self.assertEqual(set(component.keys()), set(reference.keys()))
311 elif compName == "transmissionCurve":
312 self.assertEqual(component.getThroughputAtBounds(), reference.getThroughputAtBounds())
313 elif compName == "detector":
314 c_amps = {a.getName() for a in component.getAmplifiers()}
315 r_amps = {a.getName() for a in reference.getAmplifiers()}
316 self.assertEqual(c_amps, r_amps)
317 elif compName == "summaryStats":
318 self.assertEqual(component.psfSigma, reference.psfSigma)
319 else:
320 raise RuntimeError(f"Unexpected component '{compName}' encountered in test")
322 # Full Exposure with parameters
323 inBBox = Box2I(minimum=Point2I(3, 3), maximum=Point2I(21, 16))
324 parameters = dict(bbox=inBBox, origin=LOCAL)
325 subset = self.butler.get(datasetTypeName, dataId, parameters=parameters)
326 outBBox = subset.getBBox()
327 self.assertEqual(inBBox, outBBox)
328 self.assertImagesEqual(subset.getImage(), exposure.subset(inBBox, origin=LOCAL).getImage())
330 return ref
332 def putFits(self, exposure, datasetTypeName, visit):
333 """Put different datasetTypes and return information."""
334 dataId = {"visit": visit, "instrument": "DummyCam", "physical_filter": "d-r"}
335 refC = self.butler.put(exposure, datasetTypeName, dataId)
336 uriC = self.butler.getURI(refC)
337 stat = os.stat(uriC.ospath)
338 size = stat.st_size
339 # We can't use butler's Exposure storage class metadata component,
340 # because that intentionally strips keywords that science code
341 # shouldn't ever read (to at least weaken our assumptions that we write
342 # to FITS). Instead use a lower-level method on the URI for this test.
343 meta = readMetadata(uriC.ospath)
344 return meta, size
346 def testCompression(self):
347 """Test that we can write compressed and uncompressed FITS."""
348 example = os.path.join(TESTDIR, "data", "small.fits")
349 exposure = lsst.afw.image.ExposureF(example)
351 # Write a lossless compressed
352 metaC, sizeC = self.putFits(exposure, "lossless", 42)
353 self.assertEqual(metaC["TTYPE1"], "COMPRESSED_DATA")
354 self.assertEqual(metaC["ZCMPTYPE"], "GZIP_2")
356 # Write an uncompressed FITS file
357 metaN, sizeN = self.putFits(exposure, "uncompressed", 43)
358 self.assertNotIn("ZCMPTYPE", metaN)
360 # Write an uncompressed FITS file
361 metaL, sizeL = self.putFits(exposure, "lossy", 44)
362 self.assertEqual(metaL["TTYPE1"], "COMPRESSED_DATA")
363 self.assertEqual(metaL["ZCMPTYPE"], "RICE_1")
365 self.assertNotEqual(sizeC, sizeN)
366 # Data file is so small that Lossy and Compressed are dominated
367 # by the extra compression tables
368 self.assertEqual(sizeL, sizeC)
370 def testExposureFormatterAmpParameter(self):
371 """Test the FitsExposureFormatter implementation of the Exposure
372 StorageClass's 'amp' and 'detector' parameters.
373 """
374 # Our example exposure file has a realistic detector (looks like HSC),
375 # but the image itself doesn't match it. So we just load the detector,
376 # and use it to make our own more useful images and put them.
377 detector = ExposureFitsReader(os.path.join(TESTDIR, "data", "calexp.fits")).readDetector()
378 trimmed_full = make_ramp_exposure_trimmed(detector)
379 untrimmed_full = make_ramp_exposure_untrimmed(detector)
380 trimmed_ref = self.butler.put(trimmed_full, "int_exp_trimmed")
381 untrimmed_ref = self.butler.put(untrimmed_full, "int_exp_untrimmed")
382 for n, amp in enumerate(detector):
383 # Try to read each amp as it is on disk, with a variety of
384 # parameters that should all do the same thing.
385 for amp_parameter in [amp, amp.getName(), n]:
386 for parameters in [{"amp": amp_parameter}, {"amp": amp_parameter, "detector": detector}]:
387 with self.subTest(parameters=parameters):
388 test_trimmed = self.butler.get(trimmed_ref, parameters=parameters)
389 test_untrimmed = self.butler.get(untrimmed_ref, parameters=parameters)
390 self.assertImagesEqual(test_trimmed.image, trimmed_full[amp.getBBox()].image)
391 self.assertImagesEqual(test_untrimmed.image, untrimmed_full[amp.getRawBBox()].image)
392 self.assertEqual(len(test_trimmed.getDetector()), 1)
393 self.assertEqual(len(test_untrimmed.getDetector()), 1)
394 self.assertAmplifiersEqual(test_trimmed.getDetector()[0], amp)
395 self.assertAmplifiersEqual(test_untrimmed.getDetector()[0], amp)
396 # Try to read various transformed versions of the original amp,
397 # to make sure flips and offsets are applied correctly.
398 # First flip X only.
399 amp_t1 = amp.rebuild().transform(outFlipX=True).finish()
400 test_t1_trimmed = self.butler.get(trimmed_ref, parameters={"amp": amp_t1})
401 self.assertImagesEqual(
402 test_t1_trimmed.image, flipImage(trimmed_full[amp.getBBox()].image, flipLR=True, flipTB=False)
403 )
404 self.assertAmplifiersEqual(test_t1_trimmed.getDetector()[0], amp_t1)
405 test_t1_untrimmed = self.butler.get(untrimmed_ref, parameters={"amp": amp_t1})
406 self.assertImagesEqual(
407 test_t1_untrimmed.image,
408 flipImage(untrimmed_full[amp.getRawBBox()].image, flipLR=True, flipTB=False),
409 )
410 self.assertAmplifiersEqual(test_t1_trimmed.getDetector()[0], amp_t1)
411 # Flip Y only.
412 amp_t2 = amp.rebuild().transform(outFlipY=True).finish()
413 test_t2_trimmed = self.butler.get(trimmed_ref, parameters={"amp": amp_t2})
414 self.assertImagesEqual(
415 test_t2_trimmed.image, flipImage(trimmed_full[amp.getBBox()].image, flipLR=False, flipTB=True)
416 )
417 self.assertAmplifiersEqual(test_t2_trimmed.getDetector()[0], amp_t2)
418 test_t2_untrimmed = self.butler.get(untrimmed_ref, parameters={"amp": amp_t2})
419 self.assertImagesEqual(
420 test_t2_untrimmed.image,
421 flipImage(untrimmed_full[amp.getRawBBox()].image, flipLR=False, flipTB=True),
422 )
423 self.assertAmplifiersEqual(test_t2_trimmed.getDetector()[0], amp_t2)
424 # Add an XY offset only.
425 amp_t3 = amp.rebuild().transform(outOffset=Extent2I(5, 4)).finish()
426 test_t3_trimmed = self.butler.get(trimmed_ref, parameters={"amp": amp_t3})
427 self.assertImagesEqual(test_t3_trimmed.image, trimmed_full[amp.getBBox()].image)
428 self.assertAmplifiersEqual(test_t3_trimmed.getDetector()[0], amp_t3)
429 test_t3_untrimmed = self.butler.get(untrimmed_ref, parameters={"amp": amp_t3})
430 self.assertImagesEqual(test_t3_untrimmed.image, untrimmed_full[amp.getRawBBox()].image)
431 self.assertAmplifiersEqual(test_t3_trimmed.getDetector()[0], amp_t3)
434if __name__ == "__main__": 434 ↛ 435line 434 didn't jump to line 435, because the condition on line 434 was never true
435 unittest.main()