Coverage for tests/test_butlerFits.py: 18%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24import os
25import unittest
26import tempfile
27import shutil
29from typing import TYPE_CHECKING
31import lsst.utils.tests
33import lsst.pex.config
34import lsst.afw.image
35from lsst.afw.image import ExposureFitsReader, LOCAL
36from lsst.afw.fits import readMetadata
37from lsst.afw.math import flipImage
38import lsst.afw.cameraGeom.testUtils # for test asserts injected into TestCase
39from lsst.geom import Box2I, Extent2I, Point2I
40from lsst.base import Packages
41from lsst.daf.base import PropertyList, PropertySet
43from lsst.daf.butler import Config
44from lsst.daf.butler import StorageClassFactory
45from lsst.daf.butler import DatasetType
46from lsst.daf.butler.tests import DatasetTestHelper, makeTestRepo, addDatasetType, makeTestCollection
48from lsst.obs.base.exposureAssembler import ExposureAssembler
49from lsst.obs.base.tests import make_ramp_exposure_trimmed, make_ramp_exposure_untrimmed
51if TYPE_CHECKING: 51 ↛ 52line 51 didn't jump to line 52, because the condition on line 51 was never true
52 from lsst.daf.butler import DatasetRef
54TESTDIR = os.path.dirname(__file__)
56BUTLER_CONFIG = """
57storageClasses:
58 ExposureCompositeF:
59 inheritsFrom: ExposureF
60datastore:
61 # Want to check disassembly so can't use InMemory
62 cls: lsst.daf.butler.datastores.fileDatastore.FileDatastore
63 formatters:
64 ExposureCompositeF: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
65 lossless:
66 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
67 parameters:
68 recipe: lossless
69 uncompressed:
70 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
71 parameters:
72 recipe: noCompression
73 lossy:
74 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
75 parameters:
76 recipe: lossyBasic
77 composites:
78 disassembled:
79 ExposureCompositeF: True
80"""
82# Components present in the test file
83COMPONENTS = {"wcs", "image", "mask", "coaddInputs", "psf", "visitInfo", "variance", "metadata", "photoCalib",
84 "filterLabel", "validPolygon", "transmissionCurve", "detector", "apCorrMap", "summaryStats"}
85READ_COMPONENTS = {"bbox", "xy0", "dimensions", "filter"}
88class SimpleConfig(lsst.pex.config.Config):
89 """Config to use in tests for butler put/get"""
90 i = lsst.pex.config.Field("integer test", int)
91 c = lsst.pex.config.Field("string", str)
94class ButlerFitsTests(DatasetTestHelper, lsst.utils.tests.TestCase):
96 @classmethod
97 def setUpClass(cls):
98 """Create a new butler once only."""
100 cls.storageClassFactory = StorageClassFactory()
102 cls.root = tempfile.mkdtemp(dir=TESTDIR)
104 dataIds = {
105 "instrument": ["DummyCam"],
106 "physical_filter": ["d-r"],
107 "visit": [42, 43, 44],
108 }
110 # Ensure that we test in a directory that will include some
111 # metacharacters
112 subdir = "sub?#dir"
113 butlerRoot = os.path.join(cls.root, subdir)
115 cls.creatorButler = makeTestRepo(butlerRoot, dataIds, config=Config.fromYaml(BUTLER_CONFIG))
117 # Create dataset types used by the tests
118 for datasetTypeName, storageClassName in (("calexp", "ExposureF"),
119 ("unknown", "ExposureCompositeF"),
120 ("testCatalog", "SourceCatalog"),
121 ("lossless", "ExposureF"),
122 ("uncompressed", "ExposureF"),
123 ("lossy", "ExposureF"),
124 ):
125 storageClass = cls.storageClassFactory.getStorageClass(storageClassName)
126 addDatasetType(cls.creatorButler, datasetTypeName, set(dataIds), storageClass)
128 # And some dataset types that have no dimensions for easy testing
129 for datasetTypeName, storageClassName in (("ps", "PropertySet"),
130 ("pl", "PropertyList"),
131 ("pkg", "Packages"),
132 ("config", "Config"),
133 ("int_exp_trimmed", "ExposureI"),
134 ("int_exp_untrimmed", "ExposureI"),
135 ):
136 storageClass = cls.storageClassFactory.getStorageClass(storageClassName)
137 addDatasetType(cls.creatorButler, datasetTypeName, {}, storageClass)
139 @classmethod
140 def tearDownClass(cls):
141 if cls.root is not None:
142 shutil.rmtree(cls.root, ignore_errors=True)
144 def setUp(self):
145 self.butler = makeTestCollection(self.creatorButler)
147 def makeExampleCatalog(self) -> lsst.afw.table.SourceCatalog:
148 catalogPath = os.path.join(TESTDIR, "data", "source_catalog.fits")
149 return lsst.afw.table.SourceCatalog.readFits(catalogPath)
151 def assertCatalogEqual(self, inputCatalog: lsst.afw.table.SourceCatalog,
152 outputCatalog: lsst.afw.table.SourceCatalog) -> None:
153 self.assertIsInstance(outputCatalog, lsst.afw.table.SourceCatalog)
154 inputTable = inputCatalog.getTable()
155 inputRecord = inputCatalog[0]
156 outputTable = outputCatalog.getTable()
157 outputRecord = outputCatalog[0]
158 self.assertEqual(inputRecord.getPsfInstFlux(), outputRecord.getPsfInstFlux())
159 self.assertEqual(inputRecord.getPsfFluxFlag(), outputRecord.getPsfFluxFlag())
160 self.assertEqual(inputTable.getSchema().getAliasMap().get("slot_Centroid"),
161 outputTable.getSchema().getAliasMap().get("slot_Centroid"))
162 self.assertEqual(inputRecord.getCentroid(), outputRecord.getCentroid())
163 self.assertFloatsAlmostEqual(
164 inputRecord.getCentroidErr()[0, 0],
165 outputRecord.getCentroidErr()[0, 0], rtol=1e-6)
166 self.assertFloatsAlmostEqual(
167 inputRecord.getCentroidErr()[1, 1],
168 outputRecord.getCentroidErr()[1, 1], rtol=1e-6)
169 self.assertEqual(inputTable.getSchema().getAliasMap().get("slot_Shape"),
170 outputTable.getSchema().getAliasMap().get("slot_Shape"))
171 self.assertFloatsAlmostEqual(
172 inputRecord.getShapeErr()[0, 0],
173 outputRecord.getShapeErr()[0, 0], rtol=1e-6)
174 self.assertFloatsAlmostEqual(
175 inputRecord.getShapeErr()[1, 1],
176 outputRecord.getShapeErr()[1, 1], rtol=1e-6)
177 self.assertFloatsAlmostEqual(
178 inputRecord.getShapeErr()[2, 2],
179 outputRecord.getShapeErr()[2, 2], rtol=1e-6)
181 def runFundamentalTypeTest(self, datasetTypeName, entity):
182 """Put and get the supplied entity and compare."""
183 ref = self.butler.put(entity, datasetTypeName)
184 butler_ps = self.butler.get(ref)
185 self.assertEqual(butler_ps, entity)
187 # Break the contact by ensuring that we are writing YAML
188 uri = self.butler.getURI(ref)
189 self.assertTrue(uri.path.endswith(".yaml"), f"Check extension of {uri}")
191 def testFundamentalTypes(self) -> None:
192 """Ensure that some fundamental stack types round trip."""
193 ps = PropertySet()
194 ps["a.b"] = 5
195 ps["c.d.e"] = "string"
196 self.runFundamentalTypeTest("ps", ps)
198 pl = PropertyList()
199 pl["A"] = 1
200 pl.setComment("A", "An int comment")
201 pl["B"] = "string"
202 pl.setComment("B", "A string comment")
203 self.runFundamentalTypeTest("pl", pl)
205 pkg = Packages.fromSystem()
206 self.runFundamentalTypeTest("pkg", pkg)
208 def testPexConfig(self) -> None:
209 """Test that we can put and get pex_config Configs"""
210 c = SimpleConfig(i=10, c="hello")
211 self.assertEqual(c.i, 10)
212 ref = self.butler.put(c, "config")
213 butler_c = self.butler.get(ref)
214 self.assertEqual(c, butler_c)
215 self.assertIsInstance(butler_c, SimpleConfig)
217 def testFitsCatalog(self) -> None:
218 """Test reading of a FITS catalog"""
219 catalog = self.makeExampleCatalog()
220 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"}
221 ref = self.butler.put(catalog, "testCatalog", dataId)
222 stored = self.butler.get(ref)
223 self.assertCatalogEqual(catalog, stored)
225 def testExposureCompositePutGetConcrete(self) -> None:
226 """Test composite with no disassembly"""
227 ref = self.runExposureCompositePutGetTest("calexp")
229 uri = self.butler.getURI(ref)
230 self.assertTrue(uri.exists(), f"Checking URI {uri} existence")
232 def testExposureCompositePutGetVirtual(self) -> None:
233 """Testing composite disassembly"""
234 ref = self.runExposureCompositePutGetTest("unknown")
236 primary, components = self.butler.getURIs(ref)
237 self.assertIsNone(primary)
238 self.assertEqual(set(components), COMPONENTS)
239 for compName, uri in components.items():
240 self.assertTrue(uri.exists(),
241 f"Checking URI {uri} existence for component {compName}")
243 def runExposureCompositePutGetTest(self, datasetTypeName: str) -> DatasetRef:
244 example = os.path.join(TESTDIR, "data", "calexp.fits")
245 exposure = lsst.afw.image.ExposureF(example)
247 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"}
248 ref = self.butler.put(exposure, datasetTypeName, dataId)
250 # Get the full thing
251 composite = self.butler.get(datasetTypeName, dataId)
253 # There is no assert for Exposure so just look at maskedImage
254 self.assertMaskedImagesEqual(composite.maskedImage, exposure.maskedImage)
256 # Helper for extracting components
257 assembler = ExposureAssembler(ref.datasetType.storageClass)
259 # Check all possible components that can be read
260 allComponents = set()
261 allComponents.update(COMPONENTS, READ_COMPONENTS)
263 # Get each component from butler independently
264 for compName in allComponents:
265 compTypeName = DatasetType.nameWithComponent(datasetTypeName, compName)
266 component = self.butler.get(compTypeName, dataId)
268 reference = assembler.getComponent(exposure, compName)
270 self.assertIsInstance(component, type(reference), f"Checking type of component {compName}")
272 if compName in ("image", "variance"):
273 self.assertImagesEqual(component, reference)
274 elif compName == "mask":
275 self.assertMasksEqual(component, reference)
276 elif compName == "wcs":
277 self.assertWcsAlmostEqualOverBBox(component, reference, exposure.getBBox())
278 elif compName == "coaddInputs":
279 self.assertEqual(len(component.visits), len(reference.visits),
280 f"cf visits {component.visits}")
281 self.assertEqual(len(component.ccds), len(reference.ccds),
282 f"cf CCDs {component.ccds}")
283 elif compName == "psf":
284 # Equality for PSF does not work
285 pass
286 elif compName == "filter":
287 self.assertEqual(component.getCanonicalName(), reference.getCanonicalName())
288 elif compName == "filterLabel":
289 self.assertEqual(component, reference)
290 elif compName == "visitInfo":
291 self.assertEqual(component.getExposureId(), reference.getExposureId(),
292 "VisitInfo comparison")
293 elif compName == "metadata":
294 # The component metadata has extra fields in it so cannot
295 # compare directly.
296 for k, v in reference.items():
297 self.assertEqual(component[k], v)
298 elif compName == "photoCalib":
299 # This example has a
300 # "spatially constant with mean: inf error: nan" entry
301 # which does not compare directly.
302 self.assertEqual(str(component), str(reference))
303 self.assertIn("spatially constant with mean: 1.99409", str(component),
304 "Checking photoCalib")
305 elif compName in ("bbox", "xy0", "dimensions", "validPolygon"):
306 self.assertEqual(component, reference)
307 elif compName == "apCorrMap":
308 self.assertEqual(set(component.keys()), set(reference.keys()))
309 elif compName == "transmissionCurve":
310 self.assertEqual(component.getThroughputAtBounds(),
311 reference.getThroughputAtBounds())
312 elif compName == "detector":
313 c_amps = {a.getName() for a in component.getAmplifiers()}
314 r_amps = {a.getName() for a in reference.getAmplifiers()}
315 self.assertEqual(c_amps, r_amps)
316 elif compName == 'summaryStats':
317 self.assertEqual(component.psfSigma, reference.psfSigma)
318 else:
319 raise RuntimeError(f"Unexpected component '{compName}' encountered in test")
321 # Full Exposure with parameters
322 inBBox = Box2I(minimum=Point2I(3, 3), maximum=Point2I(21, 16))
323 parameters = dict(bbox=inBBox, origin=LOCAL)
324 subset = self.butler.get(datasetTypeName, dataId, parameters=parameters)
325 outBBox = subset.getBBox()
326 self.assertEqual(inBBox, outBBox)
327 self.assertImagesEqual(subset.getImage(), exposure.subset(inBBox, origin=LOCAL).getImage())
329 return ref
331 def putFits(self, exposure, datasetTypeName, visit):
332 """Put different datasetTypes and return information."""
333 dataId = {"visit": visit, "instrument": "DummyCam", "physical_filter": "d-r"}
334 refC = self.butler.put(exposure, datasetTypeName, dataId)
335 uriC = self.butler.getURI(refC)
336 stat = os.stat(uriC.ospath)
337 size = stat.st_size
338 # We can't use butler's Exposure storage class metadata component,
339 # because that intentionally strips keywords that science code
340 # shouldn't ever read (to at least weaken our assumptions that we write
341 # to FITS). Instead use a lower-level method on the URI for this test.
342 meta = readMetadata(uriC.ospath)
343 return meta, size
345 def testCompression(self):
346 """Test that we can write compressed and uncompressed FITS."""
347 example = os.path.join(TESTDIR, "data", "small.fits")
348 exposure = lsst.afw.image.ExposureF(example)
350 # Write a lossless compressed
351 metaC, sizeC = self.putFits(exposure, "lossless", 42)
352 self.assertEqual(metaC["TTYPE1"], "COMPRESSED_DATA")
353 self.assertEqual(metaC["ZCMPTYPE"], "GZIP_2")
355 # Write an uncompressed FITS file
356 metaN, sizeN = self.putFits(exposure, "uncompressed", 43)
357 self.assertNotIn("ZCMPTYPE", metaN)
359 # Write an uncompressed FITS file
360 metaL, sizeL = self.putFits(exposure, "lossy", 44)
361 self.assertEqual(metaL["TTYPE1"], "COMPRESSED_DATA")
362 self.assertEqual(metaL["ZCMPTYPE"], "RICE_1")
364 self.assertNotEqual(sizeC, sizeN)
365 # Data file is so small that Lossy and Compressed are dominated
366 # by the extra compression tables
367 self.assertEqual(sizeL, sizeC)
369 def testExposureFormatterAmpParameter(self):
370 """Test the FitsExposureFormatter implementation of the Exposure
371 StorageClass's 'amp' and 'detector' parameters.
372 """
373 # Our example exposure file has a realistic detector (looks like HSC),
374 # but the image itself doesn't match it. So we just load the detector,
375 # and use it to make our own more useful images and put them.
376 detector = ExposureFitsReader(os.path.join(TESTDIR, "data", "calexp.fits")).readDetector()
377 trimmed_full = make_ramp_exposure_trimmed(detector)
378 untrimmed_full = make_ramp_exposure_untrimmed(detector)
379 trimmed_ref = self.butler.put(trimmed_full, "int_exp_trimmed")
380 untrimmed_ref = self.butler.put(untrimmed_full, "int_exp_untrimmed")
381 for n, amp in enumerate(detector):
382 # Try to read each amp as it is on disk, with a variety of
383 # parameters that should all do the same thing.
384 for amp_parameter in [amp, amp.getName(), n]:
385 for parameters in [{"amp": amp_parameter}, {"amp": amp_parameter, "detector": detector}]:
386 with self.subTest(parameters=parameters):
387 test_trimmed = self.butler.getDirect(trimmed_ref, parameters=parameters)
388 test_untrimmed = self.butler.getDirect(untrimmed_ref, parameters=parameters)
389 self.assertImagesEqual(test_trimmed.image, trimmed_full[amp.getBBox()].image)
390 self.assertImagesEqual(test_untrimmed.image, untrimmed_full[amp.getRawBBox()].image)
391 self.assertEqual(len(test_trimmed.getDetector()), 1)
392 self.assertEqual(len(test_untrimmed.getDetector()), 1)
393 self.assertAmplifiersEqual(test_trimmed.getDetector()[0], amp)
394 self.assertAmplifiersEqual(test_untrimmed.getDetector()[0], amp)
395 # Try to read various transformed versions of the original amp,
396 # to make sure flips and offsets are applied correctly.
397 # First flip X only.
398 amp_t1 = amp.rebuild().transform(outFlipX=True).finish()
399 test_t1_trimmed = self.butler.getDirect(trimmed_ref, parameters={"amp": amp_t1})
400 self.assertImagesEqual(test_t1_trimmed.image,
401 flipImage(trimmed_full[amp.getBBox()].image,
402 flipLR=True, flipTB=False))
403 self.assertAmplifiersEqual(test_t1_trimmed.getDetector()[0], amp_t1)
404 test_t1_untrimmed = self.butler.getDirect(untrimmed_ref, parameters={"amp": amp_t1})
405 self.assertImagesEqual(test_t1_untrimmed.image,
406 flipImage(untrimmed_full[amp.getRawBBox()].image,
407 flipLR=True, flipTB=False))
408 self.assertAmplifiersEqual(test_t1_trimmed.getDetector()[0], amp_t1)
409 # Flip Y only.
410 amp_t2 = amp.rebuild().transform(outFlipY=True).finish()
411 test_t2_trimmed = self.butler.getDirect(trimmed_ref, parameters={"amp": amp_t2})
412 self.assertImagesEqual(test_t2_trimmed.image,
413 flipImage(trimmed_full[amp.getBBox()].image,
414 flipLR=False, flipTB=True))
415 self.assertAmplifiersEqual(test_t2_trimmed.getDetector()[0], amp_t2)
416 test_t2_untrimmed = self.butler.getDirect(untrimmed_ref, parameters={"amp": amp_t2})
417 self.assertImagesEqual(test_t2_untrimmed.image,
418 flipImage(untrimmed_full[amp.getRawBBox()].image,
419 flipLR=False, flipTB=True))
420 self.assertAmplifiersEqual(test_t2_trimmed.getDetector()[0], amp_t2)
421 # Add an XY offset only.
422 amp_t3 = amp.rebuild().transform(outOffset=Extent2I(5, 4)).finish()
423 test_t3_trimmed = self.butler.getDirect(trimmed_ref, parameters={"amp": amp_t3})
424 self.assertImagesEqual(test_t3_trimmed.image,
425 trimmed_full[amp.getBBox()].image)
426 self.assertAmplifiersEqual(test_t3_trimmed.getDetector()[0], amp_t3)
427 test_t3_untrimmed = self.butler.getDirect(untrimmed_ref, parameters={"amp": amp_t3})
428 self.assertImagesEqual(test_t3_untrimmed.image,
429 untrimmed_full[amp.getRawBBox()].image)
430 self.assertAmplifiersEqual(test_t3_trimmed.getDetector()[0], amp_t3)
433if __name__ == "__main__": 433 ↛ 434line 433 didn't jump to line 434, because the condition on line 433 was never true
434 unittest.main()