Coverage for tests/test_butlerFits.py: 17%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24import os
25import unittest
26import tempfile
27import shutil
29from typing import TYPE_CHECKING
31import lsst.utils.tests
33import lsst.pex.config
34import lsst.afw.image
35from lsst.afw.image import ExposureFitsReader, LOCAL
36from lsst.afw.fits import readMetadata
37from lsst.afw.math import flipImage
38import lsst.afw.cameraGeom.testUtils # for test asserts injected into TestCase
39from lsst.geom import Box2I, Extent2I, Point2I
40from lsst.base import Packages
41from lsst.daf.base import PropertyList, PropertySet
43from lsst.daf.butler import Config
44from lsst.daf.butler import StorageClassFactory
45from lsst.daf.butler import DatasetType
46from lsst.daf.butler.tests import DatasetTestHelper, makeTestRepo, addDatasetType, makeTestCollection
48from lsst.obs.base.exposureAssembler import ExposureAssembler
49from lsst.obs.base.tests import make_ramp_exposure_trimmed, make_ramp_exposure_untrimmed
51if TYPE_CHECKING: 51 ↛ 52line 51 didn't jump to line 52, because the condition on line 51 was never true
52 from lsst.daf.butler import DatasetRef
54TESTDIR = os.path.dirname(__file__)
56BUTLER_CONFIG = """
57storageClasses:
58 ExposureCompositeF:
59 inheritsFrom: ExposureF
60datastore:
61 # Want to check disassembly so can't use InMemory
62 cls: lsst.daf.butler.datastores.fileDatastore.FileDatastore
63 formatters:
64 ExposureCompositeF: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
65 lossless:
66 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
67 parameters:
68 recipe: lossless
69 uncompressed:
70 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
71 parameters:
72 recipe: noCompression
73 lossy:
74 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
75 parameters:
76 recipe: lossyBasic
77 composites:
78 disassembled:
79 ExposureCompositeF: True
80"""
82# Components present in the test file
83COMPONENTS = {"wcs", "image", "mask", "coaddInputs", "psf", "visitInfo", "variance", "metadata", "photoCalib",
84 "filterLabel", "validPolygon", "transmissionCurve", "detector", "apCorrMap", "summaryStats",
85 "id",
86 }
87READ_COMPONENTS = {"bbox", "xy0", "dimensions", "filter"}
90class SimpleConfig(lsst.pex.config.Config):
91 """Config to use in tests for butler put/get"""
92 i = lsst.pex.config.Field("integer test", int)
93 c = lsst.pex.config.Field("string", str)
96class ButlerFitsTests(DatasetTestHelper, lsst.utils.tests.TestCase):
98 @classmethod
99 def setUpClass(cls):
100 """Create a new butler once only."""
102 cls.storageClassFactory = StorageClassFactory()
104 cls.root = tempfile.mkdtemp(dir=TESTDIR)
106 dataIds = {
107 "instrument": ["DummyCam"],
108 "physical_filter": ["d-r"],
109 "visit": [42, 43, 44],
110 }
112 # Ensure that we test in a directory that will include some
113 # metacharacters
114 subdir = "sub?#dir"
115 butlerRoot = os.path.join(cls.root, subdir)
117 cls.creatorButler = makeTestRepo(butlerRoot, dataIds, config=Config.fromYaml(BUTLER_CONFIG))
119 # Create dataset types used by the tests
120 for datasetTypeName, storageClassName in (("calexp", "ExposureF"),
121 ("unknown", "ExposureCompositeF"),
122 ("testCatalog", "SourceCatalog"),
123 ("lossless", "ExposureF"),
124 ("uncompressed", "ExposureF"),
125 ("lossy", "ExposureF"),
126 ):
127 storageClass = cls.storageClassFactory.getStorageClass(storageClassName)
128 addDatasetType(cls.creatorButler, datasetTypeName, set(dataIds), storageClass)
130 # And some dataset types that have no dimensions for easy testing
131 for datasetTypeName, storageClassName in (("ps", "PropertySet"),
132 ("pl", "PropertyList"),
133 ("pkg", "Packages"),
134 ("config", "Config"),
135 ("int_exp_trimmed", "ExposureI"),
136 ("int_exp_untrimmed", "ExposureI"),
137 ):
138 storageClass = cls.storageClassFactory.getStorageClass(storageClassName)
139 addDatasetType(cls.creatorButler, datasetTypeName, {}, storageClass)
141 @classmethod
142 def tearDownClass(cls):
143 if cls.root is not None:
144 shutil.rmtree(cls.root, ignore_errors=True)
146 def setUp(self):
147 self.butler = makeTestCollection(self.creatorButler)
149 def makeExampleCatalog(self) -> lsst.afw.table.SourceCatalog:
150 catalogPath = os.path.join(TESTDIR, "data", "source_catalog.fits")
151 return lsst.afw.table.SourceCatalog.readFits(catalogPath)
153 def assertCatalogEqual(self, inputCatalog: lsst.afw.table.SourceCatalog,
154 outputCatalog: lsst.afw.table.SourceCatalog) -> None:
155 self.assertIsInstance(outputCatalog, lsst.afw.table.SourceCatalog)
156 inputTable = inputCatalog.getTable()
157 inputRecord = inputCatalog[0]
158 outputTable = outputCatalog.getTable()
159 outputRecord = outputCatalog[0]
160 self.assertEqual(inputRecord.getPsfInstFlux(), outputRecord.getPsfInstFlux())
161 self.assertEqual(inputRecord.getPsfFluxFlag(), outputRecord.getPsfFluxFlag())
162 self.assertEqual(inputTable.getSchema().getAliasMap().get("slot_Centroid"),
163 outputTable.getSchema().getAliasMap().get("slot_Centroid"))
164 self.assertEqual(inputRecord.getCentroid(), outputRecord.getCentroid())
165 self.assertFloatsAlmostEqual(
166 inputRecord.getCentroidErr()[0, 0],
167 outputRecord.getCentroidErr()[0, 0], rtol=1e-6)
168 self.assertFloatsAlmostEqual(
169 inputRecord.getCentroidErr()[1, 1],
170 outputRecord.getCentroidErr()[1, 1], rtol=1e-6)
171 self.assertEqual(inputTable.getSchema().getAliasMap().get("slot_Shape"),
172 outputTable.getSchema().getAliasMap().get("slot_Shape"))
173 self.assertFloatsAlmostEqual(
174 inputRecord.getShapeErr()[0, 0],
175 outputRecord.getShapeErr()[0, 0], rtol=1e-6)
176 self.assertFloatsAlmostEqual(
177 inputRecord.getShapeErr()[1, 1],
178 outputRecord.getShapeErr()[1, 1], rtol=1e-6)
179 self.assertFloatsAlmostEqual(
180 inputRecord.getShapeErr()[2, 2],
181 outputRecord.getShapeErr()[2, 2], rtol=1e-6)
183 def runFundamentalTypeTest(self, datasetTypeName, entity):
184 """Put and get the supplied entity and compare."""
185 ref = self.butler.put(entity, datasetTypeName)
186 butler_ps = self.butler.get(ref)
187 self.assertEqual(butler_ps, entity)
189 # Break the contact by ensuring that we are writing YAML
190 uri = self.butler.getURI(ref)
191 self.assertTrue(uri.path.endswith(".yaml"), f"Check extension of {uri}")
193 def testFundamentalTypes(self) -> None:
194 """Ensure that some fundamental stack types round trip."""
195 ps = PropertySet()
196 ps["a.b"] = 5
197 ps["c.d.e"] = "string"
198 self.runFundamentalTypeTest("ps", ps)
200 pl = PropertyList()
201 pl["A"] = 1
202 pl.setComment("A", "An int comment")
203 pl["B"] = "string"
204 pl.setComment("B", "A string comment")
205 self.runFundamentalTypeTest("pl", pl)
207 pkg = Packages.fromSystem()
208 self.runFundamentalTypeTest("pkg", pkg)
210 def testPexConfig(self) -> None:
211 """Test that we can put and get pex_config Configs"""
212 c = SimpleConfig(i=10, c="hello")
213 self.assertEqual(c.i, 10)
214 ref = self.butler.put(c, "config")
215 butler_c = self.butler.get(ref)
216 self.assertEqual(c, butler_c)
217 self.assertIsInstance(butler_c, SimpleConfig)
219 def testFitsCatalog(self) -> None:
220 """Test reading of a FITS catalog"""
221 catalog = self.makeExampleCatalog()
222 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"}
223 ref = self.butler.put(catalog, "testCatalog", dataId)
224 stored = self.butler.get(ref)
225 self.assertCatalogEqual(catalog, stored)
227 def testExposureCompositePutGetConcrete(self) -> None:
228 """Test composite with no disassembly"""
229 ref = self.runExposureCompositePutGetTest("calexp")
231 uri = self.butler.getURI(ref)
232 self.assertTrue(uri.exists(), f"Checking URI {uri} existence")
234 def testExposureCompositePutGetVirtual(self) -> None:
235 """Testing composite disassembly"""
236 ref = self.runExposureCompositePutGetTest("unknown")
238 primary, components = self.butler.getURIs(ref)
239 self.assertIsNone(primary)
240 self.assertEqual(set(components), COMPONENTS)
241 for compName, uri in components.items():
242 self.assertTrue(uri.exists(),
243 f"Checking URI {uri} existence for component {compName}")
245 def runExposureCompositePutGetTest(self, datasetTypeName: str) -> DatasetRef:
246 example = os.path.join(TESTDIR, "data", "calexp.fits")
247 exposure = lsst.afw.image.ExposureF(example)
249 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"}
250 ref = self.butler.put(exposure, datasetTypeName, dataId)
252 # Get the full thing
253 composite = self.butler.get(datasetTypeName, dataId)
255 # There is no assert for Exposure so just look at maskedImage
256 self.assertMaskedImagesEqual(composite.maskedImage, exposure.maskedImage)
258 # Helper for extracting components
259 assembler = ExposureAssembler(ref.datasetType.storageClass)
261 # Check all possible components that can be read
262 allComponents = set()
263 allComponents.update(COMPONENTS, READ_COMPONENTS)
265 # Get each component from butler independently
266 for compName in allComponents:
267 compTypeName = DatasetType.nameWithComponent(datasetTypeName, compName)
268 component = self.butler.get(compTypeName, dataId)
270 reference = assembler.getComponent(exposure, compName)
272 self.assertIsInstance(component, type(reference), f"Checking type of component {compName}")
274 if compName in ("image", "variance"):
275 self.assertImagesEqual(component, reference)
276 elif compName == "mask":
277 self.assertMasksEqual(component, reference)
278 elif compName == "wcs":
279 self.assertWcsAlmostEqualOverBBox(component, reference, exposure.getBBox())
280 elif compName == "coaddInputs":
281 self.assertEqual(len(component.visits), len(reference.visits),
282 f"cf visits {component.visits}")
283 self.assertEqual(len(component.ccds), len(reference.ccds),
284 f"cf CCDs {component.ccds}")
285 elif compName == "psf":
286 # Equality for PSF does not work
287 pass
288 elif compName == "filter":
289 self.assertEqual(component.getCanonicalName(), reference.getCanonicalName())
290 elif compName == "filterLabel":
291 self.assertEqual(component, reference)
292 elif compName == "id":
293 self.assertEqual(component, reference)
294 elif compName == "visitInfo":
295 self.assertEqual(component, reference,
296 "VisitInfo comparison")
297 elif compName == "metadata":
298 # The component metadata has extra fields in it so cannot
299 # compare directly.
300 for k, v in reference.items():
301 self.assertEqual(component[k], v)
302 elif compName == "photoCalib":
303 # This example has a
304 # "spatially constant with mean: inf error: nan" entry
305 # which does not compare directly.
306 self.assertEqual(str(component), str(reference))
307 self.assertIn("spatially constant with mean: 1.99409", str(component),
308 "Checking photoCalib")
309 elif compName in ("bbox", "xy0", "dimensions", "validPolygon"):
310 self.assertEqual(component, reference)
311 elif compName == "apCorrMap":
312 self.assertEqual(set(component.keys()), set(reference.keys()))
313 elif compName == "transmissionCurve":
314 self.assertEqual(component.getThroughputAtBounds(),
315 reference.getThroughputAtBounds())
316 elif compName == "detector":
317 c_amps = {a.getName() for a in component.getAmplifiers()}
318 r_amps = {a.getName() for a in reference.getAmplifiers()}
319 self.assertEqual(c_amps, r_amps)
320 elif compName == 'summaryStats':
321 self.assertEqual(component.psfSigma, reference.psfSigma)
322 else:
323 raise RuntimeError(f"Unexpected component '{compName}' encountered in test")
325 # Full Exposure with parameters
326 inBBox = Box2I(minimum=Point2I(3, 3), maximum=Point2I(21, 16))
327 parameters = dict(bbox=inBBox, origin=LOCAL)
328 subset = self.butler.get(datasetTypeName, dataId, parameters=parameters)
329 outBBox = subset.getBBox()
330 self.assertEqual(inBBox, outBBox)
331 self.assertImagesEqual(subset.getImage(), exposure.subset(inBBox, origin=LOCAL).getImage())
333 return ref
335 def putFits(self, exposure, datasetTypeName, visit):
336 """Put different datasetTypes and return information."""
337 dataId = {"visit": visit, "instrument": "DummyCam", "physical_filter": "d-r"}
338 refC = self.butler.put(exposure, datasetTypeName, dataId)
339 uriC = self.butler.getURI(refC)
340 stat = os.stat(uriC.ospath)
341 size = stat.st_size
342 # We can't use butler's Exposure storage class metadata component,
343 # because that intentionally strips keywords that science code
344 # shouldn't ever read (to at least weaken our assumptions that we write
345 # to FITS). Instead use a lower-level method on the URI for this test.
346 meta = readMetadata(uriC.ospath)
347 return meta, size
349 def testCompression(self):
350 """Test that we can write compressed and uncompressed FITS."""
351 example = os.path.join(TESTDIR, "data", "small.fits")
352 exposure = lsst.afw.image.ExposureF(example)
354 # Write a lossless compressed
355 metaC, sizeC = self.putFits(exposure, "lossless", 42)
356 self.assertEqual(metaC["TTYPE1"], "COMPRESSED_DATA")
357 self.assertEqual(metaC["ZCMPTYPE"], "GZIP_2")
359 # Write an uncompressed FITS file
360 metaN, sizeN = self.putFits(exposure, "uncompressed", 43)
361 self.assertNotIn("ZCMPTYPE", metaN)
363 # Write an uncompressed FITS file
364 metaL, sizeL = self.putFits(exposure, "lossy", 44)
365 self.assertEqual(metaL["TTYPE1"], "COMPRESSED_DATA")
366 self.assertEqual(metaL["ZCMPTYPE"], "RICE_1")
368 self.assertNotEqual(sizeC, sizeN)
369 # Data file is so small that Lossy and Compressed are dominated
370 # by the extra compression tables
371 self.assertEqual(sizeL, sizeC)
373 def testExposureFormatterAmpParameter(self):
374 """Test the FitsExposureFormatter implementation of the Exposure
375 StorageClass's 'amp' and 'detector' parameters.
376 """
377 # Our example exposure file has a realistic detector (looks like HSC),
378 # but the image itself doesn't match it. So we just load the detector,
379 # and use it to make our own more useful images and put them.
380 detector = ExposureFitsReader(os.path.join(TESTDIR, "data", "calexp.fits")).readDetector()
381 trimmed_full = make_ramp_exposure_trimmed(detector)
382 untrimmed_full = make_ramp_exposure_untrimmed(detector)
383 trimmed_ref = self.butler.put(trimmed_full, "int_exp_trimmed")
384 untrimmed_ref = self.butler.put(untrimmed_full, "int_exp_untrimmed")
385 for n, amp in enumerate(detector):
386 # Try to read each amp as it is on disk, with a variety of
387 # parameters that should all do the same thing.
388 for amp_parameter in [amp, amp.getName(), n]:
389 for parameters in [{"amp": amp_parameter}, {"amp": amp_parameter, "detector": detector}]:
390 with self.subTest(parameters=parameters):
391 test_trimmed = self.butler.getDirect(trimmed_ref, parameters=parameters)
392 test_untrimmed = self.butler.getDirect(untrimmed_ref, parameters=parameters)
393 self.assertImagesEqual(test_trimmed.image, trimmed_full[amp.getBBox()].image)
394 self.assertImagesEqual(test_untrimmed.image, untrimmed_full[amp.getRawBBox()].image)
395 self.assertEqual(len(test_trimmed.getDetector()), 1)
396 self.assertEqual(len(test_untrimmed.getDetector()), 1)
397 self.assertAmplifiersEqual(test_trimmed.getDetector()[0], amp)
398 self.assertAmplifiersEqual(test_untrimmed.getDetector()[0], amp)
399 # Try to read various transformed versions of the original amp,
400 # to make sure flips and offsets are applied correctly.
401 # First flip X only.
402 amp_t1 = amp.rebuild().transform(outFlipX=True).finish()
403 test_t1_trimmed = self.butler.getDirect(trimmed_ref, parameters={"amp": amp_t1})
404 self.assertImagesEqual(test_t1_trimmed.image,
405 flipImage(trimmed_full[amp.getBBox()].image,
406 flipLR=True, flipTB=False))
407 self.assertAmplifiersEqual(test_t1_trimmed.getDetector()[0], amp_t1)
408 test_t1_untrimmed = self.butler.getDirect(untrimmed_ref, parameters={"amp": amp_t1})
409 self.assertImagesEqual(test_t1_untrimmed.image,
410 flipImage(untrimmed_full[amp.getRawBBox()].image,
411 flipLR=True, flipTB=False))
412 self.assertAmplifiersEqual(test_t1_trimmed.getDetector()[0], amp_t1)
413 # Flip Y only.
414 amp_t2 = amp.rebuild().transform(outFlipY=True).finish()
415 test_t2_trimmed = self.butler.getDirect(trimmed_ref, parameters={"amp": amp_t2})
416 self.assertImagesEqual(test_t2_trimmed.image,
417 flipImage(trimmed_full[amp.getBBox()].image,
418 flipLR=False, flipTB=True))
419 self.assertAmplifiersEqual(test_t2_trimmed.getDetector()[0], amp_t2)
420 test_t2_untrimmed = self.butler.getDirect(untrimmed_ref, parameters={"amp": amp_t2})
421 self.assertImagesEqual(test_t2_untrimmed.image,
422 flipImage(untrimmed_full[amp.getRawBBox()].image,
423 flipLR=False, flipTB=True))
424 self.assertAmplifiersEqual(test_t2_trimmed.getDetector()[0], amp_t2)
425 # Add an XY offset only.
426 amp_t3 = amp.rebuild().transform(outOffset=Extent2I(5, 4)).finish()
427 test_t3_trimmed = self.butler.getDirect(trimmed_ref, parameters={"amp": amp_t3})
428 self.assertImagesEqual(test_t3_trimmed.image,
429 trimmed_full[amp.getBBox()].image)
430 self.assertAmplifiersEqual(test_t3_trimmed.getDetector()[0], amp_t3)
431 test_t3_untrimmed = self.butler.getDirect(untrimmed_ref, parameters={"amp": amp_t3})
432 self.assertImagesEqual(test_t3_untrimmed.image,
433 untrimmed_full[amp.getRawBBox()].image)
434 self.assertAmplifiersEqual(test_t3_trimmed.getDetector()[0], amp_t3)
437if __name__ == "__main__": 437 ↛ 438line 437 didn't jump to line 438, because the condition on line 437 was never true
438 unittest.main()