Coverage for tests/test_butlerFits.py: 16%
221 statements
« prev ^ index » next coverage.py v6.4.4, created at 2022-09-02 02:23 -0700
« prev ^ index » next coverage.py v6.4.4, created at 2022-09-02 02:23 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24import os
25import shutil
26import tempfile
27import unittest
28from typing import TYPE_CHECKING
30import lsst.afw.cameraGeom.testUtils # for test asserts injected into TestCase
31import lsst.afw.image
32import lsst.pex.config
33import lsst.utils.tests
34from lsst.afw.fits import readMetadata
35from lsst.afw.image import LOCAL, ExposureFitsReader
36from lsst.afw.math import flipImage
37from lsst.daf.base import PropertyList, PropertySet
38from lsst.daf.butler import Config, DatasetType, StorageClassFactory
39from lsst.daf.butler.tests import DatasetTestHelper, addDatasetType, makeTestCollection, makeTestRepo
40from lsst.geom import Box2I, Extent2I, Point2I
41from lsst.obs.base.exposureAssembler import ExposureAssembler
42from lsst.obs.base.tests import make_ramp_exposure_trimmed, make_ramp_exposure_untrimmed
44if TYPE_CHECKING: 44 ↛ 45line 44 didn't jump to line 45, because the condition on line 44 was never true
45 from lsst.daf.butler import DatasetRef
47TESTDIR = os.path.dirname(__file__)
49BUTLER_CONFIG = """
50storageClasses:
51 ExposureCompositeF:
52 inheritsFrom: ExposureF
53datastore:
54 # Want to check disassembly so can't use InMemory
55 cls: lsst.daf.butler.datastores.fileDatastore.FileDatastore
56 formatters:
57 ExposureCompositeF: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
58 lossless:
59 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
60 parameters:
61 recipe: lossless
62 uncompressed:
63 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
64 parameters:
65 recipe: noCompression
66 lossy:
67 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
68 parameters:
69 recipe: lossyBasic
70 composites:
71 disassembled:
72 ExposureCompositeF: True
73"""
75# Components present in the test file
76COMPONENTS = {
77 "wcs",
78 "image",
79 "mask",
80 "coaddInputs",
81 "psf",
82 "visitInfo",
83 "variance",
84 "metadata",
85 "photoCalib",
86 "filter",
87 "validPolygon",
88 "transmissionCurve",
89 "detector",
90 "apCorrMap",
91 "summaryStats",
92 "id",
93}
94READ_COMPONENTS = {
95 "bbox",
96 "xy0",
97 "dimensions",
98 "filterLabel",
99}
102class ButlerFitsTests(DatasetTestHelper, lsst.utils.tests.TestCase):
103 @classmethod
104 def setUpClass(cls):
105 """Create a new butler once only."""
107 cls.storageClassFactory = StorageClassFactory()
109 cls.root = tempfile.mkdtemp(dir=TESTDIR)
111 dataIds = {
112 "instrument": ["DummyCam"],
113 "physical_filter": ["d-r"],
114 "visit": [42, 43, 44],
115 }
117 # Ensure that we test in a directory that will include some
118 # metacharacters
119 subdir = "sub?#dir"
120 butlerRoot = os.path.join(cls.root, subdir)
122 cls.creatorButler = makeTestRepo(butlerRoot, dataIds, config=Config.fromYaml(BUTLER_CONFIG))
124 # Create dataset types used by the tests
125 for datasetTypeName, storageClassName in (
126 ("calexp", "ExposureF"),
127 ("unknown", "ExposureCompositeF"),
128 ("testCatalog", "SourceCatalog"),
129 ("lossless", "ExposureF"),
130 ("uncompressed", "ExposureF"),
131 ("lossy", "ExposureF"),
132 ):
133 storageClass = cls.storageClassFactory.getStorageClass(storageClassName)
134 addDatasetType(cls.creatorButler, datasetTypeName, set(dataIds), storageClass)
136 # And some dataset types that have no dimensions for easy testing
137 for datasetTypeName, storageClassName in (
138 ("ps", "PropertySet"),
139 ("pl", "PropertyList"),
140 ("int_exp_trimmed", "ExposureI"),
141 ("int_exp_untrimmed", "ExposureI"),
142 ):
143 storageClass = cls.storageClassFactory.getStorageClass(storageClassName)
144 addDatasetType(cls.creatorButler, datasetTypeName, {}, storageClass)
146 @classmethod
147 def tearDownClass(cls):
148 if cls.root is not None:
149 shutil.rmtree(cls.root, ignore_errors=True)
151 def setUp(self):
152 self.butler = makeTestCollection(self.creatorButler)
154 def makeExampleCatalog(self) -> lsst.afw.table.SourceCatalog:
155 catalogPath = os.path.join(TESTDIR, "data", "source_catalog.fits")
156 return lsst.afw.table.SourceCatalog.readFits(catalogPath)
158 def assertCatalogEqual(
159 self, inputCatalog: lsst.afw.table.SourceCatalog, outputCatalog: lsst.afw.table.SourceCatalog
160 ) -> None:
161 self.assertIsInstance(outputCatalog, lsst.afw.table.SourceCatalog)
162 inputTable = inputCatalog.getTable()
163 inputRecord = inputCatalog[0]
164 outputTable = outputCatalog.getTable()
165 outputRecord = outputCatalog[0]
166 self.assertEqual(inputRecord.getPsfInstFlux(), outputRecord.getPsfInstFlux())
167 self.assertEqual(inputRecord.getPsfFluxFlag(), outputRecord.getPsfFluxFlag())
168 self.assertEqual(
169 inputTable.getSchema().getAliasMap().get("slot_Centroid"),
170 outputTable.getSchema().getAliasMap().get("slot_Centroid"),
171 )
172 self.assertEqual(inputRecord.getCentroid(), outputRecord.getCentroid())
173 self.assertFloatsAlmostEqual(
174 inputRecord.getCentroidErr()[0, 0], outputRecord.getCentroidErr()[0, 0], rtol=1e-6
175 )
176 self.assertFloatsAlmostEqual(
177 inputRecord.getCentroidErr()[1, 1], outputRecord.getCentroidErr()[1, 1], rtol=1e-6
178 )
179 self.assertEqual(
180 inputTable.getSchema().getAliasMap().get("slot_Shape"),
181 outputTable.getSchema().getAliasMap().get("slot_Shape"),
182 )
183 self.assertFloatsAlmostEqual(
184 inputRecord.getShapeErr()[0, 0], outputRecord.getShapeErr()[0, 0], rtol=1e-6
185 )
186 self.assertFloatsAlmostEqual(
187 inputRecord.getShapeErr()[1, 1], outputRecord.getShapeErr()[1, 1], rtol=1e-6
188 )
189 self.assertFloatsAlmostEqual(
190 inputRecord.getShapeErr()[2, 2], outputRecord.getShapeErr()[2, 2], rtol=1e-6
191 )
193 def runFundamentalTypeTest(self, datasetTypeName, entity):
194 """Put and get the supplied entity and compare."""
195 ref = self.butler.put(entity, datasetTypeName)
196 butler_ps = self.butler.get(ref)
197 self.assertEqual(butler_ps, entity)
199 # Break the contact by ensuring that we are writing YAML
200 uri = self.butler.getURI(ref)
201 self.assertEqual(uri.getExtension(), ".yaml", f"Check extension of {uri}")
203 def testFundamentalTypes(self) -> None:
204 """Ensure that some fundamental stack types round trip."""
205 ps = PropertySet()
206 ps["a.b"] = 5
207 ps["c.d.e"] = "string"
208 self.runFundamentalTypeTest("ps", ps)
210 pl = PropertyList()
211 pl["A"] = 1
212 pl.setComment("A", "An int comment")
213 pl["B"] = "string"
214 pl.setComment("B", "A string comment")
215 self.runFundamentalTypeTest("pl", pl)
217 def testFitsCatalog(self) -> None:
218 """Test reading of a FITS catalog"""
219 catalog = self.makeExampleCatalog()
220 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"}
221 ref = self.butler.put(catalog, "testCatalog", dataId)
222 stored = self.butler.get(ref)
223 self.assertCatalogEqual(catalog, stored)
225 def testExposureCompositePutGetConcrete(self) -> None:
226 """Test composite with no disassembly"""
227 ref = self.runExposureCompositePutGetTest("calexp")
229 uri = self.butler.getURI(ref)
230 self.assertTrue(uri.exists(), f"Checking URI {uri} existence")
232 def testExposureCompositePutGetVirtual(self) -> None:
233 """Testing composite disassembly"""
234 ref = self.runExposureCompositePutGetTest("unknown")
236 primary, components = self.butler.getURIs(ref)
237 self.assertIsNone(primary)
238 self.assertEqual(set(components), COMPONENTS)
239 for compName, uri in components.items():
240 self.assertTrue(uri.exists(), f"Checking URI {uri} existence for component {compName}")
242 def runExposureCompositePutGetTest(self, datasetTypeName: str) -> DatasetRef:
243 example = os.path.join(TESTDIR, "data", "calexp.fits")
244 exposure = lsst.afw.image.ExposureF(example)
246 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"}
247 ref = self.butler.put(exposure, datasetTypeName, dataId)
249 # Get the full thing
250 composite = self.butler.get(datasetTypeName, dataId)
252 # There is no assert for Exposure so just look at maskedImage
253 self.assertMaskedImagesEqual(composite.maskedImage, exposure.maskedImage)
255 # Helper for extracting components
256 assembler = ExposureAssembler(ref.datasetType.storageClass)
258 # Check all possible components that can be read
259 allComponents = set()
260 allComponents.update(COMPONENTS, READ_COMPONENTS)
262 # Get each component from butler independently
263 for compName in allComponents:
264 compTypeName = DatasetType.nameWithComponent(datasetTypeName, compName)
265 component = self.butler.get(compTypeName, dataId)
267 reference = assembler.getComponent(exposure, compName)
269 self.assertIsInstance(component, type(reference), f"Checking type of component {compName}")
271 if compName in ("image", "variance"):
272 self.assertImagesEqual(component, reference)
273 elif compName == "mask":
274 self.assertMasksEqual(component, reference)
275 elif compName == "wcs":
276 self.assertWcsAlmostEqualOverBBox(component, reference, exposure.getBBox())
277 elif compName == "coaddInputs":
278 self.assertEqual(
279 len(component.visits), len(reference.visits), f"cf visits {component.visits}"
280 )
281 self.assertEqual(len(component.ccds), len(reference.ccds), f"cf CCDs {component.ccds}")
282 elif compName == "psf":
283 # Equality for PSF does not work
284 pass
285 elif compName == "filter":
286 self.assertEqual(component, reference)
287 elif compName == "filterLabel":
288 self.assertEqual(component, reference)
289 elif compName == "id":
290 self.assertEqual(component, reference)
291 elif compName == "visitInfo":
292 self.assertEqual(component, reference, "VisitInfo comparison")
293 elif compName == "metadata":
294 # The component metadata has extra fields in it so cannot
295 # compare directly.
296 for k, v in reference.items():
297 self.assertEqual(component[k], v)
298 elif compName == "photoCalib":
299 # This example has a
300 # "spatially constant with mean: inf error: nan" entry
301 # which does not compare directly.
302 self.assertEqual(str(component), str(reference))
303 self.assertIn("spatially constant with mean: 1.99409", str(component), "Checking photoCalib")
304 elif compName in ("bbox", "xy0", "dimensions", "validPolygon"):
305 self.assertEqual(component, reference)
306 elif compName == "apCorrMap":
307 self.assertEqual(set(component.keys()), set(reference.keys()))
308 elif compName == "transmissionCurve":
309 self.assertEqual(component.getThroughputAtBounds(), reference.getThroughputAtBounds())
310 elif compName == "detector":
311 c_amps = {a.getName() for a in component.getAmplifiers()}
312 r_amps = {a.getName() for a in reference.getAmplifiers()}
313 self.assertEqual(c_amps, r_amps)
314 elif compName == "summaryStats":
315 self.assertEqual(component.psfSigma, reference.psfSigma)
316 else:
317 raise RuntimeError(f"Unexpected component '{compName}' encountered in test")
319 # Full Exposure with parameters
320 inBBox = Box2I(minimum=Point2I(3, 3), maximum=Point2I(21, 16))
321 parameters = dict(bbox=inBBox, origin=LOCAL)
322 subset = self.butler.get(datasetTypeName, dataId, parameters=parameters)
323 outBBox = subset.getBBox()
324 self.assertEqual(inBBox, outBBox)
325 self.assertImagesEqual(subset.getImage(), exposure.subset(inBBox, origin=LOCAL).getImage())
327 return ref
329 def putFits(self, exposure, datasetTypeName, visit):
330 """Put different datasetTypes and return information."""
331 dataId = {"visit": visit, "instrument": "DummyCam", "physical_filter": "d-r"}
332 refC = self.butler.put(exposure, datasetTypeName, dataId)
333 uriC = self.butler.getURI(refC)
334 stat = os.stat(uriC.ospath)
335 size = stat.st_size
336 # We can't use butler's Exposure storage class metadata component,
337 # because that intentionally strips keywords that science code
338 # shouldn't ever read (to at least weaken our assumptions that we write
339 # to FITS). Instead use a lower-level method on the URI for this test.
340 meta = readMetadata(uriC.ospath)
341 return meta, size
343 def testCompression(self):
344 """Test that we can write compressed and uncompressed FITS."""
345 example = os.path.join(TESTDIR, "data", "small.fits")
346 exposure = lsst.afw.image.ExposureF(example)
348 # Write a lossless compressed
349 metaC, sizeC = self.putFits(exposure, "lossless", 42)
350 self.assertEqual(metaC["TTYPE1"], "COMPRESSED_DATA")
351 self.assertEqual(metaC["ZCMPTYPE"], "GZIP_2")
353 # Write an uncompressed FITS file
354 metaN, sizeN = self.putFits(exposure, "uncompressed", 43)
355 self.assertNotIn("ZCMPTYPE", metaN)
357 # Write an uncompressed FITS file
358 metaL, sizeL = self.putFits(exposure, "lossy", 44)
359 self.assertEqual(metaL["TTYPE1"], "COMPRESSED_DATA")
360 self.assertEqual(metaL["ZCMPTYPE"], "RICE_1")
362 self.assertNotEqual(sizeC, sizeN)
363 # Data file is so small that Lossy and Compressed are dominated
364 # by the extra compression tables
365 self.assertEqual(sizeL, sizeC)
367 def testExposureFormatterAmpParameter(self):
368 """Test the FitsExposureFormatter implementation of the Exposure
369 StorageClass's 'amp' and 'detector' parameters.
370 """
371 # Our example exposure file has a realistic detector (looks like HSC),
372 # but the image itself doesn't match it. So we just load the detector,
373 # and use it to make our own more useful images and put them.
374 detector = ExposureFitsReader(os.path.join(TESTDIR, "data", "calexp.fits")).readDetector()
375 trimmed_full = make_ramp_exposure_trimmed(detector)
376 untrimmed_full = make_ramp_exposure_untrimmed(detector)
377 trimmed_ref = self.butler.put(trimmed_full, "int_exp_trimmed")
378 untrimmed_ref = self.butler.put(untrimmed_full, "int_exp_untrimmed")
379 for n, amp in enumerate(detector):
380 # Try to read each amp as it is on disk, with a variety of
381 # parameters that should all do the same thing.
382 for amp_parameter in [amp, amp.getName(), n]:
383 for parameters in [{"amp": amp_parameter}, {"amp": amp_parameter, "detector": detector}]:
384 with self.subTest(parameters=parameters):
385 test_trimmed = self.butler.getDirect(trimmed_ref, parameters=parameters)
386 test_untrimmed = self.butler.getDirect(untrimmed_ref, parameters=parameters)
387 self.assertImagesEqual(test_trimmed.image, trimmed_full[amp.getBBox()].image)
388 self.assertImagesEqual(test_untrimmed.image, untrimmed_full[amp.getRawBBox()].image)
389 self.assertEqual(len(test_trimmed.getDetector()), 1)
390 self.assertEqual(len(test_untrimmed.getDetector()), 1)
391 self.assertAmplifiersEqual(test_trimmed.getDetector()[0], amp)
392 self.assertAmplifiersEqual(test_untrimmed.getDetector()[0], amp)
393 # Try to read various transformed versions of the original amp,
394 # to make sure flips and offsets are applied correctly.
395 # First flip X only.
396 amp_t1 = amp.rebuild().transform(outFlipX=True).finish()
397 test_t1_trimmed = self.butler.getDirect(trimmed_ref, parameters={"amp": amp_t1})
398 self.assertImagesEqual(
399 test_t1_trimmed.image, flipImage(trimmed_full[amp.getBBox()].image, flipLR=True, flipTB=False)
400 )
401 self.assertAmplifiersEqual(test_t1_trimmed.getDetector()[0], amp_t1)
402 test_t1_untrimmed = self.butler.getDirect(untrimmed_ref, parameters={"amp": amp_t1})
403 self.assertImagesEqual(
404 test_t1_untrimmed.image,
405 flipImage(untrimmed_full[amp.getRawBBox()].image, flipLR=True, flipTB=False),
406 )
407 self.assertAmplifiersEqual(test_t1_trimmed.getDetector()[0], amp_t1)
408 # Flip Y only.
409 amp_t2 = amp.rebuild().transform(outFlipY=True).finish()
410 test_t2_trimmed = self.butler.getDirect(trimmed_ref, parameters={"amp": amp_t2})
411 self.assertImagesEqual(
412 test_t2_trimmed.image, flipImage(trimmed_full[amp.getBBox()].image, flipLR=False, flipTB=True)
413 )
414 self.assertAmplifiersEqual(test_t2_trimmed.getDetector()[0], amp_t2)
415 test_t2_untrimmed = self.butler.getDirect(untrimmed_ref, parameters={"amp": amp_t2})
416 self.assertImagesEqual(
417 test_t2_untrimmed.image,
418 flipImage(untrimmed_full[amp.getRawBBox()].image, flipLR=False, flipTB=True),
419 )
420 self.assertAmplifiersEqual(test_t2_trimmed.getDetector()[0], amp_t2)
421 # Add an XY offset only.
422 amp_t3 = amp.rebuild().transform(outOffset=Extent2I(5, 4)).finish()
423 test_t3_trimmed = self.butler.getDirect(trimmed_ref, parameters={"amp": amp_t3})
424 self.assertImagesEqual(test_t3_trimmed.image, trimmed_full[amp.getBBox()].image)
425 self.assertAmplifiersEqual(test_t3_trimmed.getDetector()[0], amp_t3)
426 test_t3_untrimmed = self.butler.getDirect(untrimmed_ref, parameters={"amp": amp_t3})
427 self.assertImagesEqual(test_t3_untrimmed.image, untrimmed_full[amp.getRawBBox()].image)
428 self.assertAmplifiersEqual(test_t3_trimmed.getDetector()[0], amp_t3)
431if __name__ == "__main__": 431 ↛ 432line 431 didn't jump to line 432, because the condition on line 431 was never true
432 unittest.main()