Coverage for tests/test_butlerFits.py: 16%
221 statements
« prev ^ index » next coverage.py v6.4, created at 2022-06-02 03:54 -0700
« prev ^ index » next coverage.py v6.4, created at 2022-06-02 03:54 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24import os
25import shutil
26import tempfile
27import unittest
28from typing import TYPE_CHECKING
30import lsst.afw.cameraGeom.testUtils # for test asserts injected into TestCase
31import lsst.afw.image
32import lsst.pex.config
33import lsst.utils.tests
34from lsst.afw.fits import readMetadata
35from lsst.afw.image import LOCAL, ExposureFitsReader
36from lsst.afw.math import flipImage
37from lsst.daf.base import PropertyList, PropertySet
38from lsst.daf.butler import Config, DatasetType, StorageClassFactory
39from lsst.daf.butler.tests import DatasetTestHelper, addDatasetType, makeTestCollection, makeTestRepo
40from lsst.geom import Box2I, Extent2I, Point2I
41from lsst.obs.base.exposureAssembler import ExposureAssembler
42from lsst.obs.base.tests import make_ramp_exposure_trimmed, make_ramp_exposure_untrimmed
44if TYPE_CHECKING: 44 ↛ 45line 44 didn't jump to line 45, because the condition on line 44 was never true
45 from lsst.daf.butler import DatasetRef
47TESTDIR = os.path.dirname(__file__)
49BUTLER_CONFIG = """
50storageClasses:
51 ExposureCompositeF:
52 inheritsFrom: ExposureF
53datastore:
54 # Want to check disassembly so can't use InMemory
55 cls: lsst.daf.butler.datastores.fileDatastore.FileDatastore
56 formatters:
57 ExposureCompositeF: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
58 lossless:
59 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
60 parameters:
61 recipe: lossless
62 uncompressed:
63 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
64 parameters:
65 recipe: noCompression
66 lossy:
67 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
68 parameters:
69 recipe: lossyBasic
70 composites:
71 disassembled:
72 ExposureCompositeF: True
73"""
75# Components present in the test file
76COMPONENTS = {
77 "wcs",
78 "image",
79 "mask",
80 "coaddInputs",
81 "psf",
82 "visitInfo",
83 "variance",
84 "metadata",
85 "photoCalib",
86 "filterLabel",
87 "validPolygon",
88 "transmissionCurve",
89 "detector",
90 "apCorrMap",
91 "summaryStats",
92 "id",
93}
94READ_COMPONENTS = {"bbox", "xy0", "dimensions", "filter"}
97class ButlerFitsTests(DatasetTestHelper, lsst.utils.tests.TestCase):
98 @classmethod
99 def setUpClass(cls):
100 """Create a new butler once only."""
102 cls.storageClassFactory = StorageClassFactory()
104 cls.root = tempfile.mkdtemp(dir=TESTDIR)
106 dataIds = {
107 "instrument": ["DummyCam"],
108 "physical_filter": ["d-r"],
109 "visit": [42, 43, 44],
110 }
112 # Ensure that we test in a directory that will include some
113 # metacharacters
114 subdir = "sub?#dir"
115 butlerRoot = os.path.join(cls.root, subdir)
117 cls.creatorButler = makeTestRepo(butlerRoot, dataIds, config=Config.fromYaml(BUTLER_CONFIG))
119 # Create dataset types used by the tests
120 for datasetTypeName, storageClassName in (
121 ("calexp", "ExposureF"),
122 ("unknown", "ExposureCompositeF"),
123 ("testCatalog", "SourceCatalog"),
124 ("lossless", "ExposureF"),
125 ("uncompressed", "ExposureF"),
126 ("lossy", "ExposureF"),
127 ):
128 storageClass = cls.storageClassFactory.getStorageClass(storageClassName)
129 addDatasetType(cls.creatorButler, datasetTypeName, set(dataIds), storageClass)
131 # And some dataset types that have no dimensions for easy testing
132 for datasetTypeName, storageClassName in (
133 ("ps", "PropertySet"),
134 ("pl", "PropertyList"),
135 ("int_exp_trimmed", "ExposureI"),
136 ("int_exp_untrimmed", "ExposureI"),
137 ):
138 storageClass = cls.storageClassFactory.getStorageClass(storageClassName)
139 addDatasetType(cls.creatorButler, datasetTypeName, {}, storageClass)
141 @classmethod
142 def tearDownClass(cls):
143 if cls.root is not None:
144 shutil.rmtree(cls.root, ignore_errors=True)
146 def setUp(self):
147 self.butler = makeTestCollection(self.creatorButler)
149 def makeExampleCatalog(self) -> lsst.afw.table.SourceCatalog:
150 catalogPath = os.path.join(TESTDIR, "data", "source_catalog.fits")
151 return lsst.afw.table.SourceCatalog.readFits(catalogPath)
153 def assertCatalogEqual(
154 self, inputCatalog: lsst.afw.table.SourceCatalog, outputCatalog: lsst.afw.table.SourceCatalog
155 ) -> None:
156 self.assertIsInstance(outputCatalog, lsst.afw.table.SourceCatalog)
157 inputTable = inputCatalog.getTable()
158 inputRecord = inputCatalog[0]
159 outputTable = outputCatalog.getTable()
160 outputRecord = outputCatalog[0]
161 self.assertEqual(inputRecord.getPsfInstFlux(), outputRecord.getPsfInstFlux())
162 self.assertEqual(inputRecord.getPsfFluxFlag(), outputRecord.getPsfFluxFlag())
163 self.assertEqual(
164 inputTable.getSchema().getAliasMap().get("slot_Centroid"),
165 outputTable.getSchema().getAliasMap().get("slot_Centroid"),
166 )
167 self.assertEqual(inputRecord.getCentroid(), outputRecord.getCentroid())
168 self.assertFloatsAlmostEqual(
169 inputRecord.getCentroidErr()[0, 0], outputRecord.getCentroidErr()[0, 0], rtol=1e-6
170 )
171 self.assertFloatsAlmostEqual(
172 inputRecord.getCentroidErr()[1, 1], outputRecord.getCentroidErr()[1, 1], rtol=1e-6
173 )
174 self.assertEqual(
175 inputTable.getSchema().getAliasMap().get("slot_Shape"),
176 outputTable.getSchema().getAliasMap().get("slot_Shape"),
177 )
178 self.assertFloatsAlmostEqual(
179 inputRecord.getShapeErr()[0, 0], outputRecord.getShapeErr()[0, 0], rtol=1e-6
180 )
181 self.assertFloatsAlmostEqual(
182 inputRecord.getShapeErr()[1, 1], outputRecord.getShapeErr()[1, 1], rtol=1e-6
183 )
184 self.assertFloatsAlmostEqual(
185 inputRecord.getShapeErr()[2, 2], outputRecord.getShapeErr()[2, 2], rtol=1e-6
186 )
188 def runFundamentalTypeTest(self, datasetTypeName, entity):
189 """Put and get the supplied entity and compare."""
190 ref = self.butler.put(entity, datasetTypeName)
191 butler_ps = self.butler.get(ref)
192 self.assertEqual(butler_ps, entity)
194 # Break the contact by ensuring that we are writing YAML
195 uri = self.butler.getURI(ref)
196 self.assertEqual(uri.getExtension(), ".yaml", f"Check extension of {uri}")
198 def testFundamentalTypes(self) -> None:
199 """Ensure that some fundamental stack types round trip."""
200 ps = PropertySet()
201 ps["a.b"] = 5
202 ps["c.d.e"] = "string"
203 self.runFundamentalTypeTest("ps", ps)
205 pl = PropertyList()
206 pl["A"] = 1
207 pl.setComment("A", "An int comment")
208 pl["B"] = "string"
209 pl.setComment("B", "A string comment")
210 self.runFundamentalTypeTest("pl", pl)
212 def testFitsCatalog(self) -> None:
213 """Test reading of a FITS catalog"""
214 catalog = self.makeExampleCatalog()
215 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"}
216 ref = self.butler.put(catalog, "testCatalog", dataId)
217 stored = self.butler.get(ref)
218 self.assertCatalogEqual(catalog, stored)
220 def testExposureCompositePutGetConcrete(self) -> None:
221 """Test composite with no disassembly"""
222 ref = self.runExposureCompositePutGetTest("calexp")
224 uri = self.butler.getURI(ref)
225 self.assertTrue(uri.exists(), f"Checking URI {uri} existence")
227 def testExposureCompositePutGetVirtual(self) -> None:
228 """Testing composite disassembly"""
229 ref = self.runExposureCompositePutGetTest("unknown")
231 primary, components = self.butler.getURIs(ref)
232 self.assertIsNone(primary)
233 self.assertEqual(set(components), COMPONENTS)
234 for compName, uri in components.items():
235 self.assertTrue(uri.exists(), f"Checking URI {uri} existence for component {compName}")
237 def runExposureCompositePutGetTest(self, datasetTypeName: str) -> DatasetRef:
238 example = os.path.join(TESTDIR, "data", "calexp.fits")
239 exposure = lsst.afw.image.ExposureF(example)
241 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"}
242 ref = self.butler.put(exposure, datasetTypeName, dataId)
244 # Get the full thing
245 composite = self.butler.get(datasetTypeName, dataId)
247 # There is no assert for Exposure so just look at maskedImage
248 self.assertMaskedImagesEqual(composite.maskedImage, exposure.maskedImage)
250 # Helper for extracting components
251 assembler = ExposureAssembler(ref.datasetType.storageClass)
253 # Check all possible components that can be read
254 allComponents = set()
255 allComponents.update(COMPONENTS, READ_COMPONENTS)
257 # Get each component from butler independently
258 for compName in allComponents:
259 compTypeName = DatasetType.nameWithComponent(datasetTypeName, compName)
260 component = self.butler.get(compTypeName, dataId)
262 reference = assembler.getComponent(exposure, compName)
264 self.assertIsInstance(component, type(reference), f"Checking type of component {compName}")
266 if compName in ("image", "variance"):
267 self.assertImagesEqual(component, reference)
268 elif compName == "mask":
269 self.assertMasksEqual(component, reference)
270 elif compName == "wcs":
271 self.assertWcsAlmostEqualOverBBox(component, reference, exposure.getBBox())
272 elif compName == "coaddInputs":
273 self.assertEqual(
274 len(component.visits), len(reference.visits), f"cf visits {component.visits}"
275 )
276 self.assertEqual(len(component.ccds), len(reference.ccds), f"cf CCDs {component.ccds}")
277 elif compName == "psf":
278 # Equality for PSF does not work
279 pass
280 elif compName == "filter":
281 self.assertEqual(component.getCanonicalName(), reference.getCanonicalName())
282 elif compName == "filterLabel":
283 self.assertEqual(component, reference)
284 elif compName == "id":
285 self.assertEqual(component, reference)
286 elif compName == "visitInfo":
287 self.assertEqual(component, reference, "VisitInfo comparison")
288 elif compName == "metadata":
289 # The component metadata has extra fields in it so cannot
290 # compare directly.
291 for k, v in reference.items():
292 self.assertEqual(component[k], v)
293 elif compName == "photoCalib":
294 # This example has a
295 # "spatially constant with mean: inf error: nan" entry
296 # which does not compare directly.
297 self.assertEqual(str(component), str(reference))
298 self.assertIn("spatially constant with mean: 1.99409", str(component), "Checking photoCalib")
299 elif compName in ("bbox", "xy0", "dimensions", "validPolygon"):
300 self.assertEqual(component, reference)
301 elif compName == "apCorrMap":
302 self.assertEqual(set(component.keys()), set(reference.keys()))
303 elif compName == "transmissionCurve":
304 self.assertEqual(component.getThroughputAtBounds(), reference.getThroughputAtBounds())
305 elif compName == "detector":
306 c_amps = {a.getName() for a in component.getAmplifiers()}
307 r_amps = {a.getName() for a in reference.getAmplifiers()}
308 self.assertEqual(c_amps, r_amps)
309 elif compName == "summaryStats":
310 self.assertEqual(component.psfSigma, reference.psfSigma)
311 else:
312 raise RuntimeError(f"Unexpected component '{compName}' encountered in test")
314 # Full Exposure with parameters
315 inBBox = Box2I(minimum=Point2I(3, 3), maximum=Point2I(21, 16))
316 parameters = dict(bbox=inBBox, origin=LOCAL)
317 subset = self.butler.get(datasetTypeName, dataId, parameters=parameters)
318 outBBox = subset.getBBox()
319 self.assertEqual(inBBox, outBBox)
320 self.assertImagesEqual(subset.getImage(), exposure.subset(inBBox, origin=LOCAL).getImage())
322 return ref
324 def putFits(self, exposure, datasetTypeName, visit):
325 """Put different datasetTypes and return information."""
326 dataId = {"visit": visit, "instrument": "DummyCam", "physical_filter": "d-r"}
327 refC = self.butler.put(exposure, datasetTypeName, dataId)
328 uriC = self.butler.getURI(refC)
329 stat = os.stat(uriC.ospath)
330 size = stat.st_size
331 # We can't use butler's Exposure storage class metadata component,
332 # because that intentionally strips keywords that science code
333 # shouldn't ever read (to at least weaken our assumptions that we write
334 # to FITS). Instead use a lower-level method on the URI for this test.
335 meta = readMetadata(uriC.ospath)
336 return meta, size
338 def testCompression(self):
339 """Test that we can write compressed and uncompressed FITS."""
340 example = os.path.join(TESTDIR, "data", "small.fits")
341 exposure = lsst.afw.image.ExposureF(example)
343 # Write a lossless compressed
344 metaC, sizeC = self.putFits(exposure, "lossless", 42)
345 self.assertEqual(metaC["TTYPE1"], "COMPRESSED_DATA")
346 self.assertEqual(metaC["ZCMPTYPE"], "GZIP_2")
348 # Write an uncompressed FITS file
349 metaN, sizeN = self.putFits(exposure, "uncompressed", 43)
350 self.assertNotIn("ZCMPTYPE", metaN)
352 # Write an uncompressed FITS file
353 metaL, sizeL = self.putFits(exposure, "lossy", 44)
354 self.assertEqual(metaL["TTYPE1"], "COMPRESSED_DATA")
355 self.assertEqual(metaL["ZCMPTYPE"], "RICE_1")
357 self.assertNotEqual(sizeC, sizeN)
358 # Data file is so small that Lossy and Compressed are dominated
359 # by the extra compression tables
360 self.assertEqual(sizeL, sizeC)
362 def testExposureFormatterAmpParameter(self):
363 """Test the FitsExposureFormatter implementation of the Exposure
364 StorageClass's 'amp' and 'detector' parameters.
365 """
366 # Our example exposure file has a realistic detector (looks like HSC),
367 # but the image itself doesn't match it. So we just load the detector,
368 # and use it to make our own more useful images and put them.
369 detector = ExposureFitsReader(os.path.join(TESTDIR, "data", "calexp.fits")).readDetector()
370 trimmed_full = make_ramp_exposure_trimmed(detector)
371 untrimmed_full = make_ramp_exposure_untrimmed(detector)
372 trimmed_ref = self.butler.put(trimmed_full, "int_exp_trimmed")
373 untrimmed_ref = self.butler.put(untrimmed_full, "int_exp_untrimmed")
374 for n, amp in enumerate(detector):
375 # Try to read each amp as it is on disk, with a variety of
376 # parameters that should all do the same thing.
377 for amp_parameter in [amp, amp.getName(), n]:
378 for parameters in [{"amp": amp_parameter}, {"amp": amp_parameter, "detector": detector}]:
379 with self.subTest(parameters=parameters):
380 test_trimmed = self.butler.getDirect(trimmed_ref, parameters=parameters)
381 test_untrimmed = self.butler.getDirect(untrimmed_ref, parameters=parameters)
382 self.assertImagesEqual(test_trimmed.image, trimmed_full[amp.getBBox()].image)
383 self.assertImagesEqual(test_untrimmed.image, untrimmed_full[amp.getRawBBox()].image)
384 self.assertEqual(len(test_trimmed.getDetector()), 1)
385 self.assertEqual(len(test_untrimmed.getDetector()), 1)
386 self.assertAmplifiersEqual(test_trimmed.getDetector()[0], amp)
387 self.assertAmplifiersEqual(test_untrimmed.getDetector()[0], amp)
388 # Try to read various transformed versions of the original amp,
389 # to make sure flips and offsets are applied correctly.
390 # First flip X only.
391 amp_t1 = amp.rebuild().transform(outFlipX=True).finish()
392 test_t1_trimmed = self.butler.getDirect(trimmed_ref, parameters={"amp": amp_t1})
393 self.assertImagesEqual(
394 test_t1_trimmed.image, flipImage(trimmed_full[amp.getBBox()].image, flipLR=True, flipTB=False)
395 )
396 self.assertAmplifiersEqual(test_t1_trimmed.getDetector()[0], amp_t1)
397 test_t1_untrimmed = self.butler.getDirect(untrimmed_ref, parameters={"amp": amp_t1})
398 self.assertImagesEqual(
399 test_t1_untrimmed.image,
400 flipImage(untrimmed_full[amp.getRawBBox()].image, flipLR=True, flipTB=False),
401 )
402 self.assertAmplifiersEqual(test_t1_trimmed.getDetector()[0], amp_t1)
403 # Flip Y only.
404 amp_t2 = amp.rebuild().transform(outFlipY=True).finish()
405 test_t2_trimmed = self.butler.getDirect(trimmed_ref, parameters={"amp": amp_t2})
406 self.assertImagesEqual(
407 test_t2_trimmed.image, flipImage(trimmed_full[amp.getBBox()].image, flipLR=False, flipTB=True)
408 )
409 self.assertAmplifiersEqual(test_t2_trimmed.getDetector()[0], amp_t2)
410 test_t2_untrimmed = self.butler.getDirect(untrimmed_ref, parameters={"amp": amp_t2})
411 self.assertImagesEqual(
412 test_t2_untrimmed.image,
413 flipImage(untrimmed_full[amp.getRawBBox()].image, flipLR=False, flipTB=True),
414 )
415 self.assertAmplifiersEqual(test_t2_trimmed.getDetector()[0], amp_t2)
416 # Add an XY offset only.
417 amp_t3 = amp.rebuild().transform(outOffset=Extent2I(5, 4)).finish()
418 test_t3_trimmed = self.butler.getDirect(trimmed_ref, parameters={"amp": amp_t3})
419 self.assertImagesEqual(test_t3_trimmed.image, trimmed_full[amp.getBBox()].image)
420 self.assertAmplifiersEqual(test_t3_trimmed.getDetector()[0], amp_t3)
421 test_t3_untrimmed = self.butler.getDirect(untrimmed_ref, parameters={"amp": amp_t3})
422 self.assertImagesEqual(test_t3_untrimmed.image, untrimmed_full[amp.getRawBBox()].image)
423 self.assertAmplifiersEqual(test_t3_trimmed.getDetector()[0], amp_t3)
426if __name__ == "__main__": 426 ↛ 427line 426 didn't jump to line 427, because the condition on line 426 was never true
427 unittest.main()