Coverage for tests/test_butlerFits.py : 19%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24import os
25import unittest
26import tempfile
27import shutil
29from typing import TYPE_CHECKING
31import lsst.utils.tests
33import lsst.pex.config
34import lsst.afw.image
35from lsst.afw.image import LOCAL
36from lsst.geom import Box2I, Point2I
37from lsst.base import Packages
38from lsst.daf.base import PropertyList, PropertySet
40from lsst.daf.butler import Config
41from lsst.daf.butler import StorageClassFactory
42from lsst.daf.butler import DatasetType
43from lsst.daf.butler.tests import DatasetTestHelper, makeTestRepo, addDatasetType, makeTestCollection
45from lsst.obs.base.exposureAssembler import ExposureAssembler
47if TYPE_CHECKING: 47 ↛ 48line 47 didn't jump to line 48, because the condition on line 47 was never true
48 from lsst.daf.butler import DatasetRef
50TESTDIR = os.path.dirname(__file__)
52BUTLER_CONFIG = """
53storageClasses:
54 ExposureCompositeF:
55 inheritsFrom: ExposureF
56datastore:
57 # Want to check disassembly so can't use InMemory
58 cls: lsst.daf.butler.datastores.posixDatastore.PosixDatastore
59 formatters:
60 ExposureCompositeF: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
61 lossless:
62 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
63 parameters:
64 recipe: lossless
65 uncompressed:
66 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
67 parameters:
68 recipe: noCompression
69 lossy:
70 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
71 parameters:
72 recipe: lossyBasic
73 composites:
74 disassembled:
75 ExposureCompositeF: True
76"""
78# Components present in the test file
79COMPONENTS = {"wcs", "image", "mask", "coaddInputs", "psf", "visitInfo", "variance", "metadata", "photoCalib",
80 "filter", "validPolygon", "transmissionCurve", "detector", "apCorrMap"}
81READ_COMPONENTS = {"bbox", "xy0", "dimensions"}
84class SimpleConfig(lsst.pex.config.Config):
85 """Config to use in tests for butler put/get"""
86 i = lsst.pex.config.Field("integer test", int)
87 c = lsst.pex.config.Field("string", str)
90class ButlerFitsTests(DatasetTestHelper, lsst.utils.tests.TestCase):
92 @classmethod
93 def setUpClass(cls):
94 """Create a new butler once only."""
96 cls.storageClassFactory = StorageClassFactory()
98 cls.root = tempfile.mkdtemp(dir=TESTDIR)
100 dataIds = {
101 "instrument": ["DummyCam"],
102 "physical_filter": ["d-r"],
103 "visit": [42, 43, 44],
104 }
106 # Ensure that we test in a directory that will include some
107 # metacharacters
108 subdir = "sub?#dir"
109 butlerRoot = os.path.join(cls.root, subdir)
111 cls.creatorButler = makeTestRepo(butlerRoot, dataIds, config=Config.fromYaml(BUTLER_CONFIG))
113 # Create dataset types used by the tests
114 for datasetTypeName, storageClassName in (("calexp", "ExposureF"),
115 ("unknown", "ExposureCompositeF"),
116 ("testCatalog", "SourceCatalog"),
117 ("lossless", "ExposureF"),
118 ("uncompressed", "ExposureF"),
119 ("lossy", "ExposureF"),
120 ):
121 storageClass = cls.storageClassFactory.getStorageClass(storageClassName)
122 addDatasetType(cls.creatorButler, datasetTypeName, set(dataIds), storageClass)
124 # And some dataset types that have no dimensions for easy testing
125 for datasetTypeName, storageClassName in (("ps", "PropertySet"),
126 ("pl", "PropertyList"),
127 ("pkg", "Packages"),
128 ("config", "Config"),
129 ):
130 storageClass = cls.storageClassFactory.getStorageClass(storageClassName)
131 addDatasetType(cls.creatorButler, datasetTypeName, {}, storageClass)
133 @classmethod
134 def tearDownClass(cls):
135 if cls.root is not None:
136 shutil.rmtree(cls.root, ignore_errors=True)
138 def setUp(self):
139 self.butler = makeTestCollection(self.creatorButler)
141 def makeExampleCatalog(self) -> lsst.afw.table.SourceCatalog:
142 catalogPath = os.path.join(TESTDIR, "data", "source_catalog.fits")
143 return lsst.afw.table.SourceCatalog.readFits(catalogPath)
145 def assertCatalogEqual(self, inputCatalog: lsst.afw.table.SourceCatalog,
146 outputCatalog: lsst.afw.table.SourceCatalog) -> None:
147 self.assertIsInstance(outputCatalog, lsst.afw.table.SourceCatalog)
148 inputTable = inputCatalog.getTable()
149 inputRecord = inputCatalog[0]
150 outputTable = outputCatalog.getTable()
151 outputRecord = outputCatalog[0]
152 self.assertEqual(inputRecord.getPsfInstFlux(), outputRecord.getPsfInstFlux())
153 self.assertEqual(inputRecord.getPsfFluxFlag(), outputRecord.getPsfFluxFlag())
154 self.assertEqual(inputTable.getSchema().getAliasMap().get("slot_Centroid"),
155 outputTable.getSchema().getAliasMap().get("slot_Centroid"))
156 self.assertEqual(inputRecord.getCentroid(), outputRecord.getCentroid())
157 self.assertFloatsAlmostEqual(
158 inputRecord.getCentroidErr()[0, 0],
159 outputRecord.getCentroidErr()[0, 0], rtol=1e-6)
160 self.assertFloatsAlmostEqual(
161 inputRecord.getCentroidErr()[1, 1],
162 outputRecord.getCentroidErr()[1, 1], rtol=1e-6)
163 self.assertEqual(inputTable.getSchema().getAliasMap().get("slot_Shape"),
164 outputTable.getSchema().getAliasMap().get("slot_Shape"))
165 self.assertFloatsAlmostEqual(
166 inputRecord.getShapeErr()[0, 0],
167 outputRecord.getShapeErr()[0, 0], rtol=1e-6)
168 self.assertFloatsAlmostEqual(
169 inputRecord.getShapeErr()[1, 1],
170 outputRecord.getShapeErr()[1, 1], rtol=1e-6)
171 self.assertFloatsAlmostEqual(
172 inputRecord.getShapeErr()[2, 2],
173 outputRecord.getShapeErr()[2, 2], rtol=1e-6)
175 def runFundamentalTypeTest(self, datasetTypeName, entity):
176 """Put and get the supplied entity and compare."""
177 ref = self.butler.put(entity, datasetTypeName)
178 butler_ps = self.butler.get(ref)
179 self.assertEqual(butler_ps, entity)
181 # Break the contact by ensuring that we are writing YAML
182 uri = self.butler.getURI(ref)
183 self.assertTrue(uri.path.endswith(".yaml"), f"Check extension of {uri}")
185 def testFundamentalTypes(self) -> None:
186 """Ensure that some fundamental stack types round trip."""
187 ps = PropertySet()
188 ps["a.b"] = 5
189 ps["c.d.e"] = "string"
190 self.runFundamentalTypeTest("ps", ps)
192 pl = PropertyList()
193 pl["A"] = 1
194 pl.setComment("A", "An int comment")
195 pl["B"] = "string"
196 pl.setComment("B", "A string comment")
197 self.runFundamentalTypeTest("pl", pl)
199 pkg = Packages.fromSystem()
200 self.runFundamentalTypeTest("pkg", pkg)
202 def testPexConfig(self) -> None:
203 """Test that we can put and get pex_config Configs"""
204 c = SimpleConfig(i=10, c="hello")
205 self.assertEqual(c.i, 10)
206 ref = self.butler.put(c, "config")
207 butler_c = self.butler.get(ref)
208 self.assertEqual(c, butler_c)
209 self.assertIsInstance(butler_c, SimpleConfig)
211 def testFitsCatalog(self) -> None:
212 """Test reading of a FITS catalog"""
213 catalog = self.makeExampleCatalog()
214 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"}
215 ref = self.butler.put(catalog, "testCatalog", dataId)
216 stored = self.butler.get(ref)
217 self.assertCatalogEqual(catalog, stored)
219 def testExposureCompositePutGetConcrete(self) -> None:
220 """Test composite with no disassembly"""
221 ref = self.runExposureCompositePutGetTest("calexp")
223 uri = self.butler.getURI(ref)
224 self.assertTrue(uri.exists(), f"Checking URI {uri} existence")
226 def testExposureCompositePutGetVirtual(self) -> None:
227 """Testing composite disassembly"""
228 ref = self.runExposureCompositePutGetTest("unknown")
230 primary, components = self.butler.getURIs(ref)
231 self.assertIsNone(primary)
232 self.assertEqual(set(components), COMPONENTS)
233 for compName, uri in components.items():
234 self.assertTrue(uri.exists(),
235 f"Checking URI {uri} existence for component {compName}")
237 def runExposureCompositePutGetTest(self, datasetTypeName: str) -> DatasetRef:
238 example = os.path.join(TESTDIR, "data", "calexp.fits")
239 exposure = lsst.afw.image.ExposureF(example)
241 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"}
242 ref = self.butler.put(exposure, datasetTypeName, dataId)
244 # Get the full thing
245 composite = self.butler.get(datasetTypeName, dataId)
247 # There is no assert for Exposure so just look at maskedImage
248 self.assertMaskedImagesEqual(composite.maskedImage, exposure.maskedImage)
250 # Helper for extracting components
251 assembler = ExposureAssembler(ref.datasetType.storageClass)
253 # Check all possible components that can be read
254 allComponents = set()
255 allComponents.update(COMPONENTS, READ_COMPONENTS)
257 # Get each component from butler independently
258 for compName in allComponents:
259 compTypeName = DatasetType.nameWithComponent(datasetTypeName, compName)
260 component = self.butler.get(compTypeName, dataId)
262 reference = assembler.getComponent(exposure, compName)
264 self.assertIsInstance(component, type(reference), f"Checking type of component {compName}")
266 if compName in ("image", "variance"):
267 self.assertImagesEqual(component, reference)
268 elif compName == "mask":
269 self.assertMasksEqual(component, reference)
270 elif compName == "wcs":
271 self.assertWcsAlmostEqualOverBBox(component, reference, exposure.getBBox())
272 elif compName == "coaddInputs":
273 self.assertEqual(len(component.visits), len(reference.visits),
274 f"cf visits {component.visits}")
275 self.assertEqual(len(component.ccds), len(reference.ccds),
276 f"cf CCDs {component.ccds}")
277 elif compName == "psf":
278 # Equality for PSF does not work
279 pass
280 elif compName == "filter":
281 self.assertEqual(component.getCanonicalName(), reference.getCanonicalName())
282 elif compName == "visitInfo":
283 self.assertEqual(component.getExposureId(), reference.getExposureId(),
284 "VisitInfo comparison")
285 elif compName == "metadata":
286 # The component metadata has extra fields in it so cannot
287 # compare directly.
288 for k, v in reference.items():
289 self.assertEqual(component[k], v)
290 elif compName == "photoCalib":
291 # This example has a
292 # "spatially constant with mean: inf error: nan" entry
293 # which does not compare directly.
294 self.assertEqual(str(component), str(reference))
295 self.assertIn("spatially constant with mean: 1.99409", str(component),
296 "Checking photoCalib")
297 elif compName in ("bbox", "xy0", "dimensions", "validPolygon"):
298 self.assertEqual(component, reference)
299 elif compName == "apCorrMap":
300 self.assertEqual(set(component.keys()), set(reference.keys()))
301 elif compName == "transmissionCurve":
302 self.assertEqual(component.getThroughputAtBounds(),
303 reference.getThroughputAtBounds())
304 elif compName == "detector":
305 c_amps = {a.getName() for a in component.getAmplifiers()}
306 r_amps = {a.getName() for a in reference.getAmplifiers()}
307 self.assertEqual(c_amps, r_amps)
308 else:
309 raise RuntimeError(f"Unexpected component '{compName}' encountered in test")
311 # Full Exposure with parameters
312 inBBox = Box2I(minimum=Point2I(3, 3), maximum=Point2I(21, 16))
313 parameters = dict(bbox=inBBox, origin=LOCAL)
314 subset = self.butler.get(datasetTypeName, dataId, parameters=parameters)
315 outBBox = subset.getBBox()
316 self.assertEqual(inBBox, outBBox)
317 self.assertImagesEqual(subset.getImage(), exposure.subset(inBBox, origin=LOCAL).getImage())
319 return ref
321 def putFits(self, exposure, datasetTypeName, visit):
322 """Put different datasetTypes and return information."""
323 dataId = {"visit": visit, "instrument": "DummyCam", "physical_filter": "d-r"}
324 refC = self.butler.put(exposure, datasetTypeName, dataId)
325 uriC = self.butler.getURI(refC)
326 stat = os.stat(uriC.ospath)
327 size = stat.st_size
328 metaDatasetTypeName = DatasetType.nameWithComponent(datasetTypeName, "metadata")
329 meta = self.butler.get(metaDatasetTypeName, dataId)
330 return meta, size
332 def testCompression(self):
333 """Test that we can write compressed and uncompressed FITS."""
334 example = os.path.join(TESTDIR, "data", "small.fits")
335 exposure = lsst.afw.image.ExposureF(example)
337 # Write a lossless compressed
338 metaC, sizeC = self.putFits(exposure, "lossless", 42)
339 self.assertEqual(metaC["TTYPE1"], "COMPRESSED_DATA")
340 self.assertEqual(metaC["ZCMPTYPE"], "GZIP_2")
342 # Write an uncompressed FITS file
343 metaN, sizeN = self.putFits(exposure, "uncompressed", 43)
344 self.assertNotIn("ZCMPTYPE", metaN)
346 # Write an uncompressed FITS file
347 metaL, sizeL = self.putFits(exposure, "lossy", 44)
348 self.assertEqual(metaL["TTYPE1"], "COMPRESSED_DATA")
349 self.assertEqual(metaL["ZCMPTYPE"], "RICE_1")
351 self.assertNotEqual(sizeC, sizeN)
352 # Data file is so small that Lossy and Compressed are dominated
353 # by the extra compression tables
354 self.assertEqual(sizeL, sizeC)
357if __name__ == "__main__": 357 ↛ 358line 357 didn't jump to line 358, because the condition on line 357 was never true
358 unittest.main()