Coverage for tests/test_butlerFits.py : 19%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24import os
25import unittest
26import tempfile
27import shutil
29from typing import TYPE_CHECKING
31import lsst.utils.tests
33import lsst.afw.image
34from lsst.afw.image import LOCAL
35from lsst.geom import Box2I, Point2I
36from lsst.base import Packages
37from lsst.daf.base import PropertyList, PropertySet
39from lsst.daf.butler import Config
40from lsst.daf.butler import StorageClassFactory
41from lsst.daf.butler import DatasetType
42from lsst.daf.butler.tests import DatasetTestHelper, makeTestRepo, addDatasetType, makeTestCollection
44from lsst.obs.base.exposureAssembler import ExposureAssembler
46if TYPE_CHECKING: 46 ↛ 47line 46 didn't jump to line 47, because the condition on line 46 was never true
47 from lsst.daf.butler import DatasetRef
49TESTDIR = os.path.dirname(__file__)
51BUTLER_CONFIG = """
52storageClasses:
53 ExposureCompositeF:
54 inheritsFrom: ExposureF
55datastore:
56 # Want to check disassembly so can't use InMemory
57 cls: lsst.daf.butler.datastores.posixDatastore.PosixDatastore
58 formatters:
59 ExposureCompositeF: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
60 lossless:
61 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
62 parameters:
63 recipe: lossless
64 uncompressed:
65 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
66 parameters:
67 recipe: noCompression
68 lossy:
69 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
70 parameters:
71 recipe: lossyBasic
72 composites:
73 disassembled:
74 ExposureCompositeF: True
75"""
77# Components present in the test file
78COMPONENTS = {"wcs", "image", "mask", "coaddInputs", "psf", "visitInfo", "variance", "metadata", "photoCalib",
79 "filter"}
80READ_COMPONENTS = {"bbox", "xy0", "dimensions"}
83class ButlerFitsTests(DatasetTestHelper, lsst.utils.tests.TestCase):
85 @classmethod
86 def setUpClass(cls):
87 """Create a new butler once only."""
89 cls.storageClassFactory = StorageClassFactory()
91 cls.root = tempfile.mkdtemp(dir=TESTDIR)
93 dataIds = {
94 "instrument": ["DummyCam"],
95 "physical_filter": ["d-r"],
96 "visit": [42, 43, 44],
97 }
99 # Ensure that we test in a directory that will include some
100 # metacharacters
101 subdir = "sub?#dir"
102 butlerRoot = os.path.join(cls.root, subdir)
104 cls.creatorButler = makeTestRepo(butlerRoot, dataIds, config=Config.fromYaml(BUTLER_CONFIG))
106 # Create dataset types used by the tests
107 for datasetTypeName, storageClassName in (("calexp", "ExposureF"),
108 ("unknown", "ExposureCompositeF"),
109 ("testCatalog", "SourceCatalog"),
110 ("lossless", "ExposureF"),
111 ("uncompressed", "ExposureF"),
112 ("lossy", "ExposureF"),
113 ):
114 storageClass = cls.storageClassFactory.getStorageClass(storageClassName)
115 addDatasetType(cls.creatorButler, datasetTypeName, set(dataIds), storageClass)
117 # And some dataset types that have no dimensions for easy testing
118 for datasetTypeName, storageClassName in (("ps", "PropertySet"),
119 ("pl", "PropertyList"),
120 ("pkg", "Packages")
121 ):
122 storageClass = cls.storageClassFactory.getStorageClass(storageClassName)
123 addDatasetType(cls.creatorButler, datasetTypeName, {}, storageClass)
125 @classmethod
126 def tearDownClass(cls):
127 if cls.root is not None:
128 shutil.rmtree(cls.root, ignore_errors=True)
130 def setUp(self):
131 self.butler = makeTestCollection(self.creatorButler)
133 def makeExampleCatalog(self) -> lsst.afw.table.SourceCatalog:
134 catalogPath = os.path.join(TESTDIR, "data", "source_catalog.fits")
135 return lsst.afw.table.SourceCatalog.readFits(catalogPath)
137 def assertCatalogEqual(self, inputCatalog: lsst.afw.table.SourceCatalog,
138 outputCatalog: lsst.afw.table.SourceCatalog) -> None:
139 self.assertIsInstance(outputCatalog, lsst.afw.table.SourceCatalog)
140 inputTable = inputCatalog.getTable()
141 inputRecord = inputCatalog[0]
142 outputTable = outputCatalog.getTable()
143 outputRecord = outputCatalog[0]
144 self.assertEqual(inputRecord.getPsfInstFlux(), outputRecord.getPsfInstFlux())
145 self.assertEqual(inputRecord.getPsfFluxFlag(), outputRecord.getPsfFluxFlag())
146 self.assertEqual(inputTable.getSchema().getAliasMap().get("slot_Centroid"),
147 outputTable.getSchema().getAliasMap().get("slot_Centroid"))
148 self.assertEqual(inputRecord.getCentroid(), outputRecord.getCentroid())
149 self.assertFloatsAlmostEqual(
150 inputRecord.getCentroidErr()[0, 0],
151 outputRecord.getCentroidErr()[0, 0], rtol=1e-6)
152 self.assertFloatsAlmostEqual(
153 inputRecord.getCentroidErr()[1, 1],
154 outputRecord.getCentroidErr()[1, 1], rtol=1e-6)
155 self.assertEqual(inputTable.getSchema().getAliasMap().get("slot_Shape"),
156 outputTable.getSchema().getAliasMap().get("slot_Shape"))
157 self.assertFloatsAlmostEqual(
158 inputRecord.getShapeErr()[0, 0],
159 outputRecord.getShapeErr()[0, 0], rtol=1e-6)
160 self.assertFloatsAlmostEqual(
161 inputRecord.getShapeErr()[1, 1],
162 outputRecord.getShapeErr()[1, 1], rtol=1e-6)
163 self.assertFloatsAlmostEqual(
164 inputRecord.getShapeErr()[2, 2],
165 outputRecord.getShapeErr()[2, 2], rtol=1e-6)
167 def runFundamentalTypeTest(self, datasetTypeName, entity):
168 """Put and get the supplied entity and compare."""
169 ref = self.butler.put(entity, datasetTypeName)
170 butler_ps = self.butler.get(ref)
171 self.assertEqual(butler_ps, entity)
173 # Break the contact by ensuring that we are writing YAML
174 uri = self.butler.getURI(ref)
175 self.assertTrue(uri.path.endswith(".yaml"), f"Check extension of {uri}")
177 def testFundamentalTypes(self) -> None:
178 """Ensure that some fundamental stack types round trip."""
179 ps = PropertySet()
180 ps["a.b"] = 5
181 ps["c.d.e"] = "string"
182 self.runFundamentalTypeTest("ps", ps)
184 pl = PropertyList()
185 pl["A"] = 1
186 pl.setComment("A", "An int comment")
187 pl["B"] = "string"
188 pl.setComment("B", "A string comment")
189 self.runFundamentalTypeTest("pl", pl)
191 pkg = Packages.fromSystem()
192 self.runFundamentalTypeTest("pkg", pkg)
194 def testFitsCatalog(self) -> None:
195 """Test reading of a FITS catalog"""
196 catalog = self.makeExampleCatalog()
197 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"}
198 ref = self.butler.put(catalog, "testCatalog", dataId)
199 stored = self.butler.get(ref)
200 self.assertCatalogEqual(catalog, stored)
202 def testExposureCompositePutGetConcrete(self) -> None:
203 """Test composite with no disassembly"""
204 ref = self.runExposureCompositePutGetTest("calexp")
206 uri = self.butler.getURI(ref)
207 self.assertTrue(uri.exists(), f"Checking URI {uri} existence")
209 def testExposureCompositePutGetVirtual(self) -> None:
210 """Testing composite disassembly"""
211 ref = self.runExposureCompositePutGetTest("unknown")
213 primary, components = self.butler.getURIs(ref)
214 self.assertIsNone(primary)
215 self.assertEqual(set(components), COMPONENTS)
216 for compName, uri in components.items():
217 self.assertTrue(uri.exists(),
218 f"Checking URI {uri} existence for component {compName}")
220 def runExposureCompositePutGetTest(self, datasetTypeName: str) -> DatasetRef:
221 example = os.path.join(TESTDIR, "data", "small.fits")
222 exposure = lsst.afw.image.ExposureF(example)
224 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"}
225 ref = self.butler.put(exposure, datasetTypeName, dataId)
227 # Get the full thing
228 composite = self.butler.get(datasetTypeName, dataId)
230 # There is no assert for Exposure so just look at maskedImage
231 self.assertMaskedImagesEqual(composite.maskedImage, exposure.maskedImage)
233 # Helper for extracting components
234 assembler = ExposureAssembler(ref.datasetType.storageClass)
236 # Check all possible components that can be read
237 allComponents = set()
238 allComponents.update(COMPONENTS, READ_COMPONENTS)
240 # Get each component from butler independently
241 for compName in allComponents:
242 compTypeName = DatasetType.nameWithComponent(datasetTypeName, compName)
243 component = self.butler.get(compTypeName, dataId)
245 reference = assembler.getComponent(exposure, compName)
246 self.assertIsInstance(component, type(reference), f"Checking type of component {compName}")
248 if compName in ("image", "variance"):
249 self.assertImagesEqual(component, reference)
250 elif compName == "mask":
251 self.assertMasksEqual(component, reference)
252 elif compName == "wcs":
253 self.assertWcsAlmostEqualOverBBox(component, reference, exposure.getBBox())
254 elif compName == "coaddInputs":
255 self.assertEqual(len(component.visits), len(reference.visits),
256 f"cf visits {component.visits}")
257 self.assertEqual(len(component.ccds), len(reference.ccds),
258 f"cf CCDs {component.ccds}")
259 elif compName == "psf":
260 # Equality for PSF does not work
261 pass
262 elif compName == "filter":
263 self.assertEqual(component.getCanonicalName(), reference.getCanonicalName())
264 elif compName == "visitInfo":
265 self.assertEqual(component.getExposureId(), reference.getExposureId(),
266 "VisitInfo comparison")
267 elif compName == "metadata":
268 # The component metadata has extra fields in it so cannot
269 # compare directly.
270 for k, v in reference.items():
271 self.assertEqual(component[k], v)
272 elif compName == "photoCalib":
273 # This example has a
274 # "spatially constant with mean: inf error: nan" entry
275 # which does not compare directly.
276 self.assertEqual(str(component), str(reference))
277 self.assertIn("spatially constant with mean: inf", str(component), "Checking photoCalib")
278 elif compName in ("bbox", "xy0", "dimensions"):
279 self.assertEqual(component, reference)
280 else:
281 raise RuntimeError(f"Unexpected component '{compName}' encountered in test")
283 # With parameters
284 inBBox = Box2I(minimum=Point2I(0, 0), maximum=Point2I(3, 3))
285 parameters = dict(bbox=inBBox, origin=LOCAL)
286 subset = self.butler.get(datasetTypeName, dataId, parameters=parameters)
287 outBBox = subset.getBBox()
288 self.assertEqual(inBBox, outBBox)
289 self.assertImagesEqual(subset.getImage(), exposure.subset(inBBox, origin=LOCAL).getImage())
291 return ref
293 def putFits(self, exposure, datasetTypeName, visit):
294 """Put different datasetTypes and return information."""
295 dataId = {"visit": visit, "instrument": "DummyCam", "physical_filter": "d-r"}
296 refC = self.butler.put(exposure, datasetTypeName, dataId)
297 uriC = self.butler.getURI(refC)
298 stat = os.stat(uriC.ospath)
299 size = stat.st_size
300 metaDatasetTypeName = DatasetType.nameWithComponent(datasetTypeName, "metadata")
301 meta = self.butler.get(metaDatasetTypeName, dataId)
302 return meta, size
304 def testCompression(self):
305 """Test that we can write compressed and uncompressed FITS."""
306 example = os.path.join(TESTDIR, "data", "small.fits")
307 exposure = lsst.afw.image.ExposureF(example)
309 # Write a lossless compressed
310 metaC, sizeC = self.putFits(exposure, "lossless", 42)
311 self.assertEqual(metaC["TTYPE1"], "COMPRESSED_DATA")
312 self.assertEqual(metaC["ZCMPTYPE"], "GZIP_2")
314 # Write an uncompressed FITS file
315 metaN, sizeN = self.putFits(exposure, "uncompressed", 43)
316 self.assertNotIn("ZCMPTYPE", metaN)
318 # Write an uncompressed FITS file
319 metaL, sizeL = self.putFits(exposure, "lossy", 44)
320 self.assertEqual(metaL["TTYPE1"], "COMPRESSED_DATA")
321 self.assertEqual(metaL["ZCMPTYPE"], "RICE_1")
323 self.assertNotEqual(sizeC, sizeN)
324 # Data file is so small that Lossy and Compressed are dominated
325 # by the extra compression tables
326 self.assertEqual(sizeL, sizeC)
329if __name__ == "__main__": 329 ↛ 330line 329 didn't jump to line 330, because the condition on line 329 was never true
330 unittest.main()