Coverage for tests/test_butlerFits.py : 23%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24import os
25import unittest
26import tempfile
27import shutil
29from typing import TYPE_CHECKING
31import lsst.utils.tests
33import lsst.afw.image
34from lsst.afw.image import LOCAL
35from lsst.geom import Box2I, Point2I
37from lsst.daf.butler import Config
38from lsst.daf.butler import StorageClassFactory
39from lsst.daf.butler import DatasetType
40from lsst.daf.butler.tests import DatasetTestHelper, makeTestRepo, addDatasetType, makeTestCollection
42from lsst.obs.base.exposureAssembler import ExposureAssembler
44if TYPE_CHECKING: 44 ↛ 45line 44 didn't jump to line 45, because the condition on line 44 was never true
45 from lsst.daf.butler import DatasetRef
47TESTDIR = os.path.dirname(__file__)
49BUTLER_CONFIG = """
50storageClasses:
51 ExposureCompositeF:
52 inheritsFrom: ExposureF
53datastore:
54 # Want to check disassembly so can't use InMemory
55 cls: lsst.daf.butler.datastores.posixDatastore.PosixDatastore
56 formatters:
57 ExposureCompositeF: lsst.obs.base.fitsExposureFormatter.FitsExposureFormatter
58 composites:
59 disassembled:
60 ExposureCompositeF: True
61"""
63# Components present in the test file
64COMPONENTS = {"wcs", "image", "mask", "coaddInputs", "psf", "visitInfo", "variance", "metadata", "photoCalib"}
67class ButlerFitsTests(DatasetTestHelper, lsst.utils.tests.TestCase):
69 @classmethod
70 def setUpClass(cls):
71 """Create a new butler once only."""
73 cls.storageClassFactory = StorageClassFactory()
75 cls.root = tempfile.mkdtemp(dir=TESTDIR)
77 dataIds = {
78 "instrument": ["DummyCam"],
79 "physical_filter": ["d-r"],
80 "visit": [42],
81 }
83 cls.creatorButler = makeTestRepo(cls.root, dataIds, config=Config.fromYaml(BUTLER_CONFIG))
85 # Create dataset types used by the tests
86 for datasetTypeName, storageClassName in (("calexp", "ExposureF"),
87 ("unknown", "ExposureCompositeF"),
88 ("testCatalog", "SourceCatalog"),
89 ):
90 storageClass = cls.storageClassFactory.getStorageClass(storageClassName)
91 addDatasetType(cls.creatorButler, datasetTypeName, set(dataIds), storageClass)
93 @classmethod
94 def tearDownClass(cls):
95 if cls.root is not None:
96 shutil.rmtree(cls.root, ignore_errors=True)
98 def setUp(self):
99 self.butler = makeTestCollection(self.creatorButler)
101 def makeExampleCatalog(self) -> lsst.afw.table.SourceCatalog:
102 catalogPath = os.path.join(TESTDIR, "data", "source_catalog.fits")
103 return lsst.afw.table.SourceCatalog.readFits(catalogPath)
105 def assertCatalogEqual(self, inputCatalog: lsst.afw.table.SourceCatalog,
106 outputCatalog: lsst.afw.table.SourceCatalog) -> None:
107 self.assertIsInstance(outputCatalog, lsst.afw.table.SourceCatalog)
108 inputTable = inputCatalog.getTable()
109 inputRecord = inputCatalog[0]
110 outputTable = outputCatalog.getTable()
111 outputRecord = outputCatalog[0]
112 self.assertEqual(inputRecord.getPsfInstFlux(), outputRecord.getPsfInstFlux())
113 self.assertEqual(inputRecord.getPsfFluxFlag(), outputRecord.getPsfFluxFlag())
114 self.assertEqual(inputTable.getSchema().getAliasMap().get("slot_Centroid"),
115 outputTable.getSchema().getAliasMap().get("slot_Centroid"))
116 self.assertEqual(inputRecord.getCentroid(), outputRecord.getCentroid())
117 self.assertFloatsAlmostEqual(
118 inputRecord.getCentroidErr()[0, 0],
119 outputRecord.getCentroidErr()[0, 0], rtol=1e-6)
120 self.assertFloatsAlmostEqual(
121 inputRecord.getCentroidErr()[1, 1],
122 outputRecord.getCentroidErr()[1, 1], rtol=1e-6)
123 self.assertEqual(inputTable.getSchema().getAliasMap().get("slot_Shape"),
124 outputTable.getSchema().getAliasMap().get("slot_Shape"))
125 self.assertFloatsAlmostEqual(
126 inputRecord.getShapeErr()[0, 0],
127 outputRecord.getShapeErr()[0, 0], rtol=1e-6)
128 self.assertFloatsAlmostEqual(
129 inputRecord.getShapeErr()[1, 1],
130 outputRecord.getShapeErr()[1, 1], rtol=1e-6)
131 self.assertFloatsAlmostEqual(
132 inputRecord.getShapeErr()[2, 2],
133 outputRecord.getShapeErr()[2, 2], rtol=1e-6)
135 def testFitsCatalog(self) -> None:
136 catalog = self.makeExampleCatalog()
137 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"}
138 ref = self.butler.put(catalog, "testCatalog", dataId)
139 stored = self.butler.get(ref)
140 self.assertCatalogEqual(catalog, stored)
142 def testExposureCompositePutGetConcrete(self) -> None:
143 ref = self.runExposureCompositePutGetTest("calexp")
145 uri = self.butler.getURI(ref)
146 self.assertTrue(os.path.exists(uri.path), f"Checking URI {uri} existence")
148 def testExposureCompositePutGetVirtual(self) -> None:
149 ref = self.runExposureCompositePutGetTest("unknown")
151 primary, components = self.butler.getURIs(ref)
152 self.assertIsNone(primary)
153 self.assertEqual(set(components), COMPONENTS)
154 for compName, uri in components.items():
155 self.assertTrue(os.path.exists(uri.path),
156 f"Checking URI {uri} existence for component {compName}")
158 def runExposureCompositePutGetTest(self, datasetTypeName: str) -> DatasetRef:
159 example = os.path.join(TESTDIR, "data", "small.fits")
160 exposure = lsst.afw.image.ExposureF(example)
162 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"}
163 ref = self.butler.put(exposure, datasetTypeName, dataId)
165 # Get the full thing
166 composite = self.butler.get(datasetTypeName, dataId)
168 # There is no assert for Exposure so just look at maskedImage
169 self.assertMaskedImagesEqual(composite.maskedImage, exposure.maskedImage)
171 # Helper for extracting components
172 assembler = ExposureAssembler(ref.datasetType.storageClass)
174 # Get each component from butler independently
175 for compName in COMPONENTS:
176 compTypeName = DatasetType.nameWithComponent(datasetTypeName, compName)
177 component = self.butler.get(compTypeName, dataId)
179 reference = assembler.getComponent(exposure, compName)
180 self.assertIsInstance(component, type(reference), f"Checking type of component {compName}")
182 if compName in ("image", "variance"):
183 self.assertImagesEqual(component, reference)
184 elif compName == "mask":
185 self.assertMasksEqual(component, reference)
186 elif compName == "wcs":
187 self.assertWcsAlmostEqualOverBBox(component, reference, exposure.getBBox())
188 elif compName == "coaddInputs":
189 self.assertEqual(len(component.visits), len(reference.visits),
190 f"cf visits {component.visits}")
191 self.assertEqual(len(component.ccds), len(reference.ccds),
192 f"cf CCDs {component.ccds}")
193 elif compName == "psf":
194 # Equality for PSF does not work
195 pass
196 elif compName == "visitInfo":
197 self.assertEqual(component.getExposureId(), reference.getExposureId(),
198 f"VisitInfo comparison")
199 elif compName == "metadata":
200 # The component metadata has extra fields in it so cannot
201 # compare directly.
202 for k, v in reference.items():
203 self.assertEqual(component[k], v)
204 elif compName == "photoCalib":
205 # This example has a
206 # "spatially constant with mean: inf error: nan" entry
207 # which does not compare directly.
208 self.assertEqual(str(component), str(reference))
209 self.assertIn("spatially constant with mean: inf", str(component), "Checking photoCalib")
210 else:
211 raise RuntimeError(f"Unexpected component '{compName}' encountered in test")
213 # With parameters
214 inBBox = Box2I(minimum=Point2I(0, 0), maximum=Point2I(3, 3))
215 parameters = dict(bbox=inBBox, origin=LOCAL)
216 subset = self.butler.get(datasetTypeName, dataId, parameters=parameters)
217 outBBox = subset.getBBox()
218 self.assertEqual(inBBox, outBBox)
219 self.assertImagesEqual(subset.getImage(), exposure.subset(inBBox, origin=LOCAL).getImage())
221 return ref
224if __name__ == "__main__": 224 ↛ 225line 224 didn't jump to line 225, because the condition on line 224 was never true
225 unittest.main()