Coverage for tests/test_testRepo.py: 22%
117 statements
« prev ^ index » next coverage.py v7.4.1, created at 2024-02-01 11:20 +0000
« prev ^ index » next coverage.py v7.4.1, created at 2024-02-01 11:20 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28"""Unit tests for `lsst.daf.butler.tests.testRepo`, a module for creating
29test repositories or butlers.
30"""
32import os
33import shutil
34import unittest
36import lsst.daf.butler
37from lsst.daf.butler.tests import (
38 MetricsExample,
39 addDataIdValue,
40 addDatasetType,
41 expandUniqueId,
42 makeTestCollection,
43 makeTestRepo,
44 registerMetricsExample,
45)
46from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir, safeTestTempDir
48TESTDIR = os.path.abspath(os.path.dirname(__file__))
51class ButlerTestRepoTestCase(unittest.TestCase):
52 """Simpler test than below without setUpClass getting in the way."""
54 def setUp(self):
55 self.root = makeTestTempDir(TESTDIR)
57 def tearDown(self):
58 removeTestTempDir(self.root)
60 def testMakeTestRepo(self):
61 dataIds = {
62 "instrument": ["DummyCam"],
63 "physical_filter": ["d-r"],
64 "exposure": [42, 43, 44],
65 "visit": [42, 43, 44],
66 }
68 butler = makeTestRepo(self.root, dataIds)
70 records = list(butler.registry.queryDimensionRecords("visit"))
71 self.assertEqual(len(records), 3)
74class ButlerUtilsTestSuite(unittest.TestCase):
75 """Test the butler test utilities."""
77 @classmethod
78 def setUpClass(cls):
79 # Repository should be re-created for each test case, but
80 # this has a prohibitive run-time cost at present
81 cls.root = makeTestTempDir(TESTDIR)
83 cls.creatorButler = makeTestRepo(cls.root)
84 addDataIdValue(cls.creatorButler, "instrument", "notACam")
85 addDataIdValue(cls.creatorButler, "instrument", "dummyCam")
86 addDataIdValue(cls.creatorButler, "physical_filter", "k2020", band="k", instrument="notACam")
87 addDataIdValue(cls.creatorButler, "physical_filter", "l2019", instrument="dummyCam")
88 addDataIdValue(cls.creatorButler, "visit", 101, instrument="notACam", physical_filter="k2020")
89 addDataIdValue(cls.creatorButler, "visit", 102, instrument="notACam", physical_filter="k2020")
90 addDataIdValue(cls.creatorButler, "detector", 5)
91 # Leave skymap/patch/tract undefined so that tests can assume
92 # they're missing.
94 registerMetricsExample(cls.creatorButler)
95 addDatasetType(cls.creatorButler, "DataType1", {"instrument"}, "StructuredDataNoComponents")
96 addDatasetType(cls.creatorButler, "DataType2", {"instrument", "visit"}, "StructuredData")
98 @classmethod
99 def tearDownClass(cls):
100 # TODO: use addClassCleanup rather than tearDownClass in Python 3.8
101 # to keep the addition and removal together and make it more robust
102 removeTestTempDir(cls.root)
104 def setUp(self):
105 # TestCase.id() is unique for each test method
106 self.butler = makeTestCollection(self.creatorButler, uniqueId=self.id())
108 def testButlerValid(self):
109 self.butler.validateConfiguration()
111 def testButlerKwargs(self):
112 # outfile has the most obvious effects of any Butler.makeRepo keyword
113 with safeTestTempDir(TESTDIR) as temp:
114 path = os.path.join(temp, "oddConfig.json")
115 makeTestRepo(temp, {}, outfile=path)
116 self.assertTrue(os.path.isfile(path))
118 def _checkButlerDimension(self, dimensions, query, expected):
119 result = list(self.butler.registry.queryDataIds(dimensions, where=query, check=False))
120 self.assertEqual(len(result), 1)
121 self.assertIn(result[0].required, expected)
123 def testButlerDimensions(self):
124 self._checkButlerDimension(
125 {"instrument"}, "instrument='notACam'", [{"instrument": "notACam"}, {"instrument": "dummyCam"}]
126 )
127 self._checkButlerDimension(
128 {"visit", "instrument"},
129 "visit=101",
130 [{"instrument": "notACam", "visit": 101}, {"instrument": "dummyCam", "visit": 101}],
131 )
132 self._checkButlerDimension(
133 {"visit", "instrument"},
134 "visit=102",
135 [{"instrument": "notACam", "visit": 102}, {"instrument": "dummyCam", "visit": 102}],
136 )
137 self._checkButlerDimension(
138 {"detector", "instrument"},
139 "detector=5",
140 [{"instrument": "notACam", "detector": 5}, {"instrument": "dummyCam", "detector": 5}],
141 )
143 def testAddDataIdValue(self):
144 addDataIdValue(self.butler, "visit", 1, instrument="notACam", physical_filter="k2020")
145 self._checkButlerDimension(
146 {"visit", "instrument"}, "visit=1", [{"instrument": "notACam", "visit": 1}]
147 )
148 addDataIdValue(self.butler, "visit", 2, instrument="dummyCam", physical_filter="l2019")
149 self._checkButlerDimension(
150 {"visit", "instrument"}, "visit=2", [{"instrument": "dummyCam", "visit": 2}]
151 )
153 with self.assertRaises(ValueError):
154 addDataIdValue(self.butler, "NotADimension", 42)
155 with self.assertRaises(ValueError):
156 addDataIdValue(self.butler, "detector", "nonNumeric")
157 with self.assertRaises(ValueError):
158 addDataIdValue(self.butler, "detector", 101, nonsenseField="string")
160 # Keywords imply different instruments
161 with self.assertRaises(RuntimeError):
162 addDataIdValue(self.butler, "exposure", 101, instrument="dummyCam", physical_filter="k2020")
164 # No skymap defined
165 with self.assertRaises(RuntimeError):
166 addDataIdValue(self.butler, "tract", 42)
167 # Didn't create skymap "map" first.
168 with self.assertRaises(RuntimeError):
169 addDataIdValue(self.butler, "tract", 43, skymap="map")
171 def testAddDatasetType(self):
172 # 1 for StructuredDataNoComponents, 1 for StructuredData (components
173 # not included).
174 with self.assertWarns(FutureWarning):
175 self.assertEqual(len(list(self.butler.registry.queryDatasetTypes(components=False))), 2)
177 # Testing the DatasetType objects is not practical, because all tests
178 # need a DimensionUniverse. So just check that we have the dataset
179 # types we expect.
180 self.butler.get_dataset_type("DataType1")
181 self.butler.get_dataset_type("DataType2")
183 with self.assertRaises(ValueError):
184 addDatasetType(self.butler, "DataType3", {"4thDimension"}, "NumpyArray")
185 with self.assertRaises(ValueError):
186 addDatasetType(self.butler, "DataType3", {"instrument"}, "UnstorableType")
188 def testRegisterMetricsExample(self):
189 id1 = {"instrument": "notACam"}
190 id2 = expandUniqueId(self.butler, {"visit": 101})
191 data = MetricsExample(summary={"answer": 42, "question": "unknown"})
193 self.butler.put(data, "DataType1", id1)
194 self.assertEqual(self.butler.get("DataType1", id1), data)
196 self.butler.put(data, "DataType2", id2)
197 self.assertEqual(self.butler.get("DataType2", id2), data)
198 self.assertEqual(self.butler.get("DataType2.summary", id2), data.summary)
200 def testRegisterMetricsExampleChained(self):
201 """Regression test for registerMetricsExample having no effect
202 on ChainedDatastore.
203 """
204 temp = makeTestTempDir(TESTDIR)
205 try:
206 config = lsst.daf.butler.Config()
207 config["datastore", "cls"] = "lsst.daf.butler.datastores.chainedDatastore.ChainedDatastore"
208 config["datastore", "datastores"] = [
209 {
210 "cls": "lsst.daf.butler.datastores.fileDatastore.FileDatastore",
211 }
212 ]
214 repo = lsst.daf.butler.Butler.makeRepo(temp, config=config)
215 butler = lsst.daf.butler.Butler.from_config(repo, run="chainedExample")
216 registerMetricsExample(butler)
217 addDatasetType(butler, "DummyType", {}, "StructuredDataNoComponents")
219 data = MetricsExample(summary={})
220 # Should not raise
221 butler.put(data, "DummyType")
222 finally:
223 shutil.rmtree(temp, ignore_errors=True)
225 def testUniqueButler(self):
226 dataId = {"instrument": "notACam"}
227 ref = self.butler.put(MetricsExample({"answer": 42, "question": "unknown"}), "DataType1", dataId)
228 self.assertTrue(self.butler.exists("DataType1", dataId))
229 self.assertTrue(self.butler.exists(ref))
231 newButler = makeTestCollection(self.creatorButler)
233 # Can not be found in the new default collection.
234 self.assertFalse(newButler.exists("DataType1", dataId))
236 # The ref does exist in the new butler though.
237 self.assertTrue(newButler.exists(ref))
239 def testExpandUniqueId(self):
240 self.assertEqual(
241 expandUniqueId(self.butler, {"instrument": "notACam"}).required, {"instrument": "notACam"}
242 )
243 self.assertIn(
244 expandUniqueId(self.butler, {"visit": 101}).required,
245 [{"instrument": "notACam", "visit": 101}, {"instrument": "dummyCam", "visit": 101}],
246 )
247 self.assertIn(
248 expandUniqueId(self.butler, {"detector": 5}).required,
249 [{"instrument": "notACam", "detector": 5}, {"instrument": "dummyCam", "detector": 5}],
250 )
251 self.assertIn(
252 expandUniqueId(self.butler, {"physical_filter": "k2020"}).required,
253 [
254 {"instrument": "notACam", "physical_filter": "k2020"},
255 {"instrument": "notACam", "physical_filter": "k2020"},
256 ],
257 )
258 with self.assertRaises(ValueError):
259 expandUniqueId(self.butler, {"tract": 42})
262if __name__ == "__main__":
263 unittest.main()