Coverage for tests/test_testRepo.py: 23%
116 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-06 10:53 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-06 10:53 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28"""Unit tests for `lsst.daf.butler.tests.testRepo`, a module for creating
29test repositories or butlers.
30"""
32import os
33import shutil
34import unittest
36import lsst.daf.butler
37from lsst.daf.butler.tests import (
38 MetricsExample,
39 addDataIdValue,
40 addDatasetType,
41 expandUniqueId,
42 makeTestCollection,
43 makeTestRepo,
44 registerMetricsExample,
45)
46from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir, safeTestTempDir
48TESTDIR = os.path.abspath(os.path.dirname(__file__))
51class ButlerTestRepoTestCase(unittest.TestCase):
52 """Simpler test than below without setUpClass getting in the way."""
54 def setUp(self):
55 self.root = makeTestTempDir(TESTDIR)
57 def tearDown(self):
58 removeTestTempDir(self.root)
60 def testMakeTestRepo(self):
61 dataIds = {
62 "instrument": ["DummyCam"],
63 "physical_filter": ["d-r"],
64 "exposure": [42, 43, 44],
65 "visit": [42, 43, 44],
66 }
68 butler = makeTestRepo(self.root, dataIds)
70 records = list(butler.registry.queryDimensionRecords("visit"))
71 self.assertEqual(len(records), 3)
74class ButlerUtilsTestSuite(unittest.TestCase):
75 """Test the butler test utilities."""
77 @classmethod
78 def setUpClass(cls):
79 # Repository should be re-created for each test case, but
80 # this has a prohibitive run-time cost at present
81 cls.root = makeTestTempDir(TESTDIR)
83 cls.creatorButler = makeTestRepo(cls.root)
84 addDataIdValue(cls.creatorButler, "instrument", "notACam")
85 addDataIdValue(cls.creatorButler, "instrument", "dummyCam")
86 addDataIdValue(cls.creatorButler, "physical_filter", "k2020", band="k", instrument="notACam")
87 addDataIdValue(cls.creatorButler, "physical_filter", "l2019", instrument="dummyCam")
88 addDataIdValue(cls.creatorButler, "visit", 101, instrument="notACam", physical_filter="k2020")
89 addDataIdValue(cls.creatorButler, "visit", 102, instrument="notACam", physical_filter="k2020")
90 addDataIdValue(cls.creatorButler, "detector", 5)
91 # Leave skymap/patch/tract undefined so that tests can assume
92 # they're missing.
94 registerMetricsExample(cls.creatorButler)
95 addDatasetType(cls.creatorButler, "DataType1", {"instrument"}, "StructuredDataNoComponents")
96 addDatasetType(cls.creatorButler, "DataType2", {"instrument", "visit"}, "StructuredData")
98 @classmethod
99 def tearDownClass(cls):
100 # TODO: use addClassCleanup rather than tearDownClass in Python 3.8
101 # to keep the addition and removal together and make it more robust
102 removeTestTempDir(cls.root)
104 def setUp(self):
105 # TestCase.id() is unique for each test method
106 self.butler = makeTestCollection(self.creatorButler, uniqueId=self.id())
108 def testButlerValid(self):
109 self.butler.validateConfiguration()
111 def testButlerKwargs(self):
112 # outfile has the most obvious effects of any Butler.makeRepo keyword
113 with safeTestTempDir(TESTDIR) as temp:
114 path = os.path.join(temp, "oddConfig.json")
115 makeTestRepo(temp, {}, outfile=path)
116 self.assertTrue(os.path.isfile(path))
118 def _checkButlerDimension(self, dimensions, query, expected):
119 result = list(self.butler.registry.queryDataIds(dimensions, where=query, check=False))
120 self.assertEqual(len(result), 1)
121 self.assertIn(result[0].required, expected)
123 def testButlerDimensions(self):
124 self._checkButlerDimension(
125 {"instrument"}, "instrument='notACam'", [{"instrument": "notACam"}, {"instrument": "dummyCam"}]
126 )
127 self._checkButlerDimension(
128 {"visit", "instrument"},
129 "visit=101",
130 [{"instrument": "notACam", "visit": 101}, {"instrument": "dummyCam", "visit": 101}],
131 )
132 self._checkButlerDimension(
133 {"visit", "instrument"},
134 "visit=102",
135 [{"instrument": "notACam", "visit": 102}, {"instrument": "dummyCam", "visit": 102}],
136 )
137 self._checkButlerDimension(
138 {"detector", "instrument"},
139 "detector=5",
140 [{"instrument": "notACam", "detector": 5}, {"instrument": "dummyCam", "detector": 5}],
141 )
143 def testAddDataIdValue(self):
144 addDataIdValue(self.butler, "visit", 1, instrument="notACam", physical_filter="k2020")
145 self._checkButlerDimension(
146 {"visit", "instrument"}, "visit=1", [{"instrument": "notACam", "visit": 1}]
147 )
148 addDataIdValue(self.butler, "visit", 2, instrument="dummyCam", physical_filter="l2019")
149 self._checkButlerDimension(
150 {"visit", "instrument"}, "visit=2", [{"instrument": "dummyCam", "visit": 2}]
151 )
153 with self.assertRaises(ValueError):
154 addDataIdValue(self.butler, "NotADimension", 42)
155 with self.assertRaises(ValueError):
156 addDataIdValue(self.butler, "detector", "nonNumeric")
157 with self.assertRaises(ValueError):
158 addDataIdValue(self.butler, "detector", 101, nonsenseField="string")
160 # Keywords imply different instruments
161 with self.assertRaises(RuntimeError):
162 addDataIdValue(self.butler, "exposure", 101, instrument="dummyCam", physical_filter="k2020")
164 # No skymap defined
165 with self.assertRaises(RuntimeError):
166 addDataIdValue(self.butler, "tract", 42)
167 # Didn't create skymap "map" first.
168 with self.assertRaises(RuntimeError):
169 addDataIdValue(self.butler, "tract", 43, skymap="map")
171 def testAddDatasetType(self):
172 # 1 for StructuredDataNoComponents, 1 for StructuredData (components
173 # not included).
174 self.assertEqual(len(list(self.butler.registry.queryDatasetTypes(components=False))), 2)
176 # Testing the DatasetType objects is not practical, because all tests
177 # need a DimensionUniverse. So just check that we have the dataset
178 # types we expect.
179 self.butler.get_dataset_type("DataType1")
180 self.butler.get_dataset_type("DataType2")
182 with self.assertRaises(ValueError):
183 addDatasetType(self.butler, "DataType3", {"4thDimension"}, "NumpyArray")
184 with self.assertRaises(ValueError):
185 addDatasetType(self.butler, "DataType3", {"instrument"}, "UnstorableType")
187 def testRegisterMetricsExample(self):
188 id1 = {"instrument": "notACam"}
189 id2 = expandUniqueId(self.butler, {"visit": 101})
190 data = MetricsExample(summary={"answer": 42, "question": "unknown"})
192 self.butler.put(data, "DataType1", id1)
193 self.assertEqual(self.butler.get("DataType1", id1), data)
195 self.butler.put(data, "DataType2", id2)
196 self.assertEqual(self.butler.get("DataType2", id2), data)
197 self.assertEqual(self.butler.get("DataType2.summary", id2), data.summary)
199 def testRegisterMetricsExampleChained(self):
200 """Regression test for registerMetricsExample having no effect
201 on ChainedDatastore.
202 """
203 temp = makeTestTempDir(TESTDIR)
204 try:
205 config = lsst.daf.butler.Config()
206 config["datastore", "cls"] = "lsst.daf.butler.datastores.chainedDatastore.ChainedDatastore"
207 config["datastore", "datastores"] = [
208 {
209 "cls": "lsst.daf.butler.datastores.fileDatastore.FileDatastore",
210 }
211 ]
213 repo = lsst.daf.butler.Butler.makeRepo(temp, config=config)
214 butler = lsst.daf.butler.Butler.from_config(repo, run="chainedExample")
215 registerMetricsExample(butler)
216 addDatasetType(butler, "DummyType", {}, "StructuredDataNoComponents")
218 data = MetricsExample(summary={})
219 # Should not raise
220 butler.put(data, "DummyType")
221 finally:
222 shutil.rmtree(temp, ignore_errors=True)
224 def testUniqueButler(self):
225 dataId = {"instrument": "notACam"}
226 ref = self.butler.put(MetricsExample({"answer": 42, "question": "unknown"}), "DataType1", dataId)
227 self.assertTrue(self.butler.exists("DataType1", dataId))
228 self.assertTrue(self.butler.exists(ref))
230 newButler = makeTestCollection(self.creatorButler)
232 # Can not be found in the new default collection.
233 self.assertFalse(newButler.exists("DataType1", dataId))
235 # The ref does exist in the new butler though.
236 self.assertTrue(newButler.exists(ref))
238 def testExpandUniqueId(self):
239 self.assertEqual(
240 expandUniqueId(self.butler, {"instrument": "notACam"}).required, {"instrument": "notACam"}
241 )
242 self.assertIn(
243 expandUniqueId(self.butler, {"visit": 101}).required,
244 [{"instrument": "notACam", "visit": 101}, {"instrument": "dummyCam", "visit": 101}],
245 )
246 self.assertIn(
247 expandUniqueId(self.butler, {"detector": 5}).required,
248 [{"instrument": "notACam", "detector": 5}, {"instrument": "dummyCam", "detector": 5}],
249 )
250 self.assertIn(
251 expandUniqueId(self.butler, {"physical_filter": "k2020"}).required,
252 [
253 {"instrument": "notACam", "physical_filter": "k2020"},
254 {"instrument": "notACam", "physical_filter": "k2020"},
255 ],
256 )
257 with self.assertRaises(ValueError):
258 expandUniqueId(self.butler, {"tract": 42})
261if __name__ == "__main__":
262 unittest.main()