Coverage for tests/test_testRepo.py: 23%
116 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-08-05 01:26 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-08-05 01:26 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Unit tests for `lsst.daf.butler.tests.testRepo`, a module for creating
23test repositories or butlers.
24"""
26import os
27import shutil
28import unittest
30import lsst.daf.butler
31from lsst.daf.butler.tests import (
32 MetricsExample,
33 addDataIdValue,
34 addDatasetType,
35 expandUniqueId,
36 makeTestCollection,
37 makeTestRepo,
38 registerMetricsExample,
39)
40from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir, safeTestTempDir
42TESTDIR = os.path.abspath(os.path.dirname(__file__))
45class ButlerTestRepoTestCase(unittest.TestCase):
46 """Simpler test than below without setUpClass getting in the way."""
48 def setUp(self):
49 self.root = makeTestTempDir(TESTDIR)
51 def tearDown(self):
52 removeTestTempDir(self.root)
54 def testMakeTestRepo(self):
55 dataIds = {
56 "instrument": ["DummyCam"],
57 "physical_filter": ["d-r"],
58 "exposure": [42, 43, 44],
59 "visit": [42, 43, 44],
60 }
62 butler = makeTestRepo(self.root, dataIds)
64 records = list(butler.registry.queryDimensionRecords("visit"))
65 self.assertEqual(len(records), 3)
68class ButlerUtilsTestSuite(unittest.TestCase):
69 """Test the butler test utilities."""
71 @classmethod
72 def setUpClass(cls):
73 # Repository should be re-created for each test case, but
74 # this has a prohibitive run-time cost at present
75 cls.root = makeTestTempDir(TESTDIR)
77 cls.creatorButler = makeTestRepo(cls.root)
78 addDataIdValue(cls.creatorButler, "instrument", "notACam")
79 addDataIdValue(cls.creatorButler, "instrument", "dummyCam")
80 addDataIdValue(cls.creatorButler, "physical_filter", "k2020", band="k", instrument="notACam")
81 addDataIdValue(cls.creatorButler, "physical_filter", "l2019", instrument="dummyCam")
82 addDataIdValue(cls.creatorButler, "visit", 101, instrument="notACam", physical_filter="k2020")
83 addDataIdValue(cls.creatorButler, "visit", 102, instrument="notACam", physical_filter="k2020")
84 addDataIdValue(cls.creatorButler, "detector", 5)
85 # Leave skymap/patch/tract undefined so that tests can assume
86 # they're missing.
88 registerMetricsExample(cls.creatorButler)
89 addDatasetType(cls.creatorButler, "DataType1", {"instrument"}, "StructuredDataNoComponents")
90 addDatasetType(cls.creatorButler, "DataType2", {"instrument", "visit"}, "StructuredData")
92 @classmethod
93 def tearDownClass(cls):
94 # TODO: use addClassCleanup rather than tearDownClass in Python 3.8
95 # to keep the addition and removal together and make it more robust
96 removeTestTempDir(cls.root)
98 def setUp(self):
99 # TestCase.id() is unique for each test method
100 self.butler = makeTestCollection(self.creatorButler, uniqueId=self.id())
102 def testButlerValid(self):
103 self.butler.validateConfiguration()
105 def testButlerKwargs(self):
106 # outfile has the most obvious effects of any Butler.makeRepo keyword
107 with safeTestTempDir(TESTDIR) as temp:
108 path = os.path.join(temp, "oddConfig.json")
109 makeTestRepo(temp, {}, outfile=path)
110 self.assertTrue(os.path.isfile(path))
112 def _checkButlerDimension(self, dimensions, query, expected):
113 result = list(self.butler.registry.queryDataIds(dimensions, where=query, check=False))
114 self.assertEqual(len(result), 1)
115 self.assertIn(result[0].byName(), expected)
117 def testButlerDimensions(self):
118 self._checkButlerDimension(
119 {"instrument"}, "instrument='notACam'", [{"instrument": "notACam"}, {"instrument": "dummyCam"}]
120 )
121 self._checkButlerDimension(
122 {"visit", "instrument"},
123 "visit=101",
124 [{"instrument": "notACam", "visit": 101}, {"instrument": "dummyCam", "visit": 101}],
125 )
126 self._checkButlerDimension(
127 {"visit", "instrument"},
128 "visit=102",
129 [{"instrument": "notACam", "visit": 102}, {"instrument": "dummyCam", "visit": 102}],
130 )
131 self._checkButlerDimension(
132 {"detector", "instrument"},
133 "detector=5",
134 [{"instrument": "notACam", "detector": 5}, {"instrument": "dummyCam", "detector": 5}],
135 )
137 def testAddDataIdValue(self):
138 addDataIdValue(self.butler, "visit", 1, instrument="notACam", physical_filter="k2020")
139 self._checkButlerDimension(
140 {"visit", "instrument"}, "visit=1", [{"instrument": "notACam", "visit": 1}]
141 )
142 addDataIdValue(self.butler, "visit", 2, instrument="dummyCam", physical_filter="l2019")
143 self._checkButlerDimension(
144 {"visit", "instrument"}, "visit=2", [{"instrument": "dummyCam", "visit": 2}]
145 )
147 with self.assertRaises(ValueError):
148 addDataIdValue(self.butler, "NotADimension", 42)
149 with self.assertRaises(ValueError):
150 addDataIdValue(self.butler, "detector", "nonNumeric")
151 with self.assertRaises(ValueError):
152 addDataIdValue(self.butler, "detector", 101, nonsenseField="string")
154 # Keywords imply different instruments
155 with self.assertRaises(RuntimeError):
156 addDataIdValue(self.butler, "exposure", 101, instrument="dummyCam", physical_filter="k2020")
158 # No skymap defined
159 with self.assertRaises(RuntimeError):
160 addDataIdValue(self.butler, "tract", 42)
161 # Didn't create skymap "map" first.
162 with self.assertRaises(RuntimeError):
163 addDataIdValue(self.butler, "tract", 43, skymap="map")
165 def testAddDatasetType(self):
166 # 1 for StructuredDataNoComponents, 1 for StructuredData (components
167 # not included).
168 self.assertEqual(len(list(self.butler.registry.queryDatasetTypes(components=False))), 2)
170 # Testing the DatasetType objects is not practical, because all tests
171 # need a DimensionUniverse. So just check that we have the dataset
172 # types we expect.
173 self.butler.registry.getDatasetType("DataType1")
174 self.butler.registry.getDatasetType("DataType2")
176 with self.assertRaises(ValueError):
177 addDatasetType(self.butler, "DataType3", {"4thDimension"}, "NumpyArray")
178 with self.assertRaises(ValueError):
179 addDatasetType(self.butler, "DataType3", {"instrument"}, "UnstorableType")
181 def testRegisterMetricsExample(self):
182 id1 = {"instrument": "notACam"}
183 id2 = expandUniqueId(self.butler, {"visit": 101})
184 data = MetricsExample(summary={"answer": 42, "question": "unknown"})
186 self.butler.put(data, "DataType1", id1)
187 self.assertEqual(self.butler.get("DataType1", id1), data)
189 self.butler.put(data, "DataType2", id2)
190 self.assertEqual(self.butler.get("DataType2", id2), data)
191 self.assertEqual(self.butler.get("DataType2.summary", id2), data.summary)
193 def testRegisterMetricsExampleChained(self):
194 """Regression test for registerMetricsExample having no effect
195 on ChainedDatastore.
196 """
197 temp = makeTestTempDir(TESTDIR)
198 try:
199 config = lsst.daf.butler.Config()
200 config["datastore", "cls"] = "lsst.daf.butler.datastores.chainedDatastore.ChainedDatastore"
201 config["datastore", "datastores"] = [
202 {
203 "cls": "lsst.daf.butler.datastores.fileDatastore.FileDatastore",
204 }
205 ]
207 repo = lsst.daf.butler.Butler.makeRepo(temp, config=config)
208 butler = lsst.daf.butler.Butler(repo, run="chainedExample")
209 registerMetricsExample(butler)
210 addDatasetType(butler, "DummyType", {}, "StructuredDataNoComponents")
212 data = MetricsExample(summary={})
213 # Should not raise
214 butler.put(data, "DummyType")
215 finally:
216 shutil.rmtree(temp, ignore_errors=True)
218 def testUniqueButler(self):
219 dataId = {"instrument": "notACam"}
220 ref = self.butler.put(MetricsExample({"answer": 42, "question": "unknown"}), "DataType1", dataId)
221 self.assertTrue(self.butler.exists("DataType1", dataId))
222 self.assertTrue(self.butler.exists(ref))
224 newButler = makeTestCollection(self.creatorButler)
226 # Can not be found in the new default collection.
227 self.assertFalse(newButler.exists("DataType1", dataId))
229 # The ref does exist in the new butler though.
230 self.assertTrue(newButler.exists(ref))
232 def testExpandUniqueId(self):
233 self.assertEqual(
234 dict(expandUniqueId(self.butler, {"instrument": "notACam"})), {"instrument": "notACam"}
235 )
236 self.assertIn(
237 dict(expandUniqueId(self.butler, {"visit": 101})),
238 [{"instrument": "notACam", "visit": 101}, {"instrument": "dummyCam", "visit": 101}],
239 )
240 self.assertIn(
241 dict(expandUniqueId(self.butler, {"detector": 5})),
242 [{"instrument": "notACam", "detector": 5}, {"instrument": "dummyCam", "detector": 5}],
243 )
244 self.assertIn(
245 dict(expandUniqueId(self.butler, {"physical_filter": "k2020"})),
246 [
247 {"instrument": "notACam", "physical_filter": "k2020"},
248 {"instrument": "notACam", "physical_filter": "k2020"},
249 ],
250 )
251 with self.assertRaises(ValueError):
252 expandUniqueId(self.butler, {"tract": 42})
255if __name__ == "__main__":
256 unittest.main()