Coverage for tests/test_testRepo.py: 25%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Unit tests for `lsst.daf.butler.tests.testRepo`, a module for creating
23test repositories or butlers.
24"""
26import os
27import shutil
28import unittest
30import lsst.daf.butler
31from lsst.daf.butler.tests import (
32 MetricsExample,
33 addDataIdValue,
34 addDatasetType,
35 expandUniqueId,
36 makeTestCollection,
37 makeTestRepo,
38 registerMetricsExample,
39)
40from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir, safeTestTempDir
42TESTDIR = os.path.abspath(os.path.dirname(__file__))
45class ButlerUtilsTestSuite(unittest.TestCase):
46 @classmethod
47 def setUpClass(cls):
48 # Repository should be re-created for each test case, but
49 # this has a prohibitive run-time cost at present
50 cls.root = makeTestTempDir(TESTDIR)
52 cls.creatorButler = makeTestRepo(cls.root)
53 addDataIdValue(cls.creatorButler, "instrument", "notACam")
54 addDataIdValue(cls.creatorButler, "instrument", "dummyCam")
55 addDataIdValue(cls.creatorButler, "physical_filter", "k2020", band="k", instrument="notACam")
56 addDataIdValue(cls.creatorButler, "physical_filter", "l2019", instrument="dummyCam")
57 addDataIdValue(cls.creatorButler, "visit", 101, instrument="notACam", physical_filter="k2020")
58 addDataIdValue(cls.creatorButler, "visit", 102, instrument="notACam", physical_filter="k2020")
59 addDataIdValue(cls.creatorButler, "detector", 5)
60 # Leave skymap/patch/tract undefined so that tests can assume
61 # they're missing.
63 registerMetricsExample(cls.creatorButler)
64 addDatasetType(cls.creatorButler, "DataType1", {"instrument"}, "StructuredDataNoComponents")
65 addDatasetType(cls.creatorButler, "DataType2", {"instrument", "visit"}, "StructuredData")
67 @classmethod
68 def tearDownClass(cls):
69 # TODO: use addClassCleanup rather than tearDownClass in Python 3.8
70 # to keep the addition and removal together and make it more robust
71 removeTestTempDir(cls.root)
73 def setUp(self):
74 # TestCase.id() is unique for each test method
75 self.butler = makeTestCollection(self.creatorButler, uniqueId=self.id())
77 def testButlerValid(self):
78 self.butler.validateConfiguration()
80 def testButlerKwargs(self):
81 # outfile has the most obvious effects of any Butler.makeRepo keyword
82 with safeTestTempDir(TESTDIR) as temp:
83 path = os.path.join(temp, "oddConfig.json")
84 makeTestRepo(temp, {}, outfile=path)
85 self.assertTrue(os.path.isfile(path))
87 def _checkButlerDimension(self, dimensions, query, expected):
88 result = list(self.butler.registry.queryDataIds(dimensions, where=query, check=False))
89 self.assertEqual(len(result), 1)
90 self.assertIn(dict(result[0]), expected)
92 def testButlerDimensions(self):
93 self._checkButlerDimension(
94 {"instrument"}, "instrument='notACam'", [{"instrument": "notACam"}, {"instrument": "dummyCam"}]
95 )
96 self._checkButlerDimension(
97 {"visit", "instrument"},
98 "visit=101",
99 [{"instrument": "notACam", "visit": 101}, {"instrument": "dummyCam", "visit": 101}],
100 )
101 self._checkButlerDimension(
102 {"visit", "instrument"},
103 "visit=102",
104 [{"instrument": "notACam", "visit": 102}, {"instrument": "dummyCam", "visit": 102}],
105 )
106 self._checkButlerDimension(
107 {"detector", "instrument"},
108 "detector=5",
109 [{"instrument": "notACam", "detector": 5}, {"instrument": "dummyCam", "detector": 5}],
110 )
112 def testAddDataIdValue(self):
113 addDataIdValue(self.butler, "visit", 1, instrument="notACam", physical_filter="k2020")
114 self._checkButlerDimension(
115 {"visit", "instrument"}, "visit=1", [{"instrument": "notACam", "visit": 1}]
116 )
117 addDataIdValue(self.butler, "visit", 2, instrument="dummyCam", physical_filter="l2019")
118 self._checkButlerDimension(
119 {"visit", "instrument"}, "visit=2", [{"instrument": "dummyCam", "visit": 2}]
120 )
122 with self.assertRaises(ValueError):
123 addDataIdValue(self.butler, "NotADimension", 42)
124 with self.assertRaises(ValueError):
125 addDataIdValue(self.butler, "detector", "nonNumeric")
126 with self.assertRaises(ValueError):
127 addDataIdValue(self.butler, "detector", 101, nonsenseField="string")
129 # Keywords imply different instruments
130 with self.assertRaises(RuntimeError):
131 addDataIdValue(self.butler, "exposure", 101, instrument="dummyCam", physical_filter="k2020")
133 # No skymap defined
134 with self.assertRaises(RuntimeError):
135 addDataIdValue(self.butler, "tract", 42)
136 with self.assertRaises(RuntimeError):
137 addDataIdValue(self.butler, "tract", 43, skymap="map")
139 def testAddDatasetType(self):
140 # 1 for StructuredDataNoComponents, 4 for StructuredData
141 self.assertEqual(len(list(self.butler.registry.queryDatasetTypes(components=True))), 5)
143 # Testing the DatasetType objects is not practical, because all tests
144 # need a DimensionUniverse. So just check that we have the dataset
145 # types we expect.
146 self.butler.registry.getDatasetType("DataType1")
147 self.butler.registry.getDatasetType("DataType2")
149 with self.assertRaises(ValueError):
150 addDatasetType(self.butler, "DataType3", {"4thDimension"}, "NumpyArray")
151 with self.assertRaises(ValueError):
152 addDatasetType(self.butler, "DataType3", {"instrument"}, "UnstorableType")
154 def testRegisterMetricsExample(self):
155 id1 = {"instrument": "notACam"}
156 id2 = expandUniqueId(self.butler, {"visit": 101})
157 data = MetricsExample(summary={"answer": 42, "question": "unknown"})
159 self.butler.put(data, "DataType1", id1)
160 self.assertEqual(self.butler.get("DataType1", id1), data)
162 self.butler.put(data, "DataType2", id2)
163 self.assertEqual(self.butler.get("DataType2", id2), data)
164 self.assertEqual(self.butler.get("DataType2.summary", id2), data.summary)
166 def testRegisterMetricsExampleChained(self):
167 """Regression test for registerMetricsExample having no effect
168 on ChainedDatastore.
169 """
170 temp = makeTestTempDir(TESTDIR)
171 try:
172 config = lsst.daf.butler.Config()
173 config["datastore", "cls"] = "lsst.daf.butler.datastores.chainedDatastore.ChainedDatastore"
174 config["datastore", "datastores"] = [
175 {
176 "cls": "lsst.daf.butler.datastores.fileDatastore.FileDatastore",
177 }
178 ]
180 repo = lsst.daf.butler.Butler.makeRepo(temp, config=config)
181 butler = lsst.daf.butler.Butler(repo, run="chainedExample")
182 registerMetricsExample(butler)
183 addDatasetType(butler, "DummyType", {}, "StructuredDataNoComponents")
185 data = MetricsExample(summary={})
186 # Should not raise
187 butler.put(data, "DummyType")
188 finally:
189 shutil.rmtree(temp, ignore_errors=True)
191 def testUniqueButler(self):
192 dataId = {"instrument": "notACam"}
193 self.butler.put(MetricsExample({"answer": 42, "question": "unknown"}), "DataType1", dataId)
194 self.assertTrue(self.butler.datasetExists("DataType1", dataId))
196 newButler = makeTestCollection(self.creatorButler)
197 with self.assertRaises(LookupError):
198 newButler.datasetExists("DataType1", dataId)
200 def testExpandUniqueId(self):
201 self.assertEqual(
202 dict(expandUniqueId(self.butler, {"instrument": "notACam"})), {"instrument": "notACam"}
203 )
204 self.assertIn(
205 dict(expandUniqueId(self.butler, {"visit": 101})),
206 [{"instrument": "notACam", "visit": 101}, {"instrument": "dummyCam", "visit": 101}],
207 )
208 self.assertIn(
209 dict(expandUniqueId(self.butler, {"detector": 5})),
210 [{"instrument": "notACam", "detector": 5}, {"instrument": "dummyCam", "detector": 5}],
211 )
212 self.assertIn(
213 dict(expandUniqueId(self.butler, {"physical_filter": "k2020"})),
214 [
215 {"instrument": "notACam", "physical_filter": "k2020"},
216 {"instrument": "notACam", "physical_filter": "k2020"},
217 ],
218 )
219 with self.assertRaises(ValueError):
220 expandUniqueId(self.butler, {"tract": 42})
223if __name__ == "__main__": 223 ↛ 224line 223 didn't jump to line 224, because the condition on line 223 was never true
224 unittest.main()