Coverage for tests/test_datasets.py : 10%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import unittest
23import pickle
25from lsst.daf.butler import (
26 DataCoordinate,
27 DatasetType,
28 DatasetRef,
29 DimensionUniverse,
30 StorageClass,
31 StorageClassFactory,
32)
34"""Tests for datasets module.
35"""
38class DatasetTypeTestCase(unittest.TestCase):
39 """Test for DatasetType.
40 """
41 def setUp(self):
42 self.universe = DimensionUniverse()
44 def testConstructor(self):
45 """Test construction preserves values.
47 Note that construction doesn't check for valid storageClass.
48 This can only be verified for a particular schema.
49 """
50 datasetTypeName = "test"
51 storageClass = StorageClass("test_StructuredData")
52 dimensions = self.universe.extract(("instrument", "visit"))
53 datasetType = DatasetType(datasetTypeName, dimensions, storageClass)
54 self.assertEqual(datasetType.name, datasetTypeName)
55 self.assertEqual(datasetType.storageClass, storageClass)
56 self.assertEqual(datasetType.dimensions, dimensions)
58 def testConstructor2(self):
59 """Test construction from StorageClass name.
60 """
61 datasetTypeName = "test"
62 storageClass = StorageClass("test_constructor2")
63 StorageClassFactory().registerStorageClass(storageClass)
64 dimensions = self.universe.extract(("instrument", "visit"))
65 datasetType = DatasetType(datasetTypeName, dimensions, "test_constructor2")
66 self.assertEqual(datasetType.name, datasetTypeName)
67 self.assertEqual(datasetType.storageClass, storageClass)
68 self.assertEqual(datasetType.dimensions, dimensions)
70 def testNameValidation(self):
71 """Test that dataset type names only contain certain characters
72 in certain positions.
73 """
74 dimensions = self.universe.extract(("instrument", "visit"))
75 storageClass = StorageClass("test_StructuredData")
76 goodNames = ("a", "A", "z1", "Z1", "a_1B", "A_1b")
77 badNames = ("1", "_", "a%b", "B+Z", "T[0]")
78 for name in goodNames:
79 self.assertEqual(DatasetType(name, dimensions, storageClass).name, name)
80 for suffix in goodNames:
81 full = f"{name}.{suffix}"
82 self.assertEqual(DatasetType(full, dimensions, storageClass).name, full)
83 for suffix in badNames:
84 full = f"{name}.{suffix}"
85 with self.subTest(full=full):
86 with self.assertRaises(ValueError):
87 DatasetType(full, dimensions, storageClass)
88 for name in badNames:
89 with self.subTest(name=name):
90 with self.assertRaises(ValueError):
91 DatasetType(name, dimensions, storageClass)
93 def testEquality(self):
94 storageA = StorageClass("test_a")
95 storageB = StorageClass("test_b")
96 dimensionsA = self.universe.extract(["instrument"])
97 dimensionsB = self.universe.extract(["skymap"])
98 self.assertEqual(DatasetType("a", dimensionsA, storageA,),
99 DatasetType("a", dimensionsA, storageA,))
100 self.assertEqual(DatasetType("a", dimensionsA, "test_a",),
101 DatasetType("a", dimensionsA, storageA,))
102 self.assertEqual(DatasetType("a", dimensionsA, storageA,),
103 DatasetType("a", dimensionsA, "test_a",))
104 self.assertEqual(DatasetType("a", dimensionsA, "test_a",),
105 DatasetType("a", dimensionsA, "test_a",))
106 self.assertNotEqual(DatasetType("a", dimensionsA, storageA,),
107 DatasetType("b", dimensionsA, storageA,))
108 self.assertNotEqual(DatasetType("a", dimensionsA, storageA,),
109 DatasetType("b", dimensionsA, "test_a",))
110 self.assertNotEqual(DatasetType("a", dimensionsA, storageA,),
111 DatasetType("a", dimensionsA, storageB,))
112 self.assertNotEqual(DatasetType("a", dimensionsA, storageA,),
113 DatasetType("a", dimensionsA, "test_b",))
114 self.assertNotEqual(DatasetType("a", dimensionsA, storageA,),
115 DatasetType("a", dimensionsB, storageA,))
116 self.assertNotEqual(DatasetType("a", dimensionsA, storageA,),
117 DatasetType("a", dimensionsB, "test_a",))
119 def testHashability(self):
120 """Test `DatasetType.__hash__`.
122 This test is performed by checking that `DatasetType` entries can
123 be inserted into a `set` and that unique values of its
124 (`name`, `storageClass`, `dimensions`) parameters result in separate
125 entries (and equal ones don't).
127 This does not check for uniformity of hashing or the actual values
128 of the hash function.
129 """
130 types = []
131 unique = 0
132 storageC = StorageClass("test_c")
133 storageD = StorageClass("test_d")
134 for name in ["a", "b"]:
135 for storageClass in [storageC, storageD]:
136 for dimensions in [("instrument", ), ("skymap", )]:
137 datasetType = DatasetType(name, self.universe.extract(dimensions), storageClass)
138 datasetTypeCopy = DatasetType(name, self.universe.extract(dimensions), storageClass)
139 types.extend((datasetType, datasetTypeCopy))
140 unique += 1 # datasetType should always equal its copy
141 self.assertEqual(len(set(types)), unique) # all other combinations are unique
143 # also check that hashes of instances constructed with StorageClass
144 # name matches hashes of instances constructed with instances
145 dimensions = self.universe.extract(["instrument"])
146 self.assertEqual(hash(DatasetType("a", dimensions, storageC)),
147 hash(DatasetType("a", dimensions, "test_c")))
148 self.assertEqual(hash(DatasetType("a", dimensions, "test_c")),
149 hash(DatasetType("a", dimensions, "test_c")))
150 self.assertNotEqual(hash(DatasetType("a", dimensions, storageC)),
151 hash(DatasetType("a", dimensions, "test_d")))
152 self.assertNotEqual(hash(DatasetType("a", dimensions, storageD)),
153 hash(DatasetType("a", dimensions, "test_c")))
154 self.assertNotEqual(hash(DatasetType("a", dimensions, "test_c")),
155 hash(DatasetType("a", dimensions, "test_d")))
157 def testPickle(self):
158 """Test pickle support.
159 """
160 storageClass = StorageClass("test_pickle")
161 datasetTypeName = "test"
162 dimensions = self.universe.extract(("instrument", "visit"))
163 # Un-pickling requires that storage class is registered with factory.
164 StorageClassFactory().registerStorageClass(storageClass)
165 datasetType = DatasetType(datasetTypeName, dimensions, storageClass)
166 datasetTypeOut = pickle.loads(pickle.dumps(datasetType))
167 self.assertIsInstance(datasetTypeOut, DatasetType)
168 self.assertEqual(datasetType.name, datasetTypeOut.name)
169 self.assertEqual(datasetType.dimensions.names, datasetTypeOut.dimensions.names)
170 self.assertEqual(datasetType.storageClass, datasetTypeOut.storageClass)
172 def test_composites(self):
173 """Test components within composite DatasetTypes."""
174 storageClassA = StorageClass("compA")
175 storageClassB = StorageClass("compB")
176 storageClass = StorageClass("test_composite", components={"compA": storageClassA,
177 "compB": storageClassB})
178 self.assertTrue(storageClass.isComposite())
179 self.assertFalse(storageClassA.isComposite())
180 self.assertFalse(storageClassB.isComposite())
182 dimensions = self.universe.extract(("instrument", "visit"))
184 datasetTypeComposite = DatasetType("composite", dimensions, storageClass)
185 datasetTypeComponentA = DatasetType("composite.compA", dimensions, storageClassA)
186 datasetTypeComponentB = DatasetType("composite.compB", dimensions, storageClassB)
188 self.assertTrue(datasetTypeComposite.isComposite())
189 self.assertFalse(datasetTypeComponentA.isComposite())
190 self.assertTrue(datasetTypeComponentB.isComponent())
191 self.assertFalse(datasetTypeComposite.isComponent())
193 self.assertEqual(datasetTypeComposite.name, "composite")
194 self.assertEqual(datasetTypeComponentA.name, "composite.compA")
195 self.assertEqual(datasetTypeComponentB.component(), "compB")
196 self.assertEqual(datasetTypeComposite.nameAndComponent(), ("composite", None))
197 self.assertEqual(datasetTypeComponentA.nameAndComponent(), ("composite", "compA"))
200class DatasetRefTestCase(unittest.TestCase):
201 """Test for DatasetRef.
202 """
204 def setUp(self):
205 self.universe = DimensionUniverse()
206 datasetTypeName = "test"
207 self.componentStorageClass1 = StorageClass("Component1")
208 self.componentStorageClass2 = StorageClass("Component2")
209 self.parentStorageClass = StorageClass("Parent", components={"a": self.componentStorageClass1,
210 "b": self.componentStorageClass2})
211 dimensions = self.universe.extract(("instrument", "visit"))
212 self.dataId = dict(instrument="DummyCam", visit=42)
213 self.datasetType = DatasetType(datasetTypeName, dimensions, self.parentStorageClass)
215 def testConstructor(self):
216 """Test that construction preserves and validates values.
217 """
218 # Construct an unresolved ref.
219 ref = DatasetRef(self.datasetType, self.dataId)
220 self.assertEqual(ref.datasetType, self.datasetType)
221 self.assertEqual(ref.dataId, DataCoordinate.standardize(self.dataId, universe=self.universe),
222 msg=ref.dataId)
223 self.assertIsInstance(ref.dataId, DataCoordinate)
224 self.assertIsNone(ref.components)
225 # Constructing an unresolved ref with run and/or components should
226 # fail.
227 run = "somerun"
228 with self.assertRaises(ValueError):
229 DatasetRef(self.datasetType, self.dataId, run=run)
230 components = {
231 "a": DatasetRef(self.datasetType.makeComponentDatasetType("a"), self.dataId, id=2, run=run)
232 }
233 with self.assertRaises(ValueError):
234 DatasetRef(self.datasetType, self.dataId, components=components)
235 # Passing a data ID that is missing dimensions should fail.
236 with self.assertRaises(KeyError):
237 DatasetRef(self.datasetType, {"instrument": "DummyCam"})
238 # Constructing a resolved ref should preserve run and components,
239 # as well as everything else.
240 ref = DatasetRef(self.datasetType, self.dataId, id=1, run=run, components=components)
241 self.assertEqual(ref.datasetType, self.datasetType)
242 self.assertEqual(ref.dataId, DataCoordinate.standardize(self.dataId, universe=self.universe),
243 msg=ref.dataId)
244 self.assertIsInstance(ref.dataId, DataCoordinate)
245 self.assertEqual(ref.id, 1)
246 self.assertEqual(ref.run, run)
247 self.assertEqual(ref.components, components)
248 # Constructing a resolved ref with bad component storage classes
249 # should fail.
250 with self.assertRaises(ValueError):
251 DatasetRef(self.datasetType, self.dataId, id=1, run=run, components={"b": components["a"]})
252 # Constructing a resolved ref with unresolved components should fail.
253 with self.assertRaises(ValueError):
254 DatasetRef(self.datasetType, self.dataId, id=1, run=run,
255 components={"a": components["a"].unresolved()})
256 # Constructing a resolved ref with bad component names should fail.
257 with self.assertRaises(ValueError):
258 DatasetRef(self.datasetType, self.dataId, id=1, run=run,
259 components={"c": components["a"]})
261 def testResolving(self):
262 ref = DatasetRef(self.datasetType, self.dataId, id=1, run="somerun")
263 unresolvedRef = ref.unresolved()
264 self.assertIsNotNone(ref.id)
265 self.assertIsNone(unresolvedRef.id)
266 self.assertIsNone(unresolvedRef.run)
267 self.assertIsNone(unresolvedRef.components)
268 self.assertNotEqual(ref, unresolvedRef)
269 self.assertEqual(ref.unresolved(), unresolvedRef)
270 self.assertEqual(ref.datasetType, unresolvedRef.datasetType)
271 self.assertEqual(ref.dataId, unresolvedRef.dataId)
272 reresolvedRef = unresolvedRef.resolved(id=1, run="somerun")
273 self.assertEqual(ref, reresolvedRef)
274 self.assertEqual(reresolvedRef.unresolved(), unresolvedRef)
275 self.assertIsNotNone(reresolvedRef.run)
276 self.assertIsNotNone(reresolvedRef.components)
278 def testPickle(self):
279 ref = DatasetRef(self.datasetType, self.dataId, id=1, run="somerun")
280 s = pickle.dumps(ref)
281 self.assertEqual(pickle.loads(s), ref)
284if __name__ == "__main__": 284 ↛ 285line 284 didn't jump to line 285, because the condition on line 284 was never true
285 unittest.main()