Coverage for tests/test_datasets.py : 9%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import unittest
23import pickle
24import copy
26from lsst.daf.butler import (
27 DataCoordinate,
28 DatasetType,
29 DatasetRef,
30 DimensionUniverse,
31 StorageClass,
32 StorageClassFactory,
33)
35"""Tests for datasets module.
36"""
39class DatasetTypeTestCase(unittest.TestCase):
40 """Test for DatasetType.
41 """
42 def setUp(self):
43 self.universe = DimensionUniverse()
45 def testConstructor(self):
46 """Test construction preserves values.
48 Note that construction doesn't check for valid storageClass.
49 This can only be verified for a particular schema.
50 """
51 datasetTypeName = "test"
52 storageClass = StorageClass("test_StructuredData")
53 dimensions = self.universe.extract(("instrument", "visit"))
54 datasetType = DatasetType(datasetTypeName, dimensions, storageClass)
55 self.assertEqual(datasetType.name, datasetTypeName)
56 self.assertEqual(datasetType.storageClass, storageClass)
57 self.assertEqual(datasetType.dimensions, dimensions)
59 with self.assertRaises(ValueError, msg="Construct component without parent storage class"):
60 DatasetType(DatasetType.nameWithComponent(datasetTypeName, "comp"),
61 dimensions, storageClass)
62 with self.assertRaises(ValueError, msg="Construct non-component with parent storage class"):
63 DatasetType(datasetTypeName,
64 dimensions, storageClass, parentStorageClass="NotAllowed")
66 def testConstructor2(self):
67 """Test construction from StorageClass name.
68 """
69 datasetTypeName = "test"
70 storageClass = StorageClass("test_constructor2")
71 StorageClassFactory().registerStorageClass(storageClass)
72 dimensions = self.universe.extract(("instrument", "visit"))
73 datasetType = DatasetType(datasetTypeName, dimensions, "test_constructor2")
74 self.assertEqual(datasetType.name, datasetTypeName)
75 self.assertEqual(datasetType.storageClass, storageClass)
76 self.assertEqual(datasetType.dimensions, dimensions)
78 def testNameValidation(self):
79 """Test that dataset type names only contain certain characters
80 in certain positions.
81 """
82 dimensions = self.universe.extract(("instrument", "visit"))
83 goodNames = ("a", "A", "z1", "Z1", "a_1B", "A_1b")
84 badNames = ("1", "_", "a%b", "B+Z", "T[0]")
86 # Construct storage class with all the good names included as
87 # components so that we can test internal consistency
88 storageClass = StorageClass("test_StructuredData",
89 components={n: StorageClass("component") for n in goodNames})
91 for name in goodNames:
92 composite = DatasetType(name, dimensions, storageClass)
93 self.assertEqual(composite.name, name)
94 for suffix in goodNames:
95 full = DatasetType.nameWithComponent(name, suffix)
96 component = composite.makeComponentDatasetType(suffix)
97 self.assertEqual(component.name, full)
98 self.assertEqual(component.parentStorageClass.name, "test_StructuredData")
99 for suffix in badNames:
100 full = DatasetType.nameWithComponent(name, suffix)
101 with self.subTest(full=full):
102 with self.assertRaises(ValueError):
103 DatasetType(full, dimensions, storageClass)
104 for name in badNames:
105 with self.subTest(name=name):
106 with self.assertRaises(ValueError):
107 DatasetType(name, dimensions, storageClass)
109 def testEquality(self):
110 storageA = StorageClass("test_a")
111 storageB = StorageClass("test_b")
112 parent = StorageClass("test")
113 dimensionsA = self.universe.extract(["instrument"])
114 dimensionsB = self.universe.extract(["skymap"])
115 self.assertEqual(DatasetType("a", dimensionsA, storageA,),
116 DatasetType("a", dimensionsA, storageA,))
117 self.assertEqual(DatasetType("a", dimensionsA, "test_a",),
118 DatasetType("a", dimensionsA, storageA,))
119 self.assertEqual(DatasetType("a", dimensionsA, storageA,),
120 DatasetType("a", dimensionsA, "test_a",))
121 self.assertEqual(DatasetType("a", dimensionsA, "test_a",),
122 DatasetType("a", dimensionsA, "test_a",))
123 self.assertEqual(DatasetType("a.b", dimensionsA, "test_b", parentStorageClass=parent),
124 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass=parent))
125 self.assertEqual(DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="parent"),
126 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="parent"))
127 self.assertNotEqual(DatasetType("a", dimensionsA, storageA,),
128 DatasetType("b", dimensionsA, storageA,))
129 self.assertNotEqual(DatasetType("a", dimensionsA, storageA,),
130 DatasetType("b", dimensionsA, "test_a",))
131 self.assertNotEqual(DatasetType("a", dimensionsA, storageA,),
132 DatasetType("a", dimensionsA, storageB,))
133 self.assertNotEqual(DatasetType("a", dimensionsA, storageA,),
134 DatasetType("a", dimensionsA, "test_b",))
135 self.assertNotEqual(DatasetType("a", dimensionsA, storageA,),
136 DatasetType("a", dimensionsB, storageA,))
137 self.assertNotEqual(DatasetType("a", dimensionsA, storageA,),
138 DatasetType("a", dimensionsB, "test_a",))
139 self.assertNotEqual(DatasetType("a.b", dimensionsA, "test_b", parentStorageClass=storageA),
140 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass=storageB))
141 self.assertNotEqual(DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="storageA"),
142 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="storageB"))
144 def testParentPlaceholder(self):
145 """Test that a parent placeholder can be replaced."""
146 storageComp = StorageClass("component")
147 storageParent = StorageClass("Parent")
148 dimensions = self.universe.extract(["instrument"])
149 component = DatasetType("a.b", dimensions, storageComp,
150 parentStorageClass=DatasetType.PlaceholderParentStorageClass)
151 self.assertIsNotNone(component.parentStorageClass)
153 with self.assertRaises(ValueError):
154 component.finalizeParentStorageClass("parent")
156 component.finalizeParentStorageClass(storageParent)
157 self.assertEqual(component.parentStorageClass, storageParent)
159 component = DatasetType("a.b", dimensions, storageComp,
160 parentStorageClass=storageParent)
162 with self.assertRaises(ValueError):
163 # Can not replace unless a placeholder
164 component.finalizeParentStorageClass(storageComp)
166 datasetType = DatasetType("a", dimensions, storageParent)
167 with self.assertRaises(ValueError):
168 # Can not add parent if not component
169 datasetType.finalizeParentStorageClass(storageComp)
171 def testHashability(self):
172 """Test `DatasetType.__hash__`.
174 This test is performed by checking that `DatasetType` entries can
175 be inserted into a `set` and that unique values of its
176 (`name`, `storageClass`, `dimensions`) parameters result in separate
177 entries (and equal ones don't).
179 This does not check for uniformity of hashing or the actual values
180 of the hash function.
181 """
182 types = []
183 unique = 0
184 storageC = StorageClass("test_c")
185 storageD = StorageClass("test_d")
186 for name in ["a", "b"]:
187 for storageClass in [storageC, storageD]:
188 for dimensions in [("instrument", ), ("skymap", )]:
189 datasetType = DatasetType(name, self.universe.extract(dimensions), storageClass)
190 datasetTypeCopy = DatasetType(name, self.universe.extract(dimensions), storageClass)
191 types.extend((datasetType, datasetTypeCopy))
192 unique += 1 # datasetType should always equal its copy
193 self.assertEqual(len(set(types)), unique) # all other combinations are unique
195 # also check that hashes of instances constructed with StorageClass
196 # name matches hashes of instances constructed with instances
197 dimensions = self.universe.extract(["instrument"])
198 self.assertEqual(hash(DatasetType("a", dimensions, storageC)),
199 hash(DatasetType("a", dimensions, "test_c")))
200 self.assertEqual(hash(DatasetType("a", dimensions, "test_c")),
201 hash(DatasetType("a", dimensions, "test_c")))
202 self.assertNotEqual(hash(DatasetType("a", dimensions, storageC)),
203 hash(DatasetType("a", dimensions, "test_d")))
204 self.assertNotEqual(hash(DatasetType("a", dimensions, storageD)),
205 hash(DatasetType("a", dimensions, "test_c")))
206 self.assertNotEqual(hash(DatasetType("a", dimensions, "test_c")),
207 hash(DatasetType("a", dimensions, "test_d")))
209 def testDeepCopy(self):
210 """Test that we can copy a dataset type."""
211 storageClass = StorageClass("test_copy")
212 datasetTypeName = "test"
213 dimensions = self.universe.extract(("instrument", "visit"))
214 datasetType = DatasetType(datasetTypeName, dimensions, storageClass)
215 dcopy = copy.deepcopy(datasetType)
216 self.assertEqual(dcopy, datasetType)
218 # And again with a composite
219 componentStorageClass = StorageClass("copy_component")
220 componentDatasetType = DatasetType(DatasetType.nameWithComponent(datasetTypeName, "comp"),
221 dimensions, componentStorageClass,
222 parentStorageClass=storageClass)
223 dcopy = copy.deepcopy(componentDatasetType)
224 self.assertEqual(dcopy, componentDatasetType)
226 def testPickle(self):
227 """Test pickle support.
228 """
229 storageClass = StorageClass("test_pickle")
230 datasetTypeName = "test"
231 dimensions = self.universe.extract(("instrument", "visit"))
232 # Un-pickling requires that storage class is registered with factory.
233 StorageClassFactory().registerStorageClass(storageClass)
234 datasetType = DatasetType(datasetTypeName, dimensions, storageClass)
235 datasetTypeOut = pickle.loads(pickle.dumps(datasetType))
236 self.assertIsInstance(datasetTypeOut, DatasetType)
237 self.assertEqual(datasetType.name, datasetTypeOut.name)
238 self.assertEqual(datasetType.dimensions.names, datasetTypeOut.dimensions.names)
239 self.assertEqual(datasetType.storageClass, datasetTypeOut.storageClass)
240 self.assertIsNone(datasetTypeOut.parentStorageClass)
242 # And again with a composite
243 componentStorageClass = StorageClass("pickle_component")
244 StorageClassFactory().registerStorageClass(componentStorageClass)
245 componentDatasetType = DatasetType(DatasetType.nameWithComponent(datasetTypeName, "comp"),
246 dimensions, componentStorageClass,
247 parentStorageClass=storageClass)
248 datasetTypeOut = pickle.loads(pickle.dumps(componentDatasetType))
249 self.assertIsInstance(datasetTypeOut, DatasetType)
250 self.assertEqual(componentDatasetType.name, datasetTypeOut.name)
251 self.assertEqual(componentDatasetType.dimensions.names, datasetTypeOut.dimensions.names)
252 self.assertEqual(componentDatasetType.storageClass, datasetTypeOut.storageClass)
253 self.assertEqual(componentDatasetType.parentStorageClass, datasetTypeOut.parentStorageClass)
254 self.assertEqual(datasetTypeOut.parentStorageClass.name,
255 storageClass.name)
256 self.assertEqual(datasetTypeOut, componentDatasetType)
258 # Now with a string and not a real storage class to test that
259 # pickling doesn't force the StorageClass to be resolved
260 componentDatasetType = DatasetType(DatasetType.nameWithComponent(datasetTypeName, "comp"),
261 dimensions, "StrangeComponent",
262 parentStorageClass="UnknownParent")
263 datasetTypeOut = pickle.loads(pickle.dumps(componentDatasetType))
264 self.assertEqual(datasetTypeOut, componentDatasetType)
265 self.assertEqual(datasetTypeOut._parentStorageClassName,
266 componentDatasetType._parentStorageClassName)
268 # Now with a storage class that is created by the factory
269 factoryStorageClassClass = StorageClassFactory.makeNewStorageClass("ParentClass")
270 factoryComponentStorageClassClass = StorageClassFactory.makeNewStorageClass("ComponentClass")
271 componentDatasetType = DatasetType(DatasetType.nameWithComponent(datasetTypeName, "comp"),
272 dimensions, factoryComponentStorageClassClass(),
273 parentStorageClass=factoryStorageClassClass())
274 datasetTypeOut = pickle.loads(pickle.dumps(componentDatasetType))
275 self.assertEqual(datasetTypeOut, componentDatasetType)
276 self.assertEqual(datasetTypeOut._parentStorageClassName,
277 componentDatasetType._parentStorageClassName)
279 def test_composites(self):
280 """Test components within composite DatasetTypes."""
281 storageClassA = StorageClass("compA")
282 storageClassB = StorageClass("compB")
283 storageClass = StorageClass("test_composite", components={"compA": storageClassA,
284 "compB": storageClassB})
285 self.assertTrue(storageClass.isComposite())
286 self.assertFalse(storageClassA.isComposite())
287 self.assertFalse(storageClassB.isComposite())
289 dimensions = self.universe.extract(("instrument", "visit"))
291 datasetTypeComposite = DatasetType("composite", dimensions, storageClass)
292 datasetTypeComponentA = datasetTypeComposite.makeComponentDatasetType("compA")
293 datasetTypeComponentB = datasetTypeComposite.makeComponentDatasetType("compB")
295 self.assertTrue(datasetTypeComposite.isComposite())
296 self.assertFalse(datasetTypeComponentA.isComposite())
297 self.assertTrue(datasetTypeComponentB.isComponent())
298 self.assertFalse(datasetTypeComposite.isComponent())
300 self.assertEqual(datasetTypeComposite.name, "composite")
301 self.assertEqual(datasetTypeComponentA.name, "composite.compA")
302 self.assertEqual(datasetTypeComponentB.component(), "compB")
303 self.assertEqual(datasetTypeComposite.nameAndComponent(), ("composite", None))
304 self.assertEqual(datasetTypeComponentA.nameAndComponent(), ("composite", "compA"))
306 self.assertEqual(datasetTypeComponentA.parentStorageClass, storageClass)
307 self.assertEqual(datasetTypeComponentB.parentStorageClass, storageClass)
308 self.assertIsNone(datasetTypeComposite.parentStorageClass)
311class DatasetRefTestCase(unittest.TestCase):
312 """Test for DatasetRef.
313 """
315 def setUp(self):
316 self.universe = DimensionUniverse()
317 datasetTypeName = "test"
318 self.componentStorageClass1 = StorageClass("Component1")
319 self.componentStorageClass2 = StorageClass("Component2")
320 self.parentStorageClass = StorageClass("Parent", components={"a": self.componentStorageClass1,
321 "b": self.componentStorageClass2})
322 dimensions = self.universe.extract(("instrument", "visit"))
323 self.dataId = dict(instrument="DummyCam", visit=42)
324 self.datasetType = DatasetType(datasetTypeName, dimensions, self.parentStorageClass)
326 def testConstructor(self):
327 """Test that construction preserves and validates values.
328 """
329 # Construct an unresolved ref.
330 ref = DatasetRef(self.datasetType, self.dataId)
331 self.assertEqual(ref.datasetType, self.datasetType)
332 self.assertEqual(ref.dataId, DataCoordinate.standardize(self.dataId, universe=self.universe),
333 msg=ref.dataId)
334 self.assertIsInstance(ref.dataId, DataCoordinate)
335 # Constructing an unresolved ref with run and/or components should
336 # fail.
337 run = "somerun"
338 with self.assertRaises(ValueError):
339 DatasetRef(self.datasetType, self.dataId, run=run)
340 # Passing a data ID that is missing dimensions should fail.
341 with self.assertRaises(KeyError):
342 DatasetRef(self.datasetType, {"instrument": "DummyCam"})
343 # Constructing a resolved ref should preserve run as well as everything
344 # else.
345 ref = DatasetRef(self.datasetType, self.dataId, id=1, run=run)
346 self.assertEqual(ref.datasetType, self.datasetType)
347 self.assertEqual(ref.dataId, DataCoordinate.standardize(self.dataId, universe=self.universe),
348 msg=ref.dataId)
349 self.assertIsInstance(ref.dataId, DataCoordinate)
350 self.assertEqual(ref.id, 1)
351 self.assertEqual(ref.run, run)
353 def testResolving(self):
354 ref = DatasetRef(self.datasetType, self.dataId, id=1, run="somerun")
355 unresolvedRef = ref.unresolved()
356 self.assertIsNotNone(ref.id)
357 self.assertIsNone(unresolvedRef.id)
358 self.assertIsNone(unresolvedRef.run)
359 self.assertNotEqual(ref, unresolvedRef)
360 self.assertEqual(ref.unresolved(), unresolvedRef)
361 self.assertEqual(ref.datasetType, unresolvedRef.datasetType)
362 self.assertEqual(ref.dataId, unresolvedRef.dataId)
363 reresolvedRef = unresolvedRef.resolved(id=1, run="somerun")
364 self.assertEqual(ref, reresolvedRef)
365 self.assertEqual(reresolvedRef.unresolved(), unresolvedRef)
366 self.assertIsNotNone(reresolvedRef.run)
368 def testPickle(self):
369 ref = DatasetRef(self.datasetType, self.dataId, id=1, run="somerun")
370 s = pickle.dumps(ref)
371 self.assertEqual(pickle.loads(s), ref)
374if __name__ == "__main__": 374 ↛ 375line 374 didn't jump to line 375, because the condition on line 374 was never true
375 unittest.main()