Coverage for tests/test_datasets.py: 100%
568 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-07-24 02:19 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28import copy
29import os
30import pickle
31import unittest
32import uuid
34from lsst.daf.butler import (
35 DataCoordinate,
36 DatasetProvenance,
37 DatasetRef,
38 DatasetType,
39 DimensionConfig,
40 DimensionUniverse,
41 FileDataset,
42 InconsistentUniverseError,
43 SerializedDatasetRefContainerV1,
44 StorageClass,
45 StorageClassFactory,
46 UnknownComponentError,
47)
48from lsst.daf.butler.datastore.stored_file_info import StoredFileInfo
49from lsst.daf.butler.datastores.file_datastore.retrieve_artifacts import ZipIndex
50from lsst.daf.butler.formatters.yaml import YamlFormatter
51from lsst.resources import ResourcePath
53TESTDIR = os.path.abspath(os.path.dirname(__file__))
55"""Tests for datasets module.
56"""
59class DatasetTypeTestCase(unittest.TestCase):
60 """Test for DatasetType."""
62 def setUp(self) -> None:
63 self.universe = DimensionUniverse()
65 def testConstructor(self) -> None:
66 """Test construction preserves values.
68 Note that construction doesn't check for valid storageClass.
69 This can only be verified for a particular schema.
70 """
71 datasetTypeName = "test"
72 storageClass = StorageClass("test_StructuredData")
73 dimensions = self.universe.conform(("visit", "instrument"))
74 datasetType = DatasetType(datasetTypeName, dimensions, storageClass)
75 self.assertEqual(datasetType.name, datasetTypeName)
76 self.assertEqual(datasetType.storageClass, storageClass)
77 self.assertEqual(datasetType.dimensions, dimensions)
79 with self.assertRaises(ValueError, msg="Construct component without parent storage class"):
80 DatasetType(DatasetType.nameWithComponent(datasetTypeName, "comp"), dimensions, storageClass)
81 with self.assertRaises(ValueError, msg="Construct non-component with parent storage class"):
82 DatasetType(datasetTypeName, dimensions, storageClass, parentStorageClass="NotAllowed")
84 def testConstructor2(self) -> None:
85 """Test construction from StorageClass name."""
86 datasetTypeName = "test"
87 storageClass = StorageClass("test_constructor2")
88 StorageClassFactory().registerStorageClass(storageClass)
89 dimensions = self.universe.conform(("instrument", "visit"))
90 datasetType = DatasetType(datasetTypeName, dimensions, "test_constructor2")
91 self.assertEqual(datasetType.name, datasetTypeName)
92 self.assertEqual(datasetType.storageClass, storageClass)
93 self.assertEqual(datasetType.dimensions, dimensions)
95 def testNameValidation(self) -> None:
96 """Test that dataset type names only contain certain characters
97 in certain positions.
98 """
99 dimensions = self.universe.conform(("instrument", "visit"))
100 goodNames = ("a", "A", "z1", "Z1", "a_1B", "A_1b", "_a")
101 badNames = ("1", "a%b", "B+Z", "T[0]")
103 # Construct storage class with all the good names included as
104 # components so that we can test internal consistency
105 storageClass = StorageClass(
106 "test_StructuredData", components={n: StorageClass("component") for n in goodNames}
107 )
109 for name in goodNames:
110 composite = DatasetType(name, dimensions, storageClass)
111 self.assertEqual(composite.name, name)
112 for suffix in goodNames:
113 full = DatasetType.nameWithComponent(name, suffix)
114 component = composite.makeComponentDatasetType(suffix)
115 self.assertEqual(component.name, full)
116 assert component.parentStorageClass is not None
117 self.assertEqual(component.parentStorageClass.name, "test_StructuredData")
118 for suffix in badNames:
119 full = DatasetType.nameWithComponent(name, suffix)
120 with self.subTest(full=full):
121 with self.assertRaises(ValueError):
122 DatasetType(full, dimensions, storageClass)
123 for name in badNames:
124 with self.subTest(name=name):
125 with self.assertRaises(ValueError):
126 DatasetType(name, dimensions, storageClass)
128 def testEquality(self) -> None:
129 storageA = StorageClass("test_a")
130 storageB = StorageClass("test_b")
131 parent = StorageClass("test")
132 dimensionsA = self.universe.conform(["instrument"])
133 dimensionsB = self.universe.conform(["skymap"])
134 self.assertEqual(
135 DatasetType(
136 "a",
137 dimensionsA,
138 storageA,
139 ),
140 DatasetType(
141 "a",
142 dimensionsA,
143 storageA,
144 ),
145 )
146 self.assertEqual(
147 DatasetType(
148 "a",
149 dimensionsA,
150 "test_a",
151 ),
152 DatasetType(
153 "a",
154 dimensionsA,
155 storageA,
156 ),
157 )
158 self.assertEqual(
159 DatasetType(
160 "a",
161 dimensionsA,
162 storageA,
163 ),
164 DatasetType(
165 "a",
166 dimensionsA,
167 "test_a",
168 ),
169 )
170 self.assertEqual(
171 DatasetType(
172 "a",
173 dimensionsA,
174 "test_a",
175 ),
176 DatasetType(
177 "a",
178 dimensionsA,
179 "test_a",
180 ),
181 )
182 self.assertEqual(
183 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass=parent),
184 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass=parent),
185 )
186 self.assertEqual(
187 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="parent"),
188 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="parent"),
189 )
190 self.assertNotEqual(
191 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="parent", isCalibration=True),
192 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="parent", isCalibration=False),
193 )
194 self.assertNotEqual(
195 DatasetType(
196 "a",
197 dimensionsA,
198 storageA,
199 ),
200 DatasetType(
201 "b",
202 dimensionsA,
203 storageA,
204 ),
205 )
206 self.assertNotEqual(
207 DatasetType(
208 "a",
209 dimensionsA,
210 storageA,
211 ),
212 DatasetType(
213 "b",
214 dimensionsA,
215 "test_a",
216 ),
217 )
218 self.assertNotEqual(
219 DatasetType(
220 "a",
221 dimensionsA,
222 storageA,
223 ),
224 DatasetType(
225 "a",
226 dimensionsA,
227 storageB,
228 ),
229 )
230 self.assertNotEqual(
231 DatasetType(
232 "a",
233 dimensionsA,
234 storageA,
235 ),
236 DatasetType(
237 "a",
238 dimensionsA,
239 "test_b",
240 ),
241 )
242 self.assertNotEqual(
243 DatasetType(
244 "a",
245 dimensionsA,
246 storageA,
247 ),
248 DatasetType(
249 "a",
250 dimensionsB,
251 storageA,
252 ),
253 )
254 self.assertNotEqual(
255 DatasetType(
256 "a",
257 dimensionsA,
258 storageA,
259 ),
260 DatasetType(
261 "a",
262 dimensionsB,
263 "test_a",
264 ),
265 )
266 self.assertNotEqual(
267 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass=storageA),
268 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass=storageB),
269 )
270 self.assertNotEqual(
271 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="storageA"),
272 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="storageB"),
273 )
275 def testCompatibility(self) -> None:
276 storageA = StorageClass("test_a", pytype=set, converters={"list": "builtins.set"})
277 storageB = StorageClass("test_b", pytype=list)
278 storageC = StorageClass("test_c", pytype=dict)
279 self.assertTrue(storageA.can_convert(storageB))
280 dimensionsA = self.universe.conform(["instrument"])
282 dA = DatasetType("a", dimensionsA, storageA)
283 dA2 = DatasetType("a", dimensionsA, storageB)
284 self.assertNotEqual(dA, dA2)
285 self.assertTrue(dA.is_compatible_with(dA))
286 self.assertTrue(dA.is_compatible_with(dA2))
287 self.assertFalse(dA2.is_compatible_with(dA))
289 dA3 = DatasetType("a", dimensionsA, storageC)
290 self.assertFalse(dA.is_compatible_with(dA3))
292 def testOverrideStorageClass(self) -> None:
293 storageA = StorageClass("test_a", pytype=list, converters={"dict": "builtins.list"})
294 storageB = StorageClass("test_b", pytype=dict, converters={"list": "dict"})
295 dimensions = self.universe.conform(["instrument"])
297 dA = DatasetType("a", dimensions, storageA)
298 dB = dA.overrideStorageClass(storageB)
299 self.assertNotEqual(dA, dB)
300 self.assertEqual(dB.storageClass, storageB)
302 round_trip = dB.overrideStorageClass(storageA)
303 self.assertEqual(round_trip, dA)
305 # Check that parents move over. Assign a pytype to avoid using
306 # object in later tests.
307 parent = StorageClass("composite", pytype=tuple, components={"a": storageA, "c": storageA})
308 dP = DatasetType("comp", dimensions, parent)
309 dP_A = dP.makeComponentDatasetType("a")
310 dp_B = dP_A.overrideStorageClass(storageB)
311 self.assertEqual(dp_B.storageClass, storageB)
312 self.assertEqual(dp_B.parentStorageClass, parent)
314 # Check that components are checked for compatibility but parents
315 # can be different.
316 parent2 = StorageClass(
317 "composite2",
318 pytype=frozenset,
319 components={"a": storageB, "c": storageB},
320 )
321 dP2 = DatasetType("comp", dimensions, parent2)
322 # Components are compatible even though parents aren't.
323 self.assertFalse(dP.is_compatible_with(dP2))
324 self.assertTrue(dP2.makeComponentDatasetType("a").is_compatible_with(dP_A))
326 def testJson(self) -> None:
327 storageA = StorageClass("test_a")
328 dimensionsA = self.universe.conform(["instrument"])
329 self.assertEqual(
330 DatasetType(
331 "a",
332 dimensionsA,
333 storageA,
334 ),
335 DatasetType.from_json(
336 DatasetType(
337 "a",
338 dimensionsA,
339 storageA,
340 ).to_json(),
341 self.universe,
342 ),
343 )
344 self.assertEqual(
345 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="parent"),
346 DatasetType.from_json(
347 DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="parent").to_json(),
348 self.universe,
349 ),
350 )
352 def testSorting(self) -> None:
353 """Can we sort a DatasetType"""
354 storage = StorageClass("test_a")
355 dimensions = self.universe.conform(["instrument"])
357 d_a = DatasetType("a", dimensions, storage)
358 d_f = DatasetType("f", dimensions, storage)
359 d_p = DatasetType("p", dimensions, storage)
361 sort = sorted([d_p, d_f, d_a])
362 self.assertEqual(sort, [d_a, d_f, d_p])
364 # Now with strings
365 with self.assertRaises(TypeError):
366 sort = sorted(["z", d_p, "c", d_f, d_a, "d"]) # type: ignore [list-item]
368 def testHashability(self) -> None:
369 """Test `DatasetType.__hash__`.
371 This test is performed by checking that `DatasetType` entries can
372 be inserted into a `set` and that unique values of its
373 (`name`, `storageClass`, `dimensions`) parameters result in separate
374 entries (and equal ones don't).
376 This does not check for uniformity of hashing or the actual values
377 of the hash function.
378 """
379 types: list[DatasetType] = []
380 unique = 0
381 storageC = StorageClass("test_c")
382 storageD = StorageClass("test_d")
383 for name in ["a", "b"]:
384 for storageClass in [storageC, storageD]:
385 for dims in [("instrument",), ("skymap",)]:
386 datasetType = DatasetType(name, self.universe.conform(dims), storageClass)
387 datasetTypeCopy = DatasetType(name, self.universe.conform(dims), storageClass)
388 types.extend((datasetType, datasetTypeCopy))
389 unique += 1 # datasetType should always equal its copy
390 self.assertEqual(len(set(types)), unique) # all other combinations are unique
392 # also check that hashes of instances constructed with StorageClass
393 # name matches hashes of instances constructed with instances
394 dimensions = self.universe.conform(["instrument"])
395 self.assertEqual(
396 hash(DatasetType("a", dimensions, storageC)), hash(DatasetType("a", dimensions, "test_c"))
397 )
398 self.assertEqual(
399 hash(DatasetType("a", dimensions, "test_c")), hash(DatasetType("a", dimensions, "test_c"))
400 )
401 self.assertNotEqual(
402 hash(DatasetType("a", dimensions, storageC)), hash(DatasetType("a", dimensions, "test_d"))
403 )
404 self.assertNotEqual(
405 hash(DatasetType("a", dimensions, storageD)), hash(DatasetType("a", dimensions, "test_c"))
406 )
407 self.assertNotEqual(
408 hash(DatasetType("a", dimensions, "test_c")), hash(DatasetType("a", dimensions, "test_d"))
409 )
411 def testDeepCopy(self) -> None:
412 """Test that we can copy a dataset type."""
413 storageClass = StorageClass("test_copy")
414 datasetTypeName = "test"
415 dimensions = self.universe.conform(("instrument", "visit"))
416 datasetType = DatasetType(datasetTypeName, dimensions, storageClass)
417 dcopy = copy.deepcopy(datasetType)
418 self.assertEqual(dcopy, datasetType)
420 # Now with calibration flag set
421 datasetType = DatasetType(datasetTypeName, dimensions, storageClass, isCalibration=True)
422 dcopy = copy.deepcopy(datasetType)
423 self.assertEqual(dcopy, datasetType)
424 self.assertTrue(dcopy.isCalibration())
426 # And again with a composite
427 componentStorageClass = StorageClass("copy_component")
428 componentDatasetType = DatasetType(
429 DatasetType.nameWithComponent(datasetTypeName, "comp"),
430 dimensions,
431 componentStorageClass,
432 parentStorageClass=storageClass,
433 )
434 dcopy = copy.deepcopy(componentDatasetType)
435 self.assertEqual(dcopy, componentDatasetType)
437 def testPickle(self) -> None:
438 """Test pickle support."""
439 storageClass = StorageClass("test_pickle")
440 datasetTypeName = "test"
441 dimensions = self.universe.conform(("instrument", "visit"))
442 # Un-pickling requires that storage class is registered with factory.
443 StorageClassFactory().registerStorageClass(storageClass)
444 datasetType = DatasetType(datasetTypeName, dimensions, storageClass)
445 datasetTypeOut = pickle.loads(pickle.dumps(datasetType))
446 self.assertIsInstance(datasetTypeOut, DatasetType)
447 self.assertEqual(datasetType.name, datasetTypeOut.name)
448 self.assertEqual(datasetType.dimensions, datasetTypeOut.dimensions)
449 self.assertEqual(datasetType.storageClass, datasetTypeOut.storageClass)
450 self.assertIsNone(datasetTypeOut.parentStorageClass)
451 self.assertIs(datasetType.isCalibration(), datasetTypeOut.isCalibration())
452 self.assertFalse(datasetTypeOut.isCalibration())
454 datasetType = DatasetType(datasetTypeName, dimensions, storageClass, isCalibration=True)
455 datasetTypeOut = pickle.loads(pickle.dumps(datasetType))
456 self.assertIs(datasetType.isCalibration(), datasetTypeOut.isCalibration())
457 self.assertTrue(datasetTypeOut.isCalibration())
459 # And again with a composite
460 componentStorageClass = StorageClass("pickle_component")
461 StorageClassFactory().registerStorageClass(componentStorageClass)
462 componentDatasetType = DatasetType(
463 DatasetType.nameWithComponent(datasetTypeName, "comp"),
464 dimensions,
465 componentStorageClass,
466 parentStorageClass=storageClass,
467 )
468 datasetTypeOut = pickle.loads(pickle.dumps(componentDatasetType))
469 self.assertIsInstance(datasetTypeOut, DatasetType)
470 self.assertEqual(componentDatasetType.name, datasetTypeOut.name)
471 self.assertEqual(componentDatasetType.dimensions.names, datasetTypeOut.dimensions.names)
472 self.assertEqual(componentDatasetType.storageClass, datasetTypeOut.storageClass)
473 self.assertEqual(componentDatasetType.parentStorageClass, datasetTypeOut.parentStorageClass)
474 self.assertEqual(datasetTypeOut.parentStorageClass.name, storageClass.name)
475 self.assertEqual(datasetTypeOut, componentDatasetType)
477 # Now with a string and not a real storage class to test that
478 # pickling doesn't force the StorageClass to be resolved
479 componentDatasetType = DatasetType(
480 DatasetType.nameWithComponent(datasetTypeName, "comp"),
481 dimensions,
482 "StrangeComponent",
483 parentStorageClass="UnknownParent",
484 )
485 datasetTypeOut = pickle.loads(pickle.dumps(componentDatasetType))
486 self.assertEqual(datasetTypeOut, componentDatasetType)
487 self.assertEqual(datasetTypeOut._parentStorageClassName, componentDatasetType._parentStorageClassName)
489 def test_composites(self) -> None:
490 """Test components within composite DatasetTypes."""
491 storageClassA = StorageClass("compA")
492 storageClassB = StorageClass("compB")
493 storageClass = StorageClass(
494 "test_composite", components={"compA": storageClassA, "compB": storageClassB}
495 )
496 self.assertTrue(storageClass.isComposite())
497 self.assertFalse(storageClassA.isComposite())
498 self.assertFalse(storageClassB.isComposite())
500 dimensions = self.universe.conform(("instrument", "visit"))
502 datasetTypeComposite = DatasetType("composite", dimensions, storageClass)
503 datasetTypeComponentA = datasetTypeComposite.makeComponentDatasetType("compA")
504 datasetTypeComponentB = datasetTypeComposite.makeComponentDatasetType("compB")
506 self.assertTrue(datasetTypeComposite.isComposite())
507 self.assertFalse(datasetTypeComponentA.isComposite())
508 self.assertTrue(datasetTypeComponentB.isComponent())
509 self.assertFalse(datasetTypeComposite.isComponent())
511 self.assertEqual(datasetTypeComposite.name, "composite")
512 self.assertEqual(datasetTypeComponentA.name, "composite.compA")
513 self.assertEqual(datasetTypeComponentB.component(), "compB")
514 self.assertEqual(datasetTypeComposite.nameAndComponent(), ("composite", None))
515 self.assertEqual(datasetTypeComponentA.nameAndComponent(), ("composite", "compA"))
517 self.assertEqual(datasetTypeComponentA.parentStorageClass, storageClass)
518 self.assertEqual(datasetTypeComponentB.parentStorageClass, storageClass)
519 self.assertIsNone(datasetTypeComposite.parentStorageClass)
521 with self.assertRaises(UnknownComponentError):
522 datasetTypeComposite.makeComponentDatasetType("compF")
524 with self.assertRaises(UnknownComponentError):
525 datasetTypeComposite.componentTypeName("unknown")
528class DatasetRefTestCase(unittest.TestCase):
529 """Test for DatasetRef."""
531 def setUp(self) -> None:
532 self.universe = DimensionUniverse()
533 datasetTypeName = "test"
534 self.componentStorageClass1 = StorageClass("Component1")
535 self.componentStorageClass2 = StorageClass("Component2")
536 self.parentStorageClass = StorageClass(
537 "Parent", components={"a": self.componentStorageClass1, "b": self.componentStorageClass2}
538 )
539 sc_factory = StorageClassFactory()
540 sc_factory.registerStorageClass(self.componentStorageClass1)
541 sc_factory.registerStorageClass(self.componentStorageClass2)
542 sc_factory.registerStorageClass(self.parentStorageClass)
543 dimensions = self.universe.conform(("instrument", "visit"))
544 self.dataId = DataCoordinate.standardize(
545 dict(instrument="DummyCam", visit=42), universe=self.universe
546 )
547 self.datasetType = DatasetType(datasetTypeName, dimensions, self.parentStorageClass)
549 def _make_datastore_records(self, ref: DatasetRef, *paths: str) -> DatasetRef:
550 """Return an updated dataset ref with datastore records."""
551 opaque_table_name = "datastore_records"
552 datastore_records = {
553 opaque_table_name: [
554 StoredFileInfo(
555 formatter="",
556 path=path,
557 storageClass=ref.datasetType.storageClass,
558 component=None,
559 checksum=None,
560 file_size=1,
561 )
562 for path in paths
563 ]
564 }
565 return ref.replace(datastore_records=datastore_records)
567 def testConstructor(self) -> None:
568 """Test that construction preserves and validates values."""
569 # Constructing a ref requires a run.
570 with self.assertRaises(TypeError):
571 DatasetRef(self.datasetType, self.dataId, id=uuid.uuid4()) # type: ignore [call-arg]
573 # Constructing an unresolved ref with run and/or components should
574 # issue a ref with an id.
575 run = "somerun"
576 ref = DatasetRef(self.datasetType, self.dataId, run=run)
577 self.assertEqual(ref.datasetType, self.datasetType)
578 self.assertEqual(
579 ref.dataId, DataCoordinate.standardize(self.dataId, universe=self.universe), msg=ref.dataId
580 )
581 self.assertIsNotNone(ref.id)
583 # Passing a data ID that is missing dimensions should fail.
584 # Create a full DataCoordinate to ensure that we are testing the
585 # right thing.
586 dimensions = self.universe.conform(("instrument",))
587 dataId = DataCoordinate.standardize(instrument="DummyCam", dimensions=dimensions)
588 with self.assertRaises(KeyError):
589 DatasetRef(self.datasetType, dataId, run="run")
590 # Constructing a resolved ref should preserve run as well as everything
591 # else.
592 id_ = uuid.uuid4()
593 ref = DatasetRef(self.datasetType, self.dataId, id=id_, run=run)
594 self.assertEqual(ref.datasetType, self.datasetType)
595 self.assertEqual(
596 ref.dataId, DataCoordinate.standardize(self.dataId, universe=self.universe), msg=ref.dataId
597 )
598 self.assertIsInstance(ref.dataId, DataCoordinate)
599 self.assertEqual(ref.id, id_)
600 self.assertEqual(ref.run, run)
602 with self.assertRaises(ValueError):
603 DatasetRef(self.datasetType, self.dataId, run=run, id_generation_mode=42) # type: ignore
605 def testSorting(self) -> None:
606 """Can we sort a DatasetRef"""
607 # All refs have the same run.
608 dimensions = self.universe.conform(("instrument", "visit"))
609 ref1 = DatasetRef(
610 self.datasetType,
611 DataCoordinate.standardize(instrument="DummyCam", visit=1, dimensions=dimensions),
612 run="run",
613 )
614 ref2 = DatasetRef(
615 self.datasetType,
616 DataCoordinate.standardize(instrument="DummyCam", visit=10, dimensions=dimensions),
617 run="run",
618 )
619 ref3 = DatasetRef(
620 self.datasetType,
621 DataCoordinate.standardize(instrument="DummyCam", visit=22, dimensions=dimensions),
622 run="run",
623 )
625 # Enable detailed diff report
626 self.maxDiff = None
628 # This will sort them on visit number
629 sort = sorted([ref3, ref1, ref2])
630 self.assertEqual(sort, [ref1, ref2, ref3], msg=f"Got order: {[r.dataId for r in sort]}")
632 # Now include different runs.
633 ref1 = DatasetRef(
634 self.datasetType,
635 DataCoordinate.standardize(instrument="DummyCam", visit=43, dimensions=dimensions),
636 run="b",
637 )
638 self.assertEqual(ref1.run, "b")
639 ref4 = DatasetRef(
640 self.datasetType,
641 DataCoordinate.standardize(instrument="DummyCam", visit=10, dimensions=dimensions),
642 run="b",
643 )
644 ref2 = DatasetRef(
645 self.datasetType,
646 DataCoordinate.standardize(instrument="DummyCam", visit=4, dimensions=dimensions),
647 run="a",
648 )
649 ref3 = DatasetRef(
650 self.datasetType,
651 DataCoordinate.standardize(instrument="DummyCam", visit=104, dimensions=dimensions),
652 run="c",
653 )
655 # This will sort them on run before visit
656 sort = sorted([ref3, ref1, ref2, ref4])
657 self.assertEqual(sort, [ref2, ref4, ref1, ref3], msg=f"Got order: {[r.dataId for r in sort]}")
659 # Now with strings
660 with self.assertRaises(TypeError):
661 sort = sorted(["z", ref1, "c"]) # type: ignore [list-item]
663 def testOverrideStorageClass(self) -> None:
664 storageA = StorageClass("test_a", pytype=list)
666 ref = DatasetRef(self.datasetType, self.dataId, run="somerun")
668 ref_new = ref.overrideStorageClass(storageA)
669 self.assertNotEqual(ref, ref_new)
670 self.assertEqual(ref_new.datasetType.storageClass, storageA)
671 self.assertEqual(ref_new.overrideStorageClass(ref.datasetType.storageClass), ref)
672 self.assertTrue(ref.is_compatible_with(ref_new))
673 with self.assertRaises(AttributeError):
674 ref_new.is_compatible_with(None) # type: ignore
676 # Check different code paths of incompatibility.
677 ref_incompat = DatasetRef(ref.datasetType, ref.dataId, run="somerun2", id=ref.id)
678 self.assertFalse(ref.is_compatible_with(ref_incompat)) # bad run
679 ref_incompat = DatasetRef(ref.datasetType, ref.dataId, run="somerun")
680 self.assertFalse(ref.is_compatible_with(ref_incompat)) # bad ID
682 incompatible_sc = StorageClass("my_int", pytype=int)
683 with self.assertRaises(ValueError):
684 # Do not test against "ref" because it has a default storage class
685 # of "object" which is compatible with everything.
686 ref_new.overrideStorageClass(incompatible_sc)
688 def testReplace(self) -> None:
689 """Test for `DatasetRef.replace` method."""
690 ref = DatasetRef(self.datasetType, self.dataId, run="somerun")
692 ref2 = ref.replace(run="somerun2")
693 self.assertEqual(ref2.run, "somerun2")
694 self.assertIsNotNone(ref2.id)
695 self.assertNotEqual(ref2.id, ref.id)
697 ref3 = ref.replace(run="somerun3", id=ref2.id)
698 self.assertEqual(ref3.run, "somerun3")
699 self.assertEqual(ref3.id, ref2.id)
701 ref4 = ref.replace(id=ref2.id)
702 self.assertEqual(ref4.run, "somerun")
703 self.assertEqual(ref4.id, ref2.id)
705 ref5 = ref.replace()
706 self.assertEqual(ref5.run, "somerun")
707 self.assertEqual(ref5, ref)
709 self.assertIsNone(ref5._datastore_records)
710 ref5 = ref5.replace(datastore_records={})
711 self.assertEqual(ref5._datastore_records, {})
712 ref5 = ref5.replace(datastore_records=None)
713 self.assertIsNone(ref5._datastore_records)
715 def testPickle(self) -> None:
716 ref = DatasetRef(self.datasetType, self.dataId, run="somerun")
717 s = pickle.dumps(ref)
718 self.assertEqual(pickle.loads(s), ref)
720 def testJson(self) -> None:
721 ref = DatasetRef(self.datasetType, self.dataId, run="somerun")
722 s = ref.to_json()
723 self.assertEqual(DatasetRef.from_json(s, universe=self.universe), ref)
725 # Also test ref with datastore records, serialization does not
726 # preserve those.
727 ref = self._make_datastore_records(ref, "/path1", "/path2")
728 s = ref.to_json()
729 ref2 = DatasetRef.from_json(s, universe=self.universe)
730 self.assertEqual(ref2, ref)
731 self.assertIsNone(ref2._datastore_records)
733 def testFileDataset(self) -> None:
734 ref = DatasetRef(self.datasetType, self.dataId, run="somerun")
735 file_dataset = FileDataset(path="something.yaml", refs=ref)
736 self.assertEqual(file_dataset.refs, [ref])
738 ref2 = DatasetRef(self.datasetType, self.dataId, run="somerun2")
739 with self.assertRaises(ValueError):
740 FileDataset(path="other.yaml", refs=[ref, ref2])
742 serialized = file_dataset.to_simple()
743 self.assertEqual(serialized.path, "something.yaml")
744 self.assertEqual(len(serialized.refs), 1)
745 serialized_ref = serialized.refs[ref.id]
746 self.assertEqual(serialized_ref.run, ref.run)
747 self.assertEqual(serialized_ref.dataset_type_name, self.datasetType.name)
748 self.assertEqual(serialized_ref.data_id, dict(ref.dataId.mapping))
750 def load_dataset_type(name: str) -> DatasetType:
751 self.assertEqual(name, ref.datasetType.name)
752 return self.datasetType
754 deserialized = file_dataset.from_simple(
755 serialized, universe=ref.dimensions.universe, dataset_type_loader=load_dataset_type
756 )
757 self.assertEqual(deserialized.formatter, file_dataset.formatter)
758 self.assertEqual(deserialized.refs, file_dataset.refs)
759 self.assertEqual(deserialized.path, file_dataset.path)
761 file_dataset.formatter = "lsst.daf.butler.formatters.json.JsonFormatter"
762 file_dataset_from_string_formatter = FileDataset.from_simple(
763 file_dataset.to_simple(), dataset_type_loader=load_dataset_type, universe=ref.dimensions.universe
764 )
765 self.assertEqual(
766 file_dataset_from_string_formatter.formatter, "lsst.daf.butler.formatters.json.JsonFormatter"
767 )
769 file_dataset.formatter = YamlFormatter
770 file_dataset_from_class_formatter = FileDataset.from_simple(
771 file_dataset.to_simple(), dataset_type_loader=load_dataset_type, universe=ref.dimensions.universe
772 )
773 self.assertEqual(
774 file_dataset_from_class_formatter.formatter, "lsst.daf.butler.formatters.yaml.YamlFormatter"
775 )
777 def test_container(self) -> None:
778 ref1 = DatasetRef(self.datasetType, self.dataId, run="somerun")
779 ref2 = ref1.replace(run="somerun2")
781 container = SerializedDatasetRefContainerV1.from_refs([ref1, ref2])
782 self.assertEqual(len(container), 2)
784 new_refs = container.to_refs(universe=self.universe)
785 self.assertEqual(new_refs, [ref1, ref2])
787 def test_dataset_provenance(self) -> None:
788 """Test that dataset provenance can be stored."""
789 dimensions = self.universe.conform(("instrument", "visit"))
790 ref1 = DatasetRef(self.datasetType, self.dataId, run="somerun")
791 ref2 = DatasetRef(
792 self.datasetType,
793 DataCoordinate.standardize(instrument="DummyCam", visit=10, dimensions=dimensions),
794 run="run",
795 )
796 ref3 = DatasetRef(
797 self.datasetType,
798 DataCoordinate.standardize(instrument="DummyCam", visit=22, dimensions=dimensions),
799 run="run",
800 )
802 quantum_id = uuid.uuid4()
803 prov = DatasetProvenance(quantum_id=quantum_id)
804 prov.add_input(ref2)
805 prov.add_input(ref3)
806 prov.add_input(ref2) # no-op which should leave ref2 still ahead of ref3 in output.
807 extra_id = uuid.uuid4()
808 prov.add_extra_provenance(
809 ref2.id, {"extra_string": "value", "extra_number": 42, "extra_id": extra_id}
810 )
812 with self.assertRaises(ValueError):
813 prov.add_extra_provenance(ref2.id, {"extra_string": "value", "extra_number": 42, "id": extra_id})
815 with self.assertRaises(ValueError):
816 # Unknown dataset.
817 prov.add_extra_provenance(ref1.id, {"extra": 42})
819 expected = {
820 "id": ref1.id,
821 "datasettype": "test",
822 "dataid.instrument": "DummyCam",
823 "dataid.visit": 42,
824 "run": "somerun",
825 "quantum": quantum_id,
826 "n_inputs": 2,
827 "input.0.datasettype": "test",
828 "input.0.run": "run",
829 "input.0.id": ref2.id,
830 "input.0.extra_number": 42,
831 "input.0.extra_string": "value",
832 "input.0.extra_id": extra_id,
833 "input.1.datasettype": "test",
834 "input.1.run": "run",
835 "input.1.id": ref3.id,
836 }
838 prov_dict = prov.to_flat_dict(ref1, sep=".")
839 self.assertEqual(prov_dict, expected)
840 DatasetProvenance.strip_provenance_from_flat_dict(prov_dict)
841 self.assertEqual(prov_dict, {})
843 expected = {
844 "id": ref1.id,
845 "datasettype": "test",
846 "dataid.instrument": "DummyCam",
847 "dataid.visit": 42,
848 "run": "somerun",
849 "quantum": quantum_id,
850 "n_inputs": 2,
851 }
853 prov_dict = prov.to_flat_dict(ref1, sep=".", max_inputs=1)
854 self.assertEqual(prov_dict, expected)
855 DatasetProvenance.strip_provenance_from_flat_dict(prov_dict)
856 self.assertEqual(prov_dict, {})
858 expected = {
859 "id": ref1.id,
860 "datasettype": "test",
861 "dataid.instrument": "DummyCam",
862 "dataid.visit": 42,
863 "run": "somerun",
864 "quantum": quantum_id,
865 "n_inputs": 2,
866 "input.0.id": ref2.id,
867 "input.0.extra_number": 42,
868 "input.0.extra_string": "value",
869 "input.0.extra_id": extra_id,
870 "input.1.id": ref3.id,
871 }
873 prov_dict = prov.to_flat_dict(ref1, sep=".", store_minimalist_inputs=True)
874 self.assertEqual(prov_dict, expected)
875 DatasetProvenance.strip_provenance_from_flat_dict(prov_dict)
876 self.assertEqual(prov_dict, {})
878 prov_dict = prov.to_flat_dict(ref1, prefix="", sep=".", simple_types=True)
879 self.assertEqual(prov_dict["id"], str(ref1.id))
880 self.assertEqual(prov_dict["quantum"], str(quantum_id))
881 self.assertEqual(prov_dict["input.0.id"], str(ref2.id))
882 self.assertEqual(prov_dict["input.0.extra_id"], str(extra_id))
883 DatasetProvenance.strip_provenance_from_flat_dict(prov_dict)
884 self.assertEqual(prov_dict, {})
886 for prefix, sep in (
887 ("LSST BUTLER 🔭", " "), # Unicode in prefix.
888 ("LSST*BUTLER 🔭", " "), # regex character.
889 ("LSST*BUTLER", "+"), # two regex characters.
890 ("LSST_BUTLER", "\\"), # backslash for extra difficulty.
891 ("LSST BUTLER 🔭", "→"), # Unicode separator.
892 ):
893 prov_dict = prov.to_flat_dict(ref1, prefix=prefix, sep=sep)
894 self.assertIn(f"{prefix}{sep}RUN", prov_dict)
895 self.assertIn(f"{prefix}{sep}INPUT{sep}0{sep}EXTRA_NUMBER", prov_dict)
896 self.assertEqual(prov_dict[f"{prefix}{sep}RUN"], "somerun")
897 self.assertEqual(prov_dict[f"{prefix}{sep}INPUT{sep}0{sep}EXTRA_NUMBER"], 42)
898 DatasetProvenance.strip_provenance_from_flat_dict(prov_dict)
899 self.assertEqual(prov_dict, {})
901 # Prefix has no case so lower case assumed.
902 prov_dict = prov.to_flat_dict(ref1, prefix="🔭 LSST BUTLER", sep="→")
903 self.assertIn("🔭 LSST BUTLER→run", prov_dict)
904 self.assertIn("🔭 LSST BUTLER→input→0→extra_number", prov_dict)
905 self.assertEqual(prov_dict["🔭 LSST BUTLER→run"], "somerun")
906 self.assertEqual(prov_dict["🔭 LSST BUTLER→input→0→extra_number"], 42)
907 DatasetProvenance.strip_provenance_from_flat_dict(prov_dict)
908 self.assertEqual(prov_dict, {})
910 # Prefix has no case but force upper.
911 prov_dict = prov.to_flat_dict(ref1, prefix="🔭 LSST BUTLER", sep="→", use_upper=True)
912 self.assertIn("🔭 LSST BUTLER→RUN", prov_dict)
913 self.assertIn("🔭 LSST BUTLER→INPUT→0→EXTRA_NUMBER", prov_dict)
914 self.assertEqual(prov_dict["🔭 LSST BUTLER→RUN"], "somerun")
915 self.assertEqual(prov_dict["🔭 LSST BUTLER→INPUT→0→EXTRA_NUMBER"], 42)
916 DatasetProvenance.strip_provenance_from_flat_dict(prov_dict)
917 self.assertEqual(prov_dict, {})
919 prov_dict = prov.to_flat_dict(None, prefix="butler", sep=" ")
920 self.assertNotIn("butler run", prov_dict)
921 self.assertIn("butler quantum", prov_dict)
922 DatasetProvenance.strip_provenance_from_flat_dict(prov_dict)
923 self.assertEqual(prov_dict, {})
925 # Check that an empty provenance returns empty dict with no ref.
926 prov2 = DatasetProvenance()
927 prov_dict = prov2.to_flat_dict(None)
928 self.assertEqual(prov_dict, {})
929 DatasetProvenance.strip_provenance_from_flat_dict(prov_dict)
930 self.assertEqual(prov_dict, {})
932 # Check that an empty provenance with a ref returns info just for
933 # that ref. Use separator that needs escaping in a regex.
934 prov_dict = prov2.to_flat_dict(ref1, prefix="", sep="*")
935 expected = {
936 "id": ref1.id,
937 "datasettype": "test",
938 "dataid*instrument": "DummyCam",
939 "dataid*visit": 42,
940 "run": "somerun",
941 "n_inputs": 0,
942 }
943 self.assertEqual(prov_dict, expected)
944 DatasetProvenance.strip_provenance_from_flat_dict(prov_dict)
945 self.assertEqual(prov_dict, {})
947 # Test with empty provenance with ref that has no dataId.
948 datasetType = DatasetType("empty", self.universe.empty, self.parentStorageClass)
949 empty_ref = DatasetRef(datasetType, {}, "empty_run")
950 prov3 = DatasetProvenance()
951 prov_dict = prov3.to_flat_dict(empty_ref)
952 expected = {
953 "id": empty_ref.id,
954 "datasettype": "empty",
955 "run": "empty_run",
956 "n_inputs": 0,
957 }
958 self.assertEqual(prov_dict, expected)
959 DatasetProvenance.strip_provenance_from_flat_dict(prov_dict)
960 self.assertEqual(prov_dict, {})
962 prov_dict = prov3.to_flat_dict(empty_ref, prefix="x-yz", sep="-")
963 expected = {
964 "x-yz-id": empty_ref.id,
965 "x-yz-datasettype": "empty",
966 "x-yz-run": "empty_run",
967 "x-yz-n_inputs": 0,
968 }
969 self.assertEqual(prov_dict, expected)
970 DatasetProvenance.strip_provenance_from_flat_dict(prov_dict)
971 self.assertEqual(prov_dict, {})
973 with self.assertRaises(ValueError):
974 prov3.to_flat_dict(empty_ref, sep="##")
975 with self.assertRaises(ValueError):
976 prov3.to_flat_dict(empty_ref, sep="a")
977 with self.assertRaises(ValueError):
978 prov3.to_flat_dict(empty_ref, sep="1")
979 with self.assertRaises(ValueError):
980 prov3.to_flat_dict(empty_ref, sep="_")
981 with self.assertRaises(ValueError):
982 prov3.to_flat_dict(empty_ref, sep="Σ")
984 # Dictionary with inconsistent prefixes and separators.
985 test_dicts = (
986 {
987 "xyz-dataid.instrument": "LATISS",
988 },
989 {
990 "xyz-dataid-detector": 10,
991 "abc-dataid-instrument": "LATISS",
992 },
993 {
994 "abc.input.0.id": "id",
995 "xyz.input.0.run": "run",
996 "abc.dataid.instrument": "latiss",
997 },
998 {
999 "abc.input.0.id": "id0",
1000 "abc input 0 id": "id1",
1001 },
1002 )
1003 for prov_dict in test_dicts:
1004 with self.assertRaises(ValueError):
1005 DatasetProvenance.strip_provenance_from_flat_dict(prov_dict)
1008class ConformToUniverseTestCase(unittest.TestCase):
1009 """Tests for DatasetType.conform_to and DatasetRef.conform_to."""
1011 def setUp(self) -> None:
1012 self.universe = DimensionUniverse()
1013 self.storageClass = StorageClass("test_conform_StructuredData")
1015 def _make_universe(
1016 self, version_offset: int, **element_overrides: dict[str, object] | None
1017 ) -> DimensionUniverse:
1018 """Make a universe derived from the default one with a different
1019 version and optional per-element configuration overrides. An
1020 override of `None` removes the element entirely.
1021 """
1022 config = DimensionConfig()
1023 config["version"] = config["version"] + version_offset
1024 for element, overrides in element_overrides.items():
1025 if overrides is None:
1026 del config["elements", element]
1027 else:
1028 for key, value in overrides.items():
1029 config["elements", element, key] = value
1030 return DimensionUniverse(config)
1032 def _load_old_universe(self, version: int) -> DimensionUniverse:
1033 """Load a historical daf_butler universe from its frozen
1034 configuration.
1035 """
1036 config = DimensionConfig(
1037 f"resource://lsst.daf.butler/configs/old_dimensions/daf_butler_universe{version}.yaml"
1038 )
1039 return DimensionUniverse(config)
1041 def test_dataset_type_same_universe(self) -> None:
1042 """Test that conforming to the dataset type's own universe returns
1043 the same object.
1044 """
1045 dataset_type = DatasetType(
1046 "test", self.universe.conform(["detector"]), self.storageClass, isCalibration=True
1047 )
1048 self.assertIs(dataset_type.conform_to(self.universe), dataset_type)
1050 def test_dataset_type_compatible_universe(self) -> None:
1051 """Test conforming a dataset type to a different but compatible
1052 universe.
1053 """
1054 other = self._make_universe(1000)
1055 dataset_type = DatasetType(
1056 "test", self.universe.conform(["detector"]), self.storageClass, isCalibration=True
1057 )
1058 conformed = dataset_type.conform_to(other)
1059 self.assertIs(conformed.dimensions.universe, other)
1060 self.assertEqual(conformed.dimensions.names, dataset_type.dimensions.names)
1061 self.assertEqual(conformed.storageClass_name, dataset_type.storageClass_name)
1062 self.assertTrue(conformed.isCalibration())
1063 self.assertEqual(conformed, dataset_type)
1065 def test_dataset_type_old_universe(self) -> None:
1066 """Test conforming a dataset type between two real historical
1067 universes in which the relevant dimension group is unchanged.
1068 """
1069 # Both universes are pinned so that this test is unaffected by what
1070 # the default universe happens to be.
1071 universe7 = self._load_old_universe(7)
1072 universe8 = self._load_old_universe(8)
1073 dataset_type = DatasetType("test", universe7.conform(["visit"]), self.storageClass)
1074 conformed = dataset_type.conform_to(universe8)
1075 self.assertIs(conformed.dimensions.universe, universe8)
1076 self.assertEqual(conformed.dimensions.names, dataset_type.dimensions.names)
1077 # And back again.
1078 round_tripped = conformed.conform_to(universe7)
1079 self.assertIs(round_tripped.dimensions.universe, universe7)
1080 self.assertEqual(round_tripped, dataset_type)
1082 def test_dataset_type_old_universe_incompatible(self) -> None:
1083 """Test conforming a dataset type between two real historical
1084 universes in which the relevant dimension group changed.
1085 """
1086 # In universe 2 a visit group did not include day_obs, so it conforms
1087 # to a larger group in universe 8.
1088 universe2 = self._load_old_universe(2)
1089 universe8 = self._load_old_universe(8)
1090 dataset_type = DatasetType("test", universe2.conform(["visit"]), self.storageClass)
1091 with self.assertRaisesRegex(InconsistentUniverseError, "different from the conforming set"):
1092 dataset_type.conform_to(universe8)
1094 def test_dataset_type_component(self) -> None:
1095 """Test conforming a component dataset type."""
1096 component_storage_class = StorageClass("test_conform_Component")
1097 parent_storage_class = StorageClass(
1098 "test_conform_Parent",
1099 components={"a": component_storage_class, "b": component_storage_class},
1100 )
1101 other = self._make_universe(1000)
1102 dataset_type = DatasetType(
1103 "test.a",
1104 self.universe.conform(["detector"]),
1105 component_storage_class,
1106 parentStorageClass=parent_storage_class,
1107 )
1108 conformed = dataset_type.conform_to(other)
1109 self.assertIs(conformed.dimensions.universe, other)
1110 self.assertEqual(conformed, dataset_type)
1112 def test_dataset_type_missing_dimension(self) -> None:
1113 """Test that conforming to a universe that lacks one of the dataset
1114 type's dimensions fails.
1115 """
1116 other = self._make_universe(1001, subfilter=None)
1117 dataset_type = DatasetType("test", self.universe.conform(["subfilter"]), self.storageClass)
1118 with self.assertRaisesRegex(InconsistentUniverseError, "do not all exist"):
1119 dataset_type.conform_to(other)
1121 def test_dataset_type_changed_required(self) -> None:
1122 """Test that conforming to a universe in which the same dimension
1123 names have a different required/implied split fails.
1124 """
1125 # In this universe a physical_filter group has the same dimension
1126 # names as in the default universe, but band is required rather than
1127 # implied.
1128 other = self._make_universe(1002, physical_filter={"requires": ["instrument", "band"], "implies": []})
1129 dataset_type = DatasetType("test", self.universe.conform(["physical_filter"]), self.storageClass)
1130 with self.assertRaisesRegex(InconsistentUniverseError, "different from the conforming set"):
1131 dataset_type.conform_to(other)
1133 def test_dataset_type_different_namespace(self) -> None:
1134 """Test that conforming to a universe with a different namespace
1135 fails.
1136 """
1137 config = DimensionConfig()
1138 config["namespace"] = "test_conform"
1139 other = DimensionUniverse(config)
1140 dataset_type = DatasetType("test", self.universe.conform(["detector"]), self.storageClass)
1141 with self.assertRaisesRegex(InconsistentUniverseError, "different namespace"):
1142 dataset_type.conform_to(other)
1144 def test_dataset_ref(self) -> None:
1145 """Test conforming a dataset ref to a different but compatible
1146 universe.
1147 """
1148 other = self._make_universe(1000)
1149 dataset_type = DatasetType("test", self.universe.conform(["physical_filter"]), self.storageClass)
1150 data_id = DataCoordinate.standardize(
1151 {"instrument": "DummyCam", "physical_filter": "d-r", "band": "r"}, universe=self.universe
1152 )
1153 self.assertTrue(data_id.hasFull())
1154 ref = DatasetRef(dataset_type, data_id, run="somerun")
1155 self.assertIs(ref.conform_to(self.universe), ref)
1156 conformed = ref.conform_to(other)
1157 self.assertIs(conformed.datasetType.dimensions.universe, other)
1158 self.assertEqual(conformed.id, ref.id)
1159 self.assertEqual(conformed.run, ref.run)
1160 self.assertEqual(dict(conformed.dataId.required), dict(ref.dataId.required))
1163class ZipIndexTestCase(unittest.TestCase):
1164 """Test that a ZipIndex can be read."""
1166 def test_v1(self):
1167 """Read a v1 serialization."""
1168 path = os.path.join(TESTDIR, "data", "zip_index.json")
1169 with open(path) as fd:
1170 index = ZipIndex.model_validate_json(fd.read())
1172 self.assertEqual(index.index_version, "V1")
1173 self.assertEqual(len(index), 17)
1174 self.assertEqual(len(index.refs), 4)
1176 # Reconstruct the refs using the required universe.
1177 universe_version = index.refs.universe_version
1178 namespace = index.refs.universe_namespace
1179 universe_path = ResourcePath(
1180 f"resource://lsst.daf.butler/configs/old_dimensions/{namespace}_universe{universe_version}.yaml"
1181 )
1182 dimension_config = DimensionConfig(universe_path)
1183 universe = DimensionUniverse(dimension_config)
1184 refs = index.refs.to_refs(universe=universe)
1185 self.assertEqual(len(refs), 4)
1186 self.assertFalse(refs[0].dataId.hasRecords())
1188 # Read an index with records attached.
1189 path = os.path.join(TESTDIR, "data", "zip_index_with_records.json")
1190 with open(path) as fd:
1191 index = ZipIndex.model_validate_json(fd.read())
1192 refs = index.refs.to_refs(universe=universe)
1193 self.assertEqual(len(refs), 2)
1194 self.assertTrue(refs[0].dataId.hasRecords())
1197if __name__ == "__main__":
1198 unittest.main()