Coverage for tests/test_dimensions.py: 11%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import copy
23import itertools
24import os
25import pickle
26import unittest
27from dataclasses import dataclass
28from random import Random
29from typing import Iterator, Optional
31from lsst.daf.butler import (
32 DataCoordinate,
33 DataCoordinateSequence,
34 DataCoordinateSet,
35 Dimension,
36 DimensionConfig,
37 DimensionGraph,
38 DimensionUniverse,
39 NamedKeyDict,
40 NamedValueSet,
41 Registry,
42 SpatialRegionDatabaseRepresentation,
43 TimespanDatabaseRepresentation,
44 YamlRepoImportBackend,
45)
46from lsst.daf.butler.registry import RegistryConfig
48DIMENSION_DATA_FILE = os.path.normpath(
49 os.path.join(os.path.dirname(__file__), "data", "registry", "hsc-rc2-subset.yaml")
50)
53def loadDimensionData() -> DataCoordinateSequence:
54 """Load dimension data from an export file included in the code repository.
56 Returns
57 -------
58 dataIds : `DataCoordinateSet`
59 A set containing all data IDs in the export file.
60 """
61 # Create an in-memory SQLite database and Registry just to import the YAML
62 # data and retreive it as a set of DataCoordinate objects.
63 config = RegistryConfig()
64 config["db"] = "sqlite://"
65 registry = Registry.createFromConfig(config)
66 with open(DIMENSION_DATA_FILE, "r") as stream:
67 backend = YamlRepoImportBackend(stream, registry)
68 backend.register()
69 backend.load(datastore=None)
70 dimensions = DimensionGraph(registry.dimensions, names=["visit", "detector", "tract", "patch"])
71 return registry.queryDataIds(dimensions).expanded().toSequence()
74class DimensionTestCase(unittest.TestCase):
75 """Tests for dimensions.
77 All tests here rely on the content of ``config/dimensions.yaml``, either
78 to test that the definitions there are read in properly or just as generic
79 data for testing various operations.
80 """
82 def setUp(self):
83 self.universe = DimensionUniverse()
85 def checkGraphInvariants(self, graph):
86 elements = list(graph.elements)
87 for n, element in enumerate(elements):
88 # Ordered comparisons on graphs behave like sets.
89 self.assertLessEqual(element.graph, graph)
90 # Ordered comparisons on elements correspond to the ordering within
91 # a DimensionUniverse (topological, with deterministic
92 # tiebreakers).
93 for other in elements[:n]:
94 self.assertLess(other, element)
95 self.assertLessEqual(other, element)
96 for other in elements[n + 1 :]:
97 self.assertGreater(other, element)
98 self.assertGreaterEqual(other, element)
99 if isinstance(element, Dimension):
100 self.assertEqual(element.graph.required, element.required)
101 self.assertEqual(DimensionGraph(self.universe, graph.required), graph)
102 self.assertCountEqual(
103 graph.required,
104 [
105 dimension
106 for dimension in graph.dimensions
107 if not any(dimension in other.graph.implied for other in graph.elements)
108 ],
109 )
110 self.assertCountEqual(graph.implied, graph.dimensions - graph.required)
111 self.assertCountEqual(
112 graph.dimensions, [element for element in graph.elements if isinstance(element, Dimension)]
113 )
114 self.assertCountEqual(graph.dimensions, itertools.chain(graph.required, graph.implied))
115 # Check primary key traversal order: each element should follow any it
116 # requires, and element that is implied by any other in the graph
117 # follow at least one of those.
118 seen = NamedValueSet()
119 for element in graph.primaryKeyTraversalOrder:
120 with self.subTest(required=graph.required, implied=graph.implied, element=element):
121 seen.add(element)
122 self.assertLessEqual(element.graph.required, seen)
123 if element in graph.implied:
124 self.assertTrue(any(element in s.implied for s in seen))
125 self.assertCountEqual(seen, graph.elements)
127 def testConfigPresent(self):
128 config = self.universe.dimensionConfig
129 self.assertIsInstance(config, DimensionConfig)
131 def testConfigRead(self):
132 self.assertEqual(
133 self.universe.getStaticDimensions().names,
134 {
135 "instrument",
136 "visit",
137 "visit_system",
138 "exposure",
139 "detector",
140 "physical_filter",
141 "band",
142 "subfilter",
143 "skymap",
144 "tract",
145 "patch",
146 }
147 | {f"htm{level}" for level in range(25)},
148 )
150 def testGraphs(self):
151 self.checkGraphInvariants(self.universe.empty)
152 for element in self.universe.getStaticElements():
153 self.checkGraphInvariants(element.graph)
155 def testInstrumentDimensions(self):
156 graph = DimensionGraph(self.universe, names=("exposure", "detector", "visit"))
157 self.assertCountEqual(
158 graph.dimensions.names,
159 ("instrument", "exposure", "detector", "visit", "physical_filter", "band", "visit_system"),
160 )
161 self.assertCountEqual(graph.required.names, ("instrument", "exposure", "detector", "visit"))
162 self.assertCountEqual(graph.implied.names, ("physical_filter", "band", "visit_system"))
163 self.assertCountEqual(
164 graph.elements.names - graph.dimensions.names, ("visit_detector_region", "visit_definition")
165 )
166 self.assertCountEqual(graph.governors.names, {"instrument"})
168 def testCalibrationDimensions(self):
169 graph = DimensionGraph(self.universe, names=("physical_filter", "detector"))
170 self.assertCountEqual(graph.dimensions.names, ("instrument", "detector", "physical_filter", "band"))
171 self.assertCountEqual(graph.required.names, ("instrument", "detector", "physical_filter"))
172 self.assertCountEqual(graph.implied.names, ("band",))
173 self.assertCountEqual(graph.elements.names, graph.dimensions.names)
174 self.assertCountEqual(graph.governors.names, {"instrument"})
176 def testObservationDimensions(self):
177 graph = DimensionGraph(self.universe, names=("exposure", "detector", "visit"))
178 self.assertCountEqual(
179 graph.dimensions.names,
180 ("instrument", "detector", "visit", "exposure", "physical_filter", "band", "visit_system"),
181 )
182 self.assertCountEqual(graph.required.names, ("instrument", "detector", "exposure", "visit"))
183 self.assertCountEqual(graph.implied.names, ("physical_filter", "band", "visit_system"))
184 self.assertCountEqual(
185 graph.elements.names - graph.dimensions.names, ("visit_detector_region", "visit_definition")
186 )
187 self.assertCountEqual(graph.spatial.names, ("observation_regions",))
188 self.assertCountEqual(graph.temporal.names, ("observation_timespans",))
189 self.assertCountEqual(graph.governors.names, {"instrument"})
190 self.assertEqual(graph.spatial.names, {"observation_regions"})
191 self.assertEqual(graph.temporal.names, {"observation_timespans"})
192 self.assertEqual(next(iter(graph.spatial)).governor, self.universe["instrument"])
193 self.assertEqual(next(iter(graph.temporal)).governor, self.universe["instrument"])
195 def testSkyMapDimensions(self):
196 graph = DimensionGraph(self.universe, names=("patch",))
197 self.assertCountEqual(graph.dimensions.names, ("skymap", "tract", "patch"))
198 self.assertCountEqual(graph.required.names, ("skymap", "tract", "patch"))
199 self.assertCountEqual(graph.implied.names, ())
200 self.assertCountEqual(graph.elements.names, graph.dimensions.names)
201 self.assertCountEqual(graph.spatial.names, ("skymap_regions",))
202 self.assertCountEqual(graph.governors.names, {"skymap"})
203 self.assertEqual(graph.spatial.names, {"skymap_regions"})
204 self.assertEqual(next(iter(graph.spatial)).governor, self.universe["skymap"])
206 def testSubsetCalculation(self):
207 """Test that independent spatial and temporal options are computed
208 correctly.
209 """
210 graph = DimensionGraph(
211 self.universe, names=("visit", "detector", "tract", "patch", "htm7", "exposure")
212 )
213 self.assertCountEqual(graph.spatial.names, ("observation_regions", "skymap_regions", "htm"))
214 self.assertCountEqual(graph.temporal.names, ("observation_timespans",))
216 def testSchemaGeneration(self):
217 tableSpecs = NamedKeyDict({})
218 for element in self.universe.getStaticElements():
219 if element.hasTable and element.viewOf is None:
220 tableSpecs[element] = element.RecordClass.fields.makeTableSpec(
221 RegionReprClass=SpatialRegionDatabaseRepresentation,
222 TimespanReprClass=TimespanDatabaseRepresentation.Compound,
223 )
224 for element, tableSpec in tableSpecs.items():
225 for dep in element.required:
226 with self.subTest(element=element.name, dep=dep.name):
227 if dep != element:
228 self.assertIn(dep.name, tableSpec.fields)
229 self.assertEqual(tableSpec.fields[dep.name].dtype, dep.primaryKey.dtype)
230 self.assertEqual(tableSpec.fields[dep.name].length, dep.primaryKey.length)
231 self.assertEqual(tableSpec.fields[dep.name].nbytes, dep.primaryKey.nbytes)
232 self.assertFalse(tableSpec.fields[dep.name].nullable)
233 self.assertTrue(tableSpec.fields[dep.name].primaryKey)
234 else:
235 self.assertIn(element.primaryKey.name, tableSpec.fields)
236 self.assertEqual(
237 tableSpec.fields[element.primaryKey.name].dtype, dep.primaryKey.dtype
238 )
239 self.assertEqual(
240 tableSpec.fields[element.primaryKey.name].length, dep.primaryKey.length
241 )
242 self.assertEqual(
243 tableSpec.fields[element.primaryKey.name].nbytes, dep.primaryKey.nbytes
244 )
245 self.assertFalse(tableSpec.fields[element.primaryKey.name].nullable)
246 self.assertTrue(tableSpec.fields[element.primaryKey.name].primaryKey)
247 for dep in element.implied:
248 with self.subTest(element=element.name, dep=dep.name):
249 self.assertIn(dep.name, tableSpec.fields)
250 self.assertEqual(tableSpec.fields[dep.name].dtype, dep.primaryKey.dtype)
251 self.assertFalse(tableSpec.fields[dep.name].primaryKey)
252 for foreignKey in tableSpec.foreignKeys:
253 self.assertIn(foreignKey.table, tableSpecs)
254 self.assertIn(foreignKey.table, element.graph.dimensions.names)
255 self.assertEqual(len(foreignKey.source), len(foreignKey.target))
256 for source, target in zip(foreignKey.source, foreignKey.target):
257 self.assertIn(source, tableSpec.fields.names)
258 self.assertIn(target, tableSpecs[foreignKey.table].fields.names)
259 self.assertEqual(
260 tableSpec.fields[source].dtype, tableSpecs[foreignKey.table].fields[target].dtype
261 )
262 self.assertEqual(
263 tableSpec.fields[source].length, tableSpecs[foreignKey.table].fields[target].length
264 )
265 self.assertEqual(
266 tableSpec.fields[source].nbytes, tableSpecs[foreignKey.table].fields[target].nbytes
267 )
269 def testPickling(self):
270 # Pickling and copying should always yield the exact same object within
271 # a single process (cross-process is impossible to test here).
272 universe1 = DimensionUniverse()
273 universe2 = pickle.loads(pickle.dumps(universe1))
274 universe3 = copy.copy(universe1)
275 universe4 = copy.deepcopy(universe1)
276 self.assertIs(universe1, universe2)
277 self.assertIs(universe1, universe3)
278 self.assertIs(universe1, universe4)
279 for element1 in universe1.getStaticElements():
280 element2 = pickle.loads(pickle.dumps(element1))
281 self.assertIs(element1, element2)
282 graph1 = element1.graph
283 graph2 = pickle.loads(pickle.dumps(graph1))
284 self.assertIs(graph1, graph2)
287@dataclass
288class SplitByStateFlags:
289 """A struct that separates data IDs with different states but the same
290 values.
291 """
293 minimal: Optional[DataCoordinateSequence] = None
294 """Data IDs that only contain values for required dimensions.
296 `DataCoordinateSequence.hasFull()` will return `True` for this if and only
297 if ``minimal.graph.implied`` has no elements.
298 `DataCoordinate.hasRecords()` will always return `False`.
299 """
301 complete: Optional[DataCoordinateSequence] = None
302 """Data IDs that contain values for all dimensions.
304 `DataCoordinateSequence.hasFull()` will always `True` and
305 `DataCoordinate.hasRecords()` will always return `True` for this attribute.
306 """
308 expanded: Optional[DataCoordinateSequence] = None
309 """Data IDs that contain values for all dimensions as well as records.
311 `DataCoordinateSequence.hasFull()` and `DataCoordinate.hasRecords()` will
312 always return `True` for this attribute.
313 """
315 def chain(self, n: Optional[int] = None) -> Iterator:
316 """Iterate over the data IDs of different types.
318 Parameters
319 ----------
320 n : `int`, optional
321 If provided (`None` is default), iterate over only the ``nth``
322 data ID in each attribute.
324 Yields
325 ------
326 dataId : `DataCoordinate`
327 A data ID from one of the attributes in this struct.
328 """
329 if n is None:
330 s = slice(None, None)
331 else:
332 s = slice(n, n + 1)
333 if self.minimal is not None:
334 yield from self.minimal[s]
335 if self.complete is not None:
336 yield from self.complete[s]
337 if self.expanded is not None:
338 yield from self.expanded[s]
341class DataCoordinateTestCase(unittest.TestCase):
343 RANDOM_SEED = 10
345 @classmethod
346 def setUpClass(cls):
347 cls.allDataIds = loadDimensionData()
349 def setUp(self):
350 self.rng = Random(self.RANDOM_SEED)
352 def randomDataIds(self, n: int, dataIds: Optional[DataCoordinateSequence] = None):
353 """Select random data IDs from those loaded from test data.
355 Parameters
356 ----------
357 n : `int`
358 Number of data IDs to select.
359 dataIds : `DataCoordinateSequence`, optional
360 Data IDs to select from. Defaults to ``self.allDataIds``.
362 Returns
363 -------
364 selected : `DataCoordinateSequence`
365 ``n`` Data IDs randomly selected from ``dataIds`` with replacement.
366 """
367 if dataIds is None:
368 dataIds = self.allDataIds
369 return DataCoordinateSequence(
370 self.rng.sample(dataIds, n),
371 graph=dataIds.graph,
372 hasFull=dataIds.hasFull(),
373 hasRecords=dataIds.hasRecords(),
374 check=False,
375 )
377 def randomDimensionSubset(self, n: int = 3, graph: Optional[DimensionGraph] = None) -> DimensionGraph:
378 """Generate a random `DimensionGraph` that has a subset of the
379 dimensions in a given one.
381 Parameters
382 ----------
383 n : `int`
384 Number of dimensions to select, before automatic expansion by
385 `DimensionGraph`.
386 dataIds : `DimensionGraph`, optional
387 Dimensions to select from. Defaults to ``self.allDataIds.graph``.
389 Returns
390 -------
391 selected : `DimensionGraph`
392 ``n`` or more dimensions randomly selected from ``graph`` with
393 replacement.
394 """
395 if graph is None:
396 graph = self.allDataIds.graph
397 return DimensionGraph(
398 graph.universe, names=self.rng.sample(list(graph.dimensions.names), max(n, len(graph.dimensions)))
399 )
401 def splitByStateFlags(
402 self,
403 dataIds: Optional[DataCoordinateSequence] = None,
404 *,
405 expanded: bool = True,
406 complete: bool = True,
407 minimal: bool = True,
408 ) -> SplitByStateFlags:
409 """Given a sequence of data IDs, generate new equivalent sequences
410 containing less information.
412 Parameters
413 ----------
414 dataIds : `DataCoordinateSequence`, optional.
415 Data IDs to start from. Defaults to ``self.allDataIds``.
416 ``dataIds.hasRecords()`` and ``dataIds.hasFull()`` must both return
417 `True`.
418 expanded : `bool`, optional
419 If `True` (default) include the original data IDs that contain all
420 information in the result.
421 complete : `bool`, optional
422 If `True` (default) include data IDs for which ``hasFull()``
423 returns `True` but ``hasRecords()`` does not.
424 minimal : `bool`, optional
425 If `True` (default) include data IDS that only contain values for
426 required dimensions, for which ``hasFull()`` may not return `True`.
428 Returns
429 -------
430 split : `SplitByStateFlags`
431 A dataclass holding the indicated data IDs in attributes that
432 correspond to the boolean keyword arguments.
433 """
434 if dataIds is None:
435 dataIds = self.allDataIds
436 assert dataIds.hasFull() and dataIds.hasRecords()
437 result = SplitByStateFlags(expanded=dataIds)
438 if complete:
439 result.complete = DataCoordinateSequence(
440 [DataCoordinate.standardize(e.full.byName(), graph=dataIds.graph) for e in result.expanded],
441 graph=dataIds.graph,
442 )
443 self.assertTrue(result.complete.hasFull())
444 self.assertFalse(result.complete.hasRecords())
445 if minimal:
446 result.minimal = DataCoordinateSequence(
447 [DataCoordinate.standardize(e.byName(), graph=dataIds.graph) for e in result.expanded],
448 graph=dataIds.graph,
449 )
450 self.assertEqual(result.minimal.hasFull(), not dataIds.graph.implied)
451 self.assertFalse(result.minimal.hasRecords())
452 if not expanded:
453 result.expanded = None
454 return result
456 def testMappingInterface(self):
457 """Test that the mapping interface in `DataCoordinate` and (when
458 applicable) its ``full`` property are self-consistent and consistent
459 with the ``graph`` property.
460 """
461 for n in range(5):
462 dimensions = self.randomDimensionSubset()
463 dataIds = self.randomDataIds(n=1).subset(dimensions)
464 split = self.splitByStateFlags(dataIds)
465 for dataId in split.chain():
466 with self.subTest(dataId=dataId):
467 self.assertEqual(list(dataId.values()), [dataId[d] for d in dataId.keys()])
468 self.assertEqual(list(dataId.values()), [dataId[d.name] for d in dataId.keys()])
469 self.assertEqual(dataId.keys(), dataId.graph.required)
470 for dataId in itertools.chain(split.complete, split.expanded):
471 with self.subTest(dataId=dataId):
472 self.assertTrue(dataId.hasFull())
473 self.assertEqual(dataId.graph.dimensions, dataId.full.keys())
474 self.assertEqual(list(dataId.full.values()), [dataId[k] for k in dataId.graph.dimensions])
476 def testEquality(self):
477 """Test that different `DataCoordinate` instances with different state
478 flags can be compared with each other and other mappings.
479 """
480 dataIds = self.randomDataIds(n=2)
481 split = self.splitByStateFlags(dataIds)
482 # Iterate over all combinations of different states of DataCoordinate,
483 # with the same underlying data ID values.
484 for a0, b0 in itertools.combinations(split.chain(0), 2):
485 self.assertEqual(a0, b0)
486 self.assertEqual(a0, b0.byName())
487 self.assertEqual(a0.byName(), b0)
488 # Same thing, for a different data ID value.
489 for a1, b1 in itertools.combinations(split.chain(1), 2):
490 self.assertEqual(a1, b1)
491 self.assertEqual(a1, b1.byName())
492 self.assertEqual(a1.byName(), b1)
493 # Iterate over all combinations of different states of DataCoordinate,
494 # with different underlying data ID values.
495 for a0, b1 in itertools.product(split.chain(0), split.chain(1)):
496 self.assertNotEqual(a0, b1)
497 self.assertNotEqual(a1, b0)
498 self.assertNotEqual(a0, b1.byName())
499 self.assertNotEqual(a0.byName(), b1)
500 self.assertNotEqual(a1, b0.byName())
501 self.assertNotEqual(a1.byName(), b0)
503 def testStandardize(self):
504 """Test constructing a DataCoordinate from many different kinds of
505 input via `DataCoordinate.standardize` and `DataCoordinate.subset`.
506 """
507 for n in range(5):
508 dimensions = self.randomDimensionSubset()
509 dataIds = self.randomDataIds(n=1).subset(dimensions)
510 split = self.splitByStateFlags(dataIds)
511 for m, dataId in enumerate(split.chain()):
512 # Passing in any kind of DataCoordinate alone just returns
513 # that object.
514 self.assertIs(dataId, DataCoordinate.standardize(dataId))
515 # Same if we also explicitly pass the dimensions we want.
516 self.assertIs(dataId, DataCoordinate.standardize(dataId, graph=dataId.graph))
517 # Same if we pass the dimensions and some irrelevant
518 # kwargs.
519 self.assertIs(dataId, DataCoordinate.standardize(dataId, graph=dataId.graph, htm7=12))
520 # Test constructing a new data ID from this one with a
521 # subset of the dimensions.
522 # This is not possible for some combinations of
523 # dimensions if hasFull is False (see
524 # `DataCoordinate.subset` docs).
525 newDimensions = self.randomDimensionSubset(n=1, graph=dataId.graph)
526 if dataId.hasFull() or dataId.graph.required.issuperset(newDimensions.required):
527 newDataIds = [
528 dataId.subset(newDimensions),
529 DataCoordinate.standardize(dataId, graph=newDimensions),
530 DataCoordinate.standardize(dataId, graph=newDimensions, htm7=12),
531 ]
532 for newDataId in newDataIds:
533 with self.subTest(newDataId=newDataId, type=type(dataId)):
534 commonKeys = dataId.keys() & newDataId.keys()
535 self.assertTrue(commonKeys)
536 self.assertEqual(
537 [newDataId[k] for k in commonKeys],
538 [dataId[k] for k in commonKeys],
539 )
540 # This should never "downgrade" from
541 # Complete to Minimal or Expanded to Complete.
542 if dataId.hasRecords():
543 self.assertTrue(newDataId.hasRecords())
544 if dataId.hasFull():
545 self.assertTrue(newDataId.hasFull())
546 # Start from a complete data ID, and pass its values in via several
547 # different ways that should be equivalent.
548 for dataId in split.complete:
549 # Split the keys (dimension names) into two random subsets, so
550 # we can pass some as kwargs below.
551 keys1 = set(
552 self.rng.sample(list(dataId.graph.dimensions.names), len(dataId.graph.dimensions) // 2)
553 )
554 keys2 = dataId.graph.dimensions.names - keys1
555 newCompleteDataIds = [
556 DataCoordinate.standardize(dataId.full.byName(), universe=dataId.universe),
557 DataCoordinate.standardize(dataId.full.byName(), graph=dataId.graph),
558 DataCoordinate.standardize(
559 DataCoordinate.makeEmpty(dataId.graph.universe), **dataId.full.byName()
560 ),
561 DataCoordinate.standardize(
562 DataCoordinate.makeEmpty(dataId.graph.universe),
563 graph=dataId.graph,
564 **dataId.full.byName(),
565 ),
566 DataCoordinate.standardize(**dataId.full.byName(), universe=dataId.universe),
567 DataCoordinate.standardize(graph=dataId.graph, **dataId.full.byName()),
568 DataCoordinate.standardize(
569 {k: dataId[k] for k in keys1},
570 universe=dataId.universe,
571 **{k: dataId[k] for k in keys2},
572 ),
573 DataCoordinate.standardize(
574 {k: dataId[k] for k in keys1}, graph=dataId.graph, **{k: dataId[k] for k in keys2}
575 ),
576 ]
577 for newDataId in newCompleteDataIds:
578 with self.subTest(dataId=dataId, newDataId=newDataId, type=type(dataId)):
579 self.assertEqual(dataId, newDataId)
580 self.assertTrue(newDataId.hasFull())
582 def testUnion(self):
583 """Test `DataCoordinate.union`."""
584 # Make test graphs to combine; mostly random, but with a few explicit
585 # cases to make sure certain edge cases are covered.
586 graphs = [self.randomDimensionSubset(n=2) for i in range(2)]
587 graphs.append(self.allDataIds.universe["visit"].graph)
588 graphs.append(self.allDataIds.universe["detector"].graph)
589 graphs.append(self.allDataIds.universe["physical_filter"].graph)
590 graphs.append(self.allDataIds.universe["band"].graph)
591 # Iterate over all combinations, including the same graph with itself.
592 for graph1, graph2 in itertools.product(graphs, repeat=2):
593 parentDataIds = self.randomDataIds(n=1)
594 split1 = self.splitByStateFlags(parentDataIds.subset(graph1))
595 split2 = self.splitByStateFlags(parentDataIds.subset(graph2))
596 (parentDataId,) = parentDataIds
597 for lhs, rhs in itertools.product(split1.chain(), split2.chain()):
598 unioned = lhs.union(rhs)
599 with self.subTest(lhs=lhs, rhs=rhs, unioned=unioned):
600 self.assertEqual(unioned.graph, graph1.union(graph2))
601 self.assertEqual(unioned, parentDataId.subset(unioned.graph))
602 if unioned.hasFull():
603 self.assertEqual(unioned.subset(lhs.graph), lhs)
604 self.assertEqual(unioned.subset(rhs.graph), rhs)
605 if lhs.hasFull() and rhs.hasFull():
606 self.assertTrue(unioned.hasFull())
607 if lhs.graph >= unioned.graph and lhs.hasFull():
608 self.assertTrue(unioned.hasFull())
609 if lhs.hasRecords():
610 self.assertTrue(unioned.hasRecords())
611 if rhs.graph >= unioned.graph and rhs.hasFull():
612 self.assertTrue(unioned.hasFull())
613 if rhs.hasRecords():
614 self.assertTrue(unioned.hasRecords())
615 if lhs.graph.required | rhs.graph.required >= unioned.graph.dimensions:
616 self.assertTrue(unioned.hasFull())
617 if lhs.hasRecords() and rhs.hasRecords():
618 if lhs.graph.elements | rhs.graph.elements >= unioned.graph.elements:
619 self.assertTrue(unioned.hasRecords())
621 def testRegions(self):
622 """Test that data IDs for a few known dimensions have the expected
623 regions.
624 """
625 for dataId in self.randomDataIds(n=4).subset(
626 DimensionGraph(self.allDataIds.universe, names=["visit"])
627 ):
628 self.assertIsNotNone(dataId.region)
629 self.assertEqual(dataId.graph.spatial.names, {"observation_regions"})
630 self.assertEqual(dataId.region, dataId.records["visit"].region)
631 for dataId in self.randomDataIds(n=4).subset(
632 DimensionGraph(self.allDataIds.universe, names=["visit", "detector"])
633 ):
634 self.assertIsNotNone(dataId.region)
635 self.assertEqual(dataId.graph.spatial.names, {"observation_regions"})
636 self.assertEqual(dataId.region, dataId.records["visit_detector_region"].region)
637 for dataId in self.randomDataIds(n=4).subset(
638 DimensionGraph(self.allDataIds.universe, names=["tract"])
639 ):
640 self.assertIsNotNone(dataId.region)
641 self.assertEqual(dataId.graph.spatial.names, {"skymap_regions"})
642 self.assertEqual(dataId.region, dataId.records["tract"].region)
643 for dataId in self.randomDataIds(n=4).subset(
644 DimensionGraph(self.allDataIds.universe, names=["patch"])
645 ):
646 self.assertIsNotNone(dataId.region)
647 self.assertEqual(dataId.graph.spatial.names, {"skymap_regions"})
648 self.assertEqual(dataId.region, dataId.records["patch"].region)
650 def testTimespans(self):
651 """Test that data IDs for a few known dimensions have the expected
652 timespans.
653 """
654 for dataId in self.randomDataIds(n=4).subset(
655 DimensionGraph(self.allDataIds.universe, names=["visit"])
656 ):
657 self.assertIsNotNone(dataId.timespan)
658 self.assertEqual(dataId.graph.temporal.names, {"observation_timespans"})
659 self.assertEqual(dataId.timespan, dataId.records["visit"].timespan)
661 def testIterableStatusFlags(self):
662 """Test that DataCoordinateSet and DataCoordinateSequence compute
663 their hasFull and hasRecords flags correctly from their elements.
664 """
665 dataIds = self.randomDataIds(n=10)
666 split = self.splitByStateFlags(dataIds)
667 for cls in (DataCoordinateSet, DataCoordinateSequence):
668 self.assertTrue(cls(split.expanded, graph=dataIds.graph, check=True).hasFull())
669 self.assertTrue(cls(split.expanded, graph=dataIds.graph, check=False).hasFull())
670 self.assertTrue(cls(split.expanded, graph=dataIds.graph, check=True).hasRecords())
671 self.assertTrue(cls(split.expanded, graph=dataIds.graph, check=False).hasRecords())
672 self.assertTrue(cls(split.complete, graph=dataIds.graph, check=True).hasFull())
673 self.assertTrue(cls(split.complete, graph=dataIds.graph, check=False).hasFull())
674 self.assertFalse(cls(split.complete, graph=dataIds.graph, check=True).hasRecords())
675 self.assertFalse(cls(split.complete, graph=dataIds.graph, check=False).hasRecords())
676 with self.assertRaises(ValueError):
677 cls(split.complete, graph=dataIds.graph, hasRecords=True, check=True)
678 self.assertEqual(
679 cls(split.minimal, graph=dataIds.graph, check=True).hasFull(), not dataIds.graph.implied
680 )
681 self.assertEqual(
682 cls(split.minimal, graph=dataIds.graph, check=False).hasFull(), not dataIds.graph.implied
683 )
684 self.assertFalse(cls(split.minimal, graph=dataIds.graph, check=True).hasRecords())
685 self.assertFalse(cls(split.minimal, graph=dataIds.graph, check=False).hasRecords())
686 with self.assertRaises(ValueError):
687 cls(split.minimal, graph=dataIds.graph, hasRecords=True, check=True)
688 if dataIds.graph.implied:
689 with self.assertRaises(ValueError):
690 cls(split.minimal, graph=dataIds.graph, hasFull=True, check=True)
692 def testSetOperations(self):
693 """Test for self-consistency across DataCoordinateSet's operations."""
694 c = self.randomDataIds(n=10).toSet()
695 a = self.randomDataIds(n=20).toSet() | c
696 b = self.randomDataIds(n=20).toSet() | c
697 # Make sure we don't have a particularly unlucky random seed, since
698 # that would make a lot of this test uninteresting.
699 self.assertNotEqual(a, b)
700 self.assertGreater(len(a), 0)
701 self.assertGreater(len(b), 0)
702 # The rest of the tests should not depend on the random seed.
703 self.assertEqual(a, a)
704 self.assertNotEqual(a, a.toSequence())
705 self.assertEqual(a, a.toSequence().toSet())
706 self.assertEqual(a, a.toSequence().toSet())
707 self.assertEqual(b, b)
708 self.assertNotEqual(b, b.toSequence())
709 self.assertEqual(b, b.toSequence().toSet())
710 self.assertEqual(a & b, a.intersection(b))
711 self.assertLessEqual(a & b, a)
712 self.assertLessEqual(a & b, b)
713 self.assertEqual(a | b, a.union(b))
714 self.assertGreaterEqual(a | b, a)
715 self.assertGreaterEqual(a | b, b)
716 self.assertEqual(a - b, a.difference(b))
717 self.assertLessEqual(a - b, a)
718 self.assertLessEqual(b - a, b)
719 self.assertEqual(a ^ b, a.symmetric_difference(b))
720 self.assertGreaterEqual(a ^ b, (a | b) - (a & b))
723if __name__ == "__main__": 723 ↛ 724line 723 didn't jump to line 724, because the condition on line 723 was never true
724 unittest.main()