Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import unittest 

23import copy 

24from dataclasses import dataclass 

25import os 

26import pickle 

27from random import Random 

28import itertools 

29from typing import Iterator, Optional 

30 

31from lsst.daf.butler import ( 

32 DatabaseTimespanRepresentation, 

33 DataCoordinate, 

34 DataCoordinateSequence, 

35 DataCoordinateSet, 

36 Dimension, 

37 DimensionGraph, 

38 DimensionUniverse, 

39 NamedKeyDict, 

40 NamedValueSet, 

41 Registry, 

42 YamlRepoImportBackend, 

43) 

44from lsst.daf.butler.registry import RegistryConfig 

45 

46DIMENSION_DATA_FILE = os.path.normpath(os.path.join(os.path.dirname(__file__), 

47 "data", "registry", "hsc-rc2-subset.yaml")) 

48 

49 

50def loadDimensionData() -> DataCoordinateSequence: 

51 """Load dimension data from an export file included in the code repository. 

52 

53 Returns 

54 ------- 

55 dataIds : `DataCoordinateSet` 

56 A set containing all data IDs in the export file. 

57 """ 

58 # Create an in-memory SQLite database and Registry just to import the YAML 

59 # data and retreive it as a set of DataCoordinate objects. 

60 config = RegistryConfig() 

61 config["db"] = "sqlite://" 

62 registry = Registry.fromConfig(config, create=True) 

63 with open(DIMENSION_DATA_FILE, 'r') as stream: 

64 backend = YamlRepoImportBackend(stream, registry) 

65 backend.register() 

66 backend.load(datastore=None) 

67 dimensions = DimensionGraph(registry.dimensions, names=["visit", "detector", "tract", "patch"]) 

68 return registry.queryDataIds(dimensions).expanded().toSequence() 

69 

70 

71class DimensionTestCase(unittest.TestCase): 

72 """Tests for dimensions. 

73 

74 All tests here rely on the content of ``config/dimensions.yaml``, either 

75 to test that the definitions there are read in properly or just as generic 

76 data for testing various operations. 

77 """ 

78 

79 def setUp(self): 

80 self.universe = DimensionUniverse() 

81 

82 def checkGraphInvariants(self, graph): 

83 elements = list(graph.elements) 

84 for n, element in enumerate(elements): 

85 # Ordered comparisons on graphs behave like sets. 

86 self.assertLessEqual(element.graph, graph) 

87 # Ordered comparisons on elements correspond to the ordering within 

88 # a DimensionUniverse (topological, with deterministic 

89 # tiebreakers). 

90 for other in elements[:n]: 

91 self.assertLess(other, element) 

92 self.assertLessEqual(other, element) 

93 for other in elements[n + 1:]: 

94 self.assertGreater(other, element) 

95 self.assertGreaterEqual(other, element) 

96 if isinstance(element, Dimension): 

97 self.assertEqual(element.graph.required, element.required) 

98 self.assertEqual(DimensionGraph(self.universe, graph.required), graph) 

99 self.assertCountEqual(graph.required, 

100 [dimension for dimension in graph.dimensions 

101 if not any(dimension in other.graph.implied for other in graph.elements)]) 

102 self.assertCountEqual(graph.implied, graph.dimensions - graph.required) 

103 self.assertCountEqual(graph.dimensions, 

104 [element for element in graph.elements 

105 if isinstance(element, Dimension)]) 

106 self.assertCountEqual(graph.dimensions, itertools.chain(graph.required, graph.implied)) 

107 # Check primary key traversal order: each element should follow any it 

108 # requires, and element that is implied by any other in the graph 

109 # follow at least one of those. 

110 seen = NamedValueSet() 

111 for element in graph.primaryKeyTraversalOrder: 

112 with self.subTest(required=graph.required, implied=graph.implied, element=element): 

113 seen.add(element) 

114 self.assertLessEqual(element.graph.required, seen) 

115 if element in graph.implied: 

116 self.assertTrue(any(element in s.implied for s in seen)) 

117 self.assertCountEqual(seen, graph.elements) 

118 # Test encoding and decoding of DimensionGraphs to bytes. 

119 encoded = graph.encode() 

120 self.assertEqual(len(encoded), self.universe.getEncodeLength()) 

121 self.assertEqual(DimensionGraph.decode(encoded, universe=self.universe), graph) 

122 

123 def testConfigRead(self): 

124 self.assertEqual(self.universe.getStaticDimensions().names, 

125 {"instrument", "visit", "visit_system", "exposure", "detector", 

126 "physical_filter", "band", "subfilter", "calibration_label", 

127 "skymap", "tract", "patch"} | {f"htm{level}" for level in range(25)}) 

128 

129 def testGraphs(self): 

130 self.checkGraphInvariants(self.universe.empty) 

131 for element in self.universe.getStaticElements(): 

132 self.checkGraphInvariants(element.graph) 

133 

134 def testInstrumentDimensions(self): 

135 graph = DimensionGraph(self.universe, names=("exposure", "detector", "visit", "calibration_label")) 

136 self.assertCountEqual(graph.dimensions.names, 

137 ("instrument", "exposure", "detector", "calibration_label", 

138 "visit", "physical_filter", "band", "visit_system")) 

139 self.assertCountEqual(graph.required.names, ("instrument", "exposure", "detector", 

140 "calibration_label", "visit")) 

141 self.assertCountEqual(graph.implied.names, ("physical_filter", "band", "visit_system")) 

142 self.assertCountEqual(graph.elements.names - graph.dimensions.names, 

143 ("visit_detector_region", "visit_definition")) 

144 

145 def testCalibrationDimensions(self): 

146 graph = DimensionGraph(self.universe, names=("calibration_label", "physical_filter", "detector")) 

147 self.assertCountEqual(graph.dimensions.names, 

148 ("instrument", "detector", "calibration_label", 

149 "physical_filter", "band")) 

150 self.assertCountEqual(graph.required.names, ("instrument", "detector", "calibration_label", 

151 "physical_filter")) 

152 self.assertCountEqual(graph.implied.names, ("band",)) 

153 self.assertCountEqual(graph.elements.names, graph.dimensions.names) 

154 

155 def testObservationDimensions(self): 

156 graph = DimensionGraph(self.universe, names=("exposure", "detector", "visit")) 

157 self.assertCountEqual(graph.dimensions.names, ("instrument", "detector", "visit", "exposure", 

158 "physical_filter", "band", "visit_system")) 

159 self.assertCountEqual(graph.required.names, ("instrument", "detector", "exposure", "visit")) 

160 self.assertCountEqual(graph.implied.names, ("physical_filter", "band", "visit_system")) 

161 self.assertCountEqual(graph.elements.names - graph.dimensions.names, 

162 ("visit_detector_region", "visit_definition")) 

163 self.assertCountEqual(graph.spatial.names, ("visit_detector_region",)) 

164 self.assertCountEqual(graph.temporal.names, ("exposure",)) 

165 

166 def testSkyMapDimensions(self): 

167 graph = DimensionGraph(self.universe, names=("patch",)) 

168 self.assertCountEqual(graph.dimensions.names, ("skymap", "tract", "patch")) 

169 self.assertCountEqual(graph.required.names, ("skymap", "tract", "patch")) 

170 self.assertCountEqual(graph.implied.names, ()) 

171 self.assertCountEqual(graph.elements.names, graph.dimensions.names) 

172 self.assertCountEqual(graph.spatial.names, ("patch",)) 

173 

174 def testSubsetCalculation(self): 

175 """Test that independent spatial and temporal options are computed 

176 correctly. 

177 """ 

178 graph = DimensionGraph(self.universe, names=("visit", "detector", "tract", "patch", "htm7", 

179 "exposure", "calibration_label")) 

180 self.assertCountEqual(graph.spatial.names, 

181 ("visit_detector_region", "patch", "htm7")) 

182 self.assertCountEqual(graph.temporal.names, 

183 ("exposure", "calibration_label")) 

184 

185 def testSchemaGeneration(self): 

186 tableSpecs = NamedKeyDict({}) 

187 for element in self.universe.getStaticElements(): 

188 if element.hasTable and element.viewOf is None: 

189 tableSpecs[element] = element.RecordClass.fields.makeTableSpec( 

190 tsRepr=DatabaseTimespanRepresentation.Compound 

191 ) 

192 for element, tableSpec in tableSpecs.items(): 

193 for dep in element.required: 

194 with self.subTest(element=element.name, dep=dep.name): 

195 if dep != element: 

196 self.assertIn(dep.name, tableSpec.fields) 

197 self.assertEqual(tableSpec.fields[dep.name].dtype, dep.primaryKey.dtype) 

198 self.assertEqual(tableSpec.fields[dep.name].length, dep.primaryKey.length) 

199 self.assertEqual(tableSpec.fields[dep.name].nbytes, dep.primaryKey.nbytes) 

200 self.assertFalse(tableSpec.fields[dep.name].nullable) 

201 self.assertTrue(tableSpec.fields[dep.name].primaryKey) 

202 else: 

203 self.assertIn(element.primaryKey.name, tableSpec.fields) 

204 self.assertEqual(tableSpec.fields[element.primaryKey.name].dtype, 

205 dep.primaryKey.dtype) 

206 self.assertEqual(tableSpec.fields[element.primaryKey.name].length, 

207 dep.primaryKey.length) 

208 self.assertEqual(tableSpec.fields[element.primaryKey.name].nbytes, 

209 dep.primaryKey.nbytes) 

210 self.assertFalse(tableSpec.fields[element.primaryKey.name].nullable) 

211 self.assertTrue(tableSpec.fields[element.primaryKey.name].primaryKey) 

212 for dep in element.implied: 

213 with self.subTest(element=element.name, dep=dep.name): 

214 self.assertIn(dep.name, tableSpec.fields) 

215 self.assertEqual(tableSpec.fields[dep.name].dtype, dep.primaryKey.dtype) 

216 self.assertFalse(tableSpec.fields[dep.name].primaryKey) 

217 for foreignKey in tableSpec.foreignKeys: 

218 self.assertIn(foreignKey.table, tableSpecs) 

219 self.assertIn(foreignKey.table, element.graph.dimensions.names) 

220 self.assertEqual(len(foreignKey.source), len(foreignKey.target)) 

221 for source, target in zip(foreignKey.source, foreignKey.target): 

222 self.assertIn(source, tableSpec.fields.names) 

223 self.assertIn(target, tableSpecs[foreignKey.table].fields.names) 

224 self.assertEqual(tableSpec.fields[source].dtype, 

225 tableSpecs[foreignKey.table].fields[target].dtype) 

226 self.assertEqual(tableSpec.fields[source].length, 

227 tableSpecs[foreignKey.table].fields[target].length) 

228 self.assertEqual(tableSpec.fields[source].nbytes, 

229 tableSpecs[foreignKey.table].fields[target].nbytes) 

230 

231 def testPickling(self): 

232 # Pickling and copying should always yield the exact same object within 

233 # a single process (cross-process is impossible to test here). 

234 universe1 = DimensionUniverse() 

235 universe2 = pickle.loads(pickle.dumps(universe1)) 

236 universe3 = copy.copy(universe1) 

237 universe4 = copy.deepcopy(universe1) 

238 self.assertIs(universe1, universe2) 

239 self.assertIs(universe1, universe3) 

240 self.assertIs(universe1, universe4) 

241 for element1 in universe1.getStaticElements(): 

242 element2 = pickle.loads(pickle.dumps(element1)) 

243 self.assertIs(element1, element2) 

244 graph1 = element1.graph 

245 graph2 = pickle.loads(pickle.dumps(graph1)) 

246 self.assertIs(graph1, graph2) 

247 

248 

249@dataclass 

250class SplitByStateFlags: 

251 """A struct that separates data IDs with different states but the same 

252 values. 

253 """ 

254 

255 minimal: Optional[DataCoordinateSequence] = None 

256 """Data IDs that only contain values for required dimensions. 

257 

258 `DataCoordinateSequence.hasFull()` will return `True` for this if and only 

259 if ``minimal.graph.implied`` has no elements. 

260 `DataCoordinate.hasRecords()` will always return `False`. 

261 """ 

262 

263 complete: Optional[DataCoordinateSequence] = None 

264 """Data IDs that contain values for all dimensions. 

265 

266 `DataCoordinateSequence.hasFull()` will always `True` and 

267 `DataCoordinate.hasRecords()` will always return `True` for this attribute. 

268 """ 

269 

270 expanded: Optional[DataCoordinateSequence] = None 

271 """Data IDs that contain values for all dimensions as well as records. 

272 

273 `DataCoordinateSequence.hasFull()` and `DataCoordinate.hasRecords()` will 

274 always return `True` for this attribute. 

275 """ 

276 

277 def chain(self, n: Optional[int] = None) -> Iterator: 

278 """Iterate over the data IDs of different types. 

279 

280 Parameters 

281 ---------- 

282 n : `int`, optional 

283 If provided (`None` is default), iterate over only the ``nth`` 

284 data ID in each attribute. 

285 

286 Yields 

287 ------ 

288 dataId : `DataCoordinate` 

289 A data ID from one of the attributes in this struct. 

290 """ 

291 if n is None: 

292 s = slice(None, None) 

293 else: 

294 s = slice(n, n + 1) 

295 if self.minimal is not None: 

296 yield from self.minimal[s] 

297 if self.complete is not None: 

298 yield from self.complete[s] 

299 if self.expanded is not None: 

300 yield from self.expanded[s] 

301 

302 

303class DataCoordinateTestCase(unittest.TestCase): 

304 

305 RANDOM_SEED = 10 

306 

307 @classmethod 

308 def setUpClass(cls): 

309 cls.allDataIds = loadDimensionData() 

310 

311 def setUp(self): 

312 self.rng = Random(self.RANDOM_SEED) 

313 

314 def randomDataIds(self, n: int, dataIds: Optional[DataCoordinateSequence] = None): 

315 """Select random data IDs from those loaded from test data. 

316 

317 Parameters 

318 ---------- 

319 n : `int` 

320 Number of data IDs to select. 

321 dataIds : `DataCoordinateSequence`, optional 

322 Data IDs to select from. Defaults to ``self.allDataIds``. 

323 

324 Returns 

325 ------- 

326 selected : `DataCoordinateSequence` 

327 ``n`` Data IDs randomly selected from ``dataIds`` with replacement. 

328 """ 

329 if dataIds is None: 

330 dataIds = self.allDataIds 

331 return DataCoordinateSequence(self.rng.sample(dataIds, n), 

332 graph=dataIds.graph, 

333 hasFull=dataIds.hasFull(), 

334 hasRecords=dataIds.hasRecords(), 

335 check=False) 

336 

337 def randomDimensionSubset(self, n: int = 3, graph: Optional[DimensionGraph] = None) -> DimensionGraph: 

338 """Generate a random `DimensionGraph` that has a subset of the 

339 dimensions in a given one. 

340 

341 Parameters 

342 ---------- 

343 n : `int` 

344 Number of dimensions to select, before automatic expansion by 

345 `DimensionGraph`. 

346 dataIds : `DimensionGraph`, optional 

347 Dimensions to select ffrom. Defaults to ``self.allDataIds.graph``. 

348 

349 Returns 

350 ------- 

351 selected : `DimensionGraph` 

352 ``n`` or more dimensions randomly selected from ``graph`` with 

353 replacement. 

354 """ 

355 if graph is None: 

356 graph = self.allDataIds.graph 

357 return DimensionGraph( 

358 graph.universe, 

359 names=self.rng.sample(list(graph.dimensions.names), max(n, len(graph.dimensions))) 

360 ) 

361 

362 def splitByStateFlags(self, dataIds: Optional[DataCoordinateSequence] = None, *, 

363 expanded: bool = True, 

364 complete: bool = True, 

365 minimal: bool = True) -> SplitByStateFlags: 

366 """Given a sequence of data IDs, generate new equivalent sequences 

367 containing less information. 

368 

369 Parameters 

370 ---------- 

371 dataIds : `DataCoordinateSequence`, optional. 

372 Data IDs to start from. Defaults to ``self.allDataIds``. 

373 ``dataIds.hasRecords()`` and ``dataIds.hasFull()`` must both return 

374 `True`. 

375 expanded : `bool`, optional 

376 If `True` (default) include the original data IDs that contain all 

377 information in the result. 

378 complete : `bool`, optional 

379 If `True` (default) include data IDs for which ``hasFull()`` 

380 returns `True` but ``hasRecords()`` does not. 

381 minimal : `bool`, optional 

382 If `True` (default) include data IDS that only contain values for 

383 required dimensions, for which ``hasFull()`` may not return `True`. 

384 

385 Returns 

386 ------- 

387 split : `SplitByStateFlags` 

388 A dataclass holding the indicated data IDs in attributes that 

389 correspond to the boolean keyword arguments. 

390 """ 

391 if dataIds is None: 

392 dataIds = self.allDataIds 

393 assert dataIds.hasFull() and dataIds.hasRecords() 

394 result = SplitByStateFlags(expanded=dataIds) 

395 if complete: 

396 result.complete = DataCoordinateSequence( 

397 [DataCoordinate.standardize(e.full.byName(), graph=dataIds.graph) for e in result.expanded], 

398 graph=dataIds.graph 

399 ) 

400 self.assertTrue(result.complete.hasFull()) 

401 self.assertFalse(result.complete.hasRecords()) 

402 if minimal: 

403 result.minimal = DataCoordinateSequence( 

404 [DataCoordinate.standardize(e.byName(), graph=dataIds.graph) for e in result.expanded], 

405 graph=dataIds.graph 

406 ) 

407 self.assertEqual(result.minimal.hasFull(), not dataIds.graph.implied) 

408 self.assertFalse(result.minimal.hasRecords()) 

409 if not expanded: 

410 result.expanded = None 

411 return result 

412 

413 def testMappingInterface(self): 

414 """Test that the mapping interface in `DataCoordinate` and (when 

415 applicable) its ``full`` property are self-consistent and consistent 

416 with the ``graph`` property. 

417 """ 

418 for n in range(5): 

419 dimensions = self.randomDimensionSubset() 

420 dataIds = self.randomDataIds(n=1).subset(dimensions) 

421 split = self.splitByStateFlags(dataIds) 

422 for dataId in split.chain(): 

423 with self.subTest(dataId=dataId): 

424 self.assertEqual(list(dataId.values()), [dataId[d] for d in dataId.keys()]) 

425 self.assertEqual(list(dataId.values()), [dataId[d.name] for d in dataId.keys()]) 

426 self.assertEqual(dataId.keys(), dataId.graph.required) 

427 for dataId in itertools.chain(split.complete, split.expanded): 

428 with self.subTest(dataId=dataId): 

429 self.assertTrue(dataId.hasFull()) 

430 self.assertEqual(dataId.graph.dimensions, dataId.full.keys()) 

431 self.assertEqual(list(dataId.full.values()), [dataId[k] for k in dataId.graph.dimensions]) 

432 

433 def testEquality(self): 

434 """Test that different `DataCoordinate` instances with different state 

435 flags can be compared with each other and other mappings. 

436 """ 

437 dataIds = self.randomDataIds(n=2) 

438 split = self.splitByStateFlags(dataIds) 

439 # Iterate over all combinations of different states of DataCoordinate, 

440 # with the same underlying data ID values. 

441 for a0, b0 in itertools.combinations(split.chain(0), 2): 

442 self.assertEqual(a0, b0) 

443 self.assertEqual(a0, b0.byName()) 

444 self.assertEqual(a0.byName(), b0) 

445 # Same thing, for a different data ID value. 

446 for a1, b1 in itertools.combinations(split.chain(1), 2): 

447 self.assertEqual(a1, b1) 

448 self.assertEqual(a1, b1.byName()) 

449 self.assertEqual(a1.byName(), b1) 

450 # Iterate over all combinations of different states of DataCoordinate, 

451 # with different underlying data ID values. 

452 for a0, b1 in itertools.product(split.chain(0), split.chain(1)): 

453 self.assertNotEqual(a0, b1) 

454 self.assertNotEqual(a1, b0) 

455 self.assertNotEqual(a0, b1.byName()) 

456 self.assertNotEqual(a0.byName(), b1) 

457 self.assertNotEqual(a1, b0.byName()) 

458 self.assertNotEqual(a1.byName(), b0) 

459 

460 def testStandardize(self): 

461 """Test constructing a DataCoordinate from many different kinds of 

462 input via `DataCoordinate.standardize` and `DataCoordinate.subset`. 

463 """ 

464 for n in range(5): 

465 dimensions = self.randomDimensionSubset() 

466 dataIds = self.randomDataIds(n=1).subset(dimensions) 

467 split = self.splitByStateFlags(dataIds) 

468 for m, dataId in enumerate(split.chain()): 

469 # Passing in any kind of DataCoordinate alone just returns 

470 # that object. 

471 self.assertIs(dataId, DataCoordinate.standardize(dataId)) 

472 # Same if we also explicitly pass the dimensions we want. 

473 self.assertIs(dataId, DataCoordinate.standardize(dataId, graph=dataId.graph)) 

474 # Same if we pass the dimensions and some irrelevant 

475 # kwargs. 

476 self.assertIs(dataId, DataCoordinate.standardize(dataId, graph=dataId.graph, htm7=12)) 

477 # Test constructing a new data ID from this one with a 

478 # subset of the dimensions. 

479 # This is not possible for some combinations of 

480 # dimensions if hasFull is False (see 

481 # `DataCoordinate.subset` docs). 

482 newDimensions = self.randomDimensionSubset(n=1, graph=dataId.graph) 

483 if dataId.hasFull() or dataId.graph.required.issuperset(newDimensions.required): 

484 newDataIds = [ 

485 dataId.subset(newDimensions), 

486 DataCoordinate.standardize(dataId, graph=newDimensions), 

487 DataCoordinate.standardize(dataId, graph=newDimensions, htm7=12), 

488 ] 

489 for newDataId in newDataIds: 

490 with self.subTest(newDataId=newDataId, type=type(dataId)): 

491 commonKeys = dataId.keys() & newDataId.keys() 

492 self.assertTrue(commonKeys) 

493 self.assertEqual( 

494 [newDataId[k] for k in commonKeys], 

495 [dataId[k] for k in commonKeys], 

496 ) 

497 # This should never "downgrade" from 

498 # Complete to Minimal or Expanded to Complete. 

499 if dataId.hasRecords(): 

500 self.assertTrue(newDataId.hasRecords()) 

501 if dataId.hasFull(): 

502 self.assertTrue(newDataId.hasFull()) 

503 # Start from a complete data ID, and pass its values in via several 

504 # different ways that should be equivalent. 

505 for dataId in split.complete: 

506 # Split the keys (dimension names) into two random subsets, so 

507 # we can pass some as kwargs below. 

508 keys1 = set(self.rng.sample(list(dataId.graph.dimensions.names), 

509 len(dataId.graph.dimensions)//2)) 

510 keys2 = dataId.graph.dimensions.names - keys1 

511 newCompleteDataIds = [ 

512 DataCoordinate.standardize(dataId.full.byName(), universe=dataId.universe), 

513 DataCoordinate.standardize(dataId.full.byName(), graph=dataId.graph), 

514 DataCoordinate.standardize(DataCoordinate.makeEmpty(dataId.graph.universe), 

515 **dataId.full.byName()), 

516 DataCoordinate.standardize(DataCoordinate.makeEmpty(dataId.graph.universe), 

517 graph=dataId.graph, **dataId.full.byName()), 

518 DataCoordinate.standardize(**dataId.full.byName(), universe=dataId.universe), 

519 DataCoordinate.standardize(graph=dataId.graph, **dataId.full.byName()), 

520 DataCoordinate.standardize( 

521 {k: dataId[k] for k in keys1}, 

522 universe=dataId.universe, 

523 **{k: dataId[k] for k in keys2} 

524 ), 

525 DataCoordinate.standardize( 

526 {k: dataId[k] for k in keys1}, 

527 graph=dataId.graph, 

528 **{k: dataId[k] for k in keys2} 

529 ), 

530 ] 

531 for newDataId in newCompleteDataIds: 

532 with self.subTest(dataId=dataId, newDataId=newDataId, type=type(dataId)): 

533 self.assertEqual(dataId, newDataId) 

534 self.assertTrue(newDataId.hasFull()) 

535 

536 def testRegions(self): 

537 """Test that data IDs for a few known dimensions have the expected 

538 regions. 

539 """ 

540 for dataId in self.randomDataIds(n=4).subset( 

541 DimensionGraph(self.allDataIds.universe, names=["visit"])): 

542 self.assertIsNotNone(dataId.region) 

543 self.assertEqual(dataId.graph.spatial.names, {"visit"}) 

544 self.assertEqual(dataId.region, dataId.records["visit"].region) 

545 for dataId in self.randomDataIds(n=4).subset( 

546 DimensionGraph(self.allDataIds.universe, names=["visit", "detector"])): 

547 self.assertIsNotNone(dataId.region) 

548 self.assertEqual(dataId.graph.spatial.names, {"visit_detector_region"}) 

549 self.assertEqual(dataId.region, dataId.records["visit_detector_region"].region) 

550 for dataId in self.randomDataIds(n=4).subset( 

551 DimensionGraph(self.allDataIds.universe, names=["tract"])): 

552 self.assertIsNotNone(dataId.region) 

553 self.assertEqual(dataId.graph.spatial.names, {"tract"}) 

554 self.assertEqual(dataId.region, dataId.records["tract"].region) 

555 for dataId in self.randomDataIds(n=4).subset( 

556 DimensionGraph(self.allDataIds.universe, names=["patch"])): 

557 self.assertIsNotNone(dataId.region) 

558 self.assertEqual(dataId.graph.spatial.names, {"patch"}) 

559 self.assertEqual(dataId.region, dataId.records["patch"].region) 

560 

561 def testTimespans(self): 

562 """Test that data IDs for a few known dimensions have the expected 

563 timespans. 

564 """ 

565 for dataId in self.randomDataIds(n=4).subset( 

566 DimensionGraph(self.allDataIds.universe, names=["visit"])): 

567 self.assertIsNotNone(dataId.timespan) 

568 self.assertEqual(dataId.graph.temporal.names, {"visit"}) 

569 self.assertEqual(dataId.timespan, dataId.records["visit"].timespan) 

570 

571 def testIterableStatusFlags(self): 

572 """Test that DataCoordinateSet and DataCoordinateSequence compute 

573 their hasFull and hasRecords flags correctly from their elements. 

574 """ 

575 dataIds = self.randomDataIds(n=10) 

576 split = self.splitByStateFlags(dataIds) 

577 for cls in (DataCoordinateSet, DataCoordinateSequence): 

578 self.assertTrue(cls(split.expanded, graph=dataIds.graph, check=True).hasFull()) 

579 self.assertTrue(cls(split.expanded, graph=dataIds.graph, check=False).hasFull()) 

580 self.assertTrue(cls(split.expanded, graph=dataIds.graph, check=True).hasRecords()) 

581 self.assertTrue(cls(split.expanded, graph=dataIds.graph, check=False).hasRecords()) 

582 self.assertTrue(cls(split.complete, graph=dataIds.graph, check=True).hasFull()) 

583 self.assertTrue(cls(split.complete, graph=dataIds.graph, check=False).hasFull()) 

584 self.assertFalse(cls(split.complete, graph=dataIds.graph, check=True).hasRecords()) 

585 self.assertFalse(cls(split.complete, graph=dataIds.graph, check=False).hasRecords()) 

586 with self.assertRaises(ValueError): 

587 cls(split.complete, graph=dataIds.graph, hasRecords=True, check=True) 

588 self.assertEqual(cls(split.minimal, graph=dataIds.graph, check=True).hasFull(), 

589 not dataIds.graph.implied) 

590 self.assertEqual(cls(split.minimal, graph=dataIds.graph, check=False).hasFull(), 

591 not dataIds.graph.implied) 

592 self.assertFalse(cls(split.minimal, graph=dataIds.graph, check=True).hasRecords()) 

593 self.assertFalse(cls(split.minimal, graph=dataIds.graph, check=False).hasRecords()) 

594 with self.assertRaises(ValueError): 

595 cls(split.minimal, graph=dataIds.graph, hasRecords=True, check=True) 

596 if dataIds.graph.implied: 

597 with self.assertRaises(ValueError): 

598 cls(split.minimal, graph=dataIds.graph, hasFull=True, check=True) 

599 

600 def testSetOperations(self): 

601 """Test for self-consistency across DataCoordinateSet's operations. 

602 """ 

603 c = self.randomDataIds(n=10).toSet() 

604 a = self.randomDataIds(n=20).toSet() | c 

605 b = self.randomDataIds(n=20).toSet() | c 

606 # Make sure we don't have a particularly unlucky random seed, since 

607 # that would make a lot of this test uninteresting. 

608 self.assertNotEqual(a, b) 

609 self.assertGreater(len(a), 0) 

610 self.assertGreater(len(b), 0) 

611 # The rest of the tests should not depend on the random seed. 

612 self.assertEqual(a, a) 

613 self.assertNotEqual(a, a.toSequence()) 

614 self.assertEqual(a, a.toSequence().toSet()) 

615 self.assertEqual(a, a.toSequence().toSet()) 

616 self.assertEqual(b, b) 

617 self.assertNotEqual(b, b.toSequence()) 

618 self.assertEqual(b, b.toSequence().toSet()) 

619 self.assertEqual(a & b, a.intersection(b)) 

620 self.assertLessEqual(a & b, a) 

621 self.assertLessEqual(a & b, b) 

622 self.assertEqual(a | b, a.union(b)) 

623 self.assertGreaterEqual(a | b, a) 

624 self.assertGreaterEqual(a | b, b) 

625 self.assertEqual(a - b, a.difference(b)) 

626 self.assertLessEqual(a - b, a) 

627 self.assertLessEqual(b - a, b) 

628 self.assertEqual(a ^ b, a.symmetric_difference(b)) 

629 self.assertGreaterEqual(a ^ b, (a | b) - (a & b)) 

630 

631 

632if __name__ == "__main__": 632 ↛ 633line 632 didn't jump to line 633, because the condition on line 632 was never true

633 unittest.main()