Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import unittest 

23import copy 

24from dataclasses import dataclass 

25import os 

26import pickle 

27from random import Random 

28import itertools 

29from typing import Iterator, Optional 

30 

31from lsst.daf.butler import ( 

32 DatabaseTimespanRepresentation, 

33 DataCoordinate, 

34 DataCoordinateSequence, 

35 DataCoordinateSet, 

36 Dimension, 

37 DimensionGraph, 

38 DimensionUniverse, 

39 NamedKeyDict, 

40 NamedValueSet, 

41 Registry, 

42 YamlRepoImportBackend, 

43) 

44from lsst.daf.butler.registry import RegistryConfig 

45 

46DIMENSION_DATA_FILE = os.path.normpath(os.path.join(os.path.dirname(__file__), 

47 "data", "registry", "hsc-rc2-subset.yaml")) 

48 

49 

50def loadDimensionData() -> DataCoordinateSequence: 

51 """Load dimension data from an export file included in the code repository. 

52 

53 Returns 

54 ------- 

55 dataIds : `DataCoordinateSet` 

56 A set containing all data IDs in the export file. 

57 """ 

58 # Create an in-memory SQLite database and Registry just to import the YAML 

59 # data and retreive it as a set of DataCoordinate objects. 

60 config = RegistryConfig() 

61 config["db"] = "sqlite://" 

62 registry = Registry.fromConfig(config, create=True) 

63 with open(DIMENSION_DATA_FILE, 'r') as stream: 

64 backend = YamlRepoImportBackend(stream, registry) 

65 backend.register() 

66 backend.load(datastore=None) 

67 dimensions = DimensionGraph(registry.dimensions, names=["visit", "detector", "tract", "patch"]) 

68 return registry.queryDataIds(dimensions).expanded().toSequence() 

69 

70 

71class DimensionTestCase(unittest.TestCase): 

72 """Tests for dimensions. 

73 

74 All tests here rely on the content of ``config/dimensions.yaml``, either 

75 to test that the definitions there are read in properly or just as generic 

76 data for testing various operations. 

77 """ 

78 

79 def setUp(self): 

80 self.universe = DimensionUniverse() 

81 

82 def checkGraphInvariants(self, graph): 

83 elements = list(graph.elements) 

84 for n, element in enumerate(elements): 

85 # Ordered comparisons on graphs behave like sets. 

86 self.assertLessEqual(element.graph, graph) 

87 # Ordered comparisons on elements correspond to the ordering within 

88 # a DimensionUniverse (topological, with deterministic 

89 # tiebreakers). 

90 for other in elements[:n]: 

91 self.assertLess(other, element) 

92 self.assertLessEqual(other, element) 

93 for other in elements[n + 1:]: 

94 self.assertGreater(other, element) 

95 self.assertGreaterEqual(other, element) 

96 if isinstance(element, Dimension): 

97 self.assertEqual(element.graph.required, element.required) 

98 self.assertEqual(DimensionGraph(self.universe, graph.required), graph) 

99 self.assertCountEqual(graph.required, 

100 [dimension for dimension in graph.dimensions 

101 if not any(dimension in other.graph.implied for other in graph.elements)]) 

102 self.assertCountEqual(graph.implied, graph.dimensions - graph.required) 

103 self.assertCountEqual(graph.dimensions, 

104 [element for element in graph.elements 

105 if isinstance(element, Dimension)]) 

106 self.assertCountEqual(graph.dimensions, itertools.chain(graph.required, graph.implied)) 

107 # Check primary key traversal order: each element should follow any it 

108 # requires, and element that is implied by any other in the graph 

109 # follow at least one of those. 

110 seen = NamedValueSet() 

111 for element in graph.primaryKeyTraversalOrder: 

112 with self.subTest(required=graph.required, implied=graph.implied, element=element): 

113 seen.add(element) 

114 self.assertLessEqual(element.graph.required, seen) 

115 if element in graph.implied: 

116 self.assertTrue(any(element in s.implied for s in seen)) 

117 self.assertCountEqual(seen, graph.elements) 

118 # Test encoding and decoding of DimensionGraphs to bytes. 

119 encoded = graph.encode() 

120 self.assertEqual(len(encoded), self.universe.getEncodeLength()) 

121 self.assertEqual(DimensionGraph.decode(encoded, universe=self.universe), graph) 

122 

123 def testConfigRead(self): 

124 self.assertEqual(self.universe.dimensions.names, 

125 {"instrument", "visit", "visit_system", "exposure", "detector", 

126 "physical_filter", "abstract_filter", "subfilter", "calibration_label", 

127 "skymap", "tract", "patch", "htm7", "htm9"}) 

128 

129 def testGraphs(self): 

130 self.checkGraphInvariants(self.universe.empty) 

131 self.checkGraphInvariants(self.universe) 

132 for element in self.universe.elements: 

133 self.checkGraphInvariants(element.graph) 

134 

135 def testInstrumentDimensions(self): 

136 graph = DimensionGraph(self.universe, names=("exposure", "detector", "visit", "calibration_label")) 

137 self.assertCountEqual(graph.dimensions.names, 

138 ("instrument", "exposure", "detector", "calibration_label", 

139 "visit", "physical_filter", "abstract_filter", "visit_system")) 

140 self.assertCountEqual(graph.required.names, ("instrument", "exposure", "detector", 

141 "calibration_label", "visit")) 

142 self.assertCountEqual(graph.implied.names, ("physical_filter", "abstract_filter", "visit_system")) 

143 self.assertCountEqual(graph.elements.names - graph.dimensions.names, 

144 ("visit_detector_region", "visit_definition")) 

145 

146 def testCalibrationDimensions(self): 

147 graph = DimensionGraph(self.universe, names=("calibration_label", "physical_filter", "detector")) 

148 self.assertCountEqual(graph.dimensions.names, 

149 ("instrument", "detector", "calibration_label", 

150 "physical_filter", "abstract_filter")) 

151 self.assertCountEqual(graph.required.names, ("instrument", "detector", "calibration_label", 

152 "physical_filter")) 

153 self.assertCountEqual(graph.implied.names, ("abstract_filter",)) 

154 self.assertCountEqual(graph.elements.names, graph.dimensions.names) 

155 

156 def testObservationDimensions(self): 

157 graph = DimensionGraph(self.universe, names=("exposure", "detector", "visit")) 

158 self.assertCountEqual(graph.dimensions.names, ("instrument", "detector", "visit", "exposure", 

159 "physical_filter", "abstract_filter", "visit_system")) 

160 self.assertCountEqual(graph.required.names, ("instrument", "detector", "exposure", "visit")) 

161 self.assertCountEqual(graph.implied.names, ("physical_filter", "abstract_filter", "visit_system")) 

162 self.assertCountEqual(graph.elements.names - graph.dimensions.names, 

163 ("visit_detector_region", "visit_definition")) 

164 self.assertCountEqual(graph.spatial.names, ("visit_detector_region",)) 

165 self.assertCountEqual(graph.temporal.names, ("exposure",)) 

166 

167 def testSkyMapDimensions(self): 

168 graph = DimensionGraph(self.universe, names=("patch",)) 

169 self.assertCountEqual(graph.dimensions.names, ("skymap", "tract", "patch")) 

170 self.assertCountEqual(graph.required.names, ("skymap", "tract", "patch")) 

171 self.assertCountEqual(graph.implied.names, ()) 

172 self.assertCountEqual(graph.elements.names, graph.dimensions.names) 

173 self.assertCountEqual(graph.spatial.names, ("patch",)) 

174 

175 def testSubsetCalculation(self): 

176 """Test that independent spatial and temporal options are computed 

177 correctly. 

178 """ 

179 graph = DimensionGraph(self.universe, names=("visit", "detector", "tract", "patch", "htm7", 

180 "exposure", "calibration_label")) 

181 self.assertCountEqual(graph.spatial.names, 

182 ("visit_detector_region", "patch", "htm7")) 

183 self.assertCountEqual(graph.temporal.names, 

184 ("exposure", "calibration_label")) 

185 

186 def testSchemaGeneration(self): 

187 tableSpecs = NamedKeyDict({}) 

188 for element in self.universe.elements: 

189 if element.hasTable and element.viewOf is None: 

190 tableSpecs[element] = element.RecordClass.fields.makeTableSpec( 

191 tsRepr=DatabaseTimespanRepresentation.Compound 

192 ) 

193 for element, tableSpec in tableSpecs.items(): 

194 for dep in element.required: 

195 with self.subTest(element=element.name, dep=dep.name): 

196 if dep != element: 

197 self.assertIn(dep.name, tableSpec.fields) 

198 self.assertEqual(tableSpec.fields[dep.name].dtype, dep.primaryKey.dtype) 

199 self.assertEqual(tableSpec.fields[dep.name].length, dep.primaryKey.length) 

200 self.assertEqual(tableSpec.fields[dep.name].nbytes, dep.primaryKey.nbytes) 

201 self.assertFalse(tableSpec.fields[dep.name].nullable) 

202 self.assertTrue(tableSpec.fields[dep.name].primaryKey) 

203 else: 

204 self.assertIn(element.primaryKey.name, tableSpec.fields) 

205 self.assertEqual(tableSpec.fields[element.primaryKey.name].dtype, 

206 dep.primaryKey.dtype) 

207 self.assertEqual(tableSpec.fields[element.primaryKey.name].length, 

208 dep.primaryKey.length) 

209 self.assertEqual(tableSpec.fields[element.primaryKey.name].nbytes, 

210 dep.primaryKey.nbytes) 

211 self.assertFalse(tableSpec.fields[element.primaryKey.name].nullable) 

212 self.assertTrue(tableSpec.fields[element.primaryKey.name].primaryKey) 

213 for dep in element.implied: 

214 with self.subTest(element=element.name, dep=dep.name): 

215 self.assertIn(dep.name, tableSpec.fields) 

216 self.assertEqual(tableSpec.fields[dep.name].dtype, dep.primaryKey.dtype) 

217 self.assertFalse(tableSpec.fields[dep.name].primaryKey) 

218 for foreignKey in tableSpec.foreignKeys: 

219 self.assertIn(foreignKey.table, tableSpecs) 

220 self.assertIn(foreignKey.table, element.graph.dimensions.names) 

221 self.assertEqual(len(foreignKey.source), len(foreignKey.target)) 

222 for source, target in zip(foreignKey.source, foreignKey.target): 

223 self.assertIn(source, tableSpec.fields.names) 

224 self.assertIn(target, tableSpecs[foreignKey.table].fields.names) 

225 self.assertEqual(tableSpec.fields[source].dtype, 

226 tableSpecs[foreignKey.table].fields[target].dtype) 

227 self.assertEqual(tableSpec.fields[source].length, 

228 tableSpecs[foreignKey.table].fields[target].length) 

229 self.assertEqual(tableSpec.fields[source].nbytes, 

230 tableSpecs[foreignKey.table].fields[target].nbytes) 

231 

232 def testPickling(self): 

233 # Pickling and copying should always yield the exact same object within 

234 # a single process (cross-process is impossible to test here). 

235 universe1 = DimensionUniverse() 

236 universe2 = pickle.loads(pickle.dumps(universe1)) 

237 universe3 = copy.copy(universe1) 

238 universe4 = copy.deepcopy(universe1) 

239 self.assertIs(universe1, universe2) 

240 self.assertIs(universe1, universe3) 

241 self.assertIs(universe1, universe4) 

242 for element1 in universe1.elements: 

243 element2 = pickle.loads(pickle.dumps(element1)) 

244 self.assertIs(element1, element2) 

245 graph1 = element1.graph 

246 graph2 = pickle.loads(pickle.dumps(graph1)) 

247 self.assertIs(graph1, graph2) 

248 

249 

250@dataclass 

251class SplitByStateFlags: 

252 """A struct that separates data IDs with different states but the same 

253 values. 

254 """ 

255 

256 minimal: Optional[DataCoordinateSequence] = None 

257 """Data IDs that only contain values for required dimensions. 

258 

259 `DataCoordinateSequence.hasFull()` will return `True` for this if and only 

260 if ``minimal.graph.implied`` has no elements. 

261 `DataCoordinate.hasRecords()` will always return `False`. 

262 """ 

263 

264 complete: Optional[DataCoordinateSequence] = None 

265 """Data IDs that contain values for all dimensions. 

266 

267 `DataCoordinateSequence.hasFull()` will always `True` and 

268 `DataCoordinate.hasRecords()` will always return `True` for this attribute. 

269 """ 

270 

271 expanded: Optional[DataCoordinateSequence] = None 

272 """Data IDs that contain values for all dimensions as well as records. 

273 

274 `DataCoordinateSequence.hasFull()` and `DataCoordinate.hasRecords()` will 

275 always return `True` for this attribute. 

276 """ 

277 

278 def chain(self, n: Optional[int] = None) -> Iterator: 

279 """Iterate over the data IDs of different types. 

280 

281 Parameters 

282 ---------- 

283 n : `int`, optional 

284 If provided (`None` is default), iterate over only the ``nth`` 

285 data ID in each attribute. 

286 

287 Yields 

288 ------ 

289 dataId : `DataCoordinate` 

290 A data ID from one of the attributes in this struct. 

291 """ 

292 if n is None: 

293 s = slice(None, None) 

294 else: 

295 s = slice(n, n + 1) 

296 if self.minimal is not None: 

297 yield from self.minimal[s] 

298 if self.complete is not None: 

299 yield from self.complete[s] 

300 if self.expanded is not None: 

301 yield from self.expanded[s] 

302 

303 

304class DataCoordinateTestCase(unittest.TestCase): 

305 

306 RANDOM_SEED = 10 

307 

308 @classmethod 

309 def setUpClass(cls): 

310 cls.allDataIds = loadDimensionData() 

311 

312 def setUp(self): 

313 self.rng = Random(self.RANDOM_SEED) 

314 

315 def randomDataIds(self, n: int, dataIds: Optional[DataCoordinateSequence] = None): 

316 """Select random data IDs from those loaded from test data. 

317 

318 Parameters 

319 ---------- 

320 n : `int` 

321 Number of data IDs to select. 

322 dataIds : `DataCoordinateSequence`, optional 

323 Data IDs to select from. Defaults to ``self.allDataIds``. 

324 

325 Returns 

326 ------- 

327 selected : `DataCoordinateSequence` 

328 ``n`` Data IDs randomly selected from ``dataIds`` with replacement. 

329 """ 

330 if dataIds is None: 

331 dataIds = self.allDataIds 

332 return DataCoordinateSequence(self.rng.sample(dataIds, n), 

333 graph=dataIds.graph, 

334 hasFull=dataIds.hasFull(), 

335 hasRecords=dataIds.hasRecords(), 

336 check=False) 

337 

338 def randomDimensionSubset(self, n: int = 3, graph: Optional[DimensionGraph] = None) -> DimensionGraph: 

339 """Generate a random `DimensionGraph` that has a subset of the 

340 dimensions in a given one. 

341 

342 Parameters 

343 ---------- 

344 n : `int` 

345 Number of dimensions to select, before automatic expansion by 

346 `DimensionGraph`. 

347 dataIds : `DimensionGraph`, optional 

348 Dimensions to select ffrom. Defaults to ``self.allDataIds.graph``. 

349 

350 Returns 

351 ------- 

352 selected : `DimensionGraph` 

353 ``n`` or more dimensions randomly selected from ``graph`` with 

354 replacement. 

355 """ 

356 if graph is None: 

357 graph = self.allDataIds.graph 

358 return DimensionGraph( 

359 graph.universe, 

360 names=self.rng.sample(list(graph.dimensions.names), max(n, len(graph.dimensions))) 

361 ) 

362 

363 def splitByStateFlags(self, dataIds: Optional[DataCoordinateSequence] = None, *, 

364 expanded: bool = True, 

365 complete: bool = True, 

366 minimal: bool = True) -> SplitByStateFlags: 

367 """Given a sequence of data IDs, generate new equivalent sequences 

368 containing less information. 

369 

370 Parameters 

371 ---------- 

372 dataIds : `DataCoordinateSequence`, optional. 

373 Data IDs to start from. Defaults to ``self.allDataIds``. 

374 ``dataIds.hasRecords()`` and ``dataIds.hasFull()`` must both return 

375 `True`. 

376 expanded : `bool`, optional 

377 If `True` (default) include the original data IDs that contain all 

378 information in the result. 

379 complete : `bool`, optional 

380 If `True` (default) include data IDs for which ``hasFull()`` 

381 returns `True` but ``hasRecords()`` does not. 

382 minimal : `bool`, optional 

383 If `True` (default) include data IDS that only contain values for 

384 required dimensions, for which ``hasFull()`` may not return `True`. 

385 

386 Returns 

387 ------- 

388 split : `SplitByStateFlags` 

389 A dataclass holding the indicated data IDs in attributes that 

390 correspond to the boolean keyword arguments. 

391 """ 

392 if dataIds is None: 

393 dataIds = self.allDataIds 

394 assert dataIds.hasFull() and dataIds.hasRecords() 

395 result = SplitByStateFlags(expanded=dataIds) 

396 if complete: 

397 result.complete = DataCoordinateSequence( 

398 [DataCoordinate.standardize(e.full.byName(), graph=dataIds.graph) for e in result.expanded], 

399 graph=dataIds.graph 

400 ) 

401 self.assertTrue(result.complete.hasFull()) 

402 self.assertFalse(result.complete.hasRecords()) 

403 if minimal: 

404 result.minimal = DataCoordinateSequence( 

405 [DataCoordinate.standardize(e.byName(), graph=dataIds.graph) for e in result.expanded], 

406 graph=dataIds.graph 

407 ) 

408 self.assertEqual(result.minimal.hasFull(), not dataIds.graph.implied) 

409 self.assertFalse(result.minimal.hasRecords()) 

410 if not expanded: 

411 result.expanded = None 

412 return result 

413 

414 def testMappingInterface(self): 

415 """Test that the mapping interface in `DataCoordinate` and (when 

416 applicable) its ``full`` property are self-consistent and consistent 

417 with the ``graph`` property. 

418 """ 

419 for n in range(5): 

420 dimensions = self.randomDimensionSubset() 

421 dataIds = self.randomDataIds(n=1).subset(dimensions) 

422 split = self.splitByStateFlags(dataIds) 

423 for dataId in split.chain(): 

424 with self.subTest(dataId=dataId): 

425 self.assertEqual(list(dataId.values()), [dataId[d] for d in dataId.keys()]) 

426 self.assertEqual(list(dataId.values()), [dataId[d.name] for d in dataId.keys()]) 

427 self.assertEqual(dataId.keys(), dataId.graph.required) 

428 for dataId in itertools.chain(split.complete, split.expanded): 

429 with self.subTest(dataId=dataId): 

430 self.assertTrue(dataId.hasFull()) 

431 self.assertEqual(dataId.graph.dimensions, dataId.full.keys()) 

432 self.assertEqual(list(dataId.full.values()), [dataId[k] for k in dataId.graph.dimensions]) 

433 

434 def testEquality(self): 

435 """Test that different `DataCoordinate` instances with different state 

436 flags can be compared with each other and other mappings. 

437 """ 

438 dataIds = self.randomDataIds(n=2) 

439 split = self.splitByStateFlags(dataIds) 

440 # Iterate over all combinations of different states of DataCoordinate, 

441 # with the same underlying data ID values. 

442 for a0, b0 in itertools.combinations(split.chain(0), 2): 

443 self.assertEqual(a0, b0) 

444 self.assertEqual(a0, b0.byName()) 

445 self.assertEqual(a0.byName(), b0) 

446 # Same thing, for a different data ID value. 

447 for a1, b1 in itertools.combinations(split.chain(1), 2): 

448 self.assertEqual(a1, b1) 

449 self.assertEqual(a1, b1.byName()) 

450 self.assertEqual(a1.byName(), b1) 

451 # Iterate over all combinations of different states of DataCoordinate, 

452 # with different underlying data ID values. 

453 for a0, b1 in itertools.product(split.chain(0), split.chain(1)): 

454 self.assertNotEqual(a0, b1) 

455 self.assertNotEqual(a1, b0) 

456 self.assertNotEqual(a0, b1.byName()) 

457 self.assertNotEqual(a0.byName(), b1) 

458 self.assertNotEqual(a1, b0.byName()) 

459 self.assertNotEqual(a1.byName(), b0) 

460 

461 def testStandardize(self): 

462 """Test constructing a DataCoordinate from many different kinds of 

463 input via `DataCoordinate.standardize` and `DataCoordinate.subset`. 

464 """ 

465 for n in range(5): 

466 dimensions = self.randomDimensionSubset() 

467 dataIds = self.randomDataIds(n=1).subset(dimensions) 

468 split = self.splitByStateFlags(dataIds) 

469 for m, dataId in enumerate(split.chain()): 

470 # Passing in any kind of DataCoordinate alone just returns 

471 # that object. 

472 self.assertIs(dataId, DataCoordinate.standardize(dataId)) 

473 # Same if we also explicitly pass the dimensions we want. 

474 self.assertIs(dataId, DataCoordinate.standardize(dataId, graph=dataId.graph)) 

475 # Same if we pass the dimensions and some irrelevant 

476 # kwargs. 

477 self.assertIs(dataId, DataCoordinate.standardize(dataId, graph=dataId.graph, htm7=12)) 

478 # Test constructing a new data ID from this one with a 

479 # subset of the dimensions. 

480 # This is not possible for some combinations of 

481 # dimensions if hasFull is False (see 

482 # `DataCoordinate.subset` docs). 

483 newDimensions = self.randomDimensionSubset(n=1, graph=dataId.graph) 

484 if dataId.hasFull() or dataId.graph.required.issuperset(newDimensions.required): 

485 newDataIds = [ 

486 dataId.subset(newDimensions), 

487 DataCoordinate.standardize(dataId, graph=newDimensions), 

488 DataCoordinate.standardize(dataId, graph=newDimensions, htm7=12), 

489 ] 

490 for newDataId in newDataIds: 

491 with self.subTest(newDataId=newDataId, type=type(dataId)): 

492 commonKeys = dataId.keys() & newDataId.keys() 

493 self.assertTrue(commonKeys) 

494 self.assertEqual( 

495 [newDataId[k] for k in commonKeys], 

496 [dataId[k] for k in commonKeys], 

497 ) 

498 # This should never "downgrade" from 

499 # Complete to Minimal or Expanded to Complete. 

500 if dataId.hasRecords(): 

501 self.assertTrue(newDataId.hasRecords()) 

502 if dataId.hasFull(): 

503 self.assertTrue(newDataId.hasFull()) 

504 # Start from a complete data ID, and pass its values in via several 

505 # different ways that should be equivalent. 

506 for dataId in split.complete: 

507 # Split the keys (dimension names) into two random subsets, so 

508 # we can pass some as kwargs below. 

509 keys1 = set(self.rng.sample(list(dataId.graph.dimensions.names), 

510 len(dataId.graph.dimensions)//2)) 

511 keys2 = dataId.graph.dimensions.names - keys1 

512 newCompleteDataIds = [ 

513 DataCoordinate.standardize(dataId.full.byName(), universe=dataId.universe), 

514 DataCoordinate.standardize(dataId.full.byName(), graph=dataId.graph), 

515 DataCoordinate.standardize(DataCoordinate.makeEmpty(dataId.graph.universe), 

516 **dataId.full.byName()), 

517 DataCoordinate.standardize(DataCoordinate.makeEmpty(dataId.graph.universe), 

518 graph=dataId.graph, **dataId.full.byName()), 

519 DataCoordinate.standardize(**dataId.full.byName(), universe=dataId.universe), 

520 DataCoordinate.standardize(graph=dataId.graph, **dataId.full.byName()), 

521 DataCoordinate.standardize( 

522 {k: dataId[k] for k in keys1}, 

523 universe=dataId.universe, 

524 **{k: dataId[k] for k in keys2} 

525 ), 

526 DataCoordinate.standardize( 

527 {k: dataId[k] for k in keys1}, 

528 graph=dataId.graph, 

529 **{k: dataId[k] for k in keys2} 

530 ), 

531 ] 

532 for newDataId in newCompleteDataIds: 

533 with self.subTest(dataId=dataId, newDataId=newDataId, type=type(dataId)): 

534 self.assertEqual(dataId, newDataId) 

535 self.assertTrue(newDataId.hasFull()) 

536 

537 def testRegions(self): 

538 """Test that data IDs for a few known dimensions have the expected 

539 regions. 

540 """ 

541 for dataId in self.randomDataIds(n=4).subset( 

542 DimensionGraph(self.allDataIds.universe, names=["visit"])): 

543 self.assertIsNotNone(dataId.region) 

544 self.assertEqual(dataId.graph.spatial.names, {"visit"}) 

545 self.assertEqual(dataId.region, dataId.records["visit"].region) 

546 for dataId in self.randomDataIds(n=4).subset( 

547 DimensionGraph(self.allDataIds.universe, names=["visit", "detector"])): 

548 self.assertIsNotNone(dataId.region) 

549 self.assertEqual(dataId.graph.spatial.names, {"visit_detector_region"}) 

550 self.assertEqual(dataId.region, dataId.records["visit_detector_region"].region) 

551 for dataId in self.randomDataIds(n=4).subset( 

552 DimensionGraph(self.allDataIds.universe, names=["tract"])): 

553 self.assertIsNotNone(dataId.region) 

554 self.assertEqual(dataId.graph.spatial.names, {"tract"}) 

555 self.assertEqual(dataId.region, dataId.records["tract"].region) 

556 for dataId in self.randomDataIds(n=4).subset( 

557 DimensionGraph(self.allDataIds.universe, names=["patch"])): 

558 self.assertIsNotNone(dataId.region) 

559 self.assertEqual(dataId.graph.spatial.names, {"patch"}) 

560 self.assertEqual(dataId.region, dataId.records["patch"].region) 

561 

562 def testTimespans(self): 

563 """Test that data IDs for a few known dimensions have the expected 

564 timespans. 

565 """ 

566 for dataId in self.randomDataIds(n=4).subset( 

567 DimensionGraph(self.allDataIds.universe, names=["visit"])): 

568 self.assertIsNotNone(dataId.timespan) 

569 self.assertEqual(dataId.graph.temporal.names, {"visit"}) 

570 self.assertEqual(dataId.timespan, dataId.records["visit"].timespan) 

571 

572 def testIterableStatusFlags(self): 

573 """Test that DataCoordinateSet and DataCoordinateSequence compute 

574 their hasFull and hasRecords flags correctly from their elements. 

575 """ 

576 dataIds = self.randomDataIds(n=10) 

577 split = self.splitByStateFlags(dataIds) 

578 for cls in (DataCoordinateSet, DataCoordinateSequence): 

579 self.assertTrue(cls(split.expanded, graph=dataIds.graph, check=True).hasFull()) 

580 self.assertTrue(cls(split.expanded, graph=dataIds.graph, check=False).hasFull()) 

581 self.assertTrue(cls(split.expanded, graph=dataIds.graph, check=True).hasRecords()) 

582 self.assertTrue(cls(split.expanded, graph=dataIds.graph, check=False).hasRecords()) 

583 self.assertTrue(cls(split.complete, graph=dataIds.graph, check=True).hasFull()) 

584 self.assertTrue(cls(split.complete, graph=dataIds.graph, check=False).hasFull()) 

585 self.assertFalse(cls(split.complete, graph=dataIds.graph, check=True).hasRecords()) 

586 self.assertFalse(cls(split.complete, graph=dataIds.graph, check=False).hasRecords()) 

587 with self.assertRaises(ValueError): 

588 cls(split.complete, graph=dataIds.graph, hasRecords=True, check=True) 

589 self.assertEqual(cls(split.minimal, graph=dataIds.graph, check=True).hasFull(), 

590 not dataIds.graph.implied) 

591 self.assertEqual(cls(split.minimal, graph=dataIds.graph, check=False).hasFull(), 

592 not dataIds.graph.implied) 

593 self.assertFalse(cls(split.minimal, graph=dataIds.graph, check=True).hasRecords()) 

594 self.assertFalse(cls(split.minimal, graph=dataIds.graph, check=False).hasRecords()) 

595 with self.assertRaises(ValueError): 

596 cls(split.minimal, graph=dataIds.graph, hasRecords=True, check=True) 

597 if dataIds.graph.implied: 

598 with self.assertRaises(ValueError): 

599 cls(split.minimal, graph=dataIds.graph, hasFull=True, check=True) 

600 

601 def testSetOperations(self): 

602 """Test for self-consistency across DataCoordinateSet's operations. 

603 """ 

604 c = self.randomDataIds(n=10).toSet() 

605 a = self.randomDataIds(n=20).toSet() | c 

606 b = self.randomDataIds(n=20).toSet() | c 

607 # Make sure we don't have a particularly unlucky random seed, since 

608 # that would make a lot of this test uninteresting. 

609 self.assertNotEqual(a, b) 

610 self.assertGreater(len(a), 0) 

611 self.assertGreater(len(b), 0) 

612 # The rest of the tests should not depend on the random seed. 

613 self.assertEqual(a, a) 

614 self.assertNotEqual(a, a.toSequence()) 

615 self.assertEqual(a, a.toSequence().toSet()) 

616 self.assertEqual(a, a.toSequence().toSet()) 

617 self.assertEqual(b, b) 

618 self.assertNotEqual(b, b.toSequence()) 

619 self.assertEqual(b, b.toSequence().toSet()) 

620 self.assertEqual(a & b, a.intersection(b)) 

621 self.assertLessEqual(a & b, a) 

622 self.assertLessEqual(a & b, b) 

623 self.assertEqual(a | b, a.union(b)) 

624 self.assertGreaterEqual(a | b, a) 

625 self.assertGreaterEqual(a | b, b) 

626 self.assertEqual(a - b, a.difference(b)) 

627 self.assertLessEqual(a - b, a) 

628 self.assertLessEqual(b - a, b) 

629 self.assertEqual(a ^ b, a.symmetric_difference(b)) 

630 self.assertGreaterEqual(a ^ b, (a | b) - (a & b)) 

631 

632 

633if __name__ == "__main__": 633 ↛ 634line 633 didn't jump to line 634, because the condition on line 633 was never true

634 unittest.main()