Coverage for tests/test_dimensions.py: 11%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

363 statements  

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import copy 

23import itertools 

24import os 

25import pickle 

26import unittest 

27from dataclasses import dataclass 

28from random import Random 

29from typing import Iterator, Optional 

30 

31from lsst.daf.butler import ( 

32 DataCoordinate, 

33 DataCoordinateSequence, 

34 DataCoordinateSet, 

35 Dimension, 

36 DimensionConfig, 

37 DimensionGraph, 

38 DimensionUniverse, 

39 NamedKeyDict, 

40 NamedValueSet, 

41 Registry, 

42 SpatialRegionDatabaseRepresentation, 

43 TimespanDatabaseRepresentation, 

44 YamlRepoImportBackend, 

45) 

46from lsst.daf.butler.registry import RegistryConfig 

47 

48DIMENSION_DATA_FILE = os.path.normpath( 

49 os.path.join(os.path.dirname(__file__), "data", "registry", "hsc-rc2-subset.yaml") 

50) 

51 

52 

53def loadDimensionData() -> DataCoordinateSequence: 

54 """Load dimension data from an export file included in the code repository. 

55 

56 Returns 

57 ------- 

58 dataIds : `DataCoordinateSet` 

59 A set containing all data IDs in the export file. 

60 """ 

61 # Create an in-memory SQLite database and Registry just to import the YAML 

62 # data and retreive it as a set of DataCoordinate objects. 

63 config = RegistryConfig() 

64 config["db"] = "sqlite://" 

65 registry = Registry.createFromConfig(config) 

66 with open(DIMENSION_DATA_FILE, "r") as stream: 

67 backend = YamlRepoImportBackend(stream, registry) 

68 backend.register() 

69 backend.load(datastore=None) 

70 dimensions = DimensionGraph(registry.dimensions, names=["visit", "detector", "tract", "patch"]) 

71 return registry.queryDataIds(dimensions).expanded().toSequence() 

72 

73 

74class DimensionTestCase(unittest.TestCase): 

75 """Tests for dimensions. 

76 

77 All tests here rely on the content of ``config/dimensions.yaml``, either 

78 to test that the definitions there are read in properly or just as generic 

79 data for testing various operations. 

80 """ 

81 

82 def setUp(self): 

83 self.universe = DimensionUniverse() 

84 

85 def checkGraphInvariants(self, graph): 

86 elements = list(graph.elements) 

87 for n, element in enumerate(elements): 

88 # Ordered comparisons on graphs behave like sets. 

89 self.assertLessEqual(element.graph, graph) 

90 # Ordered comparisons on elements correspond to the ordering within 

91 # a DimensionUniverse (topological, with deterministic 

92 # tiebreakers). 

93 for other in elements[:n]: 

94 self.assertLess(other, element) 

95 self.assertLessEqual(other, element) 

96 for other in elements[n + 1 :]: 

97 self.assertGreater(other, element) 

98 self.assertGreaterEqual(other, element) 

99 if isinstance(element, Dimension): 

100 self.assertEqual(element.graph.required, element.required) 

101 self.assertEqual(DimensionGraph(self.universe, graph.required), graph) 

102 self.assertCountEqual( 

103 graph.required, 

104 [ 

105 dimension 

106 for dimension in graph.dimensions 

107 if not any(dimension in other.graph.implied for other in graph.elements) 

108 ], 

109 ) 

110 self.assertCountEqual(graph.implied, graph.dimensions - graph.required) 

111 self.assertCountEqual( 

112 graph.dimensions, [element for element in graph.elements if isinstance(element, Dimension)] 

113 ) 

114 self.assertCountEqual(graph.dimensions, itertools.chain(graph.required, graph.implied)) 

115 # Check primary key traversal order: each element should follow any it 

116 # requires, and element that is implied by any other in the graph 

117 # follow at least one of those. 

118 seen = NamedValueSet() 

119 for element in graph.primaryKeyTraversalOrder: 

120 with self.subTest(required=graph.required, implied=graph.implied, element=element): 

121 seen.add(element) 

122 self.assertLessEqual(element.graph.required, seen) 

123 if element in graph.implied: 

124 self.assertTrue(any(element in s.implied for s in seen)) 

125 self.assertCountEqual(seen, graph.elements) 

126 

127 def testConfigPresent(self): 

128 config = self.universe.dimensionConfig 

129 self.assertIsInstance(config, DimensionConfig) 

130 

131 def testConfigRead(self): 

132 self.assertEqual( 

133 self.universe.getStaticDimensions().names, 

134 { 

135 "instrument", 

136 "visit", 

137 "visit_system", 

138 "exposure", 

139 "detector", 

140 "physical_filter", 

141 "band", 

142 "subfilter", 

143 "skymap", 

144 "tract", 

145 "patch", 

146 } 

147 | {f"htm{level}" for level in range(25)}, 

148 ) 

149 

150 def testGraphs(self): 

151 self.checkGraphInvariants(self.universe.empty) 

152 for element in self.universe.getStaticElements(): 

153 self.checkGraphInvariants(element.graph) 

154 

155 def testInstrumentDimensions(self): 

156 graph = DimensionGraph(self.universe, names=("exposure", "detector", "visit")) 

157 self.assertCountEqual( 

158 graph.dimensions.names, 

159 ("instrument", "exposure", "detector", "visit", "physical_filter", "band", "visit_system"), 

160 ) 

161 self.assertCountEqual(graph.required.names, ("instrument", "exposure", "detector", "visit")) 

162 self.assertCountEqual(graph.implied.names, ("physical_filter", "band", "visit_system")) 

163 self.assertCountEqual( 

164 graph.elements.names - graph.dimensions.names, ("visit_detector_region", "visit_definition") 

165 ) 

166 self.assertCountEqual(graph.governors.names, {"instrument"}) 

167 

168 def testCalibrationDimensions(self): 

169 graph = DimensionGraph(self.universe, names=("physical_filter", "detector")) 

170 self.assertCountEqual(graph.dimensions.names, ("instrument", "detector", "physical_filter", "band")) 

171 self.assertCountEqual(graph.required.names, ("instrument", "detector", "physical_filter")) 

172 self.assertCountEqual(graph.implied.names, ("band",)) 

173 self.assertCountEqual(graph.elements.names, graph.dimensions.names) 

174 self.assertCountEqual(graph.governors.names, {"instrument"}) 

175 

176 def testObservationDimensions(self): 

177 graph = DimensionGraph(self.universe, names=("exposure", "detector", "visit")) 

178 self.assertCountEqual( 

179 graph.dimensions.names, 

180 ("instrument", "detector", "visit", "exposure", "physical_filter", "band", "visit_system"), 

181 ) 

182 self.assertCountEqual(graph.required.names, ("instrument", "detector", "exposure", "visit")) 

183 self.assertCountEqual(graph.implied.names, ("physical_filter", "band", "visit_system")) 

184 self.assertCountEqual( 

185 graph.elements.names - graph.dimensions.names, ("visit_detector_region", "visit_definition") 

186 ) 

187 self.assertCountEqual(graph.spatial.names, ("observation_regions",)) 

188 self.assertCountEqual(graph.temporal.names, ("observation_timespans",)) 

189 self.assertCountEqual(graph.governors.names, {"instrument"}) 

190 self.assertEqual(graph.spatial.names, {"observation_regions"}) 

191 self.assertEqual(graph.temporal.names, {"observation_timespans"}) 

192 self.assertEqual(next(iter(graph.spatial)).governor, self.universe["instrument"]) 

193 self.assertEqual(next(iter(graph.temporal)).governor, self.universe["instrument"]) 

194 

195 def testSkyMapDimensions(self): 

196 graph = DimensionGraph(self.universe, names=("patch",)) 

197 self.assertCountEqual(graph.dimensions.names, ("skymap", "tract", "patch")) 

198 self.assertCountEqual(graph.required.names, ("skymap", "tract", "patch")) 

199 self.assertCountEqual(graph.implied.names, ()) 

200 self.assertCountEqual(graph.elements.names, graph.dimensions.names) 

201 self.assertCountEqual(graph.spatial.names, ("skymap_regions",)) 

202 self.assertCountEqual(graph.governors.names, {"skymap"}) 

203 self.assertEqual(graph.spatial.names, {"skymap_regions"}) 

204 self.assertEqual(next(iter(graph.spatial)).governor, self.universe["skymap"]) 

205 

206 def testSubsetCalculation(self): 

207 """Test that independent spatial and temporal options are computed 

208 correctly. 

209 """ 

210 graph = DimensionGraph( 

211 self.universe, names=("visit", "detector", "tract", "patch", "htm7", "exposure") 

212 ) 

213 self.assertCountEqual(graph.spatial.names, ("observation_regions", "skymap_regions", "htm")) 

214 self.assertCountEqual(graph.temporal.names, ("observation_timespans",)) 

215 

216 def testSchemaGeneration(self): 

217 tableSpecs = NamedKeyDict({}) 

218 for element in self.universe.getStaticElements(): 

219 if element.hasTable and element.viewOf is None: 

220 tableSpecs[element] = element.RecordClass.fields.makeTableSpec( 

221 RegionReprClass=SpatialRegionDatabaseRepresentation, 

222 TimespanReprClass=TimespanDatabaseRepresentation.Compound, 

223 ) 

224 for element, tableSpec in tableSpecs.items(): 

225 for dep in element.required: 

226 with self.subTest(element=element.name, dep=dep.name): 

227 if dep != element: 

228 self.assertIn(dep.name, tableSpec.fields) 

229 self.assertEqual(tableSpec.fields[dep.name].dtype, dep.primaryKey.dtype) 

230 self.assertEqual(tableSpec.fields[dep.name].length, dep.primaryKey.length) 

231 self.assertEqual(tableSpec.fields[dep.name].nbytes, dep.primaryKey.nbytes) 

232 self.assertFalse(tableSpec.fields[dep.name].nullable) 

233 self.assertTrue(tableSpec.fields[dep.name].primaryKey) 

234 else: 

235 self.assertIn(element.primaryKey.name, tableSpec.fields) 

236 self.assertEqual( 

237 tableSpec.fields[element.primaryKey.name].dtype, dep.primaryKey.dtype 

238 ) 

239 self.assertEqual( 

240 tableSpec.fields[element.primaryKey.name].length, dep.primaryKey.length 

241 ) 

242 self.assertEqual( 

243 tableSpec.fields[element.primaryKey.name].nbytes, dep.primaryKey.nbytes 

244 ) 

245 self.assertFalse(tableSpec.fields[element.primaryKey.name].nullable) 

246 self.assertTrue(tableSpec.fields[element.primaryKey.name].primaryKey) 

247 for dep in element.implied: 

248 with self.subTest(element=element.name, dep=dep.name): 

249 self.assertIn(dep.name, tableSpec.fields) 

250 self.assertEqual(tableSpec.fields[dep.name].dtype, dep.primaryKey.dtype) 

251 self.assertFalse(tableSpec.fields[dep.name].primaryKey) 

252 for foreignKey in tableSpec.foreignKeys: 

253 self.assertIn(foreignKey.table, tableSpecs) 

254 self.assertIn(foreignKey.table, element.graph.dimensions.names) 

255 self.assertEqual(len(foreignKey.source), len(foreignKey.target)) 

256 for source, target in zip(foreignKey.source, foreignKey.target): 

257 self.assertIn(source, tableSpec.fields.names) 

258 self.assertIn(target, tableSpecs[foreignKey.table].fields.names) 

259 self.assertEqual( 

260 tableSpec.fields[source].dtype, tableSpecs[foreignKey.table].fields[target].dtype 

261 ) 

262 self.assertEqual( 

263 tableSpec.fields[source].length, tableSpecs[foreignKey.table].fields[target].length 

264 ) 

265 self.assertEqual( 

266 tableSpec.fields[source].nbytes, tableSpecs[foreignKey.table].fields[target].nbytes 

267 ) 

268 

269 def testPickling(self): 

270 # Pickling and copying should always yield the exact same object within 

271 # a single process (cross-process is impossible to test here). 

272 universe1 = DimensionUniverse() 

273 universe2 = pickle.loads(pickle.dumps(universe1)) 

274 universe3 = copy.copy(universe1) 

275 universe4 = copy.deepcopy(universe1) 

276 self.assertIs(universe1, universe2) 

277 self.assertIs(universe1, universe3) 

278 self.assertIs(universe1, universe4) 

279 for element1 in universe1.getStaticElements(): 

280 element2 = pickle.loads(pickle.dumps(element1)) 

281 self.assertIs(element1, element2) 

282 graph1 = element1.graph 

283 graph2 = pickle.loads(pickle.dumps(graph1)) 

284 self.assertIs(graph1, graph2) 

285 

286 

287@dataclass 

288class SplitByStateFlags: 

289 """A struct that separates data IDs with different states but the same 

290 values. 

291 """ 

292 

293 minimal: Optional[DataCoordinateSequence] = None 

294 """Data IDs that only contain values for required dimensions. 

295 

296 `DataCoordinateSequence.hasFull()` will return `True` for this if and only 

297 if ``minimal.graph.implied`` has no elements. 

298 `DataCoordinate.hasRecords()` will always return `False`. 

299 """ 

300 

301 complete: Optional[DataCoordinateSequence] = None 

302 """Data IDs that contain values for all dimensions. 

303 

304 `DataCoordinateSequence.hasFull()` will always `True` and 

305 `DataCoordinate.hasRecords()` will always return `True` for this attribute. 

306 """ 

307 

308 expanded: Optional[DataCoordinateSequence] = None 

309 """Data IDs that contain values for all dimensions as well as records. 

310 

311 `DataCoordinateSequence.hasFull()` and `DataCoordinate.hasRecords()` will 

312 always return `True` for this attribute. 

313 """ 

314 

315 def chain(self, n: Optional[int] = None) -> Iterator: 

316 """Iterate over the data IDs of different types. 

317 

318 Parameters 

319 ---------- 

320 n : `int`, optional 

321 If provided (`None` is default), iterate over only the ``nth`` 

322 data ID in each attribute. 

323 

324 Yields 

325 ------ 

326 dataId : `DataCoordinate` 

327 A data ID from one of the attributes in this struct. 

328 """ 

329 if n is None: 

330 s = slice(None, None) 

331 else: 

332 s = slice(n, n + 1) 

333 if self.minimal is not None: 

334 yield from self.minimal[s] 

335 if self.complete is not None: 

336 yield from self.complete[s] 

337 if self.expanded is not None: 

338 yield from self.expanded[s] 

339 

340 

341class DataCoordinateTestCase(unittest.TestCase): 

342 

343 RANDOM_SEED = 10 

344 

345 @classmethod 

346 def setUpClass(cls): 

347 cls.allDataIds = loadDimensionData() 

348 

349 def setUp(self): 

350 self.rng = Random(self.RANDOM_SEED) 

351 

352 def randomDataIds(self, n: int, dataIds: Optional[DataCoordinateSequence] = None): 

353 """Select random data IDs from those loaded from test data. 

354 

355 Parameters 

356 ---------- 

357 n : `int` 

358 Number of data IDs to select. 

359 dataIds : `DataCoordinateSequence`, optional 

360 Data IDs to select from. Defaults to ``self.allDataIds``. 

361 

362 Returns 

363 ------- 

364 selected : `DataCoordinateSequence` 

365 ``n`` Data IDs randomly selected from ``dataIds`` with replacement. 

366 """ 

367 if dataIds is None: 

368 dataIds = self.allDataIds 

369 return DataCoordinateSequence( 

370 self.rng.sample(dataIds, n), 

371 graph=dataIds.graph, 

372 hasFull=dataIds.hasFull(), 

373 hasRecords=dataIds.hasRecords(), 

374 check=False, 

375 ) 

376 

377 def randomDimensionSubset(self, n: int = 3, graph: Optional[DimensionGraph] = None) -> DimensionGraph: 

378 """Generate a random `DimensionGraph` that has a subset of the 

379 dimensions in a given one. 

380 

381 Parameters 

382 ---------- 

383 n : `int` 

384 Number of dimensions to select, before automatic expansion by 

385 `DimensionGraph`. 

386 dataIds : `DimensionGraph`, optional 

387 Dimensions to select from. Defaults to ``self.allDataIds.graph``. 

388 

389 Returns 

390 ------- 

391 selected : `DimensionGraph` 

392 ``n`` or more dimensions randomly selected from ``graph`` with 

393 replacement. 

394 """ 

395 if graph is None: 

396 graph = self.allDataIds.graph 

397 return DimensionGraph( 

398 graph.universe, names=self.rng.sample(list(graph.dimensions.names), max(n, len(graph.dimensions))) 

399 ) 

400 

401 def splitByStateFlags( 

402 self, 

403 dataIds: Optional[DataCoordinateSequence] = None, 

404 *, 

405 expanded: bool = True, 

406 complete: bool = True, 

407 minimal: bool = True, 

408 ) -> SplitByStateFlags: 

409 """Given a sequence of data IDs, generate new equivalent sequences 

410 containing less information. 

411 

412 Parameters 

413 ---------- 

414 dataIds : `DataCoordinateSequence`, optional. 

415 Data IDs to start from. Defaults to ``self.allDataIds``. 

416 ``dataIds.hasRecords()`` and ``dataIds.hasFull()`` must both return 

417 `True`. 

418 expanded : `bool`, optional 

419 If `True` (default) include the original data IDs that contain all 

420 information in the result. 

421 complete : `bool`, optional 

422 If `True` (default) include data IDs for which ``hasFull()`` 

423 returns `True` but ``hasRecords()`` does not. 

424 minimal : `bool`, optional 

425 If `True` (default) include data IDS that only contain values for 

426 required dimensions, for which ``hasFull()`` may not return `True`. 

427 

428 Returns 

429 ------- 

430 split : `SplitByStateFlags` 

431 A dataclass holding the indicated data IDs in attributes that 

432 correspond to the boolean keyword arguments. 

433 """ 

434 if dataIds is None: 

435 dataIds = self.allDataIds 

436 assert dataIds.hasFull() and dataIds.hasRecords() 

437 result = SplitByStateFlags(expanded=dataIds) 

438 if complete: 

439 result.complete = DataCoordinateSequence( 

440 [DataCoordinate.standardize(e.full.byName(), graph=dataIds.graph) for e in result.expanded], 

441 graph=dataIds.graph, 

442 ) 

443 self.assertTrue(result.complete.hasFull()) 

444 self.assertFalse(result.complete.hasRecords()) 

445 if minimal: 

446 result.minimal = DataCoordinateSequence( 

447 [DataCoordinate.standardize(e.byName(), graph=dataIds.graph) for e in result.expanded], 

448 graph=dataIds.graph, 

449 ) 

450 self.assertEqual(result.minimal.hasFull(), not dataIds.graph.implied) 

451 self.assertFalse(result.minimal.hasRecords()) 

452 if not expanded: 

453 result.expanded = None 

454 return result 

455 

456 def testMappingInterface(self): 

457 """Test that the mapping interface in `DataCoordinate` and (when 

458 applicable) its ``full`` property are self-consistent and consistent 

459 with the ``graph`` property. 

460 """ 

461 for n in range(5): 

462 dimensions = self.randomDimensionSubset() 

463 dataIds = self.randomDataIds(n=1).subset(dimensions) 

464 split = self.splitByStateFlags(dataIds) 

465 for dataId in split.chain(): 

466 with self.subTest(dataId=dataId): 

467 self.assertEqual(list(dataId.values()), [dataId[d] for d in dataId.keys()]) 

468 self.assertEqual(list(dataId.values()), [dataId[d.name] for d in dataId.keys()]) 

469 self.assertEqual(dataId.keys(), dataId.graph.required) 

470 for dataId in itertools.chain(split.complete, split.expanded): 

471 with self.subTest(dataId=dataId): 

472 self.assertTrue(dataId.hasFull()) 

473 self.assertEqual(dataId.graph.dimensions, dataId.full.keys()) 

474 self.assertEqual(list(dataId.full.values()), [dataId[k] for k in dataId.graph.dimensions]) 

475 

476 def testEquality(self): 

477 """Test that different `DataCoordinate` instances with different state 

478 flags can be compared with each other and other mappings. 

479 """ 

480 dataIds = self.randomDataIds(n=2) 

481 split = self.splitByStateFlags(dataIds) 

482 # Iterate over all combinations of different states of DataCoordinate, 

483 # with the same underlying data ID values. 

484 for a0, b0 in itertools.combinations(split.chain(0), 2): 

485 self.assertEqual(a0, b0) 

486 self.assertEqual(a0, b0.byName()) 

487 self.assertEqual(a0.byName(), b0) 

488 # Same thing, for a different data ID value. 

489 for a1, b1 in itertools.combinations(split.chain(1), 2): 

490 self.assertEqual(a1, b1) 

491 self.assertEqual(a1, b1.byName()) 

492 self.assertEqual(a1.byName(), b1) 

493 # Iterate over all combinations of different states of DataCoordinate, 

494 # with different underlying data ID values. 

495 for a0, b1 in itertools.product(split.chain(0), split.chain(1)): 

496 self.assertNotEqual(a0, b1) 

497 self.assertNotEqual(a1, b0) 

498 self.assertNotEqual(a0, b1.byName()) 

499 self.assertNotEqual(a0.byName(), b1) 

500 self.assertNotEqual(a1, b0.byName()) 

501 self.assertNotEqual(a1.byName(), b0) 

502 

503 def testStandardize(self): 

504 """Test constructing a DataCoordinate from many different kinds of 

505 input via `DataCoordinate.standardize` and `DataCoordinate.subset`. 

506 """ 

507 for n in range(5): 

508 dimensions = self.randomDimensionSubset() 

509 dataIds = self.randomDataIds(n=1).subset(dimensions) 

510 split = self.splitByStateFlags(dataIds) 

511 for m, dataId in enumerate(split.chain()): 

512 # Passing in any kind of DataCoordinate alone just returns 

513 # that object. 

514 self.assertIs(dataId, DataCoordinate.standardize(dataId)) 

515 # Same if we also explicitly pass the dimensions we want. 

516 self.assertIs(dataId, DataCoordinate.standardize(dataId, graph=dataId.graph)) 

517 # Same if we pass the dimensions and some irrelevant 

518 # kwargs. 

519 self.assertIs(dataId, DataCoordinate.standardize(dataId, graph=dataId.graph, htm7=12)) 

520 # Test constructing a new data ID from this one with a 

521 # subset of the dimensions. 

522 # This is not possible for some combinations of 

523 # dimensions if hasFull is False (see 

524 # `DataCoordinate.subset` docs). 

525 newDimensions = self.randomDimensionSubset(n=1, graph=dataId.graph) 

526 if dataId.hasFull() or dataId.graph.required.issuperset(newDimensions.required): 

527 newDataIds = [ 

528 dataId.subset(newDimensions), 

529 DataCoordinate.standardize(dataId, graph=newDimensions), 

530 DataCoordinate.standardize(dataId, graph=newDimensions, htm7=12), 

531 ] 

532 for newDataId in newDataIds: 

533 with self.subTest(newDataId=newDataId, type=type(dataId)): 

534 commonKeys = dataId.keys() & newDataId.keys() 

535 self.assertTrue(commonKeys) 

536 self.assertEqual( 

537 [newDataId[k] for k in commonKeys], 

538 [dataId[k] for k in commonKeys], 

539 ) 

540 # This should never "downgrade" from 

541 # Complete to Minimal or Expanded to Complete. 

542 if dataId.hasRecords(): 

543 self.assertTrue(newDataId.hasRecords()) 

544 if dataId.hasFull(): 

545 self.assertTrue(newDataId.hasFull()) 

546 # Start from a complete data ID, and pass its values in via several 

547 # different ways that should be equivalent. 

548 for dataId in split.complete: 

549 # Split the keys (dimension names) into two random subsets, so 

550 # we can pass some as kwargs below. 

551 keys1 = set( 

552 self.rng.sample(list(dataId.graph.dimensions.names), len(dataId.graph.dimensions) // 2) 

553 ) 

554 keys2 = dataId.graph.dimensions.names - keys1 

555 newCompleteDataIds = [ 

556 DataCoordinate.standardize(dataId.full.byName(), universe=dataId.universe), 

557 DataCoordinate.standardize(dataId.full.byName(), graph=dataId.graph), 

558 DataCoordinate.standardize( 

559 DataCoordinate.makeEmpty(dataId.graph.universe), **dataId.full.byName() 

560 ), 

561 DataCoordinate.standardize( 

562 DataCoordinate.makeEmpty(dataId.graph.universe), 

563 graph=dataId.graph, 

564 **dataId.full.byName(), 

565 ), 

566 DataCoordinate.standardize(**dataId.full.byName(), universe=dataId.universe), 

567 DataCoordinate.standardize(graph=dataId.graph, **dataId.full.byName()), 

568 DataCoordinate.standardize( 

569 {k: dataId[k] for k in keys1}, 

570 universe=dataId.universe, 

571 **{k: dataId[k] for k in keys2}, 

572 ), 

573 DataCoordinate.standardize( 

574 {k: dataId[k] for k in keys1}, graph=dataId.graph, **{k: dataId[k] for k in keys2} 

575 ), 

576 ] 

577 for newDataId in newCompleteDataIds: 

578 with self.subTest(dataId=dataId, newDataId=newDataId, type=type(dataId)): 

579 self.assertEqual(dataId, newDataId) 

580 self.assertTrue(newDataId.hasFull()) 

581 

582 def testUnion(self): 

583 """Test `DataCoordinate.union`.""" 

584 # Make test graphs to combine; mostly random, but with a few explicit 

585 # cases to make sure certain edge cases are covered. 

586 graphs = [self.randomDimensionSubset(n=2) for i in range(2)] 

587 graphs.append(self.allDataIds.universe["visit"].graph) 

588 graphs.append(self.allDataIds.universe["detector"].graph) 

589 graphs.append(self.allDataIds.universe["physical_filter"].graph) 

590 graphs.append(self.allDataIds.universe["band"].graph) 

591 # Iterate over all combinations, including the same graph with itself. 

592 for graph1, graph2 in itertools.product(graphs, repeat=2): 

593 parentDataIds = self.randomDataIds(n=1) 

594 split1 = self.splitByStateFlags(parentDataIds.subset(graph1)) 

595 split2 = self.splitByStateFlags(parentDataIds.subset(graph2)) 

596 (parentDataId,) = parentDataIds 

597 for lhs, rhs in itertools.product(split1.chain(), split2.chain()): 

598 unioned = lhs.union(rhs) 

599 with self.subTest(lhs=lhs, rhs=rhs, unioned=unioned): 

600 self.assertEqual(unioned.graph, graph1.union(graph2)) 

601 self.assertEqual(unioned, parentDataId.subset(unioned.graph)) 

602 if unioned.hasFull(): 

603 self.assertEqual(unioned.subset(lhs.graph), lhs) 

604 self.assertEqual(unioned.subset(rhs.graph), rhs) 

605 if lhs.hasFull() and rhs.hasFull(): 

606 self.assertTrue(unioned.hasFull()) 

607 if lhs.graph >= unioned.graph and lhs.hasFull(): 

608 self.assertTrue(unioned.hasFull()) 

609 if lhs.hasRecords(): 

610 self.assertTrue(unioned.hasRecords()) 

611 if rhs.graph >= unioned.graph and rhs.hasFull(): 

612 self.assertTrue(unioned.hasFull()) 

613 if rhs.hasRecords(): 

614 self.assertTrue(unioned.hasRecords()) 

615 if lhs.graph.required | rhs.graph.required >= unioned.graph.dimensions: 

616 self.assertTrue(unioned.hasFull()) 

617 if lhs.hasRecords() and rhs.hasRecords(): 

618 if lhs.graph.elements | rhs.graph.elements >= unioned.graph.elements: 

619 self.assertTrue(unioned.hasRecords()) 

620 

621 def testRegions(self): 

622 """Test that data IDs for a few known dimensions have the expected 

623 regions. 

624 """ 

625 for dataId in self.randomDataIds(n=4).subset( 

626 DimensionGraph(self.allDataIds.universe, names=["visit"]) 

627 ): 

628 self.assertIsNotNone(dataId.region) 

629 self.assertEqual(dataId.graph.spatial.names, {"observation_regions"}) 

630 self.assertEqual(dataId.region, dataId.records["visit"].region) 

631 for dataId in self.randomDataIds(n=4).subset( 

632 DimensionGraph(self.allDataIds.universe, names=["visit", "detector"]) 

633 ): 

634 self.assertIsNotNone(dataId.region) 

635 self.assertEqual(dataId.graph.spatial.names, {"observation_regions"}) 

636 self.assertEqual(dataId.region, dataId.records["visit_detector_region"].region) 

637 for dataId in self.randomDataIds(n=4).subset( 

638 DimensionGraph(self.allDataIds.universe, names=["tract"]) 

639 ): 

640 self.assertIsNotNone(dataId.region) 

641 self.assertEqual(dataId.graph.spatial.names, {"skymap_regions"}) 

642 self.assertEqual(dataId.region, dataId.records["tract"].region) 

643 for dataId in self.randomDataIds(n=4).subset( 

644 DimensionGraph(self.allDataIds.universe, names=["patch"]) 

645 ): 

646 self.assertIsNotNone(dataId.region) 

647 self.assertEqual(dataId.graph.spatial.names, {"skymap_regions"}) 

648 self.assertEqual(dataId.region, dataId.records["patch"].region) 

649 

650 def testTimespans(self): 

651 """Test that data IDs for a few known dimensions have the expected 

652 timespans. 

653 """ 

654 for dataId in self.randomDataIds(n=4).subset( 

655 DimensionGraph(self.allDataIds.universe, names=["visit"]) 

656 ): 

657 self.assertIsNotNone(dataId.timespan) 

658 self.assertEqual(dataId.graph.temporal.names, {"observation_timespans"}) 

659 self.assertEqual(dataId.timespan, dataId.records["visit"].timespan) 

660 

661 def testIterableStatusFlags(self): 

662 """Test that DataCoordinateSet and DataCoordinateSequence compute 

663 their hasFull and hasRecords flags correctly from their elements. 

664 """ 

665 dataIds = self.randomDataIds(n=10) 

666 split = self.splitByStateFlags(dataIds) 

667 for cls in (DataCoordinateSet, DataCoordinateSequence): 

668 self.assertTrue(cls(split.expanded, graph=dataIds.graph, check=True).hasFull()) 

669 self.assertTrue(cls(split.expanded, graph=dataIds.graph, check=False).hasFull()) 

670 self.assertTrue(cls(split.expanded, graph=dataIds.graph, check=True).hasRecords()) 

671 self.assertTrue(cls(split.expanded, graph=dataIds.graph, check=False).hasRecords()) 

672 self.assertTrue(cls(split.complete, graph=dataIds.graph, check=True).hasFull()) 

673 self.assertTrue(cls(split.complete, graph=dataIds.graph, check=False).hasFull()) 

674 self.assertFalse(cls(split.complete, graph=dataIds.graph, check=True).hasRecords()) 

675 self.assertFalse(cls(split.complete, graph=dataIds.graph, check=False).hasRecords()) 

676 with self.assertRaises(ValueError): 

677 cls(split.complete, graph=dataIds.graph, hasRecords=True, check=True) 

678 self.assertEqual( 

679 cls(split.minimal, graph=dataIds.graph, check=True).hasFull(), not dataIds.graph.implied 

680 ) 

681 self.assertEqual( 

682 cls(split.minimal, graph=dataIds.graph, check=False).hasFull(), not dataIds.graph.implied 

683 ) 

684 self.assertFalse(cls(split.minimal, graph=dataIds.graph, check=True).hasRecords()) 

685 self.assertFalse(cls(split.minimal, graph=dataIds.graph, check=False).hasRecords()) 

686 with self.assertRaises(ValueError): 

687 cls(split.minimal, graph=dataIds.graph, hasRecords=True, check=True) 

688 if dataIds.graph.implied: 

689 with self.assertRaises(ValueError): 

690 cls(split.minimal, graph=dataIds.graph, hasFull=True, check=True) 

691 

692 def testSetOperations(self): 

693 """Test for self-consistency across DataCoordinateSet's operations.""" 

694 c = self.randomDataIds(n=10).toSet() 

695 a = self.randomDataIds(n=20).toSet() | c 

696 b = self.randomDataIds(n=20).toSet() | c 

697 # Make sure we don't have a particularly unlucky random seed, since 

698 # that would make a lot of this test uninteresting. 

699 self.assertNotEqual(a, b) 

700 self.assertGreater(len(a), 0) 

701 self.assertGreater(len(b), 0) 

702 # The rest of the tests should not depend on the random seed. 

703 self.assertEqual(a, a) 

704 self.assertNotEqual(a, a.toSequence()) 

705 self.assertEqual(a, a.toSequence().toSet()) 

706 self.assertEqual(a, a.toSequence().toSet()) 

707 self.assertEqual(b, b) 

708 self.assertNotEqual(b, b.toSequence()) 

709 self.assertEqual(b, b.toSequence().toSet()) 

710 self.assertEqual(a & b, a.intersection(b)) 

711 self.assertLessEqual(a & b, a) 

712 self.assertLessEqual(a & b, b) 

713 self.assertEqual(a | b, a.union(b)) 

714 self.assertGreaterEqual(a | b, a) 

715 self.assertGreaterEqual(a | b, b) 

716 self.assertEqual(a - b, a.difference(b)) 

717 self.assertLessEqual(a - b, a) 

718 self.assertLessEqual(b - a, b) 

719 self.assertEqual(a ^ b, a.symmetric_difference(b)) 

720 self.assertGreaterEqual(a ^ b, (a | b) - (a & b)) 

721 

722 

723if __name__ == "__main__": 723 ↛ 724line 723 didn't jump to line 724, because the condition on line 723 was never true

724 unittest.main()