Coverage for tests/test_dimensions.py: 11%

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28import copy 

29import itertools 

30import math 

31import os 

32import pickle 

33import unittest 

34from collections.abc import Iterator 

35from dataclasses import dataclass 

36from random import Random 

37 

38import pydantic 

39 

40import lsst.sphgeom 

41from lsst.daf.butler import ( 

42 Config, 

43 DataCoordinate, 

44 DataCoordinateSequence, 

45 DataCoordinateSet, 

46 DatasetType, 

47 Dimension, 

48 DimensionConfig, 

49 DimensionElement, 

50 DimensionGroup, 

51 DimensionNameError, 

52 DimensionPacker, 

53 DimensionUniverse, 

54 NamedKeyDict, 

55 NamedValueSet, 

56 ddl, 

57) 

58from lsst.daf.butler.tests.utils import create_populated_sqlite_registry 

59from lsst.daf.butler.timespan_database_representation import TimespanDatabaseRepresentation 

60 

61DIMENSION_DATA_FILE = os.path.normpath( 

62 os.path.join(os.path.dirname(__file__), "data", "registry", "hsc-rc2-subset.yaml") 

63) 

64 

65 

66def loadDimensionData() -> DataCoordinateSequence: 

67 """Load dimension data from an export file included in the code repository. 

68 

69 Returns 

70 ------- 

71 dataIds : `DataCoordinateSet` 

72 A set containing all data IDs in the export file. 

73 """ 

74 # Create an in-memory SQLite database and Registry just to import the YAML 

75 # data and retrieve it as a set of DataCoordinate objects. 

76 butler = create_populated_sqlite_registry(DIMENSION_DATA_FILE) 

77 dimensions = butler.registry.dimensions.conform(["visit", "detector", "tract", "patch"]) 

78 return butler.registry.queryDataIds(dimensions).expanded().toSequence() 

79 

80 

81class ConcreteTestDimensionPacker(DimensionPacker): 

82 """A concrete `DimensionPacker` for testing its base class implementations. 

83 

84 This class just returns the detector ID as-is. 

85 """ 

86 

87 def __init__(self, fixed: DataCoordinate, dimensions: DimensionGroup): 

88 super().__init__(fixed, dimensions) 

89 self._n_detectors = fixed.records["instrument"].detector_max 

90 self._max_bits = (self._n_detectors - 1).bit_length() 

91 

92 @property 

93 def maxBits(self) -> int: 

94 # Docstring inherited from DimensionPacker.maxBits 

95 return self._max_bits 

96 

97 def _pack(self, dataId: DataCoordinate) -> int: 

98 # Docstring inherited from DimensionPacker._pack 

99 return dataId["detector"] 

100 

101 def unpack(self, packedId: int) -> DataCoordinate: 

102 # Docstring inherited from DimensionPacker.unpack 

103 return DataCoordinate.standardize( 

104 { 

105 "instrument": self.fixed["instrument"], 

106 "detector": packedId, 

107 }, 

108 dimensions=self._dimensions, 

109 ) 

110 

111 

112class DimensionTestCase(unittest.TestCase): 

113 """Tests for dimensions. 

114 

115 All tests here rely on the content of ``config/dimensions.yaml``, either 

116 to test that the definitions there are read in properly or just as generic 

117 data for testing various operations. 

118 """ 

119 

120 def setUp(self): 

121 self.universe = DimensionUniverse() 

122 

123 def checkGroupInvariants(self, group: DimensionGroup): 

124 elements = list(group.elements) 

125 for n, element_name in enumerate(elements): 

126 element = self.universe[element_name] 

127 # Ordered comparisons on graphs behave like sets. 

128 self.assertLessEqual(element.minimal_group, group) 

129 # Ordered comparisons on elements correspond to the ordering within 

130 # a DimensionUniverse (topological, with deterministic 

131 # tiebreakers). 

132 for other_name in elements[:n]: 

133 other = self.universe[other_name] 

134 self.assertLess(other, element) 

135 self.assertLessEqual(other, element) 

136 for other_name in elements[n + 1 :]: 

137 other = self.universe[other_name] 

138 self.assertGreater(other, element) 

139 self.assertGreaterEqual(other, element) 

140 if isinstance(element, Dimension): 

141 self.assertEqual(element.minimal_group.required, element.required) 

142 self.assertEqual(self.universe.conform(group.required), group) 

143 self.assertCountEqual( 

144 group.required, 

145 [ 

146 dimension_name 

147 for dimension_name in group.names 

148 if not any( 

149 dimension_name in self.universe[other_name].minimal_group.implied 

150 for other_name in group.elements 

151 ) 

152 ], 

153 ) 

154 self.assertCountEqual(group.implied, group.names - group.required) 

155 self.assertCountEqual(group.names, itertools.chain(group.required, group.implied)) 

156 # Check primary key traversal order: each element should follow any it 

157 # requires, and element that is implied by any other in the graph 

158 # follow at least one of those. 

159 seen: set[str] = set() 

160 for element_name in group.lookup_order: 

161 element = self.universe[element_name] 

162 with self.subTest(required=group.required, implied=group.implied, element=element): 

163 seen.add(element_name) 

164 self.assertLessEqual(element.minimal_group.required, seen) 

165 if element_name in group.implied: 

166 self.assertTrue(any(element_name in self.universe[s].implied for s in seen)) 

167 self.assertCountEqual(seen, group.elements) 

168 

169 def testConfigPresent(self): 

170 config = self.universe.dimensionConfig 

171 self.assertIsInstance(config, DimensionConfig) 

172 

173 def test_group_union(self) -> None: 

174 """Test unions of DimensionGroups.""" 

175 a = self.universe.conform(["visit"]) 

176 b = self.universe.conform(["detector"]) 

177 self.assertIs(a.union(b), self.universe.conform(["visit", "detector"])) 

178 self.assertIs(DimensionGroup.union(a, b), self.universe.conform(["visit", "detector"])) 

179 self.assertIs(DimensionGroup.union(universe=self.universe), self.universe.empty) 

180 with self.assertRaises(TypeError): 

181 DimensionGroup.union() 

182 

183 def test_group_set_ops(self) -> None: 

184 """Test set operations on DimensionGroups.""" 

185 a = self.universe.conform(["visit"]) 

186 b = self.universe.conform(["detector"]) 

187 c = self.universe.conform(["visit", "detector"]) 

188 d = self.universe.conform(["physical_filter"]) 

189 e = self.universe.conform(["detector", "physical_filter"]) 

190 self.assertEqual(a.union(b), c) 

191 self.assertEqual(a.union(d), a) 

192 self.assertEqual(b.union(d), e) 

193 self.assertEqual(a.difference(b), a) 

194 self.assertEqual(a.difference(c), self.universe.empty) 

195 self.assertEqual(a.difference(d), a) 

196 self.assertEqual(c.difference(a), b) 

197 self.assertEqual(a.intersection(c), a) 

198 self.assertEqual(a.intersection(d), d) 

199 self.assertEqual(a.intersection(e), d) 

200 

201 def testCompatibility(self): 

202 # Simple check that should always be true. 

203 self.assertTrue(self.universe.isCompatibleWith(self.universe)) 

204 

205 # Create a universe like the default universe but with a different 

206 # version number. 

207 clone = self.universe.dimensionConfig.copy() 

208 clone["version"] = clone["version"] + 1_000_000 # High version number 

209 universe_clone = DimensionUniverse(config=clone) 

210 with self.assertLogs("lsst.daf.butler.dimensions", "INFO") as cm: 

211 self.assertTrue(self.universe.isCompatibleWith(universe_clone)) 

212 self.assertIn("differing versions", "\n".join(cm.output)) 

213 

214 # Create completely incompatible universe. 

215 config = Config( 

216 { 

217 "version": 1, 

218 "namespace": "compat_test", 

219 "skypix": { 

220 "common": "htm7", 

221 "htm": { 

222 "class": "lsst.sphgeom.HtmPixelization", 

223 "max_level": 24, 

224 }, 

225 }, 

226 "elements": { 

227 "A": { 

228 "keys": [ 

229 { 

230 "name": "id", 

231 "type": "int", 

232 } 

233 ], 

234 "storage": { 

235 "cls": "lsst.daf.butler.registry.dimensions.table.TableDimensionRecordStorage", 

236 }, 

237 }, 

238 "B": { 

239 "keys": [ 

240 { 

241 "name": "id", 

242 "type": "int", 

243 } 

244 ], 

245 "storage": { 

246 "cls": "lsst.daf.butler.registry.dimensions.table.TableDimensionRecordStorage", 

247 }, 

248 }, 

249 }, 

250 "packers": {}, 

251 } 

252 ) 

253 universe2 = DimensionUniverse(config=config) 

254 self.assertFalse(universe2.isCompatibleWith(self.universe)) 

255 

256 def testVersion(self): 

257 self.assertEqual(self.universe.namespace, "daf_butler") 

258 # Test was added starting at version 2. 

259 self.assertGreaterEqual(self.universe.version, 2) 

260 

261 def testConfigRead(self): 

262 self.assertEqual( 

263 set(self.universe.getStaticDimensions().names), 

264 { 

265 "instrument", 

266 "visit", 

267 "visit_system", 

268 "exposure", 

269 "detector", 

270 "group", 

271 "day_obs", 

272 "physical_filter", 

273 "band", 

274 "subfilter", 

275 "skymap", 

276 "tract", 

277 "patch", 

278 } 

279 | {f"htm{level}" for level in range(1, 25)} 

280 | {f"healpix{level}" for level in range(1, 18)}, 

281 ) 

282 

283 def testGraphs(self): 

284 self.checkGroupInvariants(self.universe.empty) 

285 for element in self.universe.getStaticElements(): 

286 self.checkGroupInvariants(element.minimal_group) 

287 

288 def testInstrumentDimensions(self): 

289 group = self.universe.conform(["exposure", "detector", "visit"]) 

290 self.assertCountEqual( 

291 group.names, 

292 ("instrument", "exposure", "detector", "visit", "physical_filter", "band", "group", "day_obs"), 

293 ) 

294 self.assertCountEqual(group.required, ("instrument", "exposure", "detector", "visit")) 

295 self.assertCountEqual(group.implied, ("physical_filter", "band", "group", "day_obs")) 

296 self.assertCountEqual(group.elements - group.names, ("visit_detector_region", "visit_definition")) 

297 self.assertCountEqual(group.governors, {"instrument"}) 

298 for element in group.elements: 

299 self.assertEqual(self.universe[element].has_own_table, element != "band", element) 

300 self.assertEqual( 

301 self.universe[element].implied_union_target, 

302 "physical_filter" if element == "band" else None, 

303 element, 

304 ) 

305 self.assertEqual( 

306 self.universe[element].defines_relationships, 

307 element 

308 in ("visit", "exposure", "physical_filter", "visit_definition", "visit_system_membership"), 

309 element, 

310 ) 

311 

312 def testCalibrationDimensions(self): 

313 group = self.universe.conform(["physical_filter", "detector"]) 

314 self.assertCountEqual(group.names, ("instrument", "detector", "physical_filter", "band")) 

315 self.assertCountEqual(group.required, ("instrument", "detector", "physical_filter")) 

316 self.assertCountEqual(group.implied, ("band",)) 

317 self.assertCountEqual(group.elements, group.names) 

318 self.assertCountEqual(group.governors, {"instrument"}) 

319 self.assertIsNone(group.region_dimension) 

320 self.assertIsNone(group.timespan_dimension) 

321 

322 def testObservationDimensions(self): 

323 group = self.universe.conform(["exposure", "detector", "visit"]) 

324 self.assertCountEqual( 

325 group.names, 

326 ("instrument", "detector", "visit", "exposure", "physical_filter", "band", "group", "day_obs"), 

327 ) 

328 self.assertCountEqual(group.required, ("instrument", "detector", "exposure", "visit")) 

329 self.assertCountEqual(group.implied, ("physical_filter", "band", "group", "day_obs")) 

330 self.assertCountEqual(group.elements - group.names, ("visit_detector_region", "visit_definition")) 

331 self.assertCountEqual(group.spatial.names, ("observation_regions",)) 

332 self.assertCountEqual(group.temporal.names, ("observation_timespans",)) 

333 self.assertCountEqual(group.governors, {"instrument"}) 

334 self.assertEqual(group.region_dimension, "visit_detector_region") 

335 self.assertEqual(group.timespan_dimension, "exposure") 

336 self.assertEqual(group.spatial.names, {"observation_regions"}) 

337 self.assertEqual(group.temporal.names, {"observation_timespans"}) 

338 self.assertEqual(next(iter(group.spatial)).governor, self.universe["instrument"]) 

339 self.assertEqual(next(iter(group.temporal)).governor, self.universe["instrument"]) 

340 self.assertEqual(self.universe["visit_definition"].populated_by, self.universe["visit"]) 

341 self.assertEqual(self.universe["visit_system_membership"].populated_by, self.universe["visit"]) 

342 self.assertEqual(self.universe["visit_detector_region"].populated_by, self.universe["visit"]) 

343 self.assertEqual( 

344 self.universe.get_elements_populated_by(self.universe["visit"]), 

345 NamedValueSet( 

346 { 

347 self.universe["visit"], 

348 self.universe["visit_definition"], 

349 self.universe["visit_system_membership"], 

350 self.universe["visit_detector_region"], 

351 } 

352 ), 

353 ) 

354 

355 def testSkyMapDimensions(self): 

356 group = self.universe.conform(["patch"]) 

357 self.assertEqual(group.names, {"skymap", "tract", "patch"}) 

358 self.assertEqual(group.required, {"skymap", "tract", "patch"}) 

359 self.assertEqual(group.implied, set()) 

360 self.assertEqual(group.elements, group.names) 

361 self.assertEqual(group.governors, {"skymap"}) 

362 self.assertEqual(group.spatial.names, {"skymap_regions"}) 

363 self.assertEqual(next(iter(group.spatial)).governor, self.universe["skymap"]) 

364 

365 def testSubsetCalculation(self): 

366 """Test that independent spatial and temporal options are computed 

367 correctly. 

368 """ 

369 group = self.universe.conform(["visit", "detector", "tract", "patch", "htm7", "exposure"]) 

370 self.assertCountEqual(group.spatial.names, ("observation_regions", "skymap_regions", "htm")) 

371 self.assertCountEqual(group.temporal.names, ("observation_timespans",)) 

372 self.assertEqual(group.timespan_dimension, "exposure") 

373 # Can not choose between visit_detector_region or htm7 or tract/patch. 

374 self.assertIsNone(group.region_dimension) 

375 

376 def testSchemaGeneration(self): 

377 tableSpecs: NamedKeyDict[DimensionElement, ddl.TableSpec] = NamedKeyDict({}) 

378 for element in self.universe.getStaticElements(): 

379 if element.has_own_table: 

380 tableSpecs[element] = element.RecordClass.fields.makeTableSpec( 

381 TimespanReprClass=TimespanDatabaseRepresentation.Compound, 

382 ) 

383 for element, tableSpec in tableSpecs.items(): 

384 for dep in element.required: 

385 with self.subTest(element=element.name, dep=dep.name): 

386 if dep != element: 

387 self.assertIn(dep.name, tableSpec.fields) 

388 self.assertEqual(tableSpec.fields[dep.name].dtype, dep.primaryKey.dtype) 

389 self.assertEqual(tableSpec.fields[dep.name].length, dep.primaryKey.length) 

390 self.assertEqual(tableSpec.fields[dep.name].nbytes, dep.primaryKey.nbytes) 

391 self.assertFalse(tableSpec.fields[dep.name].nullable) 

392 self.assertTrue(tableSpec.fields[dep.name].primaryKey) 

393 else: 

394 self.assertIn(element.primaryKey.name, tableSpec.fields) 

395 self.assertEqual( 

396 tableSpec.fields[element.primaryKey.name].dtype, dep.primaryKey.dtype 

397 ) 

398 self.assertEqual( 

399 tableSpec.fields[element.primaryKey.name].length, dep.primaryKey.length 

400 ) 

401 self.assertEqual( 

402 tableSpec.fields[element.primaryKey.name].nbytes, dep.primaryKey.nbytes 

403 ) 

404 self.assertFalse(tableSpec.fields[element.primaryKey.name].nullable) 

405 self.assertTrue(tableSpec.fields[element.primaryKey.name].primaryKey) 

406 for dep in element.implied: 

407 with self.subTest(element=element.name, dep=dep.name): 

408 self.assertIn(dep.name, tableSpec.fields) 

409 self.assertEqual(tableSpec.fields[dep.name].dtype, dep.primaryKey.dtype) 

410 self.assertFalse(tableSpec.fields[dep.name].primaryKey) 

411 for foreignKey in tableSpec.foreignKeys: 

412 self.assertIn(foreignKey.table, tableSpecs) 

413 self.assertIn(foreignKey.table, element.dimensions) 

414 self.assertEqual(len(foreignKey.source), len(foreignKey.target)) 

415 for source, target in zip(foreignKey.source, foreignKey.target, strict=True): 

416 self.assertIn(source, tableSpec.fields.names) 

417 self.assertIn(target, tableSpecs[foreignKey.table].fields.names) 

418 self.assertEqual( 

419 tableSpec.fields[source].dtype, tableSpecs[foreignKey.table].fields[target].dtype 

420 ) 

421 self.assertEqual( 

422 tableSpec.fields[source].length, tableSpecs[foreignKey.table].fields[target].length 

423 ) 

424 self.assertEqual( 

425 tableSpec.fields[source].nbytes, tableSpecs[foreignKey.table].fields[target].nbytes 

426 ) 

427 

428 def testPickling(self): 

429 # Pickling and copying should always yield the exact same object within 

430 # a single process (cross-process is impossible to test here). 

431 universe1 = DimensionUniverse() 

432 universe2 = pickle.loads(pickle.dumps(universe1)) 

433 universe3 = copy.copy(universe1) 

434 universe4 = copy.deepcopy(universe1) 

435 self.assertIs(universe1, universe2) 

436 self.assertIs(universe1, universe3) 

437 self.assertIs(universe1, universe4) 

438 for element1 in universe1.getStaticElements(): 

439 element2 = pickle.loads(pickle.dumps(element1)) 

440 self.assertIs(element1, element2) 

441 group1 = element1.minimal_group 

442 group2 = pickle.loads(pickle.dumps(group1)) 

443 self.assertIs(group1, group2) 

444 

445 def testSerialization(self): 

446 # Check that dataset types round-trip correctly through serialization. 

447 dataset_type = DatasetType( 

448 "flat", 

449 dimensions=["instrument", "detector", "physical_filter", "band"], 

450 isCalibration=True, 

451 universe=self.universe, 

452 storageClass="int", 

453 ) 

454 roundtripped = DatasetType.from_simple(dataset_type.to_simple(), universe=self.universe) 

455 

456 self.assertEqual(dataset_type, roundtripped) 

457 

458 

459@dataclass 

460class SplitByStateFlags: 

461 """A struct that separates data IDs with different states but the same 

462 values. 

463 """ 

464 

465 minimal: DataCoordinateSequence | None = None 

466 """Data IDs that only contain values for required dimensions. 

467 

468 `DataCoordinateSequence.hasFull()` will return `True` for this if and only 

469 if ``minimal.dimensions.implied`` has no elements. 

470 `DataCoordinate.hasRecords()` will always return `False`. 

471 """ 

472 

473 complete: DataCoordinateSequence | None = None 

474 """Data IDs that contain values for all dimensions. 

475 

476 `DataCoordinateSequence.hasFull()` will always `True` and 

477 `DataCoordinate.hasRecords()` will always return `True` for this attribute. 

478 """ 

479 

480 expanded: DataCoordinateSequence | None = None 

481 """Data IDs that contain values for all dimensions as well as records. 

482 

483 `DataCoordinateSequence.hasFull()` and `DataCoordinate.hasRecords()` will 

484 always return `True` for this attribute. 

485 """ 

486 

487 def chain(self, n: int | None = None) -> Iterator[DataCoordinate]: 

488 """Iterate over the data IDs of different types. 

489 

490 Parameters 

491 ---------- 

492 n : `int`, optional 

493 If provided (`None` is default), iterate over only the ``nth`` 

494 data ID in each attribute. 

495 

496 Yields 

497 ------ 

498 dataId : `DataCoordinate` 

499 A data ID from one of the attributes in this struct. 

500 """ 

501 if n is None: 

502 s = slice(None, None) 

503 else: 

504 s = slice(n, n + 1) 

505 if self.minimal is not None: 

506 yield from self.minimal[s] 

507 if self.complete is not None: 

508 yield from self.complete[s] 

509 if self.expanded is not None: 

510 yield from self.expanded[s] 

511 

512 

513class DataCoordinateTestCase(unittest.TestCase): 

514 """Test `DataCoordinate`.""" 

515 

516 RANDOM_SEED = 10 

517 

518 @classmethod 

519 def setUpClass(cls): 

520 cls.allDataIds = loadDimensionData() 

521 

522 def setUp(self): 

523 self.rng = Random(self.RANDOM_SEED) 

524 

525 def randomDataIds(self, n: int, dataIds: DataCoordinateSequence | None = None): 

526 """Select random data IDs from those loaded from test data. 

527 

528 Parameters 

529 ---------- 

530 n : `int` 

531 Number of data IDs to select. 

532 dataIds : `DataCoordinateSequence`, optional 

533 Data IDs to select from. Defaults to ``self.allDataIds``. 

534 

535 Returns 

536 ------- 

537 selected : `DataCoordinateSequence` 

538 ``n`` Data IDs randomly selected from ``dataIds`` with replacement. 

539 """ 

540 if dataIds is None: 

541 dataIds = self.allDataIds 

542 return DataCoordinateSequence( 

543 self.rng.sample(dataIds, n), 

544 dimensions=dataIds.dimensions, 

545 hasFull=dataIds.hasFull(), 

546 hasRecords=dataIds.hasRecords(), 

547 check=False, 

548 ) 

549 

550 def randomDimensionSubset(self, n: int = 3, group: DimensionGroup | None = None) -> DimensionGroup: 

551 """Generate a random `DimensionGroup` that has a subset of the 

552 dimensions in a given one. 

553 

554 Parameters 

555 ---------- 

556 n : `int` 

557 Number of dimensions to select, before automatic expansion by 

558 `DimensionGroup`. 

559 group : `DimensionGroup`, optional 

560 Dimensions to select from. Defaults to 

561 ``self.allDataIds.dimensions``. 

562 

563 Returns 

564 ------- 

565 selected : `DimensionGroup` 

566 ``n`` or more dimensions randomly selected from ``group`` with 

567 replacement. 

568 """ 

569 if group is None: 

570 group = self.allDataIds.dimensions 

571 return group.universe.conform(self.rng.sample(list(group.names), max(n, len(group)))) 

572 

573 def splitByStateFlags( 

574 self, 

575 dataIds: DataCoordinateSequence | None = None, 

576 *, 

577 expanded: bool = True, 

578 complete: bool = True, 

579 minimal: bool = True, 

580 ) -> SplitByStateFlags: 

581 """Given a sequence of data IDs, generate new equivalent sequences 

582 containing less information. 

583 

584 Parameters 

585 ---------- 

586 dataIds : `DataCoordinateSequence`, optional. 

587 Data IDs to start from. Defaults to ``self.allDataIds``. 

588 ``dataIds.hasRecords()`` and ``dataIds.hasFull()`` must both return 

589 `True`. 

590 expanded : `bool`, optional 

591 If `True` (default) include the original data IDs that contain all 

592 information in the result. 

593 complete : `bool`, optional 

594 If `True` (default) include data IDs for which ``hasFull()`` 

595 returns `True` but ``hasRecords()`` does not. 

596 minimal : `bool`, optional 

597 If `True` (default) include data IDS that only contain values for 

598 required dimensions, for which ``hasFull()`` may not return `True`. 

599 

600 Returns 

601 ------- 

602 split : `SplitByStateFlags` 

603 A dataclass holding the indicated data IDs in attributes that 

604 correspond to the boolean keyword arguments. 

605 """ 

606 if dataIds is None: 

607 dataIds = self.allDataIds 

608 assert dataIds.hasFull() and dataIds.hasRecords() 

609 result = SplitByStateFlags(expanded=dataIds) 

610 if complete: 

611 result.complete = DataCoordinateSequence( 

612 [ 

613 DataCoordinate.standardize(e.mapping, dimensions=dataIds.dimensions) 

614 for e in result.expanded 

615 ], 

616 dimensions=dataIds.dimensions, 

617 ) 

618 self.assertTrue(result.complete.hasFull()) 

619 self.assertFalse(result.complete.hasRecords()) 

620 if minimal: 

621 result.minimal = DataCoordinateSequence( 

622 [ 

623 DataCoordinate.standardize(e.required, dimensions=dataIds.dimensions) 

624 for e in result.expanded 

625 ], 

626 dimensions=dataIds.dimensions, 

627 ) 

628 self.assertEqual(result.minimal.hasFull(), not dataIds.dimensions.implied) 

629 self.assertFalse(result.minimal.hasRecords()) 

630 if not expanded: 

631 result.expanded = None 

632 return result 

633 

634 def testMappingViews(self): 

635 """Test that the ``mapping`` and ``required`` attributes in 

636 `DataCoordinate` are self-consistent and consistent with the 

637 ``dimensions`` property. 

638 """ 

639 for _ in range(5): 

640 dimensions = self.randomDimensionSubset() 

641 dataIds = self.randomDataIds(n=1).subset(dimensions) 

642 split = self.splitByStateFlags(dataIds) 

643 for dataId in split.chain(): 

644 with self.subTest(dataId=dataId): 

645 self.assertEqual(dataId.required.keys(), dataId.dimensions.required) 

646 self.assertEqual( 

647 list(dataId.required.values()), [dataId[d] for d in dataId.dimensions.required] 

648 ) 

649 self.assertEqual( 

650 list(dataId.required_values), [dataId[d] for d in dataId.dimensions.required] 

651 ) 

652 self.assertEqual(dataId.required.keys(), dataId.dimensions.required) 

653 for dataId in itertools.chain(split.complete, split.expanded): 

654 with self.subTest(dataId=dataId): 

655 self.assertTrue(dataId.hasFull()) 

656 self.assertEqual(dataId.dimensions.names, dataId.mapping.keys()) 

657 self.assertEqual( 

658 list(dataId.mapping.values()), [dataId[k] for k in dataId.mapping.keys()] 

659 ) 

660 

661 def test_pickle(self): 

662 for _ in range(5): 

663 dimensions = self.randomDimensionSubset() 

664 dataIds = self.randomDataIds(n=1).subset(dimensions) 

665 split = self.splitByStateFlags(dataIds) 

666 for data_id in split.chain(): 

667 s = pickle.dumps(data_id) 

668 read_data_id: DataCoordinate = pickle.loads(s) 

669 self.assertEqual(data_id, read_data_id) 

670 self.assertEqual(data_id.hasFull(), read_data_id.hasFull()) 

671 self.assertEqual(data_id.hasRecords(), read_data_id.hasRecords()) 

672 if data_id.hasFull(): 

673 self.assertEqual(data_id.mapping, read_data_id.mapping) 

674 if data_id.hasRecords(): 

675 for element_name in data_id.dimensions.elements: 

676 self.assertEqual( 

677 data_id.records[element_name], read_data_id.records[element_name] 

678 ) 

679 

680 def test_record_attributes(self): 

681 """Test that dimension records are available as attributes on expanded 

682 data coordinates. 

683 """ 

684 for _ in range(5): 

685 dimensions = self.randomDimensionSubset() 

686 dataIds = self.randomDataIds(n=1).subset(dimensions) 

687 split = self.splitByStateFlags(dataIds) 

688 for data_id in split.expanded: 

689 for element_name in data_id.dimensions.elements: 

690 self.assertIs(getattr(data_id, element_name), data_id.records[element_name]) 

691 self.assertIn(element_name, dir(data_id)) 

692 with self.assertRaisesRegex(AttributeError, "^not_a_dimension_name$"): 

693 data_id.not_a_dimension_name 

694 for data_id in itertools.chain(split.minimal, split.complete): 

695 for element_name in data_id.dimensions.elements: 

696 with self.assertRaisesRegex(AttributeError, "only available on expanded DataCoordinates"): 

697 getattr(data_id, element_name) 

698 with self.assertRaisesRegex(AttributeError, "^not_a_dimension_name$"): 

699 data_id.not_a_dimension_name 

700 

701 def testEquality(self): 

702 """Test that different `DataCoordinate` instances with different state 

703 flags can be compared with each other and other mappings. 

704 """ 

705 dataIds = self.randomDataIds(n=2) 

706 split = self.splitByStateFlags(dataIds) 

707 # Iterate over all combinations of different states of DataCoordinate, 

708 # with the same underlying data ID values. 

709 for a0, b0 in itertools.combinations(split.chain(0), 2): 

710 self.assertEqual(a0, b0) 

711 # Same thing, for a different data ID value. 

712 for a1, b1 in itertools.combinations(split.chain(1), 2): 

713 self.assertEqual(a1, b1) 

714 # Iterate over all combinations of different states of DataCoordinate, 

715 # with different underlying data ID values. 

716 for a0, b1 in itertools.product(split.chain(0), split.chain(1)): 

717 self.assertNotEqual(a0, b1) 

718 self.assertNotEqual(a1, b0) 

719 

720 def testStandardize(self): 

721 """Test constructing a DataCoordinate from many different kinds of 

722 input via `DataCoordinate.standardize` and `DataCoordinate.subset`. 

723 """ 

724 for _ in range(5): 

725 dimensions = self.randomDimensionSubset() 

726 dataIds = self.randomDataIds(n=1).subset(dimensions)