Coverage for python/lsst/daf/butler/dimensions/_graph.py: 63%

168 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-04-10 10:14 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = ["DimensionGraph", "SerializedDimensionGraph"] 

31 

32import warnings 

33from collections.abc import Iterable, Iterator, Mapping, Set 

34from typing import TYPE_CHECKING, Any, ClassVar, TypeVar, cast 

35 

36import pydantic 

37from deprecated.sphinx import deprecated 

38from lsst.utils.classes import cached_getter, immutable 

39from lsst.utils.introspection import find_outside_stacklevel 

40 

41from .._named import NamedValueAbstractSet, NameMappingSetView 

42from .._topology import TopologicalFamily, TopologicalSpace 

43from ..json import from_json_pydantic, to_json_pydantic 

44from ._group import DimensionGroup 

45 

46if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 

47 from ..registry import Registry 

48 from ._elements import Dimension, DimensionElement 

49 from ._governor import GovernorDimension 

50 from ._skypix import SkyPixDimension 

51 from ._universe import DimensionUniverse 

52 

53 

54class SerializedDimensionGraph(pydantic.BaseModel): 

55 """Simplified model of a `DimensionGraph` suitable for serialization.""" 

56 

57 names: list[str] 

58 

59 @classmethod 

60 def direct(cls, *, names: list[str]) -> SerializedDimensionGraph: 

61 """Construct a `SerializedDimensionGraph` directly without validators. 

62 

63 Parameters 

64 ---------- 

65 names : `list` [`str`] 

66 The names of the dimensions to include. 

67 

68 Returns 

69 ------- 

70 graph : `SerializedDimensionGraph` 

71 Model representing these dimensions. 

72 

73 Notes 

74 ----- 

75 This differs from the pydantic "construct" method in that the arguments 

76 are explicitly what the model requires, and it will recurse through 

77 members, constructing them from their corresponding `direct` methods. 

78 

79 This method should only be called when the inputs are trusted. 

80 """ 

81 return cls.model_construct(names=names) 

82 

83 

84_T = TypeVar("_T", bound="DimensionElement", covariant=True) 

85 

86 

87# TODO: Remove on DM-41326. 

88_NVAS_DEPRECATION_MSG = """DimensionGraph is deprecated in favor of 

89DimensionGroup, which uses sets of str names instead of NamedValueAbstractSets 

90of Dimension or DimensionElement instances. Support for the 

91NamedValueAbstractSet interfaces on this object will be dropped after v27. 

92""" 

93 

94 

95class _DimensionGraphNamedValueSet(NameMappingSetView[_T]): 

96 def __init__(self, keys: Set[str], universe: DimensionUniverse): 

97 super().__init__({k: cast(_T, universe[k]) for k in keys}) 

98 

99 # TODO: Remove on DM-41326. 

100 @deprecated( 

101 _NVAS_DEPRECATION_MSG 

102 + "Use a dict comprehension and DimensionUniverse indexing to construct a mapping when needed.", 

103 version="v27", 

104 category=FutureWarning, 

105 ) 

106 def asMapping(self) -> Mapping[str, _T]: 

107 return super().asMapping() 

108 

109 # TODO: Remove on DM-41326. 

110 @deprecated( 

111 _NVAS_DEPRECATION_MSG + "Use DimensionUniverse for DimensionElement lookups.", 

112 version="v27", 

113 category=FutureWarning, 

114 ) 

115 def __getitem__(self, key: str | _T) -> _T: 

116 return super().__getitem__(key) 

117 

118 def __contains__(self, key: Any) -> bool: 

119 from ._elements import DimensionElement 

120 

121 if isinstance(key, DimensionElement): 

122 warnings.warn( 

123 _NVAS_DEPRECATION_MSG + "'in' expressions must use str keys.", 

124 category=FutureWarning, 

125 stacklevel=find_outside_stacklevel("lsst.daf.butler."), 

126 ) 

127 return super().__contains__(key) 

128 

129 def __iter__(self) -> Iterator[_T]: 

130 # TODO: Remove on DM-41326. 

131 warnings.warn( 

132 _NVAS_DEPRECATION_MSG 

133 + ( 

134 "In the future, iteration will yield str names; for now, use .names " 

135 "to do the same without triggering this warning." 

136 ), 

137 category=FutureWarning, 

138 stacklevel=find_outside_stacklevel("lsst.daf.butler."), 

139 ) 

140 return super().__iter__() 

141 

142 def __eq__(self, other: Any) -> bool: 

143 # TODO: Remove on DM-41326. 

144 warnings.warn( 

145 _NVAS_DEPRECATION_MSG 

146 + ( 

147 "In the future, set-equality will assume str keys; for now, use .names " 

148 "to do the same without triggering this warning." 

149 ), 

150 category=FutureWarning, 

151 stacklevel=find_outside_stacklevel("lsst.daf.butler."), 

152 ) 

153 return super().__eq__(other) 

154 

155 def __le__(self, other: Set[Any]) -> bool: 

156 # TODO: Remove on DM-41326. 

157 warnings.warn( 

158 _NVAS_DEPRECATION_MSG 

159 + ( 

160 "In the future, subset tests will assume str keys; for now, use .names " 

161 "to do the same without triggering this warning." 

162 ), 

163 category=FutureWarning, 

164 stacklevel=find_outside_stacklevel("lsst.daf.butler."), 

165 ) 

166 return super().__le__(other) 

167 

168 def __ge__(self, other: Set[Any]) -> bool: 

169 # TODO: Remove on DM-41326. 

170 warnings.warn( 

171 _NVAS_DEPRECATION_MSG 

172 + ( 

173 "In the future, superset tests will assume str keys; for now, use .names " 

174 "to do the same without triggering this warning." 

175 ), 

176 category=FutureWarning, 

177 stacklevel=find_outside_stacklevel("lsst.daf.butler."), 

178 ) 

179 return super().__ge__(other) 

180 

181 

182# TODO: Remove on DM-41326. 

183@deprecated( 

184 "DimensionGraph is deprecated in favor of DimensionGroup and will be removed after v27.", 

185 category=FutureWarning, 

186 version="v27", 

187) 

188@immutable 

189class DimensionGraph: # numpydoc ignore=PR02 

190 """An immutable, dependency-complete collection of dimensions. 

191 

192 `DimensionGraph` is deprecated in favor of `DimensionGroup` and will be 

193 removed after v27. The two types have very similar interfaces, but 

194 `DimensionGroup` does not support direct iteration and its set-like 

195 attributes are of dimension element names, not `DimensionElement` 

196 instances. `DimensionGraph` objects are still returned by certain 

197 non-deprecated methods and properties (most prominently 

198 `DatasetType.dimensions`), and to handle these cases deprecation warnings 

199 are only emitted for operations on `DimensionGraph` that are not 

200 supported by `DimensionGroup` as well. 

201 

202 Parameters 

203 ---------- 

204 universe : `DimensionUniverse` 

205 The special graph of all known dimensions of which this graph will be a 

206 subset. 

207 dimensions : iterable of `Dimension`, optional 

208 An iterable of `Dimension` instances that must be included in the 

209 graph. All (recursive) dependencies of these dimensions will also be 

210 included. At most one of ``dimensions`` and ``names`` must be 

211 provided. 

212 names : iterable of `str`, optional 

213 An iterable of the names of dimensions that must be included in the 

214 graph. All (recursive) dependencies of these dimensions will also be 

215 included. At most one of ``dimensions`` and ``names`` must be 

216 provided. 

217 conform : `bool`, optional 

218 If `True` (default), expand to include dependencies. `False` should 

219 only be used for callers that can guarantee that other arguments are 

220 already correctly expanded, and is primarily for internal use. 

221 

222 Notes 

223 ----- 

224 `DimensionGraph` should be used instead of other collections in most 

225 contexts where a collection of dimensions is required and a 

226 `DimensionUniverse` is available. Exceptions include cases where order 

227 matters (and is different from the consistent ordering defined by the 

228 `DimensionUniverse`), or complete `~collection.abc.Set` semantics are 

229 required. 

230 """ 

231 

232 _serializedType: ClassVar[type[pydantic.BaseModel]] = SerializedDimensionGraph 

233 

234 def __new__( 

235 cls, 

236 universe: DimensionUniverse, 

237 dimensions: Iterable[Dimension] | None = None, 

238 names: Iterable[str] | None = None, 

239 conform: bool = True, 

240 ) -> DimensionGraph: 

241 if names is None: 

242 if dimensions is None: 

243 group = DimensionGroup(universe) 

244 else: 

245 group = DimensionGroup(universe, {d.name for d in dimensions}, _conform=conform) 

246 else: 

247 if dimensions is not None: 

248 raise TypeError("Only one of 'dimensions' and 'names' may be provided.") 

249 group = DimensionGroup(universe, names, _conform=conform) 

250 return group._as_graph() 

251 

252 @property 

253 def universe(self) -> DimensionUniverse: 

254 """Object that manages all known dimensions.""" 

255 return self._group.universe 

256 

257 @property 

258 @deprecated( 

259 _NVAS_DEPRECATION_MSG + "Use '.names' instead of '.dimensions' or '.dimensions.names'.", 

260 version="v27", 

261 category=FutureWarning, 

262 ) 

263 @cached_getter 

264 def dimensions(self) -> NamedValueAbstractSet[Dimension]: 

265 """A true `~collections.abc.Set` of all true `Dimension` instances in 

266 the graph. 

267 """ 

268 return _DimensionGraphNamedValueSet(self._group.names, self._group.universe) 

269 

270 @property 

271 @cached_getter 

272 def elements(self) -> NamedValueAbstractSet[DimensionElement]: 

273 """A true `~collections.abc.Set` of all `DimensionElement` instances in 

274 the graph; a superset of `dimensions` (`NamedValueAbstractSet` of 

275 `DimensionElement`). 

276 """ 

277 return _DimensionGraphNamedValueSet(self._group.elements, self._group.universe) 

278 

279 @property 

280 @cached_getter 

281 def governors(self) -> NamedValueAbstractSet[GovernorDimension]: 

282 """A true `~collections.abc.Set` of all `GovernorDimension` instances 

283 in the graph. 

284 """ 

285 return _DimensionGraphNamedValueSet(self._group.governors, self._group.universe) 

286 

287 @property 

288 @cached_getter 

289 def skypix(self) -> NamedValueAbstractSet[SkyPixDimension]: 

290 """A true `~collections.abc.Set` of all `SkyPixDimension` instances 

291 in the graph. 

292 """ 

293 return _DimensionGraphNamedValueSet(self._group.skypix, self._group.universe) 

294 

295 @property 

296 @cached_getter 

297 def required(self) -> NamedValueAbstractSet[Dimension]: 

298 """The subset of `dimensions` whose elements must be directly 

299 identified via their primary keys in a data ID in order to identify the 

300 rest of the elements in the graph. 

301 """ 

302 return _DimensionGraphNamedValueSet(self._group.required, self._group.universe) 

303 

304 @property 

305 @cached_getter 

306 def implied(self) -> NamedValueAbstractSet[Dimension]: 

307 """The subset of `dimensions` whose elements need not be directly 

308 identified via their primary keys in a data ID. 

309 """ 

310 return _DimensionGraphNamedValueSet(self._group.implied, self._group.universe) 

311 

312 def __getnewargs__(self) -> tuple: 

313 return (self.universe, None, tuple(self._group.names), False) 

314 

315 def __deepcopy__(self, memo: dict) -> DimensionGraph: 

316 # DimensionGraph is recursively immutable; see note in @immutable 

317 # decorator. 

318 return self 

319 

320 @property 

321 def names(self) -> Set[str]: 

322 """Set of the names of all dimensions in the graph.""" 

323 return self._group.names 

324 

325 def to_simple(self, minimal: bool = False) -> SerializedDimensionGraph: 

326 """Convert this class to a simple python type. 

327 

328 This type is suitable for serialization. 

329 

330 Parameters 

331 ---------- 

332 minimal : `bool`, optional 

333 Use minimal serialization. Has no effect on for this class. 

334 

335 Returns 

336 ------- 

337 names : `list` 

338 The names of the dimensions. 

339 """ 

340 # Names are all we can serialize. 

341 return SerializedDimensionGraph(names=list(self.names)) 

342 

343 @classmethod 

344 def from_simple( 

345 cls, 

346 names: SerializedDimensionGraph, 

347 universe: DimensionUniverse | None = None, 

348 registry: Registry | None = None, 

349 ) -> DimensionGraph: 

350 """Construct a new object from the simplified form. 

351 

352 This is assumed to support data data returned from the `to_simple` 

353 method. 

354 

355 Parameters 

356 ---------- 

357 names : `list` of `str` 

358 The names of the dimensions. 

359 universe : `DimensionUniverse` 

360 The special graph of all known dimensions of which this graph will 

361 be a subset. Can be `None` if `Registry` is provided. 

362 registry : `lsst.daf.butler.Registry`, optional 

363 Registry from which a universe can be extracted. Can be `None` 

364 if universe is provided explicitly. 

365 

366 Returns 

367 ------- 

368 graph : `DimensionGraph` 

369 Newly-constructed object. 

370 """ 

371 if universe is None and registry is None: 

372 raise ValueError("One of universe or registry is required to convert names to a DimensionGraph") 

373 if universe is None and registry is not None: 

374 universe = registry.dimensions 

375 if universe is None: 

376 # this is for mypy 

377 raise ValueError("Unable to determine a usable universe") 

378 

379 return cls(names=names.names, universe=universe) 

380 

381 to_json = to_json_pydantic 

382 from_json: ClassVar = classmethod(from_json_pydantic) 

383 

384 def __iter__(self) -> Iterator[Dimension]: 

385 """Iterate over all dimensions in the graph. 

386 

387 (and true `Dimension` instances only). 

388 """ 

389 return iter(self.dimensions) 

390 

391 def __len__(self) -> int: 

392 """Return the number of dimensions in the graph. 

393 

394 (and true `Dimension` instances only). 

395 """ 

396 return len(self._group) 

397 

398 def __contains__(self, element: str | DimensionElement) -> bool: 

399 """Return `True` if the given element or element name is in the graph. 

400 

401 This test covers all `DimensionElement` instances in ``self.elements``, 

402 not just true `Dimension` instances). 

403 """ 

404 return element in self.elements 

405 

406 def __getitem__(self, name: str) -> DimensionElement: 

407 """Return the element with the given name. 

408 

409 This lookup covers all `DimensionElement` instances in 

410 ``self.elements``, not just true `Dimension` instances). 

411 """ 

412 return self.elements[name] 

413 

414 def get(self, name: str, default: Any = None) -> DimensionElement: 

415 """Return the element with the given name. 

416 

417 This lookup covers all `DimensionElement` instances in 

418 ``self.elements``, not just true `Dimension` instances). 

419 

420 Parameters 

421 ---------- 

422 name : `str` 

423 Name of element to return. 

424 default : `typing.Any` or `None` 

425 Default value if named element is not present. 

426 

427 Returns 

428 ------- 

429 element : `DimensionElement` or `None` 

430 The element found, or the default. 

431 """ 

432 return self.elements.get(name, default) 

433 

434 def __str__(self) -> str: 

435 return str(self.as_group()) 

436 

437 def __repr__(self) -> str: 

438 return f"DimensionGraph({str(self)})" 

439 

440 def as_group(self) -> DimensionGroup: 

441 """Return a `DimensionGroup` that represents the same set of 

442 dimensions. 

443 

444 Returns 

445 ------- 

446 group : `DimensionGroup` 

447 Group that represents the same set of dimensions. 

448 """ 

449 return self._group 

450 

451 def isdisjoint(self, other: DimensionGroup | DimensionGraph) -> bool: 

452 """Test whether the intersection of two graphs is empty. 

453 

454 Parameters 

455 ---------- 

456 other : `DimensionGroup` or `DimensionGraph` 

457 Other graph to compare with. 

458 

459 Returns 

460 ------- 

461 is_disjoint : `bool` 

462 Returns `True` if either operand is the empty. 

463 """ 

464 return self._group.isdisjoint(other.as_group()) 

465 

466 def issubset(self, other: DimensionGroup | DimensionGraph) -> bool: 

467 """Test whether all dimensions in ``self`` are also in ``other``. 

468 

469 Parameters 

470 ---------- 

471 other : `DimensionGroup` or `DimensionGraph` 

472 Other graph to compare with. 

473 

474 Returns 

475 ------- 

476 is_subset : `bool` 

477 Returns `True` if ``self`` is empty. 

478 """ 

479 return self._group <= other.as_group() 

480 

481 def issuperset(self, other: DimensionGroup | DimensionGraph) -> bool: 

482 """Test whether all dimensions in ``other`` are also in ``self``. 

483 

484 Parameters 

485 ---------- 

486 other : `DimensionGroup` or `DimensionGraph` 

487 Other graph to compare with. 

488 

489 Returns 

490 ------- 

491 is_superset : `bool` 

492 Returns `True` if ``other`` is empty. 

493 """ 

494 return self._group >= other.as_group() 

495 

496 def __eq__(self, other: Any) -> bool: 

497 """Test the arguments have exactly the same dimensions & elements.""" 

498 if isinstance(other, DimensionGraph | DimensionGroup): 

499 return self._group == other.as_group() 

500 return False 

501 

502 def __hash__(self) -> int: 

503 return hash(self.as_group()) 

504 

505 def __le__(self, other: DimensionGroup | DimensionGraph) -> bool: 

506 """Test whether ``self`` is a subset of ``other``.""" 

507 return self._group <= other.as_group() 

508 

509 def __ge__(self, other: DimensionGroup | DimensionGraph) -> bool: 

510 """Test whether ``self`` is a superset of ``other``.""" 

511 return self._group >= other.as_group() 

512 

513 def __lt__(self, other: DimensionGroup | DimensionGraph) -> bool: 

514 """Test whether ``self`` is a strict subset of ``other``.""" 

515 return self._group < other.as_group() 

516 

517 def __gt__(self, other: DimensionGroup | DimensionGraph) -> bool: 

518 """Test whether ``self`` is a strict superset of ``other``.""" 

519 return self._group > other.as_group() 

520 

521 def union(self, *others: DimensionGroup | DimensionGraph) -> DimensionGraph: 

522 """Construct a new graph with all dimensions in any of the operands. 

523 

524 Parameters 

525 ---------- 

526 *others : `DimensionGroup` or `DimensionGraph` 

527 Other graphs to join with. 

528 

529 Returns 

530 ------- 

531 union : `DimensionGraph` 

532 The union of this graph wit hall the others. 

533 

534 Notes 

535 ----- 

536 The elements of the returned graph may exceed the naive union of 

537 their elements, as some `DimensionElement` instances are included 

538 in graphs whenever multiple dimensions are present, and those 

539 dependency dimensions could have been provided by different operands. 

540 """ 

541 names = set(self.names).union(*[other.names for other in others]) 

542 return self.universe.conform(names)._as_graph() 

543 

544 def intersection(self, *others: DimensionGroup | DimensionGraph) -> DimensionGraph: 

545 """Construct a new graph with only dimensions in all of the operands. 

546 

547 Parameters 

548 ---------- 

549 *others : `DimensionGroup` or `DimensionGraph` 

550 Other graphs to use. 

551 

552 Returns 

553 ------- 

554 inter : `DimensionGraph` 

555 Intersection of all the graphs. 

556 

557 Notes 

558 ----- 

559 See also `union`. 

560 """ 

561 names = set(self.names).intersection(*[other.names for other in others]) 

562 return self.universe.conform(names)._as_graph() 

563 

564 def __or__(self, other: DimensionGroup | DimensionGraph) -> DimensionGraph: 

565 """Construct a new graph with all dimensions in any of the operands. 

566 

567 See `union`. 

568 """ 

569 return self.union(other) 

570 

571 def __and__(self, other: DimensionGroup | DimensionGraph) -> DimensionGraph: 

572 """Construct a new graph with only dimensions in all of the operands. 

573 

574 See `intersection`. 

575 """ 

576 return self.intersection(other) 

577 

578 # TODO: Remove on DM-41326. 

579 @property 

580 @deprecated( 

581 "DimensionGraph is deprecated in favor of DimensionGroup, which does not have this attribute; " 

582 "use .lookup_order. DimensionGraph will be removed after v27.", 

583 category=FutureWarning, 

584 version="v27", 

585 ) 

586 def primaryKeyTraversalOrder(self) -> tuple[DimensionElement, ...]: 

587 """A tuple of all elements in specific order. 

588 

589 The order allows records to be found given their primary keys, starting 

590 from only the primary keys of required dimensions (`tuple` [ 

591 `DimensionRecord` ]). 

592 

593 Unlike the table definition/topological order (which is what 

594 DimensionUniverse.sorted gives you), when dimension A implies dimension 

595 B, dimension A appears first. 

596 """ 

597 return tuple(self.universe[element_name] for element_name in self._group.lookup_order) 

598 

599 @property 

600 def spatial(self) -> NamedValueAbstractSet[TopologicalFamily]: 

601 """Families represented by the spatial elements in this graph.""" 

602 return self._group.spatial 

603 

604 @property 

605 def temporal(self) -> NamedValueAbstractSet[TopologicalFamily]: 

606 """Families represented by the temporal elements in this graph.""" 

607 return self._group.temporal 

608 

609 # TODO: Remove on DM-41326. 

610 @property 

611 @deprecated( 

612 "DimensionGraph is deprecated in favor of DimensionGroup, which does not have this attribute; " 

613 "use .spatial or .temporal. DimensionGraph will be removed after v27.", 

614 category=FutureWarning, 

615 version="v27", 

616 ) 

617 def topology(self) -> Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]]: 

618 """Families of elements in this graph that can participate in 

619 topological relationships. 

620 """ 

621 return self._group._space_families 

622 

623 _group: DimensionGroup