Coverage for python/lsst/daf/butler/dimensions/_graph.py: 63%

168 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-12-05 11:07 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = ["DimensionGraph", "SerializedDimensionGraph"] 

31 

32import warnings 

33from collections.abc import Iterable, Iterator, Mapping, Set 

34from typing import TYPE_CHECKING, Any, ClassVar, TypeVar, cast 

35 

36from deprecated.sphinx import deprecated 

37from lsst.daf.butler._compat import _BaseModelCompat 

38from lsst.utils.classes import cached_getter, immutable 

39from lsst.utils.introspection import find_outside_stacklevel 

40 

41from .._named import NamedValueAbstractSet, NameMappingSetView 

42from .._topology import TopologicalFamily, TopologicalSpace 

43from ..json import from_json_pydantic, to_json_pydantic 

44from ._group import DimensionGroup 

45 

46if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 

47 from ..registry import Registry 

48 from ._elements import Dimension, DimensionElement 

49 from ._governor import GovernorDimension 

50 from ._skypix import SkyPixDimension 

51 from ._universe import DimensionUniverse 

52 

53 

54class SerializedDimensionGraph(_BaseModelCompat): 

55 """Simplified model of a `DimensionGraph` suitable for serialization.""" 

56 

57 names: list[str] 

58 

59 @classmethod 

60 def direct(cls, *, names: list[str]) -> SerializedDimensionGraph: 

61 """Construct a `SerializedDimensionGraph` directly without validators. 

62 

63 This differs from the pydantic "construct" method in that the arguments 

64 are explicitly what the model requires, and it will recurse through 

65 members, constructing them from their corresponding `direct` methods. 

66 

67 This method should only be called when the inputs are trusted. 

68 """ 

69 return cls.model_construct(names=names) 

70 

71 

72_T = TypeVar("_T", bound="DimensionElement", covariant=True) 

73 

74 

75# TODO: Remove on DM-41326. 

76_NVAS_DEPRECATION_MSG = """DimensionGraph is deprecated in favor of 

77DimensionGroup, which uses sets of str names instead of NamedValueAbstractSets 

78of Dimension or DimensionElement instances. Support for the 

79NamedValueAbstractSet interfaces on this object will be dropped after v27. 

80""" 

81 

82 

83class _DimensionGraphNamedValueSet(NameMappingSetView[_T]): 

84 def __init__(self, keys: Set[str], universe: DimensionUniverse): 

85 super().__init__({k: cast(_T, universe[k]) for k in keys}) 

86 

87 # TODO: Remove on DM-41326. 

88 @deprecated( 

89 _NVAS_DEPRECATION_MSG 

90 + "Use a dict comprehension and DimensionUniverse indexing to construct a mapping when needed.", 

91 version="v27", 

92 category=FutureWarning, 

93 ) 

94 def asMapping(self) -> Mapping[str, _T]: 

95 return super().asMapping() 

96 

97 # TODO: Remove on DM-41326. 

98 @deprecated( 

99 _NVAS_DEPRECATION_MSG + "Use DimensionUniverse for DimensionElement lookups.", 

100 version="v27", 

101 category=FutureWarning, 

102 ) 

103 def __getitem__(self, key: str | _T) -> _T: 

104 return super().__getitem__(key) 

105 

106 def __contains__(self, key: Any) -> bool: 

107 from ._elements import DimensionElement 

108 

109 if isinstance(key, DimensionElement): 

110 warnings.warn( 

111 _NVAS_DEPRECATION_MSG + "'in' expressions must use str keys.", 

112 category=FutureWarning, 

113 stacklevel=find_outside_stacklevel("lsst.daf.butler."), 

114 ) 

115 return super().__contains__(key) 

116 

117 def __iter__(self) -> Iterator[_T]: 

118 # TODO: Remove on DM-41326. 

119 warnings.warn( 

120 _NVAS_DEPRECATION_MSG 

121 + ( 

122 "In the future, iteration will yield str names; for now, use .names " 

123 "to do the same without triggering this warning." 

124 ), 

125 category=FutureWarning, 

126 stacklevel=find_outside_stacklevel("lsst.daf.butler."), 

127 ) 

128 return super().__iter__() 

129 

130 def __eq__(self, other: Any) -> bool: 

131 # TODO: Remove on DM-41326. 

132 warnings.warn( 

133 _NVAS_DEPRECATION_MSG 

134 + ( 

135 "In the future, set-equality will assume str keys; for now, use .names " 

136 "to do the same without triggering this warning." 

137 ), 

138 category=FutureWarning, 

139 stacklevel=find_outside_stacklevel("lsst.daf.butler."), 

140 ) 

141 return super().__eq__(other) 

142 

143 def __le__(self, other: Set[Any]) -> bool: 

144 # TODO: Remove on DM-41326. 

145 warnings.warn( 

146 _NVAS_DEPRECATION_MSG 

147 + ( 

148 "In the future, subset tests will assume str keys; for now, use .names " 

149 "to do the same without triggering this warning." 

150 ), 

151 category=FutureWarning, 

152 stacklevel=find_outside_stacklevel("lsst.daf.butler."), 

153 ) 

154 return super().__le__(other) 

155 

156 def __ge__(self, other: Set[Any]) -> bool: 

157 # TODO: Remove on DM-41326. 

158 warnings.warn( 

159 _NVAS_DEPRECATION_MSG 

160 + ( 

161 "In the future, superset tests will assume str keys; for now, use .names " 

162 "to do the same without triggering this warning." 

163 ), 

164 category=FutureWarning, 

165 stacklevel=find_outside_stacklevel("lsst.daf.butler."), 

166 ) 

167 return super().__ge__(other) 

168 

169 

170# TODO: Remove on DM-41326. 

171@deprecated( 

172 "DimensionGraph is deprecated in favor of DimensionGroup and will be removed after v27.", 

173 category=FutureWarning, 

174 version="v27", 

175) 

176@immutable 

177class DimensionGraph: 

178 """An immutable, dependency-complete collection of dimensions. 

179 

180 `DimensionGraph` is deprecated in favor of `DimensionGroup` and will be 

181 removed after v27. The two types have very similar interfaces, but 

182 `DimensionGroup` does not support direct iteration and its set-like 

183 attributes are of dimension element names, not `DimensionElement` 

184 instances. `DimensionGraph` objects are still returned by certain 

185 non-deprecated methods and properties (most prominently 

186 `DatasetType.dimensions`), and to handle these cases deprecation warnings 

187 are only emitted for operations on `DimensionGraph` that are not 

188 supported by `DimensionGroup` as well. 

189 

190 Parameters 

191 ---------- 

192 universe : `DimensionUniverse` 

193 The special graph of all known dimensions of which this graph will be a 

194 subset. 

195 dimensions : iterable of `Dimension`, optional 

196 An iterable of `Dimension` instances that must be included in the 

197 graph. All (recursive) dependencies of these dimensions will also be 

198 included. At most one of ``dimensions`` and ``names`` must be 

199 provided. 

200 names : iterable of `str`, optional 

201 An iterable of the names of dimensions that must be included in the 

202 graph. All (recursive) dependencies of these dimensions will also be 

203 included. At most one of ``dimensions`` and ``names`` must be 

204 provided. 

205 conform : `bool`, optional 

206 If `True` (default), expand to include dependencies. `False` should 

207 only be used for callers that can guarantee that other arguments are 

208 already correctly expanded, and is primarily for internal use. 

209 

210 Notes 

211 ----- 

212 `DimensionGraph` should be used instead of other collections in most 

213 contexts where a collection of dimensions is required and a 

214 `DimensionUniverse` is available. Exceptions include cases where order 

215 matters (and is different from the consistent ordering defined by the 

216 `DimensionUniverse`), or complete `~collection.abc.Set` semantics are 

217 required. 

218 """ 

219 

220 _serializedType = SerializedDimensionGraph 

221 

222 def __new__( 

223 cls, 

224 universe: DimensionUniverse, 

225 dimensions: Iterable[Dimension] | None = None, 

226 names: Iterable[str] | None = None, 

227 conform: bool = True, 

228 ) -> DimensionGraph: 

229 if names is None: 

230 if dimensions is None: 

231 group = DimensionGroup(universe) 

232 else: 

233 group = DimensionGroup(universe, {d.name for d in dimensions}, _conform=conform) 

234 else: 

235 if dimensions is not None: 

236 raise TypeError("Only one of 'dimensions' and 'names' may be provided.") 

237 group = DimensionGroup(universe, names, _conform=conform) 

238 return group._as_graph() 

239 

240 @property 

241 def universe(self) -> DimensionUniverse: 

242 """Object that manages all known dimensions.""" 

243 return self._group.universe 

244 

245 @property 

246 @deprecated( 

247 _NVAS_DEPRECATION_MSG + "Use '.names' instead of '.dimensions' or '.dimensions.names'.", 

248 version="v27", 

249 category=FutureWarning, 

250 ) 

251 @cached_getter 

252 def dimensions(self) -> NamedValueAbstractSet[Dimension]: 

253 """A true `~collections.abc.Set` of all true `Dimension` instances in 

254 the graph. 

255 """ 

256 return _DimensionGraphNamedValueSet(self._group.names, self._group.universe) 

257 

258 @property 

259 @cached_getter 

260 def elements(self) -> NamedValueAbstractSet[DimensionElement]: 

261 """A true `~collections.abc.Set` of all `DimensionElement` instances in 

262 the graph; a superset of `dimensions` (`NamedValueAbstractSet` of 

263 `DimensionElement`). 

264 """ 

265 return _DimensionGraphNamedValueSet(self._group.elements, self._group.universe) 

266 

267 @property 

268 @cached_getter 

269 def governors(self) -> NamedValueAbstractSet[GovernorDimension]: 

270 """A true `~collections.abc.Set` of all `GovernorDimension` instances 

271 in the graph. 

272 """ 

273 return _DimensionGraphNamedValueSet(self._group.governors, self._group.universe) 

274 

275 @property 

276 @cached_getter 

277 def skypix(self) -> NamedValueAbstractSet[SkyPixDimension]: 

278 """A true `~collections.abc.Set` of all `SkyPixDimension` instances 

279 in the graph. 

280 """ 

281 return _DimensionGraphNamedValueSet(self._group.skypix, self._group.universe) 

282 

283 @property 

284 @cached_getter 

285 def required(self) -> NamedValueAbstractSet[Dimension]: 

286 """The subset of `dimensions` whose elements must be directly 

287 identified via their primary keys in a data ID in order to identify the 

288 rest of the elements in the graph. 

289 """ 

290 return _DimensionGraphNamedValueSet(self._group.required, self._group.universe) 

291 

292 @property 

293 @cached_getter 

294 def implied(self) -> NamedValueAbstractSet[Dimension]: 

295 """The subset of `dimensions` whose elements need not be directly 

296 identified via their primary keys in a data ID. 

297 """ 

298 return _DimensionGraphNamedValueSet(self._group.implied, self._group.universe) 

299 

300 def __getnewargs__(self) -> tuple: 

301 return (self.universe, None, tuple(self._group.names), False) 

302 

303 def __deepcopy__(self, memo: dict) -> DimensionGraph: 

304 # DimensionGraph is recursively immutable; see note in @immutable 

305 # decorator. 

306 return self 

307 

308 @property 

309 def names(self) -> Set[str]: 

310 """Set of the names of all dimensions in the graph.""" 

311 return self._group.names 

312 

313 def to_simple(self, minimal: bool = False) -> SerializedDimensionGraph: 

314 """Convert this class to a simple python type. 

315 

316 This type is suitable for serialization. 

317 

318 Parameters 

319 ---------- 

320 minimal : `bool`, optional 

321 Use minimal serialization. Has no effect on for this class. 

322 

323 Returns 

324 ------- 

325 names : `list` 

326 The names of the dimensions. 

327 """ 

328 # Names are all we can serialize. 

329 return SerializedDimensionGraph(names=list(self.names)) 

330 

331 @classmethod 

332 def from_simple( 

333 cls, 

334 names: SerializedDimensionGraph, 

335 universe: DimensionUniverse | None = None, 

336 registry: Registry | None = None, 

337 ) -> DimensionGraph: 

338 """Construct a new object from the simplified form. 

339 

340 This is assumed to support data data returned from the `to_simple` 

341 method. 

342 

343 Parameters 

344 ---------- 

345 names : `list` of `str` 

346 The names of the dimensions. 

347 universe : `DimensionUniverse` 

348 The special graph of all known dimensions of which this graph will 

349 be a subset. Can be `None` if `Registry` is provided. 

350 registry : `lsst.daf.butler.Registry`, optional 

351 Registry from which a universe can be extracted. Can be `None` 

352 if universe is provided explicitly. 

353 

354 Returns 

355 ------- 

356 graph : `DimensionGraph` 

357 Newly-constructed object. 

358 """ 

359 if universe is None and registry is None: 

360 raise ValueError("One of universe or registry is required to convert names to a DimensionGraph") 

361 if universe is None and registry is not None: 

362 universe = registry.dimensions 

363 if universe is None: 

364 # this is for mypy 

365 raise ValueError("Unable to determine a usable universe") 

366 

367 return cls(names=names.names, universe=universe) 

368 

369 to_json = to_json_pydantic 

370 from_json: ClassVar = classmethod(from_json_pydantic) 

371 

372 def __iter__(self) -> Iterator[Dimension]: 

373 """Iterate over all dimensions in the graph. 

374 

375 (and true `Dimension` instances only). 

376 """ 

377 return iter(self.dimensions) 

378 

379 def __len__(self) -> int: 

380 """Return the number of dimensions in the graph. 

381 

382 (and true `Dimension` instances only). 

383 """ 

384 return len(self._group) 

385 

386 def __contains__(self, element: str | DimensionElement) -> bool: 

387 """Return `True` if the given element or element name is in the graph. 

388 

389 This test covers all `DimensionElement` instances in ``self.elements``, 

390 not just true `Dimension` instances). 

391 """ 

392 return element in self.elements 

393 

394 def __getitem__(self, name: str) -> DimensionElement: 

395 """Return the element with the given name. 

396 

397 This lookup covers all `DimensionElement` instances in 

398 ``self.elements``, not just true `Dimension` instances). 

399 """ 

400 return self.elements[name] 

401 

402 def get(self, name: str, default: Any = None) -> DimensionElement: 

403 """Return the element with the given name. 

404 

405 This lookup covers all `DimensionElement` instances in 

406 ``self.elements``, not just true `Dimension` instances). 

407 """ 

408 return self.elements.get(name, default) 

409 

410 def __str__(self) -> str: 

411 return str(self.as_group()) 

412 

413 def __repr__(self) -> str: 

414 return f"DimensionGraph({str(self)})" 

415 

416 def as_group(self) -> DimensionGroup: 

417 """Return a `DimensionGroup` that represents the same set of 

418 dimensions. 

419 """ 

420 return self._group 

421 

422 def isdisjoint(self, other: DimensionGroup | DimensionGraph) -> bool: 

423 """Test whether the intersection of two graphs is empty. 

424 

425 Returns `True` if either operand is the empty. 

426 """ 

427 return self._group.isdisjoint(other.as_group()) 

428 

429 def issubset(self, other: DimensionGroup | DimensionGraph) -> bool: 

430 """Test whether all dimensions in ``self`` are also in ``other``. 

431 

432 Returns `True` if ``self`` is empty. 

433 """ 

434 return self._group <= other.as_group() 

435 

436 def issuperset(self, other: DimensionGroup | DimensionGraph) -> bool: 

437 """Test whether all dimensions in ``other`` are also in ``self``. 

438 

439 Returns `True` if ``other`` is empty. 

440 """ 

441 return self._group >= other.as_group() 

442 

443 def __eq__(self, other: Any) -> bool: 

444 """Test the arguments have exactly the same dimensions & elements.""" 

445 if isinstance(other, (DimensionGraph, DimensionGroup)): 

446 return self._group == other.as_group() 

447 return False 

448 

449 def __hash__(self) -> int: 

450 return hash(self.as_group()) 

451 

452 def __le__(self, other: DimensionGroup | DimensionGraph) -> bool: 

453 """Test whether ``self`` is a subset of ``other``.""" 

454 return self._group <= other.as_group() 

455 

456 def __ge__(self, other: DimensionGroup | DimensionGraph) -> bool: 

457 """Test whether ``self`` is a superset of ``other``.""" 

458 return self._group >= other.as_group() 

459 

460 def __lt__(self, other: DimensionGroup | DimensionGraph) -> bool: 

461 """Test whether ``self`` is a strict subset of ``other``.""" 

462 return self._group < other.as_group() 

463 

464 def __gt__(self, other: DimensionGroup | DimensionGraph) -> bool: 

465 """Test whether ``self`` is a strict superset of ``other``.""" 

466 return self._group > other.as_group() 

467 

468 def union(self, *others: DimensionGroup | DimensionGraph) -> DimensionGraph: 

469 """Construct a new graph with all dimensions in any of the operands. 

470 

471 The elements of the returned graph may exceed the naive union of 

472 their elements, as some `DimensionElement` instances are included 

473 in graphs whenever multiple dimensions are present, and those 

474 dependency dimensions could have been provided by different operands. 

475 """ 

476 names = set(self.names).union(*[other.names for other in others]) 

477 return self.universe.conform(names)._as_graph() 

478 

479 def intersection(self, *others: DimensionGroup | DimensionGraph) -> DimensionGraph: 

480 """Construct a new graph with only dimensions in all of the operands. 

481 

482 See also `union`. 

483 """ 

484 names = set(self.names).intersection(*[other.names for other in others]) 

485 return self.universe.conform(names)._as_graph() 

486 

487 def __or__(self, other: DimensionGroup | DimensionGraph) -> DimensionGraph: 

488 """Construct a new graph with all dimensions in any of the operands. 

489 

490 See `union`. 

491 """ 

492 return self.union(other) 

493 

494 def __and__(self, other: DimensionGroup | DimensionGraph) -> DimensionGraph: 

495 """Construct a new graph with only dimensions in all of the operands. 

496 

497 See `intersection`. 

498 """ 

499 return self.intersection(other) 

500 

501 # TODO: Remove on DM-41326. 

502 @property 

503 @deprecated( 

504 "DimensionGraph is deprecated in favor of DimensionGroup, which does not have this attribute; " 

505 "use .lookup_order. DimensionGraph will be removed after v27.", 

506 category=FutureWarning, 

507 version="v27", 

508 ) 

509 def primaryKeyTraversalOrder(self) -> tuple[DimensionElement, ...]: 

510 """A tuple of all elements in specific order. 

511 

512 The order allows records to be found given their primary keys, starting 

513 from only the primary keys of required dimensions (`tuple` [ 

514 `DimensionRecord` ]). 

515 

516 Unlike the table definition/topological order (which is what 

517 DimensionUniverse.sorted gives you), when dimension A implies dimension 

518 B, dimension A appears first. 

519 """ 

520 return tuple(self.universe[element_name] for element_name in self._group.lookup_order) 

521 

522 @property 

523 def spatial(self) -> NamedValueAbstractSet[TopologicalFamily]: 

524 """Families represented by the spatial elements in this graph.""" 

525 return self._group.spatial 

526 

527 @property 

528 def temporal(self) -> NamedValueAbstractSet[TopologicalFamily]: 

529 """Families represented by the temporal elements in this graph.""" 

530 return self._group.temporal 

531 

532 # TODO: Remove on DM-41326. 

533 @property 

534 @deprecated( 

535 "DimensionGraph is deprecated in favor of DimensionGroup, which does not have this attribute; " 

536 "use .spatial or .temporal. DimensionGraph will be removed after v27.", 

537 category=FutureWarning, 

538 version="v27", 

539 ) 

540 def topology(self) -> Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]]: 

541 """Families of elements in this graph that can participate in 

542 topological relationships. 

543 """ 

544 return self._group._space_families 

545 

546 _group: DimensionGroup