Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ["DimensionGraph"] 

25 

26import itertools 

27from types import MappingProxyType 

28from typing import ( 

29 AbstractSet, 

30 Any, 

31 Dict, 

32 Iterable, 

33 Iterator, 

34 Mapping, 

35 Optional, 

36 Set, 

37 Tuple, 

38 TYPE_CHECKING, 

39 Union, 

40) 

41 

42from ..named import NamedValueAbstractSet, NamedValueSet 

43from ..utils import immutable 

44from .._topology import TopologicalSpace, TopologicalFamily 

45 

46 

47if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 47 ↛ 48line 47 didn't jump to line 48, because the condition on line 47 was never true

48 from ._universe import DimensionUniverse 

49 from ._elements import DimensionElement, Dimension 

50 from ._governor import GovernorDimension 

51 

52 

53@immutable 

54class DimensionGraph: 

55 """An immutable, dependency-complete collection of dimensions. 

56 

57 `DimensionGraph` behaves in many respects like a set of `Dimension` 

58 instances that maintains several special subsets and supersets of 

59 related `DimensionElement` instances. It does not fully implement the 

60 `collections.abc.Set` interface, as its automatic expansion of dependencies 

61 would make set difference and XOR operations behave surprisingly. 

62 

63 It also provides dict-like lookup of `DimensionElement` instances from 

64 their names. 

65 

66 Parameters 

67 ---------- 

68 universe : `DimensionUniverse` 

69 The special graph of all known dimensions of which this graph will be 

70 a subset. 

71 dimensions : iterable of `Dimension`, optional 

72 An iterable of `Dimension` instances that must be included in the 

73 graph. All (recursive) dependencies of these dimensions will also 

74 be included. At most one of ``dimensions`` and ``names`` must be 

75 provided. 

76 names : iterable of `str`, optional 

77 An iterable of the names of dimensiosn that must be included in the 

78 graph. All (recursive) dependencies of these dimensions will also 

79 be included. At most one of ``dimensions`` and ``names`` must be 

80 provided. 

81 conform : `bool`, optional 

82 If `True` (default), expand to include dependencies. `False` should 

83 only be used for callers that can guarantee that other arguments are 

84 already correctly expanded, and is primarily for internal use. 

85 

86 Notes 

87 ----- 

88 `DimensionGraph` should be used instead of other collections in most 

89 contexts where a collection of dimensions is required and a 

90 `DimensionUniverse` is available. Exceptions include cases where order 

91 matters (and is different from the consistent ordering defined by the 

92 `DimensionUniverse`), or complete `~collection.abc.Set` semantics are 

93 required. 

94 """ 

95 def __new__( 

96 cls, 

97 universe: DimensionUniverse, 

98 dimensions: Optional[Iterable[Dimension]] = None, 

99 names: Optional[Iterable[str]] = None, 

100 conform: bool = True 

101 ) -> DimensionGraph: 

102 conformedNames: Set[str] 

103 if names is None: 

104 if dimensions is None: 

105 conformedNames = set() 

106 else: 

107 try: 

108 # Optimize for NamedValueSet/NamedKeyDict, though that's 

109 # not required. 

110 conformedNames = set(dimensions.names) # type: ignore 

111 except AttributeError: 

112 conformedNames = set(d.name for d in dimensions) 

113 else: 

114 if dimensions is not None: 

115 raise TypeError("Only one of 'dimensions' and 'names' may be provided.") 

116 conformedNames = set(names) 

117 if conform: 

118 universe.expandDimensionNameSet(conformedNames) 

119 # Look in the cache of existing graphs, with the expanded set of names. 

120 cacheKey = frozenset(conformedNames) 

121 self = universe._cache.get(cacheKey, None) 

122 if self is not None: 

123 return self 

124 # This is apparently a new graph. Create it, and add it to the cache. 

125 self = super().__new__(cls) 

126 universe._cache[cacheKey] = self 

127 self.universe = universe 

128 # Reorder dimensions by iterating over the universe (which is 

129 # ordered already) and extracting the ones in the set. 

130 self.dimensions = NamedValueSet(universe.sorted(conformedNames)).freeze() 

131 # Make a set that includes both the dimensions and any 

132 # DimensionElements whose dependencies are in self.dimensions. 

133 self.elements = NamedValueSet(e for e in universe.getStaticElements() 

134 if e.required.names <= self.dimensions.names).freeze() 

135 self._finish() 

136 return self 

137 

138 def _finish(self) -> None: 

139 # Make a set containing just the governor dimensions in this graph. 

140 # Need local import to avoid cycle. 

141 from ._governor import GovernorDimension 

142 self.governors = NamedValueSet( 

143 d for d in self.dimensions if isinstance(d, GovernorDimension) 

144 ).freeze() 

145 # Split dependencies up into "required" and "implied" subsets. 

146 # Note that a dimension may be required in one graph and implied in 

147 # another. 

148 required: NamedValueSet[Dimension] = NamedValueSet() 

149 implied: NamedValueSet[Dimension] = NamedValueSet() 

150 for i1, dim1 in enumerate(self.dimensions): 

151 for i2, dim2 in enumerate(self.dimensions): 

152 if dim1.name in dim2.implied.names: 

153 implied.add(dim1) 

154 break 

155 else: 

156 # If no other dimension implies dim1, it's required. 

157 required.add(dim1) 

158 self.required = required.freeze() 

159 self.implied = implied.freeze() 

160 

161 self.topology = MappingProxyType({ 

162 space: NamedValueSet(e.topology[space] for e in self.elements if space in e.topology).freeze() 

163 for space in TopologicalSpace.__members__.values() 

164 }) 

165 

166 # Build mappings from dimension to index; this is really for 

167 # DataCoordinate, but we put it in DimensionGraph because many 

168 # (many!) DataCoordinates will share the same DimensionGraph, and 

169 # we want them to be lightweight. The order here is what's convenient 

170 # for DataCoordinate: all required dimensions before all implied 

171 # dimensions. 

172 self._dataCoordinateIndices: Dict[str, int] = { 

173 name: i for i, name in enumerate(itertools.chain(self.required.names, self.implied.names)) 

174 } 

175 

176 def __getnewargs__(self) -> tuple: 

177 return (self.universe, None, tuple(self.dimensions.names), False) 

178 

179 def __deepcopy__(self, memo: dict) -> DimensionGraph: 

180 # DimensionGraph is recursively immutable; see note in @immutable 

181 # decorator. 

182 return self 

183 

184 @property 

185 def names(self) -> AbstractSet[str]: 

186 """A set of the names of all dimensions in the graph (`KeysView`). 

187 """ 

188 return self.dimensions.names 

189 

190 def __iter__(self) -> Iterator[Dimension]: 

191 """Iterate over all dimensions in the graph (and true `Dimension` 

192 instances only). 

193 """ 

194 return iter(self.dimensions) 

195 

196 def __len__(self) -> int: 

197 """Return the number of dimensions in the graph (and true `Dimension` 

198 instances only). 

199 """ 

200 return len(self.dimensions) 

201 

202 def __contains__(self, element: Union[str, DimensionElement]) -> bool: 

203 """Return `True` if the given element or element name is in the graph. 

204 

205 This test covers all `DimensionElement` instances in ``self.elements``, 

206 not just true `Dimension` instances). 

207 """ 

208 return element in self.elements 

209 

210 def __getitem__(self, name: str) -> DimensionElement: 

211 """Return the element with the given name. 

212 

213 This lookup covers all `DimensionElement` instances in 

214 ``self.elements``, not just true `Dimension` instances). 

215 """ 

216 return self.elements[name] 

217 

218 def get(self, name: str, default: Any = None) -> DimensionElement: 

219 """Return the element with the given name. 

220 

221 This lookup covers all `DimensionElement` instances in 

222 ``self.elements``, not just true `Dimension` instances). 

223 """ 

224 return self.elements.get(name, default) 

225 

226 def __str__(self) -> str: 

227 return str(self.dimensions) 

228 

229 def __repr__(self) -> str: 

230 return f"DimensionGraph({str(self)})" 

231 

232 def isdisjoint(self, other: DimensionGraph) -> bool: 

233 """Test whether the intersection of two graphs is empty. 

234 

235 Returns `True` if either operand is the empty. 

236 """ 

237 return self.dimensions.isdisjoint(other.dimensions) 

238 

239 def issubset(self, other: DimensionGraph) -> bool: 

240 """Test whether all dimensions in ``self`` are also in ``other``. 

241 

242 Returns `True` if ``self`` is empty. 

243 """ 

244 return self.dimensions <= other.dimensions 

245 

246 def issuperset(self, other: DimensionGraph) -> bool: 

247 """Test whether all dimensions in ``other`` are also in ``self``. 

248 

249 Returns `True` if ``other`` is empty. 

250 """ 

251 return self.dimensions >= other.dimensions 

252 

253 def __eq__(self, other: Any) -> bool: 

254 """Test whether ``self`` and ``other`` have exactly the same dimensions 

255 and elements. 

256 """ 

257 if isinstance(other, DimensionGraph): 

258 return self.dimensions == other.dimensions 

259 else: 

260 return False 

261 

262 def __hash__(self) -> int: 

263 return hash(tuple(self.dimensions.names)) 

264 

265 def __le__(self, other: DimensionGraph) -> bool: 

266 """Test whether ``self`` is a subset of ``other``. 

267 """ 

268 return self.dimensions <= other.dimensions 

269 

270 def __ge__(self, other: DimensionGraph) -> bool: 

271 """Test whether ``self`` is a superset of ``other``. 

272 """ 

273 return self.dimensions >= other.dimensions 

274 

275 def __lt__(self, other: DimensionGraph) -> bool: 

276 """Test whether ``self`` is a strict subset of ``other``. 

277 """ 

278 return self.dimensions < other.dimensions 

279 

280 def __gt__(self, other: DimensionGraph) -> bool: 

281 """Test whether ``self`` is a strict superset of ``other``. 

282 """ 

283 return self.dimensions > other.dimensions 

284 

285 def union(self, *others: DimensionGraph) -> DimensionGraph: 

286 """Construct a new graph containing all dimensions in any of the 

287 operands. 

288 

289 The elements of the returned graph may exceed the naive union of 

290 their elements, as some `DimensionElement` instances are included 

291 in graphs whenever multiple dimensions are present, and those 

292 dependency dimensions could have been provided by different operands. 

293 """ 

294 names = set(self.names).union(*[other.names for other in others]) 

295 return DimensionGraph(self.universe, names=names) 

296 

297 def intersection(self, *others: DimensionGraph) -> DimensionGraph: 

298 """Construct a new graph containing only dimensions in all of the 

299 operands. 

300 """ 

301 names = set(self.names).intersection(*[other.names for other in others]) 

302 return DimensionGraph(self.universe, names=names) 

303 

304 def __or__(self, other: DimensionGraph) -> DimensionGraph: 

305 """Construct a new graph containing all dimensions in any of the 

306 operands. 

307 

308 See `union`. 

309 """ 

310 return self.union(other) 

311 

312 def __and__(self, other: DimensionGraph) -> DimensionGraph: 

313 """Construct a new graph containing only dimensions in all of the 

314 operands. 

315 """ 

316 return self.intersection(other) 

317 

318 @property 

319 def primaryKeyTraversalOrder(self) -> Tuple[DimensionElement, ...]: 

320 """Return a tuple of all elements in an order allows records to be 

321 found given their primary keys, starting from only the primary keys of 

322 required dimensions (`tuple` [ `DimensionRecord` ]). 

323 

324 Unlike the table definition/topological order (which is what 

325 DimensionUniverse.sorted gives you), when dimension A implies 

326 dimension B, dimension A appears first. 

327 """ 

328 order = getattr(self, "_primaryKeyTraversalOrder", None) 

329 if order is None: 

330 done: Set[str] = set() 

331 order = [] 

332 

333 def addToOrder(element: DimensionElement) -> None: 

334 if element.name in done: 

335 return 

336 predecessors = set(element.required.names) 

337 predecessors.discard(element.name) 

338 if not done.issuperset(predecessors): 

339 return 

340 order.append(element) 

341 done.add(element.name) 

342 for other in element.implied: 

343 addToOrder(other) 

344 

345 while not done.issuperset(self.required): 

346 for dimension in self.required: 

347 addToOrder(dimension) 

348 

349 order.extend(element for element in self.elements if element.name not in done) 

350 order = tuple(order) 

351 self._primaryKeyTraversalOrder = order 

352 return order 

353 

354 @property 

355 def spatial(self) -> NamedValueAbstractSet[TopologicalFamily]: 

356 """The `~TopologicalSpace.SPATIAL` families represented by the elements 

357 in this graph. 

358 """ 

359 return self.topology[TopologicalSpace.SPATIAL] 

360 

361 @property 

362 def temporal(self) -> NamedValueAbstractSet[TopologicalFamily]: 

363 """The `~TopologicalSpace.TEMPORAL` families represented by the 

364 elements in this graph. 

365 """ 

366 return self.topology[TopologicalSpace.TEMPORAL] 

367 

368 # Class attributes below are shadowed by instance attributes, and are 

369 # present just to hold the docstrings for those instance attributes. 

370 

371 universe: DimensionUniverse 

372 """The set of all known dimensions, of which this graph is a subset 

373 (`DimensionUniverse`). 

374 """ 

375 

376 dimensions: NamedValueAbstractSet[Dimension] 

377 """A true `~collections.abc.Set` of all true `Dimension` instances in the 

378 graph (`NamedValueAbstractSet` of `Dimension`). 

379 

380 This is the set used for iteration, ``len()``, and most set-like operations 

381 on `DimensionGraph` itself. 

382 """ 

383 

384 elements: NamedValueAbstractSet[DimensionElement] 

385 """A true `~collections.abc.Set` of all `DimensionElement` instances in the 

386 graph; a superset of `dimensions` (`NamedValueAbstractSet` of 

387 `DimensionElement`). 

388 

389 This is the set used for dict-like lookups, including the ``in`` operator, 

390 on `DimensionGraph` itself. 

391 """ 

392 

393 governors: NamedValueAbstractSet[GovernorDimension] 

394 """A true `~collections.abc.Set` of all true `GovernorDimension` instances 

395 in the graph (`NamedValueAbstractSet` of `GovernorDimension`). 

396 """ 

397 

398 required: NamedValueAbstractSet[Dimension] 

399 """The subset of `dimensions` whose elments must be directly identified via 

400 their primary keys in a data ID in order to identify the rest of the 

401 elements in the graph (`NamedValueAbstractSet` of `Dimension`). 

402 """ 

403 

404 implied: NamedValueAbstractSet[Dimension] 

405 """The subset of `dimensions` whose elements need not be directly 

406 identified via their primary keys in a data ID (`NamedValueAbstractSet` of 

407 `Dimension`). 

408 """ 

409 

410 topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]] 

411 """Families of elements in this graph that can participate in topological 

412 relationships (`Mapping` from `TopologicalSpace` to 

413 `NamedValueAbstractSet` of `TopologicalFamily`). 

414 """