Coverage for python/lsst/daf/butler/core/dimensions/_universe.py: 35%

150 statements  

« prev     ^ index     » next       coverage.py v6.4.1, created at 2022-06-23 09:44 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ["DimensionUniverse"] 

25 

26import math 

27import pickle 

28from typing import ( 

29 TYPE_CHECKING, 

30 Any, 

31 ClassVar, 

32 Dict, 

33 FrozenSet, 

34 Iterable, 

35 List, 

36 Mapping, 

37 Optional, 

38 Set, 

39 Tuple, 

40 TypeVar, 

41 Union, 

42) 

43 

44from lsst.utils.classes import cached_getter, immutable 

45 

46from .._topology import TopologicalFamily, TopologicalSpace 

47from ..config import Config 

48from ..named import NamedValueAbstractSet, NamedValueSet 

49from ._config import _DEFAULT_NAMESPACE, DimensionConfig 

50from ._database import DatabaseDimensionElement 

51from ._elements import Dimension, DimensionElement 

52from ._governor import GovernorDimension 

53from ._graph import DimensionGraph 

54from ._skypix import SkyPixDimension, SkyPixSystem 

55 

56if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 56 ↛ 57line 56 didn't jump to line 57, because the condition on line 56 was never true

57 from ._coordinate import DataCoordinate 

58 from ._packer import DimensionPacker, DimensionPackerFactory 

59 from .construction import DimensionConstructionBuilder 

60 

61 

62E = TypeVar("E", bound=DimensionElement) 

63 

64 

65@immutable 

66class DimensionUniverse: 

67 """Self-consistent set of dimensions. 

68 

69 A parent class that represents a complete, self-consistent set of 

70 dimensions and their relationships. 

71 

72 `DimensionUniverse` is not a class-level singleton, but all instances are 

73 tracked in a singleton map keyed by the version number and namespace 

74 in the configuration they were loaded from. Because these universes 

75 are solely responsible for constructing `DimensionElement` instances, 

76 these are also indirectly tracked by that singleton as well. 

77 

78 Parameters 

79 ---------- 

80 config : `Config`, optional 

81 Configuration object from which dimension definitions can be extracted. 

82 Ignored if ``builder`` is provided, or if ``version`` is provided and 

83 an instance with that version already exists. 

84 version : `int`, optional 

85 Integer version for this `DimensionUniverse`. If not provided, a 

86 version will be obtained from ``builder`` or ``config``. 

87 namespace : `str`, optional 

88 Namespace of this `DimensionUniverse`, combined with the version 

89 to provide universe safety for registries that use different 

90 dimension definitions. 

91 builder : `DimensionConstructionBuilder`, optional 

92 Builder object used to initialize a new instance. Ignored if 

93 ``version`` is provided and an instance with that version already 

94 exists. Should not have had `~DimensionConstructionBuilder.finish` 

95 called; this will be called if needed by `DimensionUniverse`. 

96 """ 

97 

98 _instances: ClassVar[Dict[Tuple[int, str], DimensionUniverse]] = {} 

99 """Singleton dictionary of all instances, keyed by version. 

100 

101 For internal use only. 

102 """ 

103 

104 def __new__( 

105 cls, 

106 config: Optional[Config] = None, 

107 *, 

108 version: Optional[int] = None, 

109 namespace: Optional[str] = None, 

110 builder: Optional[DimensionConstructionBuilder] = None, 

111 ) -> DimensionUniverse: 

112 # Try to get a version first, to look for existing instances; try to 

113 # do as little work as possible at this stage. 

114 if version is None: 

115 if builder is None: 

116 config = DimensionConfig(config) 

117 version = config["version"] 

118 else: 

119 version = builder.version 

120 

121 # Then a namespace. 

122 if namespace is None: 

123 if builder is None: 

124 config = DimensionConfig(config) 

125 namespace = config.get("namespace", _DEFAULT_NAMESPACE) 

126 else: 

127 namespace = builder.namespace 

128 # if still None use the default 

129 if namespace is None: 

130 namespace = _DEFAULT_NAMESPACE 

131 

132 # See if an equivalent instance already exists. 

133 self: Optional[DimensionUniverse] = cls._instances.get((version, namespace)) 

134 if self is not None: 

135 return self 

136 

137 # Ensure we have a builder, building one from config if necessary. 

138 if builder is None: 

139 config = DimensionConfig(config) 

140 builder = config.makeBuilder() 

141 

142 # Delegate to the builder for most of the construction work. 

143 builder.finish() 

144 

145 # Create the universe instance and create core attributes, mostly 

146 # copying from builder. 

147 self = object.__new__(cls) 

148 assert self is not None 

149 self._cache = {} 

150 self._dimensions = builder.dimensions 

151 self._elements = builder.elements 

152 self._topology = builder.topology 

153 self._packers = builder.packers 

154 self.dimensionConfig = builder.config 

155 commonSkyPix = self._dimensions[builder.commonSkyPixName] 

156 assert isinstance(commonSkyPix, SkyPixDimension) 

157 self.commonSkyPix = commonSkyPix 

158 

159 # Attach self to all elements. 

160 for element in self._elements: 

161 element.universe = self 

162 

163 # Add attribute for special subsets of the graph. 

164 self.empty = DimensionGraph(self, (), conform=False) 

165 

166 # Use the version number and namespace from the config as a key in 

167 # the singleton dict containing all instances; that will let us 

168 # transfer dimension objects between processes using pickle without 

169 # actually going through real initialization, as long as a universe 

170 # with the same version and namespace has already been constructed in 

171 # the receiving process. 

172 self._version = version 

173 self._namespace = namespace 

174 cls._instances[self._version, self._namespace] = self 

175 

176 # Build mappings from element to index. These are used for 

177 # topological-sort comparison operators in DimensionElement itself. 

178 self._elementIndices = {name: i for i, name in enumerate(self._elements.names)} 

179 # Same for dimension to index, sorted topologically across required 

180 # and implied. This is used for encode/decode. 

181 self._dimensionIndices = {name: i for i, name in enumerate(self._dimensions.names)} 

182 

183 return self 

184 

185 @property 

186 def version(self) -> int: 

187 """The version number of this universe. 

188 

189 Returns 

190 ------- 

191 version : `int` 

192 An integer representing the version number of this universe. 

193 Uniquely defined when combined with the `namespace`. 

194 """ 

195 return self._version 

196 

197 @property 

198 def namespace(self) -> str: 

199 """The namespace associated with this universe. 

200 

201 Returns 

202 ------- 

203 namespace : `str` 

204 The namespace. When combined with the `version` can uniquely 

205 define this universe. 

206 """ 

207 return self._namespace 

208 

209 def __repr__(self) -> str: 

210 return f"DimensionUniverse({self._version}, {self._namespace})" 

211 

212 def __getitem__(self, name: str) -> DimensionElement: 

213 return self._elements[name] 

214 

215 def __contains__(self, name: Any) -> bool: 

216 return name in self._elements 

217 

218 def get(self, name: str, default: Optional[DimensionElement] = None) -> Optional[DimensionElement]: 

219 """Return the `DimensionElement` with the given name or a default. 

220 

221 Parameters 

222 ---------- 

223 name : `str` 

224 Name of the element. 

225 default : `DimensionElement`, optional 

226 Element to return if the named one does not exist. Defaults to 

227 `None`. 

228 

229 Returns 

230 ------- 

231 element : `DimensionElement` 

232 The named element. 

233 """ 

234 return self._elements.get(name, default) 

235 

236 def getStaticElements(self) -> NamedValueAbstractSet[DimensionElement]: 

237 """Return a set of all static elements in this universe. 

238 

239 Non-static elements that are created as needed may also exist, but 

240 these are guaranteed to have no direct relationships to other elements 

241 (though they may have spatial or temporal relationships). 

242 

243 Returns 

244 ------- 

245 elements : `NamedValueAbstractSet` [ `DimensionElement` ] 

246 A frozen set of `DimensionElement` instances. 

247 """ 

248 return self._elements 

249 

250 def getStaticDimensions(self) -> NamedValueAbstractSet[Dimension]: 

251 """Return a set of all static dimensions in this universe. 

252 

253 Non-static dimensions that are created as needed may also exist, but 

254 these are guaranteed to have no direct relationships to other elements 

255 (though they may have spatial or temporal relationships). 

256 

257 Returns 

258 ------- 

259 dimensions : `NamedValueAbstractSet` [ `Dimension` ] 

260 A frozen set of `Dimension` instances. 

261 """ 

262 return self._dimensions 

263 

264 @cached_getter 

265 def getGovernorDimensions(self) -> NamedValueAbstractSet[GovernorDimension]: 

266 """Return a set of all `GovernorDimension` instances in this universe. 

267 

268 Returns 

269 ------- 

270 governors : `NamedValueAbstractSet` [ `GovernorDimension` ] 

271 A frozen set of `GovernorDimension` instances. 

272 """ 

273 return NamedValueSet(d for d in self._dimensions if isinstance(d, GovernorDimension)).freeze() 

274 

275 @cached_getter 

276 def getDatabaseElements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]: 

277 """Return set of all `DatabaseDimensionElement` instances in universe. 

278 

279 This does not include `GovernorDimension` instances, which are backed 

280 by the database but do not inherit from `DatabaseDimensionElement`. 

281 

282 Returns 

283 ------- 

284 elements : `NamedValueAbstractSet` [ `DatabaseDimensionElement` ] 

285 A frozen set of `DatabaseDimensionElement` instances. 

286 """ 

287 return NamedValueSet(d for d in self._elements if isinstance(d, DatabaseDimensionElement)).freeze() 

288 

289 @property # type: ignore 

290 @cached_getter 

291 def skypix(self) -> NamedValueAbstractSet[SkyPixSystem]: 

292 """All skypix systems known to this universe. 

293 

294 (`NamedValueAbstractSet` [ `SkyPixSystem` ]). 

295 """ 

296 return NamedValueSet( 

297 [ 

298 family 

299 for family in self._topology[TopologicalSpace.SPATIAL] 

300 if isinstance(family, SkyPixSystem) 

301 ] 

302 ).freeze() 

303 

304 def getElementIndex(self, name: str) -> int: 

305 """Return the position of the named dimension element. 

306 

307 The position is in this universe's sorting of all elements. 

308 

309 Parameters 

310 ---------- 

311 name : `str` 

312 Name of the element. 

313 

314 Returns 

315 ------- 

316 index : `int` 

317 Sorting index for this element. 

318 """ 

319 return self._elementIndices[name] 

320 

321 def getDimensionIndex(self, name: str) -> int: 

322 """Return the position of the named dimension. 

323 

324 This position is in this universe's sorting of all dimensions. 

325 

326 Parameters 

327 ---------- 

328 name : `str` 

329 Name of the dimension. 

330 

331 Returns 

332 ------- 

333 index : `int` 

334 Sorting index for this dimension. 

335 

336 Notes 

337 ----- 

338 The dimension sort order for a universe is consistent with the element 

339 order (all dimensions are elements), and either can be used to sort 

340 dimensions if used consistently. But there are also some contexts in 

341 which contiguous dimension-only indices are necessary or at least 

342 desirable. 

343 """ 

344 return self._dimensionIndices[name] 

345 

346 def expandDimensionNameSet(self, names: Set[str]) -> None: 

347 """Expand a set of dimension names in-place. 

348 

349 Includes recursive dependencies. 

350 

351 This is an advanced interface for cases where constructing a 

352 `DimensionGraph` (which also expands required dependencies) is 

353 impossible or undesirable. 

354 

355 Parameters 

356 ---------- 

357 names : `set` [ `str` ] 

358 A true `set` of dimension names, to be expanded in-place. 

359 """ 

360 # Keep iterating until the set of names stops growing. This is not as 

361 # efficient as it could be, but we work pretty hard cache 

362 # DimensionGraph instances to keep actual construction rare, so that 

363 # shouldn't matter. 

364 oldSize = len(names) 

365 while True: 

366 # iterate over a temporary copy so we can modify the original 

367 for name in tuple(names): 

368 names.update(self._dimensions[name].required.names) 

369 names.update(self._dimensions[name].implied.names) 

370 if oldSize == len(names): 

371 break 

372 else: 

373 oldSize = len(names) 

374 

375 def extract(self, iterable: Iterable[Union[Dimension, str]]) -> DimensionGraph: 

376 """Construct graph from iterable. 

377 

378 Constructs a `DimensionGraph` from a possibly-heterogenous iterable 

379 of `Dimension` instances and string names thereof. 

380 

381 Constructing `DimensionGraph` directly from names or dimension 

382 instances is slightly more efficient when it is known in advance that 

383 the iterable is not heterogenous. 

384 

385 Parameters 

386 ---------- 

387 iterable: iterable of `Dimension` or `str` 

388 Dimensions that must be included in the returned graph (their 

389 dependencies will be as well). 

390 

391 Returns 

392 ------- 

393 graph : `DimensionGraph` 

394 A `DimensionGraph` instance containing all given dimensions. 

395 """ 

396 names = set() 

397 for item in iterable: 

398 try: 

399 names.add(item.name) # type: ignore 

400 except AttributeError: 

401 names.add(item) 

402 return DimensionGraph(universe=self, names=names) 

403 

404 def sorted(self, elements: Iterable[Union[E, str]], *, reverse: bool = False) -> List[E]: 

405 """Return a sorted version of the given iterable of dimension elements. 

406 

407 The universe's sort order is topological (an element's dependencies 

408 precede it), with an unspecified (but deterministic) approach to 

409 breaking ties. 

410 

411 Parameters 

412 ---------- 

413 elements : iterable of `DimensionElement`. 

414 Elements to be sorted. 

415 reverse : `bool`, optional 

416 If `True`, sort in the opposite order. 

417 

418 Returns 

419 ------- 

420 sorted : `list` of `DimensionElement` 

421 A sorted list containing the same elements that were given. 

422 """ 

423 s = set(elements) 

424 result = [element for element in self._elements if element in s or element.name in s] 

425 if reverse: 

426 result.reverse() 

427 # mypy thinks this can return DimensionElements even if all the user 

428 # passed it was Dimensions; we know better. 

429 return result # type: ignore 

430 

431 def makePacker(self, name: str, dataId: DataCoordinate) -> DimensionPacker: 

432 """Make a dimension packer. 

433 

434 Constructs a `DimensionPacker` that can pack data ID dictionaries 

435 into unique integers. 

436 

437 Parameters 

438 ---------- 

439 name : `str` 

440 Name of the packer, matching a key in the "packers" section of the 

441 dimension configuration. 

442 dataId : `DataCoordinate` 

443 Fully-expanded data ID that identifies the at least the "fixed" 

444 dimensions of the packer (i.e. those that are assumed/given, 

445 setting the space over which packed integer IDs are unique). 

446 ``dataId.hasRecords()`` must return `True`. 

447 """ 

448 return self._packers[name](self, dataId) 

449 

450 def getEncodeLength(self) -> int: 

451 """Return encoded size of graph. 

452 

453 Returns the size (in bytes) of the encoded size of `DimensionGraph` 

454 instances in this universe. 

455 

456 See `DimensionGraph.encode` and `DimensionGraph.decode` for more 

457 information. 

458 """ 

459 return math.ceil(len(self._dimensions) / 8) 

460 

461 @classmethod 

462 def _unpickle(cls, version: int, namespace: Optional[str] = None) -> DimensionUniverse: 

463 """Return an unpickled dimension universe. 

464 

465 Callable used for unpickling. 

466 

467 For internal use only. 

468 """ 

469 if namespace is None: 

470 # Old pickled universe. 

471 namespace = _DEFAULT_NAMESPACE 

472 try: 

473 return cls._instances[version, namespace] 

474 except KeyError as err: 

475 raise pickle.UnpicklingError( 

476 f"DimensionUniverse with version '{version}' and namespace {namespace!r} " 

477 f"not found. Note that DimensionUniverse objects are not " 

478 f"truly serialized; when using pickle to transfer them " 

479 f"between processes, an equivalent instance with the same " 

480 f"version must already exist in the receiving process." 

481 ) from err 

482 

483 def __reduce__(self) -> tuple: 

484 return (self._unpickle, (self._version, self._namespace)) 

485 

486 def __deepcopy__(self, memo: dict) -> DimensionUniverse: 

487 # DimensionUniverse is recursively immutable; see note in @immutable 

488 # decorator. 

489 return self 

490 

491 # Class attributes below are shadowed by instance attributes, and are 

492 # present just to hold the docstrings for those instance attributes. 

493 

494 empty: DimensionGraph 

495 """The `DimensionGraph` that contains no dimensions (`DimensionGraph`). 

496 """ 

497 

498 commonSkyPix: SkyPixDimension 

499 """The special skypix dimension that is used to relate all other spatial 

500 dimensions in the `Registry` database (`SkyPixDimension`). 

501 """ 

502 

503 dimensionConfig: DimensionConfig 

504 """The configuration used to create this Universe (`DimensionConfig`).""" 

505 

506 _cache: Dict[FrozenSet[str], DimensionGraph] 

507 

508 _dimensions: NamedValueAbstractSet[Dimension] 

509 

510 _elements: NamedValueAbstractSet[DimensionElement] 

511 

512 _topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]] 

513 

514 _dimensionIndices: Dict[str, int] 

515 

516 _elementIndices: Dict[str, int] 

517 

518 _packers: Dict[str, DimensionPackerFactory] 

519 

520 _version: int 

521 

522 _namespace: str