Coverage for python/lsst/daf/butler/core/dimensions/_universe.py: 40%

165 statements  

« prev     ^ index     » next       coverage.py v7.3.1, created at 2023-10-02 08:00 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = ["DimensionUniverse"] 

31 

32import logging 

33import math 

34import pickle 

35from collections import defaultdict 

36from collections.abc import Iterable, Mapping 

37from typing import TYPE_CHECKING, Any, ClassVar, TypeVar 

38 

39from deprecated.sphinx import deprecated 

40from lsst.utils.classes import cached_getter, immutable 

41 

42from .._topology import TopologicalFamily, TopologicalSpace 

43from ..config import Config 

44from ..named import NamedValueAbstractSet, NamedValueSet 

45from ._config import _DEFAULT_NAMESPACE, DimensionConfig 

46from ._database import DatabaseDimensionElement 

47from ._elements import Dimension, DimensionElement 

48from ._governor import GovernorDimension 

49from ._graph import DimensionGraph 

50from ._skypix import SkyPixDimension, SkyPixSystem 

51 

52if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 

53 from ._coordinate import DataCoordinate 

54 from ._packer import DimensionPacker, DimensionPackerFactory 

55 from .construction import DimensionConstructionBuilder 

56 

57 

58E = TypeVar("E", bound=DimensionElement) 

59_LOG = logging.getLogger(__name__) 

60 

61 

62@immutable 

63class DimensionUniverse: 

64 """Self-consistent set of dimensions. 

65 

66 A parent class that represents a complete, self-consistent set of 

67 dimensions and their relationships. 

68 

69 `DimensionUniverse` is not a class-level singleton, but all instances are 

70 tracked in a singleton map keyed by the version number and namespace 

71 in the configuration they were loaded from. Because these universes 

72 are solely responsible for constructing `DimensionElement` instances, 

73 these are also indirectly tracked by that singleton as well. 

74 

75 Parameters 

76 ---------- 

77 config : `Config`, optional 

78 Configuration object from which dimension definitions can be extracted. 

79 Ignored if ``builder`` is provided, or if ``version`` is provided and 

80 an instance with that version already exists. 

81 version : `int`, optional 

82 Integer version for this `DimensionUniverse`. If not provided, a 

83 version will be obtained from ``builder`` or ``config``. 

84 namespace : `str`, optional 

85 Namespace of this `DimensionUniverse`, combined with the version 

86 to provide universe safety for registries that use different 

87 dimension definitions. 

88 builder : `DimensionConstructionBuilder`, optional 

89 Builder object used to initialize a new instance. Ignored if 

90 ``version`` is provided and an instance with that version already 

91 exists. Should not have had `~DimensionConstructionBuilder.finish` 

92 called; this will be called if needed by `DimensionUniverse`. 

93 """ 

94 

95 _instances: ClassVar[dict[tuple[int, str], DimensionUniverse]] = {} 

96 """Singleton dictionary of all instances, keyed by version. 

97 

98 For internal use only. 

99 """ 

100 

101 def __new__( 

102 cls, 

103 config: Config | None = None, 

104 *, 

105 version: int | None = None, 

106 namespace: str | None = None, 

107 builder: DimensionConstructionBuilder | None = None, 

108 ) -> DimensionUniverse: 

109 # Try to get a version first, to look for existing instances; try to 

110 # do as little work as possible at this stage. 

111 if version is None: 

112 if builder is None: 

113 config = DimensionConfig(config) 

114 version = config["version"] 

115 else: 

116 version = builder.version 

117 

118 # Then a namespace. 

119 if namespace is None: 

120 if builder is None: 

121 config = DimensionConfig(config) 

122 namespace = config.get("namespace", _DEFAULT_NAMESPACE) 

123 else: 

124 namespace = builder.namespace 

125 # if still None use the default 

126 if namespace is None: 

127 namespace = _DEFAULT_NAMESPACE 

128 

129 # See if an equivalent instance already exists. 

130 self: DimensionUniverse | None = cls._instances.get((version, namespace)) 

131 if self is not None: 

132 return self 

133 

134 # Ensure we have a builder, building one from config if necessary. 

135 if builder is None: 

136 config = DimensionConfig(config) 

137 builder = config.makeBuilder() 

138 

139 # Delegate to the builder for most of the construction work. 

140 builder.finish() 

141 

142 # Create the universe instance and create core attributes, mostly 

143 # copying from builder. 

144 self = object.__new__(cls) 

145 assert self is not None 

146 self._cache = {} 

147 self._dimensions = builder.dimensions 

148 self._elements = builder.elements 

149 self._topology = builder.topology 

150 self._packers = builder.packers 

151 self.dimensionConfig = builder.config 

152 commonSkyPix = self._dimensions[builder.commonSkyPixName] 

153 assert isinstance(commonSkyPix, SkyPixDimension) 

154 self.commonSkyPix = commonSkyPix 

155 

156 # Attach self to all elements. 

157 for element in self._elements: 

158 element.universe = self 

159 

160 # Add attribute for special subsets of the graph. 

161 self.empty = DimensionGraph(self, (), conform=False) 

162 

163 # Use the version number and namespace from the config as a key in 

164 # the singleton dict containing all instances; that will let us 

165 # transfer dimension objects between processes using pickle without 

166 # actually going through real initialization, as long as a universe 

167 # with the same version and namespace has already been constructed in 

168 # the receiving process. 

169 self._version = version 

170 self._namespace = namespace 

171 cls._instances[self._version, self._namespace] = self 

172 

173 # Build mappings from element to index. These are used for 

174 # topological-sort comparison operators in DimensionElement itself. 

175 self._elementIndices = {name: i for i, name in enumerate(self._elements.names)} 

176 # Same for dimension to index, sorted topologically across required 

177 # and implied. This is used for encode/decode. 

178 self._dimensionIndices = {name: i for i, name in enumerate(self._dimensions.names)} 

179 

180 self._populates = defaultdict(NamedValueSet) 

181 for element in self._elements: 

182 if element.populated_by is not None: 

183 self._populates[element.populated_by.name].add(element) 

184 

185 return self 

186 

187 @property 

188 def version(self) -> int: 

189 """The version number of this universe. 

190 

191 Returns 

192 ------- 

193 version : `int` 

194 An integer representing the version number of this universe. 

195 Uniquely defined when combined with the `namespace`. 

196 """ 

197 return self._version 

198 

199 @property 

200 def namespace(self) -> str: 

201 """The namespace associated with this universe. 

202 

203 Returns 

204 ------- 

205 namespace : `str` 

206 The namespace. When combined with the `version` can uniquely 

207 define this universe. 

208 """ 

209 return self._namespace 

210 

211 def isCompatibleWith(self, other: DimensionUniverse) -> bool: 

212 """Check compatibility between this `DimensionUniverse` and another. 

213 

214 Parameters 

215 ---------- 

216 other : `DimensionUniverse` 

217 The other `DimensionUniverse` to check for compatibility 

218 

219 Returns 

220 ------- 

221 results : `bool` 

222 If the other `DimensionUniverse` is compatible with this one return 

223 `True`, else `False` 

224 """ 

225 # Different namespaces mean that these universes cannot be compatible. 

226 if self.namespace != other.namespace: 

227 return False 

228 if self.version != other.version: 

229 _LOG.info( 

230 "Universes share a namespace %r but have differing versions (%d != %d). " 

231 " This could be okay but may be responsible for dimension errors later.", 

232 self.namespace, 

233 self.version, 

234 other.version, 

235 ) 

236 

237 # For now assume compatibility if versions differ. 

238 return True 

239 

240 def __repr__(self) -> str: 

241 return f"DimensionUniverse({self._version}, {self._namespace})" 

242 

243 def __getitem__(self, name: str) -> DimensionElement: 

244 return self._elements[name] 

245 

246 def __contains__(self, name: Any) -> bool: 

247 return name in self._elements 

248 

249 def get(self, name: str, default: DimensionElement | None = None) -> DimensionElement | None: 

250 """Return the `DimensionElement` with the given name or a default. 

251 

252 Parameters 

253 ---------- 

254 name : `str` 

255 Name of the element. 

256 default : `DimensionElement`, optional 

257 Element to return if the named one does not exist. Defaults to 

258 `None`. 

259 

260 Returns 

261 ------- 

262 element : `DimensionElement` 

263 The named element. 

264 """ 

265 return self._elements.get(name, default) 

266 

267 def getStaticElements(self) -> NamedValueAbstractSet[DimensionElement]: 

268 """Return a set of all static elements in this universe. 

269 

270 Non-static elements that are created as needed may also exist, but 

271 these are guaranteed to have no direct relationships to other elements 

272 (though they may have spatial or temporal relationships). 

273 

274 Returns 

275 ------- 

276 elements : `NamedValueAbstractSet` [ `DimensionElement` ] 

277 A frozen set of `DimensionElement` instances. 

278 """ 

279 return self._elements 

280 

281 def getStaticDimensions(self) -> NamedValueAbstractSet[Dimension]: 

282 """Return a set of all static dimensions in this universe. 

283 

284 Non-static dimensions that are created as needed may also exist, but 

285 these are guaranteed to have no direct relationships to other elements 

286 (though they may have spatial or temporal relationships). 

287 

288 Returns 

289 ------- 

290 dimensions : `NamedValueAbstractSet` [ `Dimension` ] 

291 A frozen set of `Dimension` instances. 

292 """ 

293 return self._dimensions 

294 

295 @cached_getter 

296 def getGovernorDimensions(self) -> NamedValueAbstractSet[GovernorDimension]: 

297 """Return a set of all `GovernorDimension` instances in this universe. 

298 

299 Returns 

300 ------- 

301 governors : `NamedValueAbstractSet` [ `GovernorDimension` ] 

302 A frozen set of `GovernorDimension` instances. 

303 """ 

304 return NamedValueSet(d for d in self._dimensions if isinstance(d, GovernorDimension)).freeze() 

305 

306 @cached_getter 

307 def getDatabaseElements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]: 

308 """Return set of all `DatabaseDimensionElement` instances in universe. 

309 

310 This does not include `GovernorDimension` instances, which are backed 

311 by the database but do not inherit from `DatabaseDimensionElement`. 

312 

313 Returns 

314 ------- 

315 elements : `NamedValueAbstractSet` [ `DatabaseDimensionElement` ] 

316 A frozen set of `DatabaseDimensionElement` instances. 

317 """ 

318 return NamedValueSet(d for d in self._elements if isinstance(d, DatabaseDimensionElement)).freeze() 

319 

320 @property 

321 @cached_getter 

322 def skypix(self) -> NamedValueAbstractSet[SkyPixSystem]: 

323 """All skypix systems known to this universe. 

324 

325 (`NamedValueAbstractSet` [ `SkyPixSystem` ]). 

326 """ 

327 return NamedValueSet( 

328 [ 

329 family 

330 for family in self._topology[TopologicalSpace.SPATIAL] 

331 if isinstance(family, SkyPixSystem) 

332 ] 

333 ).freeze() 

334 

335 def getElementIndex(self, name: str) -> int: 

336 """Return the position of the named dimension element. 

337 

338 The position is in this universe's sorting of all elements. 

339 

340 Parameters 

341 ---------- 

342 name : `str` 

343 Name of the element. 

344 

345 Returns 

346 ------- 

347 index : `int` 

348 Sorting index for this element. 

349 """ 

350 return self._elementIndices[name] 

351 

352 def getDimensionIndex(self, name: str) -> int: 

353 """Return the position of the named dimension. 

354 

355 This position is in this universe's sorting of all dimensions. 

356 

357 Parameters 

358 ---------- 

359 name : `str` 

360 Name of the dimension. 

361 

362 Returns 

363 ------- 

364 index : `int` 

365 Sorting index for this dimension. 

366 

367 Notes 

368 ----- 

369 The dimension sort order for a universe is consistent with the element 

370 order (all dimensions are elements), and either can be used to sort 

371 dimensions if used consistently. But there are also some contexts in 

372 which contiguous dimension-only indices are necessary or at least 

373 desirable. 

374 """ 

375 return self._dimensionIndices[name] 

376 

377 def expandDimensionNameSet(self, names: set[str]) -> None: 

378 """Expand a set of dimension names in-place. 

379 

380 Includes recursive dependencies. 

381 

382 This is an advanced interface for cases where constructing a 

383 `DimensionGraph` (which also expands required dependencies) is 

384 impossible or undesirable. 

385 

386 Parameters 

387 ---------- 

388 names : `set` [ `str` ] 

389 A true `set` of dimension names, to be expanded in-place. 

390 """ 

391 # Keep iterating until the set of names stops growing. This is not as 

392 # efficient as it could be, but we work pretty hard cache 

393 # DimensionGraph instances to keep actual construction rare, so that 

394 # shouldn't matter. 

395 oldSize = len(names) 

396 while True: 

397 # iterate over a temporary copy so we can modify the original 

398 for name in tuple(names): 

399 names.update(self._dimensions[name].required.names) 

400 names.update(self._dimensions[name].implied.names) 

401 if oldSize == len(names): 

402 break 

403 else: 

404 oldSize = len(names) 

405 

406 def extract(self, iterable: Iterable[Dimension | str]) -> DimensionGraph: 

407 """Construct graph from iterable. 

408 

409 Constructs a `DimensionGraph` from a possibly-heterogenous iterable 

410 of `Dimension` instances and string names thereof. 

411 

412 Constructing `DimensionGraph` directly from names or dimension 

413 instances is slightly more efficient when it is known in advance that 

414 the iterable is not heterogenous. 

415 

416 Parameters 

417 ---------- 

418 iterable: iterable of `Dimension` or `str` 

419 Dimensions that must be included in the returned graph (their 

420 dependencies will be as well). 

421 

422 Returns 

423 ------- 

424 graph : `DimensionGraph` 

425 A `DimensionGraph` instance containing all given dimensions. 

426 """ 

427 names = set() 

428 for item in iterable: 

429 try: 

430 names.add(item.name) # type: ignore 

431 except AttributeError: 

432 names.add(item) 

433 return DimensionGraph(universe=self, names=names) 

434 

435 def sorted(self, elements: Iterable[E | str], *, reverse: bool = False) -> list[E]: 

436 """Return a sorted version of the given iterable of dimension elements. 

437 

438 The universe's sort order is topological (an element's dependencies 

439 precede it), with an unspecified (but deterministic) approach to 

440 breaking ties. 

441 

442 Parameters 

443 ---------- 

444 elements : iterable of `DimensionElement`. 

445 Elements to be sorted. 

446 reverse : `bool`, optional 

447 If `True`, sort in the opposite order. 

448 

449 Returns 

450 ------- 

451 sorted : `list` of `DimensionElement` 

452 A sorted list containing the same elements that were given. 

453 """ 

454 s = set(elements) 

455 result = [element for element in self._elements if element in s or element.name in s] 

456 if reverse: 

457 result.reverse() 

458 # mypy thinks this can return DimensionElements even if all the user 

459 # passed it was Dimensions; we know better. 

460 return result # type: ignore 

461 

462 # TODO: Remove this method on DM-38687. 

463 @deprecated( 

464 "Deprecated in favor of configurable dimension packers. Will be removed after v26.", 

465 version="v26", 

466 category=FutureWarning, 

467 ) 

468 def makePacker(self, name: str, dataId: DataCoordinate) -> DimensionPacker: 

469 """Make a dimension packer. 

470 

471 Constructs a `DimensionPacker` that can pack data ID dictionaries 

472 into unique integers. 

473 

474 Parameters 

475 ---------- 

476 name : `str` 

477 Name of the packer, matching a key in the "packers" section of the 

478 dimension configuration. 

479 dataId : `DataCoordinate` 

480 Fully-expanded data ID that identifies the at least the "fixed" 

481 dimensions of the packer (i.e. those that are assumed/given, 

482 setting the space over which packed integer IDs are unique). 

483 ``dataId.hasRecords()`` must return `True`. 

484 """ 

485 return self._packers[name](self, dataId) 

486 

487 def getEncodeLength(self) -> int: 

488 """Return encoded size of graph. 

489 

490 Returns the size (in bytes) of the encoded size of `DimensionGraph` 

491 instances in this universe. 

492 

493 See `DimensionGraph.encode` and `DimensionGraph.decode` for more 

494 information. 

495 """ 

496 return math.ceil(len(self._dimensions) / 8) 

497 

498 def get_elements_populated_by(self, dimension: Dimension) -> NamedValueAbstractSet[DimensionElement]: 

499 """Return the set of `DimensionElement` objects whose 

500 `~DimensionElement.populated_by` atttribute is the given dimension. 

501 """ 

502 return self._populates[dimension.name] 

503 

504 @classmethod 

505 def _unpickle(cls, version: int, namespace: str | None = None) -> DimensionUniverse: 

506 """Return an unpickled dimension universe. 

507 

508 Callable used for unpickling. 

509 

510 For internal use only. 

511 """ 

512 if namespace is None: 

513 # Old pickled universe. 

514 namespace = _DEFAULT_NAMESPACE 

515 try: 

516 return cls._instances[version, namespace] 

517 except KeyError as err: 

518 raise pickle.UnpicklingError( 

519 f"DimensionUniverse with version '{version}' and namespace {namespace!r} " 

520 "not found. Note that DimensionUniverse objects are not " 

521 "truly serialized; when using pickle to transfer them " 

522 "between processes, an equivalent instance with the same " 

523 "version must already exist in the receiving process." 

524 ) from err 

525 

526 def __reduce__(self) -> tuple: 

527 return (self._unpickle, (self._version, self._namespace)) 

528 

529 def __deepcopy__(self, memo: dict) -> DimensionUniverse: 

530 # DimensionUniverse is recursively immutable; see note in @immutable 

531 # decorator. 

532 return self 

533 

534 # Class attributes below are shadowed by instance attributes, and are 

535 # present just to hold the docstrings for those instance attributes. 

536 

537 empty: DimensionGraph 

538 """The `DimensionGraph` that contains no dimensions (`DimensionGraph`). 

539 """ 

540 

541 commonSkyPix: SkyPixDimension 

542 """The special skypix dimension that is used to relate all other spatial 

543 dimensions in the `Registry` database (`SkyPixDimension`). 

544 """ 

545 

546 dimensionConfig: DimensionConfig 

547 """The configuration used to create this Universe (`DimensionConfig`).""" 

548 

549 _cache: dict[frozenset[str], DimensionGraph] 

550 

551 _dimensions: NamedValueAbstractSet[Dimension] 

552 

553 _elements: NamedValueAbstractSet[DimensionElement] 

554 

555 _topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]] 

556 

557 _dimensionIndices: dict[str, int] 

558 

559 _elementIndices: dict[str, int] 

560 

561 _packers: dict[str, DimensionPackerFactory] 

562 

563 _populates: defaultdict[str, NamedValueSet[DimensionElement]] 

564 

565 _version: int 

566 

567 _namespace: str