Coverage for python/lsst/daf/butler/core/dimensions/_universe.py: 40%

165 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-08-05 01:26 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ["DimensionUniverse"] 

25 

26import logging 

27import math 

28import pickle 

29from collections import defaultdict 

30from collections.abc import Iterable, Mapping 

31from typing import TYPE_CHECKING, Any, ClassVar, TypeVar 

32 

33from deprecated.sphinx import deprecated 

34from lsst.utils.classes import cached_getter, immutable 

35 

36from .._topology import TopologicalFamily, TopologicalSpace 

37from ..config import Config 

38from ..named import NamedValueAbstractSet, NamedValueSet 

39from ._config import _DEFAULT_NAMESPACE, DimensionConfig 

40from ._database import DatabaseDimensionElement 

41from ._elements import Dimension, DimensionElement 

42from ._governor import GovernorDimension 

43from ._graph import DimensionGraph 

44from ._skypix import SkyPixDimension, SkyPixSystem 

45 

46if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 

47 from ._coordinate import DataCoordinate 

48 from ._packer import DimensionPacker, DimensionPackerFactory 

49 from .construction import DimensionConstructionBuilder 

50 

51 

52E = TypeVar("E", bound=DimensionElement) 

53_LOG = logging.getLogger(__name__) 

54 

55 

56@immutable 

57class DimensionUniverse: 

58 """Self-consistent set of dimensions. 

59 

60 A parent class that represents a complete, self-consistent set of 

61 dimensions and their relationships. 

62 

63 `DimensionUniverse` is not a class-level singleton, but all instances are 

64 tracked in a singleton map keyed by the version number and namespace 

65 in the configuration they were loaded from. Because these universes 

66 are solely responsible for constructing `DimensionElement` instances, 

67 these are also indirectly tracked by that singleton as well. 

68 

69 Parameters 

70 ---------- 

71 config : `Config`, optional 

72 Configuration object from which dimension definitions can be extracted. 

73 Ignored if ``builder`` is provided, or if ``version`` is provided and 

74 an instance with that version already exists. 

75 version : `int`, optional 

76 Integer version for this `DimensionUniverse`. If not provided, a 

77 version will be obtained from ``builder`` or ``config``. 

78 namespace : `str`, optional 

79 Namespace of this `DimensionUniverse`, combined with the version 

80 to provide universe safety for registries that use different 

81 dimension definitions. 

82 builder : `DimensionConstructionBuilder`, optional 

83 Builder object used to initialize a new instance. Ignored if 

84 ``version`` is provided and an instance with that version already 

85 exists. Should not have had `~DimensionConstructionBuilder.finish` 

86 called; this will be called if needed by `DimensionUniverse`. 

87 """ 

88 

89 _instances: ClassVar[dict[tuple[int, str], DimensionUniverse]] = {} 

90 """Singleton dictionary of all instances, keyed by version. 

91 

92 For internal use only. 

93 """ 

94 

95 def __new__( 

96 cls, 

97 config: Config | None = None, 

98 *, 

99 version: int | None = None, 

100 namespace: str | None = None, 

101 builder: DimensionConstructionBuilder | None = None, 

102 ) -> DimensionUniverse: 

103 # Try to get a version first, to look for existing instances; try to 

104 # do as little work as possible at this stage. 

105 if version is None: 

106 if builder is None: 

107 config = DimensionConfig(config) 

108 version = config["version"] 

109 else: 

110 version = builder.version 

111 

112 # Then a namespace. 

113 if namespace is None: 

114 if builder is None: 

115 config = DimensionConfig(config) 

116 namespace = config.get("namespace", _DEFAULT_NAMESPACE) 

117 else: 

118 namespace = builder.namespace 

119 # if still None use the default 

120 if namespace is None: 

121 namespace = _DEFAULT_NAMESPACE 

122 

123 # See if an equivalent instance already exists. 

124 self: DimensionUniverse | None = cls._instances.get((version, namespace)) 

125 if self is not None: 

126 return self 

127 

128 # Ensure we have a builder, building one from config if necessary. 

129 if builder is None: 

130 config = DimensionConfig(config) 

131 builder = config.makeBuilder() 

132 

133 # Delegate to the builder for most of the construction work. 

134 builder.finish() 

135 

136 # Create the universe instance and create core attributes, mostly 

137 # copying from builder. 

138 self = object.__new__(cls) 

139 assert self is not None 

140 self._cache = {} 

141 self._dimensions = builder.dimensions 

142 self._elements = builder.elements 

143 self._topology = builder.topology 

144 self._packers = builder.packers 

145 self.dimensionConfig = builder.config 

146 commonSkyPix = self._dimensions[builder.commonSkyPixName] 

147 assert isinstance(commonSkyPix, SkyPixDimension) 

148 self.commonSkyPix = commonSkyPix 

149 

150 # Attach self to all elements. 

151 for element in self._elements: 

152 element.universe = self 

153 

154 # Add attribute for special subsets of the graph. 

155 self.empty = DimensionGraph(self, (), conform=False) 

156 

157 # Use the version number and namespace from the config as a key in 

158 # the singleton dict containing all instances; that will let us 

159 # transfer dimension objects between processes using pickle without 

160 # actually going through real initialization, as long as a universe 

161 # with the same version and namespace has already been constructed in 

162 # the receiving process. 

163 self._version = version 

164 self._namespace = namespace 

165 cls._instances[self._version, self._namespace] = self 

166 

167 # Build mappings from element to index. These are used for 

168 # topological-sort comparison operators in DimensionElement itself. 

169 self._elementIndices = {name: i for i, name in enumerate(self._elements.names)} 

170 # Same for dimension to index, sorted topologically across required 

171 # and implied. This is used for encode/decode. 

172 self._dimensionIndices = {name: i for i, name in enumerate(self._dimensions.names)} 

173 

174 self._populates = defaultdict(NamedValueSet) 

175 for element in self._elements: 

176 if element.populated_by is not None: 

177 self._populates[element.populated_by.name].add(element) 

178 

179 return self 

180 

181 @property 

182 def version(self) -> int: 

183 """The version number of this universe. 

184 

185 Returns 

186 ------- 

187 version : `int` 

188 An integer representing the version number of this universe. 

189 Uniquely defined when combined with the `namespace`. 

190 """ 

191 return self._version 

192 

193 @property 

194 def namespace(self) -> str: 

195 """The namespace associated with this universe. 

196 

197 Returns 

198 ------- 

199 namespace : `str` 

200 The namespace. When combined with the `version` can uniquely 

201 define this universe. 

202 """ 

203 return self._namespace 

204 

205 def isCompatibleWith(self, other: DimensionUniverse) -> bool: 

206 """Check compatibility between this `DimensionUniverse` and another. 

207 

208 Parameters 

209 ---------- 

210 other : `DimensionUniverse` 

211 The other `DimensionUniverse` to check for compatibility 

212 

213 Returns 

214 ------- 

215 results : `bool` 

216 If the other `DimensionUniverse` is compatible with this one return 

217 `True`, else `False` 

218 """ 

219 # Different namespaces mean that these universes cannot be compatible. 

220 if self.namespace != other.namespace: 

221 return False 

222 if self.version != other.version: 

223 _LOG.info( 

224 "Universes share a namespace %r but have differing versions (%d != %d). " 

225 " This could be okay but may be responsible for dimension errors later.", 

226 self.namespace, 

227 self.version, 

228 other.version, 

229 ) 

230 

231 # For now assume compatibility if versions differ. 

232 return True 

233 

234 def __repr__(self) -> str: 

235 return f"DimensionUniverse({self._version}, {self._namespace})" 

236 

237 def __getitem__(self, name: str) -> DimensionElement: 

238 return self._elements[name] 

239 

240 def __contains__(self, name: Any) -> bool: 

241 return name in self._elements 

242 

243 def get(self, name: str, default: DimensionElement | None = None) -> DimensionElement | None: 

244 """Return the `DimensionElement` with the given name or a default. 

245 

246 Parameters 

247 ---------- 

248 name : `str` 

249 Name of the element. 

250 default : `DimensionElement`, optional 

251 Element to return if the named one does not exist. Defaults to 

252 `None`. 

253 

254 Returns 

255 ------- 

256 element : `DimensionElement` 

257 The named element. 

258 """ 

259 return self._elements.get(name, default) 

260 

261 def getStaticElements(self) -> NamedValueAbstractSet[DimensionElement]: 

262 """Return a set of all static elements in this universe. 

263 

264 Non-static elements that are created as needed may also exist, but 

265 these are guaranteed to have no direct relationships to other elements 

266 (though they may have spatial or temporal relationships). 

267 

268 Returns 

269 ------- 

270 elements : `NamedValueAbstractSet` [ `DimensionElement` ] 

271 A frozen set of `DimensionElement` instances. 

272 """ 

273 return self._elements 

274 

275 def getStaticDimensions(self) -> NamedValueAbstractSet[Dimension]: 

276 """Return a set of all static dimensions in this universe. 

277 

278 Non-static dimensions that are created as needed may also exist, but 

279 these are guaranteed to have no direct relationships to other elements 

280 (though they may have spatial or temporal relationships). 

281 

282 Returns 

283 ------- 

284 dimensions : `NamedValueAbstractSet` [ `Dimension` ] 

285 A frozen set of `Dimension` instances. 

286 """ 

287 return self._dimensions 

288 

289 @cached_getter 

290 def getGovernorDimensions(self) -> NamedValueAbstractSet[GovernorDimension]: 

291 """Return a set of all `GovernorDimension` instances in this universe. 

292 

293 Returns 

294 ------- 

295 governors : `NamedValueAbstractSet` [ `GovernorDimension` ] 

296 A frozen set of `GovernorDimension` instances. 

297 """ 

298 return NamedValueSet(d for d in self._dimensions if isinstance(d, GovernorDimension)).freeze() 

299 

300 @cached_getter 

301 def getDatabaseElements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]: 

302 """Return set of all `DatabaseDimensionElement` instances in universe. 

303 

304 This does not include `GovernorDimension` instances, which are backed 

305 by the database but do not inherit from `DatabaseDimensionElement`. 

306 

307 Returns 

308 ------- 

309 elements : `NamedValueAbstractSet` [ `DatabaseDimensionElement` ] 

310 A frozen set of `DatabaseDimensionElement` instances. 

311 """ 

312 return NamedValueSet(d for d in self._elements if isinstance(d, DatabaseDimensionElement)).freeze() 

313 

314 @property 

315 @cached_getter 

316 def skypix(self) -> NamedValueAbstractSet[SkyPixSystem]: 

317 """All skypix systems known to this universe. 

318 

319 (`NamedValueAbstractSet` [ `SkyPixSystem` ]). 

320 """ 

321 return NamedValueSet( 

322 [ 

323 family 

324 for family in self._topology[TopologicalSpace.SPATIAL] 

325 if isinstance(family, SkyPixSystem) 

326 ] 

327 ).freeze() 

328 

329 def getElementIndex(self, name: str) -> int: 

330 """Return the position of the named dimension element. 

331 

332 The position is in this universe's sorting of all elements. 

333 

334 Parameters 

335 ---------- 

336 name : `str` 

337 Name of the element. 

338 

339 Returns 

340 ------- 

341 index : `int` 

342 Sorting index for this element. 

343 """ 

344 return self._elementIndices[name] 

345 

346 def getDimensionIndex(self, name: str) -> int: 

347 """Return the position of the named dimension. 

348 

349 This position is in this universe's sorting of all dimensions. 

350 

351 Parameters 

352 ---------- 

353 name : `str` 

354 Name of the dimension. 

355 

356 Returns 

357 ------- 

358 index : `int` 

359 Sorting index for this dimension. 

360 

361 Notes 

362 ----- 

363 The dimension sort order for a universe is consistent with the element 

364 order (all dimensions are elements), and either can be used to sort 

365 dimensions if used consistently. But there are also some contexts in 

366 which contiguous dimension-only indices are necessary or at least 

367 desirable. 

368 """ 

369 return self._dimensionIndices[name] 

370 

371 def expandDimensionNameSet(self, names: set[str]) -> None: 

372 """Expand a set of dimension names in-place. 

373 

374 Includes recursive dependencies. 

375 

376 This is an advanced interface for cases where constructing a 

377 `DimensionGraph` (which also expands required dependencies) is 

378 impossible or undesirable. 

379 

380 Parameters 

381 ---------- 

382 names : `set` [ `str` ] 

383 A true `set` of dimension names, to be expanded in-place. 

384 """ 

385 # Keep iterating until the set of names stops growing. This is not as 

386 # efficient as it could be, but we work pretty hard cache 

387 # DimensionGraph instances to keep actual construction rare, so that 

388 # shouldn't matter. 

389 oldSize = len(names) 

390 while True: 

391 # iterate over a temporary copy so we can modify the original 

392 for name in tuple(names): 

393 names.update(self._dimensions[name].required.names) 

394 names.update(self._dimensions[name].implied.names) 

395 if oldSize == len(names): 

396 break 

397 else: 

398 oldSize = len(names) 

399 

400 def extract(self, iterable: Iterable[Dimension | str]) -> DimensionGraph: 

401 """Construct graph from iterable. 

402 

403 Constructs a `DimensionGraph` from a possibly-heterogenous iterable 

404 of `Dimension` instances and string names thereof. 

405 

406 Constructing `DimensionGraph` directly from names or dimension 

407 instances is slightly more efficient when it is known in advance that 

408 the iterable is not heterogenous. 

409 

410 Parameters 

411 ---------- 

412 iterable: iterable of `Dimension` or `str` 

413 Dimensions that must be included in the returned graph (their 

414 dependencies will be as well). 

415 

416 Returns 

417 ------- 

418 graph : `DimensionGraph` 

419 A `DimensionGraph` instance containing all given dimensions. 

420 """ 

421 names = set() 

422 for item in iterable: 

423 try: 

424 names.add(item.name) # type: ignore 

425 except AttributeError: 

426 names.add(item) 

427 return DimensionGraph(universe=self, names=names) 

428 

429 def sorted(self, elements: Iterable[E | str], *, reverse: bool = False) -> list[E]: 

430 """Return a sorted version of the given iterable of dimension elements. 

431 

432 The universe's sort order is topological (an element's dependencies 

433 precede it), with an unspecified (but deterministic) approach to 

434 breaking ties. 

435 

436 Parameters 

437 ---------- 

438 elements : iterable of `DimensionElement`. 

439 Elements to be sorted. 

440 reverse : `bool`, optional 

441 If `True`, sort in the opposite order. 

442 

443 Returns 

444 ------- 

445 sorted : `list` of `DimensionElement` 

446 A sorted list containing the same elements that were given. 

447 """ 

448 s = set(elements) 

449 result = [element for element in self._elements if element in s or element.name in s] 

450 if reverse: 

451 result.reverse() 

452 # mypy thinks this can return DimensionElements even if all the user 

453 # passed it was Dimensions; we know better. 

454 return result # type: ignore 

455 

456 # TODO: Remove this method on DM-38687. 

457 @deprecated( 

458 "Deprecated in favor of configurable dimension packers. Will be removed after v26.", 

459 version="v26", 

460 category=FutureWarning, 

461 ) 

462 def makePacker(self, name: str, dataId: DataCoordinate) -> DimensionPacker: 

463 """Make a dimension packer. 

464 

465 Constructs a `DimensionPacker` that can pack data ID dictionaries 

466 into unique integers. 

467 

468 Parameters 

469 ---------- 

470 name : `str` 

471 Name of the packer, matching a key in the "packers" section of the 

472 dimension configuration. 

473 dataId : `DataCoordinate` 

474 Fully-expanded data ID that identifies the at least the "fixed" 

475 dimensions of the packer (i.e. those that are assumed/given, 

476 setting the space over which packed integer IDs are unique). 

477 ``dataId.hasRecords()`` must return `True`. 

478 """ 

479 return self._packers[name](self, dataId) 

480 

481 def getEncodeLength(self) -> int: 

482 """Return encoded size of graph. 

483 

484 Returns the size (in bytes) of the encoded size of `DimensionGraph` 

485 instances in this universe. 

486 

487 See `DimensionGraph.encode` and `DimensionGraph.decode` for more 

488 information. 

489 """ 

490 return math.ceil(len(self._dimensions) / 8) 

491 

492 def get_elements_populated_by(self, dimension: Dimension) -> NamedValueAbstractSet[DimensionElement]: 

493 """Return the set of `DimensionElement` objects whose 

494 `~DimensionElement.populated_by` atttribute is the given dimension. 

495 """ 

496 return self._populates[dimension.name] 

497 

498 @classmethod 

499 def _unpickle(cls, version: int, namespace: str | None = None) -> DimensionUniverse: 

500 """Return an unpickled dimension universe. 

501 

502 Callable used for unpickling. 

503 

504 For internal use only. 

505 """ 

506 if namespace is None: 

507 # Old pickled universe. 

508 namespace = _DEFAULT_NAMESPACE 

509 try: 

510 return cls._instances[version, namespace] 

511 except KeyError as err: 

512 raise pickle.UnpicklingError( 

513 f"DimensionUniverse with version '{version}' and namespace {namespace!r} " 

514 "not found. Note that DimensionUniverse objects are not " 

515 "truly serialized; when using pickle to transfer them " 

516 "between processes, an equivalent instance with the same " 

517 "version must already exist in the receiving process." 

518 ) from err 

519 

520 def __reduce__(self) -> tuple: 

521 return (self._unpickle, (self._version, self._namespace)) 

522 

523 def __deepcopy__(self, memo: dict) -> DimensionUniverse: 

524 # DimensionUniverse is recursively immutable; see note in @immutable 

525 # decorator. 

526 return self 

527 

528 # Class attributes below are shadowed by instance attributes, and are 

529 # present just to hold the docstrings for those instance attributes. 

530 

531 empty: DimensionGraph 

532 """The `DimensionGraph` that contains no dimensions (`DimensionGraph`). 

533 """ 

534 

535 commonSkyPix: SkyPixDimension 

536 """The special skypix dimension that is used to relate all other spatial 

537 dimensions in the `Registry` database (`SkyPixDimension`). 

538 """ 

539 

540 dimensionConfig: DimensionConfig 

541 """The configuration used to create this Universe (`DimensionConfig`).""" 

542 

543 _cache: dict[frozenset[str], DimensionGraph] 

544 

545 _dimensions: NamedValueAbstractSet[Dimension] 

546 

547 _elements: NamedValueAbstractSet[DimensionElement] 

548 

549 _topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]] 

550 

551 _dimensionIndices: dict[str, int] 

552 

553 _elementIndices: dict[str, int] 

554 

555 _packers: dict[str, DimensionPackerFactory] 

556 

557 _populates: defaultdict[str, NamedValueSet[DimensionElement]] 

558 

559 _version: int 

560 

561 _namespace: str