Coverage for python/lsst/daf/butler/core/dimensions/_universe.py: 36%

156 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-02 02:16 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ["DimensionUniverse"] 

25 

26import logging 

27import math 

28import pickle 

29from typing import ( 

30 TYPE_CHECKING, 

31 Any, 

32 ClassVar, 

33 Dict, 

34 FrozenSet, 

35 Iterable, 

36 List, 

37 Mapping, 

38 Optional, 

39 Set, 

40 Tuple, 

41 TypeVar, 

42 Union, 

43) 

44 

45from deprecated.sphinx import deprecated 

46from lsst.utils.classes import cached_getter, immutable 

47 

48from .._topology import TopologicalFamily, TopologicalSpace 

49from ..config import Config 

50from ..named import NamedValueAbstractSet, NamedValueSet 

51from ._config import _DEFAULT_NAMESPACE, DimensionConfig 

52from ._database import DatabaseDimensionElement 

53from ._elements import Dimension, DimensionElement 

54from ._governor import GovernorDimension 

55from ._graph import DimensionGraph 

56from ._skypix import SkyPixDimension, SkyPixSystem 

57 

58if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 

59 from ._coordinate import DataCoordinate 

60 from ._packer import DimensionPacker, DimensionPackerFactory 

61 from .construction import DimensionConstructionBuilder 

62 

63 

64E = TypeVar("E", bound=DimensionElement) 

65_LOG = logging.getLogger(__name__) 

66 

67 

68@immutable 

69class DimensionUniverse: 

70 """Self-consistent set of dimensions. 

71 

72 A parent class that represents a complete, self-consistent set of 

73 dimensions and their relationships. 

74 

75 `DimensionUniverse` is not a class-level singleton, but all instances are 

76 tracked in a singleton map keyed by the version number and namespace 

77 in the configuration they were loaded from. Because these universes 

78 are solely responsible for constructing `DimensionElement` instances, 

79 these are also indirectly tracked by that singleton as well. 

80 

81 Parameters 

82 ---------- 

83 config : `Config`, optional 

84 Configuration object from which dimension definitions can be extracted. 

85 Ignored if ``builder`` is provided, or if ``version`` is provided and 

86 an instance with that version already exists. 

87 version : `int`, optional 

88 Integer version for this `DimensionUniverse`. If not provided, a 

89 version will be obtained from ``builder`` or ``config``. 

90 namespace : `str`, optional 

91 Namespace of this `DimensionUniverse`, combined with the version 

92 to provide universe safety for registries that use different 

93 dimension definitions. 

94 builder : `DimensionConstructionBuilder`, optional 

95 Builder object used to initialize a new instance. Ignored if 

96 ``version`` is provided and an instance with that version already 

97 exists. Should not have had `~DimensionConstructionBuilder.finish` 

98 called; this will be called if needed by `DimensionUniverse`. 

99 """ 

100 

101 _instances: ClassVar[Dict[Tuple[int, str], DimensionUniverse]] = {} 

102 """Singleton dictionary of all instances, keyed by version. 

103 

104 For internal use only. 

105 """ 

106 

107 def __new__( 

108 cls, 

109 config: Optional[Config] = None, 

110 *, 

111 version: Optional[int] = None, 

112 namespace: Optional[str] = None, 

113 builder: Optional[DimensionConstructionBuilder] = None, 

114 ) -> DimensionUniverse: 

115 # Try to get a version first, to look for existing instances; try to 

116 # do as little work as possible at this stage. 

117 if version is None: 

118 if builder is None: 

119 config = DimensionConfig(config) 

120 version = config["version"] 

121 else: 

122 version = builder.version 

123 

124 # Then a namespace. 

125 if namespace is None: 

126 if builder is None: 

127 config = DimensionConfig(config) 

128 namespace = config.get("namespace", _DEFAULT_NAMESPACE) 

129 else: 

130 namespace = builder.namespace 

131 # if still None use the default 

132 if namespace is None: 

133 namespace = _DEFAULT_NAMESPACE 

134 

135 # See if an equivalent instance already exists. 

136 self: Optional[DimensionUniverse] = cls._instances.get((version, namespace)) 

137 if self is not None: 

138 return self 

139 

140 # Ensure we have a builder, building one from config if necessary. 

141 if builder is None: 

142 config = DimensionConfig(config) 

143 builder = config.makeBuilder() 

144 

145 # Delegate to the builder for most of the construction work. 

146 builder.finish() 

147 

148 # Create the universe instance and create core attributes, mostly 

149 # copying from builder. 

150 self = object.__new__(cls) 

151 assert self is not None 

152 self._cache = {} 

153 self._dimensions = builder.dimensions 

154 self._elements = builder.elements 

155 self._topology = builder.topology 

156 self._packers = builder.packers 

157 self.dimensionConfig = builder.config 

158 commonSkyPix = self._dimensions[builder.commonSkyPixName] 

159 assert isinstance(commonSkyPix, SkyPixDimension) 

160 self.commonSkyPix = commonSkyPix 

161 

162 # Attach self to all elements. 

163 for element in self._elements: 

164 element.universe = self 

165 

166 # Add attribute for special subsets of the graph. 

167 self.empty = DimensionGraph(self, (), conform=False) 

168 

169 # Use the version number and namespace from the config as a key in 

170 # the singleton dict containing all instances; that will let us 

171 # transfer dimension objects between processes using pickle without 

172 # actually going through real initialization, as long as a universe 

173 # with the same version and namespace has already been constructed in 

174 # the receiving process. 

175 self._version = version 

176 self._namespace = namespace 

177 cls._instances[self._version, self._namespace] = self 

178 

179 # Build mappings from element to index. These are used for 

180 # topological-sort comparison operators in DimensionElement itself. 

181 self._elementIndices = {name: i for i, name in enumerate(self._elements.names)} 

182 # Same for dimension to index, sorted topologically across required 

183 # and implied. This is used for encode/decode. 

184 self._dimensionIndices = {name: i for i, name in enumerate(self._dimensions.names)} 

185 

186 return self 

187 

188 @property 

189 def version(self) -> int: 

190 """The version number of this universe. 

191 

192 Returns 

193 ------- 

194 version : `int` 

195 An integer representing the version number of this universe. 

196 Uniquely defined when combined with the `namespace`. 

197 """ 

198 return self._version 

199 

200 @property 

201 def namespace(self) -> str: 

202 """The namespace associated with this universe. 

203 

204 Returns 

205 ------- 

206 namespace : `str` 

207 The namespace. When combined with the `version` can uniquely 

208 define this universe. 

209 """ 

210 return self._namespace 

211 

212 def isCompatibleWith(self, other: DimensionUniverse) -> bool: 

213 """Check compatibility between this `DimensionUniverse` and another 

214 

215 Parameters 

216 ---------- 

217 other : `DimensionUniverse` 

218 The other `DimensionUniverse` to check for compatibility 

219 

220 Returns 

221 ------- 

222 results : `bool` 

223 If the other `DimensionUniverse` is compatible with this one return 

224 `True`, else `False` 

225 """ 

226 # Different namespaces mean that these universes cannot be compatible. 

227 if self.namespace != other.namespace: 

228 return False 

229 if self.version != other.version: 

230 _LOG.info( 

231 "Universes share a namespace %r but have differing versions (%d != %d). " 

232 " This could be okay but may be responsible for dimension errors later.", 

233 self.namespace, 

234 self.version, 

235 other.version, 

236 ) 

237 

238 # For now assume compatibility if versions differ. 

239 return True 

240 

241 def __repr__(self) -> str: 

242 return f"DimensionUniverse({self._version}, {self._namespace})" 

243 

244 def __getitem__(self, name: str) -> DimensionElement: 

245 return self._elements[name] 

246 

247 def __contains__(self, name: Any) -> bool: 

248 return name in self._elements 

249 

250 def get(self, name: str, default: Optional[DimensionElement] = None) -> Optional[DimensionElement]: 

251 """Return the `DimensionElement` with the given name or a default. 

252 

253 Parameters 

254 ---------- 

255 name : `str` 

256 Name of the element. 

257 default : `DimensionElement`, optional 

258 Element to return if the named one does not exist. Defaults to 

259 `None`. 

260 

261 Returns 

262 ------- 

263 element : `DimensionElement` 

264 The named element. 

265 """ 

266 return self._elements.get(name, default) 

267 

268 def getStaticElements(self) -> NamedValueAbstractSet[DimensionElement]: 

269 """Return a set of all static elements in this universe. 

270 

271 Non-static elements that are created as needed may also exist, but 

272 these are guaranteed to have no direct relationships to other elements 

273 (though they may have spatial or temporal relationships). 

274 

275 Returns 

276 ------- 

277 elements : `NamedValueAbstractSet` [ `DimensionElement` ] 

278 A frozen set of `DimensionElement` instances. 

279 """ 

280 return self._elements 

281 

282 def getStaticDimensions(self) -> NamedValueAbstractSet[Dimension]: 

283 """Return a set of all static dimensions in this universe. 

284 

285 Non-static dimensions that are created as needed may also exist, but 

286 these are guaranteed to have no direct relationships to other elements 

287 (though they may have spatial or temporal relationships). 

288 

289 Returns 

290 ------- 

291 dimensions : `NamedValueAbstractSet` [ `Dimension` ] 

292 A frozen set of `Dimension` instances. 

293 """ 

294 return self._dimensions 

295 

296 @cached_getter 

297 def getGovernorDimensions(self) -> NamedValueAbstractSet[GovernorDimension]: 

298 """Return a set of all `GovernorDimension` instances in this universe. 

299 

300 Returns 

301 ------- 

302 governors : `NamedValueAbstractSet` [ `GovernorDimension` ] 

303 A frozen set of `GovernorDimension` instances. 

304 """ 

305 return NamedValueSet(d for d in self._dimensions if isinstance(d, GovernorDimension)).freeze() 

306 

307 @cached_getter 

308 def getDatabaseElements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]: 

309 """Return set of all `DatabaseDimensionElement` instances in universe. 

310 

311 This does not include `GovernorDimension` instances, which are backed 

312 by the database but do not inherit from `DatabaseDimensionElement`. 

313 

314 Returns 

315 ------- 

316 elements : `NamedValueAbstractSet` [ `DatabaseDimensionElement` ] 

317 A frozen set of `DatabaseDimensionElement` instances. 

318 """ 

319 return NamedValueSet(d for d in self._elements if isinstance(d, DatabaseDimensionElement)).freeze() 

320 

321 @property 

322 @cached_getter 

323 def skypix(self) -> NamedValueAbstractSet[SkyPixSystem]: 

324 """All skypix systems known to this universe. 

325 

326 (`NamedValueAbstractSet` [ `SkyPixSystem` ]). 

327 """ 

328 return NamedValueSet( 

329 [ 

330 family 

331 for family in self._topology[TopologicalSpace.SPATIAL] 

332 if isinstance(family, SkyPixSystem) 

333 ] 

334 ).freeze() 

335 

336 def getElementIndex(self, name: str) -> int: 

337 """Return the position of the named dimension element. 

338 

339 The position is in this universe's sorting of all elements. 

340 

341 Parameters 

342 ---------- 

343 name : `str` 

344 Name of the element. 

345 

346 Returns 

347 ------- 

348 index : `int` 

349 Sorting index for this element. 

350 """ 

351 return self._elementIndices[name] 

352 

353 def getDimensionIndex(self, name: str) -> int: 

354 """Return the position of the named dimension. 

355 

356 This position is in this universe's sorting of all dimensions. 

357 

358 Parameters 

359 ---------- 

360 name : `str` 

361 Name of the dimension. 

362 

363 Returns 

364 ------- 

365 index : `int` 

366 Sorting index for this dimension. 

367 

368 Notes 

369 ----- 

370 The dimension sort order for a universe is consistent with the element 

371 order (all dimensions are elements), and either can be used to sort 

372 dimensions if used consistently. But there are also some contexts in 

373 which contiguous dimension-only indices are necessary or at least 

374 desirable. 

375 """ 

376 return self._dimensionIndices[name] 

377 

378 def expandDimensionNameSet(self, names: Set[str]) -> None: 

379 """Expand a set of dimension names in-place. 

380 

381 Includes recursive dependencies. 

382 

383 This is an advanced interface for cases where constructing a 

384 `DimensionGraph` (which also expands required dependencies) is 

385 impossible or undesirable. 

386 

387 Parameters 

388 ---------- 

389 names : `set` [ `str` ] 

390 A true `set` of dimension names, to be expanded in-place. 

391 """ 

392 # Keep iterating until the set of names stops growing. This is not as 

393 # efficient as it could be, but we work pretty hard cache 

394 # DimensionGraph instances to keep actual construction rare, so that 

395 # shouldn't matter. 

396 oldSize = len(names) 

397 while True: 

398 # iterate over a temporary copy so we can modify the original 

399 for name in tuple(names): 

400 names.update(self._dimensions[name].required.names) 

401 names.update(self._dimensions[name].implied.names) 

402 if oldSize == len(names): 

403 break 

404 else: 

405 oldSize = len(names) 

406 

407 def extract(self, iterable: Iterable[Union[Dimension, str]]) -> DimensionGraph: 

408 """Construct graph from iterable. 

409 

410 Constructs a `DimensionGraph` from a possibly-heterogenous iterable 

411 of `Dimension` instances and string names thereof. 

412 

413 Constructing `DimensionGraph` directly from names or dimension 

414 instances is slightly more efficient when it is known in advance that 

415 the iterable is not heterogenous. 

416 

417 Parameters 

418 ---------- 

419 iterable: iterable of `Dimension` or `str` 

420 Dimensions that must be included in the returned graph (their 

421 dependencies will be as well). 

422 

423 Returns 

424 ------- 

425 graph : `DimensionGraph` 

426 A `DimensionGraph` instance containing all given dimensions. 

427 """ 

428 names = set() 

429 for item in iterable: 

430 try: 

431 names.add(item.name) # type: ignore 

432 except AttributeError: 

433 names.add(item) 

434 return DimensionGraph(universe=self, names=names) 

435 

436 def sorted(self, elements: Iterable[Union[E, str]], *, reverse: bool = False) -> List[E]: 

437 """Return a sorted version of the given iterable of dimension elements. 

438 

439 The universe's sort order is topological (an element's dependencies 

440 precede it), with an unspecified (but deterministic) approach to 

441 breaking ties. 

442 

443 Parameters 

444 ---------- 

445 elements : iterable of `DimensionElement`. 

446 Elements to be sorted. 

447 reverse : `bool`, optional 

448 If `True`, sort in the opposite order. 

449 

450 Returns 

451 ------- 

452 sorted : `list` of `DimensionElement` 

453 A sorted list containing the same elements that were given. 

454 """ 

455 s = set(elements) 

456 result = [element for element in self._elements if element in s or element.name in s] 

457 if reverse: 

458 result.reverse() 

459 # mypy thinks this can return DimensionElements even if all the user 

460 # passed it was Dimensions; we know better. 

461 return result # type: ignore 

462 

463 # TODO: Remove this method on DM-38687. 

464 @deprecated( 

465 "Deprecated in favor of configurable dimension packers. Will be removed after v27.", 

466 version="v26", 

467 category=FutureWarning, 

468 ) 

469 def makePacker(self, name: str, dataId: DataCoordinate) -> DimensionPacker: 

470 """Make a dimension packer. 

471 

472 Constructs a `DimensionPacker` that can pack data ID dictionaries 

473 into unique integers. 

474 

475 Parameters 

476 ---------- 

477 name : `str` 

478 Name of the packer, matching a key in the "packers" section of the 

479 dimension configuration. 

480 dataId : `DataCoordinate` 

481 Fully-expanded data ID that identifies the at least the "fixed" 

482 dimensions of the packer (i.e. those that are assumed/given, 

483 setting the space over which packed integer IDs are unique). 

484 ``dataId.hasRecords()`` must return `True`. 

485 """ 

486 return self._packers[name](self, dataId) 

487 

488 def getEncodeLength(self) -> int: 

489 """Return encoded size of graph. 

490 

491 Returns the size (in bytes) of the encoded size of `DimensionGraph` 

492 instances in this universe. 

493 

494 See `DimensionGraph.encode` and `DimensionGraph.decode` for more 

495 information. 

496 """ 

497 return math.ceil(len(self._dimensions) / 8) 

498 

499 @classmethod 

500 def _unpickle(cls, version: int, namespace: Optional[str] = None) -> DimensionUniverse: 

501 """Return an unpickled dimension universe. 

502 

503 Callable used for unpickling. 

504 

505 For internal use only. 

506 """ 

507 if namespace is None: 

508 # Old pickled universe. 

509 namespace = _DEFAULT_NAMESPACE 

510 try: 

511 return cls._instances[version, namespace] 

512 except KeyError as err: 

513 raise pickle.UnpicklingError( 

514 f"DimensionUniverse with version '{version}' and namespace {namespace!r} " 

515 "not found. Note that DimensionUniverse objects are not " 

516 "truly serialized; when using pickle to transfer them " 

517 "between processes, an equivalent instance with the same " 

518 "version must already exist in the receiving process." 

519 ) from err 

520 

521 def __reduce__(self) -> tuple: 

522 return (self._unpickle, (self._version, self._namespace)) 

523 

524 def __deepcopy__(self, memo: dict) -> DimensionUniverse: 

525 # DimensionUniverse is recursively immutable; see note in @immutable 

526 # decorator. 

527 return self 

528 

529 # Class attributes below are shadowed by instance attributes, and are 

530 # present just to hold the docstrings for those instance attributes. 

531 

532 empty: DimensionGraph 

533 """The `DimensionGraph` that contains no dimensions (`DimensionGraph`). 

534 """ 

535 

536 commonSkyPix: SkyPixDimension 

537 """The special skypix dimension that is used to relate all other spatial 

538 dimensions in the `Registry` database (`SkyPixDimension`). 

539 """ 

540 

541 dimensionConfig: DimensionConfig 

542 """The configuration used to create this Universe (`DimensionConfig`).""" 

543 

544 _cache: Dict[FrozenSet[str], DimensionGraph] 

545 

546 _dimensions: NamedValueAbstractSet[Dimension] 

547 

548 _elements: NamedValueAbstractSet[DimensionElement] 

549 

550 _topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]] 

551 

552 _dimensionIndices: Dict[str, int] 

553 

554 _elementIndices: Dict[str, int] 

555 

556 _packers: Dict[str, DimensionPackerFactory] 

557 

558 _version: int 

559 

560 _namespace: str