Coverage for python/lsst/daf/butler/core/dimensions/_universe.py: 35%

154 statements  

« prev     ^ index     » next       coverage.py v7.2.3, created at 2023-04-22 02:18 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ["DimensionUniverse"] 

25 

26import logging 

27import math 

28import pickle 

29from typing import ( 

30 TYPE_CHECKING, 

31 Any, 

32 ClassVar, 

33 Dict, 

34 FrozenSet, 

35 Iterable, 

36 List, 

37 Mapping, 

38 Optional, 

39 Set, 

40 Tuple, 

41 TypeVar, 

42 Union, 

43) 

44 

45from lsst.utils.classes import cached_getter, immutable 

46 

47from .._topology import TopologicalFamily, TopologicalSpace 

48from ..config import Config 

49from ..named import NamedValueAbstractSet, NamedValueSet 

50from ._config import _DEFAULT_NAMESPACE, DimensionConfig 

51from ._database import DatabaseDimensionElement 

52from ._elements import Dimension, DimensionElement 

53from ._governor import GovernorDimension 

54from ._graph import DimensionGraph 

55from ._skypix import SkyPixDimension, SkyPixSystem 

56 

57if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 

58 from ._coordinate import DataCoordinate 

59 from ._packer import DimensionPacker, DimensionPackerFactory 

60 from .construction import DimensionConstructionBuilder 

61 

62 

63E = TypeVar("E", bound=DimensionElement) 

64_LOG = logging.getLogger(__name__) 

65 

66 

67@immutable 

68class DimensionUniverse: 

69 """Self-consistent set of dimensions. 

70 

71 A parent class that represents a complete, self-consistent set of 

72 dimensions and their relationships. 

73 

74 `DimensionUniverse` is not a class-level singleton, but all instances are 

75 tracked in a singleton map keyed by the version number and namespace 

76 in the configuration they were loaded from. Because these universes 

77 are solely responsible for constructing `DimensionElement` instances, 

78 these are also indirectly tracked by that singleton as well. 

79 

80 Parameters 

81 ---------- 

82 config : `Config`, optional 

83 Configuration object from which dimension definitions can be extracted. 

84 Ignored if ``builder`` is provided, or if ``version`` is provided and 

85 an instance with that version already exists. 

86 version : `int`, optional 

87 Integer version for this `DimensionUniverse`. If not provided, a 

88 version will be obtained from ``builder`` or ``config``. 

89 namespace : `str`, optional 

90 Namespace of this `DimensionUniverse`, combined with the version 

91 to provide universe safety for registries that use different 

92 dimension definitions. 

93 builder : `DimensionConstructionBuilder`, optional 

94 Builder object used to initialize a new instance. Ignored if 

95 ``version`` is provided and an instance with that version already 

96 exists. Should not have had `~DimensionConstructionBuilder.finish` 

97 called; this will be called if needed by `DimensionUniverse`. 

98 """ 

99 

100 _instances: ClassVar[Dict[Tuple[int, str], DimensionUniverse]] = {} 

101 """Singleton dictionary of all instances, keyed by version. 

102 

103 For internal use only. 

104 """ 

105 

106 def __new__( 

107 cls, 

108 config: Optional[Config] = None, 

109 *, 

110 version: Optional[int] = None, 

111 namespace: Optional[str] = None, 

112 builder: Optional[DimensionConstructionBuilder] = None, 

113 ) -> DimensionUniverse: 

114 # Try to get a version first, to look for existing instances; try to 

115 # do as little work as possible at this stage. 

116 if version is None: 

117 if builder is None: 

118 config = DimensionConfig(config) 

119 version = config["version"] 

120 else: 

121 version = builder.version 

122 

123 # Then a namespace. 

124 if namespace is None: 

125 if builder is None: 

126 config = DimensionConfig(config) 

127 namespace = config.get("namespace", _DEFAULT_NAMESPACE) 

128 else: 

129 namespace = builder.namespace 

130 # if still None use the default 

131 if namespace is None: 

132 namespace = _DEFAULT_NAMESPACE 

133 

134 # See if an equivalent instance already exists. 

135 self: Optional[DimensionUniverse] = cls._instances.get((version, namespace)) 

136 if self is not None: 

137 return self 

138 

139 # Ensure we have a builder, building one from config if necessary. 

140 if builder is None: 

141 config = DimensionConfig(config) 

142 builder = config.makeBuilder() 

143 

144 # Delegate to the builder for most of the construction work. 

145 builder.finish() 

146 

147 # Create the universe instance and create core attributes, mostly 

148 # copying from builder. 

149 self = object.__new__(cls) 

150 assert self is not None 

151 self._cache = {} 

152 self._dimensions = builder.dimensions 

153 self._elements = builder.elements 

154 self._topology = builder.topology 

155 self._packers = builder.packers 

156 self.dimensionConfig = builder.config 

157 commonSkyPix = self._dimensions[builder.commonSkyPixName] 

158 assert isinstance(commonSkyPix, SkyPixDimension) 

159 self.commonSkyPix = commonSkyPix 

160 

161 # Attach self to all elements. 

162 for element in self._elements: 

163 element.universe = self 

164 

165 # Add attribute for special subsets of the graph. 

166 self.empty = DimensionGraph(self, (), conform=False) 

167 

168 # Use the version number and namespace from the config as a key in 

169 # the singleton dict containing all instances; that will let us 

170 # transfer dimension objects between processes using pickle without 

171 # actually going through real initialization, as long as a universe 

172 # with the same version and namespace has already been constructed in 

173 # the receiving process. 

174 self._version = version 

175 self._namespace = namespace 

176 cls._instances[self._version, self._namespace] = self 

177 

178 # Build mappings from element to index. These are used for 

179 # topological-sort comparison operators in DimensionElement itself. 

180 self._elementIndices = {name: i for i, name in enumerate(self._elements.names)} 

181 # Same for dimension to index, sorted topologically across required 

182 # and implied. This is used for encode/decode. 

183 self._dimensionIndices = {name: i for i, name in enumerate(self._dimensions.names)} 

184 

185 return self 

186 

187 @property 

188 def version(self) -> int: 

189 """The version number of this universe. 

190 

191 Returns 

192 ------- 

193 version : `int` 

194 An integer representing the version number of this universe. 

195 Uniquely defined when combined with the `namespace`. 

196 """ 

197 return self._version 

198 

199 @property 

200 def namespace(self) -> str: 

201 """The namespace associated with this universe. 

202 

203 Returns 

204 ------- 

205 namespace : `str` 

206 The namespace. When combined with the `version` can uniquely 

207 define this universe. 

208 """ 

209 return self._namespace 

210 

211 def isCompatibleWith(self, other: DimensionUniverse) -> bool: 

212 """Check compatibility between this `DimensionUniverse` and another 

213 

214 Parameters 

215 ---------- 

216 other : `DimensionUniverse` 

217 The other `DimensionUniverse` to check for compatibility 

218 

219 Returns 

220 ------- 

221 results : `bool` 

222 If the other `DimensionUniverse` is compatible with this one return 

223 `True`, else `False` 

224 """ 

225 # Different namespaces mean that these universes cannot be compatible. 

226 if self.namespace != other.namespace: 

227 return False 

228 if self.version != other.version: 

229 _LOG.info( 

230 "Universes share a namespace %r but have differing versions (%d != %d). " 

231 " This could be okay but may be responsible for dimension errors later.", 

232 self.namespace, 

233 self.version, 

234 other.version, 

235 ) 

236 

237 # For now assume compatibility if versions differ. 

238 return True 

239 

240 def __repr__(self) -> str: 

241 return f"DimensionUniverse({self._version}, {self._namespace})" 

242 

243 def __getitem__(self, name: str) -> DimensionElement: 

244 return self._elements[name] 

245 

246 def __contains__(self, name: Any) -> bool: 

247 return name in self._elements 

248 

249 def get(self, name: str, default: Optional[DimensionElement] = None) -> Optional[DimensionElement]: 

250 """Return the `DimensionElement` with the given name or a default. 

251 

252 Parameters 

253 ---------- 

254 name : `str` 

255 Name of the element. 

256 default : `DimensionElement`, optional 

257 Element to return if the named one does not exist. Defaults to 

258 `None`. 

259 

260 Returns 

261 ------- 

262 element : `DimensionElement` 

263 The named element. 

264 """ 

265 return self._elements.get(name, default) 

266 

267 def getStaticElements(self) -> NamedValueAbstractSet[DimensionElement]: 

268 """Return a set of all static elements in this universe. 

269 

270 Non-static elements that are created as needed may also exist, but 

271 these are guaranteed to have no direct relationships to other elements 

272 (though they may have spatial or temporal relationships). 

273 

274 Returns 

275 ------- 

276 elements : `NamedValueAbstractSet` [ `DimensionElement` ] 

277 A frozen set of `DimensionElement` instances. 

278 """ 

279 return self._elements 

280 

281 def getStaticDimensions(self) -> NamedValueAbstractSet[Dimension]: 

282 """Return a set of all static dimensions in this universe. 

283 

284 Non-static dimensions that are created as needed may also exist, but 

285 these are guaranteed to have no direct relationships to other elements 

286 (though they may have spatial or temporal relationships). 

287 

288 Returns 

289 ------- 

290 dimensions : `NamedValueAbstractSet` [ `Dimension` ] 

291 A frozen set of `Dimension` instances. 

292 """ 

293 return self._dimensions 

294 

295 @cached_getter 

296 def getGovernorDimensions(self) -> NamedValueAbstractSet[GovernorDimension]: 

297 """Return a set of all `GovernorDimension` instances in this universe. 

298 

299 Returns 

300 ------- 

301 governors : `NamedValueAbstractSet` [ `GovernorDimension` ] 

302 A frozen set of `GovernorDimension` instances. 

303 """ 

304 return NamedValueSet(d for d in self._dimensions if isinstance(d, GovernorDimension)).freeze() 

305 

306 @cached_getter 

307 def getDatabaseElements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]: 

308 """Return set of all `DatabaseDimensionElement` instances in universe. 

309 

310 This does not include `GovernorDimension` instances, which are backed 

311 by the database but do not inherit from `DatabaseDimensionElement`. 

312 

313 Returns 

314 ------- 

315 elements : `NamedValueAbstractSet` [ `DatabaseDimensionElement` ] 

316 A frozen set of `DatabaseDimensionElement` instances. 

317 """ 

318 return NamedValueSet(d for d in self._elements if isinstance(d, DatabaseDimensionElement)).freeze() 

319 

320 @property 

321 @cached_getter 

322 def skypix(self) -> NamedValueAbstractSet[SkyPixSystem]: 

323 """All skypix systems known to this universe. 

324 

325 (`NamedValueAbstractSet` [ `SkyPixSystem` ]). 

326 """ 

327 return NamedValueSet( 

328 [ 

329 family 

330 for family in self._topology[TopologicalSpace.SPATIAL] 

331 if isinstance(family, SkyPixSystem) 

332 ] 

333 ).freeze() 

334 

335 def getElementIndex(self, name: str) -> int: 

336 """Return the position of the named dimension element. 

337 

338 The position is in this universe's sorting of all elements. 

339 

340 Parameters 

341 ---------- 

342 name : `str` 

343 Name of the element. 

344 

345 Returns 

346 ------- 

347 index : `int` 

348 Sorting index for this element. 

349 """ 

350 return self._elementIndices[name] 

351 

352 def getDimensionIndex(self, name: str) -> int: 

353 """Return the position of the named dimension. 

354 

355 This position is in this universe's sorting of all dimensions. 

356 

357 Parameters 

358 ---------- 

359 name : `str` 

360 Name of the dimension. 

361 

362 Returns 

363 ------- 

364 index : `int` 

365 Sorting index for this dimension. 

366 

367 Notes 

368 ----- 

369 The dimension sort order for a universe is consistent with the element 

370 order (all dimensions are elements), and either can be used to sort 

371 dimensions if used consistently. But there are also some contexts in 

372 which contiguous dimension-only indices are necessary or at least 

373 desirable. 

374 """ 

375 return self._dimensionIndices[name] 

376 

377 def expandDimensionNameSet(self, names: Set[str]) -> None: 

378 """Expand a set of dimension names in-place. 

379 

380 Includes recursive dependencies. 

381 

382 This is an advanced interface for cases where constructing a 

383 `DimensionGraph` (which also expands required dependencies) is 

384 impossible or undesirable. 

385 

386 Parameters 

387 ---------- 

388 names : `set` [ `str` ] 

389 A true `set` of dimension names, to be expanded in-place. 

390 """ 

391 # Keep iterating until the set of names stops growing. This is not as 

392 # efficient as it could be, but we work pretty hard cache 

393 # DimensionGraph instances to keep actual construction rare, so that 

394 # shouldn't matter. 

395 oldSize = len(names) 

396 while True: 

397 # iterate over a temporary copy so we can modify the original 

398 for name in tuple(names): 

399 names.update(self._dimensions[name].required.names) 

400 names.update(self._dimensions[name].implied.names) 

401 if oldSize == len(names): 

402 break 

403 else: 

404 oldSize = len(names) 

405 

406 def extract(self, iterable: Iterable[Union[Dimension, str]]) -> DimensionGraph: 

407 """Construct graph from iterable. 

408 

409 Constructs a `DimensionGraph` from a possibly-heterogenous iterable 

410 of `Dimension` instances and string names thereof. 

411 

412 Constructing `DimensionGraph` directly from names or dimension 

413 instances is slightly more efficient when it is known in advance that 

414 the iterable is not heterogenous. 

415 

416 Parameters 

417 ---------- 

418 iterable: iterable of `Dimension` or `str` 

419 Dimensions that must be included in the returned graph (their 

420 dependencies will be as well). 

421 

422 Returns 

423 ------- 

424 graph : `DimensionGraph` 

425 A `DimensionGraph` instance containing all given dimensions. 

426 """ 

427 names = set() 

428 for item in iterable: 

429 try: 

430 names.add(item.name) # type: ignore 

431 except AttributeError: 

432 names.add(item) 

433 return DimensionGraph(universe=self, names=names) 

434 

435 def sorted(self, elements: Iterable[Union[E, str]], *, reverse: bool = False) -> List[E]: 

436 """Return a sorted version of the given iterable of dimension elements. 

437 

438 The universe's sort order is topological (an element's dependencies 

439 precede it), with an unspecified (but deterministic) approach to 

440 breaking ties. 

441 

442 Parameters 

443 ---------- 

444 elements : iterable of `DimensionElement`. 

445 Elements to be sorted. 

446 reverse : `bool`, optional 

447 If `True`, sort in the opposite order. 

448 

449 Returns 

450 ------- 

451 sorted : `list` of `DimensionElement` 

452 A sorted list containing the same elements that were given. 

453 """ 

454 s = set(elements) 

455 result = [element for element in self._elements if element in s or element.name in s] 

456 if reverse: 

457 result.reverse() 

458 # mypy thinks this can return DimensionElements even if all the user 

459 # passed it was Dimensions; we know better. 

460 return result # type: ignore 

461 

462 def makePacker(self, name: str, dataId: DataCoordinate) -> DimensionPacker: 

463 """Make a dimension packer. 

464 

465 Constructs a `DimensionPacker` that can pack data ID dictionaries 

466 into unique integers. 

467 

468 Parameters 

469 ---------- 

470 name : `str` 

471 Name of the packer, matching a key in the "packers" section of the 

472 dimension configuration. 

473 dataId : `DataCoordinate` 

474 Fully-expanded data ID that identifies the at least the "fixed" 

475 dimensions of the packer (i.e. those that are assumed/given, 

476 setting the space over which packed integer IDs are unique). 

477 ``dataId.hasRecords()`` must return `True`. 

478 """ 

479 return self._packers[name](self, dataId) 

480 

481 def getEncodeLength(self) -> int: 

482 """Return encoded size of graph. 

483 

484 Returns the size (in bytes) of the encoded size of `DimensionGraph` 

485 instances in this universe. 

486 

487 See `DimensionGraph.encode` and `DimensionGraph.decode` for more 

488 information. 

489 """ 

490 return math.ceil(len(self._dimensions) / 8) 

491 

492 @classmethod 

493 def _unpickle(cls, version: int, namespace: Optional[str] = None) -> DimensionUniverse: 

494 """Return an unpickled dimension universe. 

495 

496 Callable used for unpickling. 

497 

498 For internal use only. 

499 """ 

500 if namespace is None: 

501 # Old pickled universe. 

502 namespace = _DEFAULT_NAMESPACE 

503 try: 

504 return cls._instances[version, namespace] 

505 except KeyError as err: 

506 raise pickle.UnpicklingError( 

507 f"DimensionUniverse with version '{version}' and namespace {namespace!r} " 

508 "not found. Note that DimensionUniverse objects are not " 

509 "truly serialized; when using pickle to transfer them " 

510 "between processes, an equivalent instance with the same " 

511 "version must already exist in the receiving process." 

512 ) from err 

513 

514 def __reduce__(self) -> tuple: 

515 return (self._unpickle, (self._version, self._namespace)) 

516 

517 def __deepcopy__(self, memo: dict) -> DimensionUniverse: 

518 # DimensionUniverse is recursively immutable; see note in @immutable 

519 # decorator. 

520 return self 

521 

522 # Class attributes below are shadowed by instance attributes, and are 

523 # present just to hold the docstrings for those instance attributes. 

524 

525 empty: DimensionGraph 

526 """The `DimensionGraph` that contains no dimensions (`DimensionGraph`). 

527 """ 

528 

529 commonSkyPix: SkyPixDimension 

530 """The special skypix dimension that is used to relate all other spatial 

531 dimensions in the `Registry` database (`SkyPixDimension`). 

532 """ 

533 

534 dimensionConfig: DimensionConfig 

535 """The configuration used to create this Universe (`DimensionConfig`).""" 

536 

537 _cache: Dict[FrozenSet[str], DimensionGraph] 

538 

539 _dimensions: NamedValueAbstractSet[Dimension] 

540 

541 _elements: NamedValueAbstractSet[DimensionElement] 

542 

543 _topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]] 

544 

545 _dimensionIndices: Dict[str, int] 

546 

547 _elementIndices: Dict[str, int] 

548 

549 _packers: Dict[str, DimensionPackerFactory] 

550 

551 _version: int 

552 

553 _namespace: str