Coverage for python/lsst/daf/butler/dimensions/

1# This file is part of daf_butler.

3# Developed for the LSST Data Management System.

4# This product includes software developed by the LSST Project

5# (http://www.lsst.org).

6# See the COPYRIGHT file at the top-level directory of this distribution

7# for details of code ownership.

9# This software is dual licensed under the GNU General Public License and also

10# under a 3-clause BSD license. Recipients may choose which of these licenses

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,

12# respectively. If you choose the GPL option then the following text applies

13# (but note that there is still no warranty even if you opt for BSD instead):

14#

15# This program is free software: you can redistribute it and/or modify

16# it under the terms of the GNU General Public License as published by

17# the Free Software Foundation, either version 3 of the License, or

18# (at your option) any later version.

19#

20# This program is distributed in the hope that it will be useful,

21# but WITHOUT ANY WARRANTY; without even the implied warranty of

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

23# GNU General Public License for more details.

24#

25# You should have received a copy of the GNU General Public License

26# along with this program. If not, see <http://www.gnu.org/licenses/>.

28from __future__ import annotations

30__all__ = ["DimensionUniverse"]

32import logging

33import pickle

34import warnings

35from collections import defaultdict

36from collections.abc import Iterable, Mapping, Sequence

37from typing import TYPE_CHECKING, Any, ClassVar, TypeVar, overload

39from lsst.utils.classes import cached_getter, immutable

41from .._config import Config

42from .._named import NamedValueAbstractSet, NamedValueSet

43from .._topology import TopologicalFamily, TopologicalSpace

44from .._utilities.thread_safe_cache import ThreadSafeCache

45from ._config import _DEFAULT_NAMESPACE, DimensionConfig

46from ._database import DatabaseDimensionElement

47from ._elements import Dimension, DimensionElement

48from ._governor import GovernorDimension

49from ._group import DimensionGroup

50from ._skypix import SkyPixDimension, SkyPixSystem

52if TYPE_CHECKING: # Imports needed only for type annotations; may be circular.

53 from .construction import DimensionConstructionBuilder

56E = TypeVar("E", bound=DimensionElement)

57_LOG = logging.getLogger(__name__)

60@immutable

61class DimensionUniverse: # numpydoc ignore=PR02

62 """Self-consistent set of dimensions.

64 A parent class that represents a complete, self-consistent set of

65 dimensions and their relationships.

67 `DimensionUniverse` is not a class-level singleton, but all instances are

68 tracked in a singleton map keyed by the version number and namespace

69 in the configuration they were loaded from. Because these universes

70 are solely responsible for constructing `DimensionElement` instances,

71 these are also indirectly tracked by that singleton as well.

73 Parameters

74 ----------

75 config : `Config`, optional

76 Configuration object from which dimension definitions can be extracted.

77 Ignored if ``builder`` is provided, or if ``version`` is provided and

78 an instance with that version already exists.

79 version : `int`, optional

80 Integer version for this `DimensionUniverse`. If not provided, a

81 version will be obtained from ``builder`` or ``config``.

82 namespace : `str`, optional

83 Namespace of this `DimensionUniverse`, combined with the version

84 to provide universe safety for registries that use different

85 dimension definitions.

86 builder : `DimensionConstructionBuilder`, optional

87 Builder object used to initialize a new instance. Ignored if

88 ``version`` is provided and an instance with that version already

89 exists. Should not have had `~DimensionConstructionBuilder.finish`

90 called; this will be called if needed by `DimensionUniverse`.

91 use_cache : `bool`, optional

92 If `True` or not provided, cache `DimensionUniverse` instances globally

93 to avoid creating more than one `DimensionUniverse` instance for a

94 given configuration.

95 """

97 _instances: ClassVar[ThreadSafeCache[tuple[int, str], DimensionUniverse]] = ThreadSafeCache()

98 """Singleton dictionary of all instances, keyed by version.

100 For internal use only.

101 """

102

103 def __new__(

104 cls,

105 config: Config | None = None,

106 *,

107 version: int | None = None,

108 namespace: str | None = None,

109 builder: DimensionConstructionBuilder | None = None,

110 use_cache: bool = True,

111 ) -> DimensionUniverse:

112 # Try to get a version first, to look for existing instances; try to

113 # do as little work as possible at this stage.

114 if version is None:

115 if builder is None:

116 config = DimensionConfig(config)

117 version = config["version"]

118 else:

119 version = builder.version

120

121 if use_cache is not True:

122 warnings.warn(

123 "use_cache parameter is no longer supported and is ignored. Will be removed after v30.",

124 category=FutureWarning,

125 stacklevel=2,

126 )

127

128 # Then a namespace.

129 if namespace is None:

130 if builder is None:

131 config = DimensionConfig(config)

132 namespace = config.get("namespace", _DEFAULT_NAMESPACE)

133 else:

134 namespace = builder.namespace

135 # if still None use the default

136 if namespace is None:

137 namespace = _DEFAULT_NAMESPACE

138

139 # See if an equivalent instance already exists.

140 existing_instance = cls._instances.get((version, namespace))

141 if existing_instance is not None:

142 return existing_instance

143

144 # Ensure we have a builder, building one from config if necessary.

145 if builder is None:

146 config = DimensionConfig(config)

147 builder = config.makeBuilder()

148

149 # Delegate to the builder for most of the construction work.

150 builder.finish()

151

152 # Create the universe instance and create core attributes, mostly

153 # copying from builder.

154 self: DimensionUniverse = object.__new__(cls)

155 assert self is not None

156 self._cached_groups = ThreadSafeCache()

157 self._dimensions = builder.dimensions

158 self._elements = builder.elements

159 self._topology = builder.topology

160 self.dimensionConfig = builder.config

161 commonSkyPix = self._dimensions[builder.commonSkyPixName]

162 assert isinstance(commonSkyPix, SkyPixDimension)

163 self.commonSkyPix = commonSkyPix

164

165 # Attach self to all elements.

166 for element in self._elements:

167 element.universe = self

168

169 # Add attribute for special subsets of the graph.

170 self._empty = DimensionGroup(self, (), _conform=False)

171

172 # Use the version number and namespace from the config as a key in

173 # the singleton dict containing all instances; that will let us

174 # transfer dimension objects between processes using pickle without

175 # actually going through real initialization, as long as a universe

176 # with the same version and namespace has already been constructed in

177 # the receiving process.

178 self._version = version

179 self._namespace = namespace

180

181 # Build mappings from element to index. These are used for

182 # topological-sort comparison operators in DimensionElement itself.

183 self._elementIndices = {name: i for i, name in enumerate(self._elements.names)}

184 # Same for dimension to index, sorted topologically across required

185 # and implied. This is used for encode/decode.

186 self._dimensionIndices = {name: i for i, name in enumerate(self._dimensions.names)}

187

188 self._populates = defaultdict(NamedValueSet)

189 for element in self._elements:

190 if element.populated_by is not None:

191 self._populates[element.populated_by.name].add(element)

192

193 return cls._instances.set_or_get((self._version, self._namespace), self)

194

195 @property

196 def version(self) -> int:

197 """The version number of this universe.

198

199 Returns

200 -------

201 version : `int`

202 An integer representing the version number of this universe.

203 Uniquely defined when combined with the `namespace`.

204 """

205 return self._version

206

207 @property

208 def namespace(self) -> str:

209 """The namespace associated with this universe.

210

211 Returns

212 -------

213 namespace : `str`

214 The namespace. When combined with the `version` can uniquely

215 define this universe.

216 """

217 return self._namespace

218

219 def isCompatibleWith(self, other: DimensionUniverse) -> bool:

220 """Check compatibility between this `DimensionUniverse` and another.

221

222 Parameters

223 ----------

224 other : `DimensionUniverse`

225 The other `DimensionUniverse` to check for compatibility.

226

227 Returns

228 -------

229 results : `bool`

230 If the other `DimensionUniverse` is compatible with this one return

231 `True`, else `False`.

232 """

233 # Different namespaces mean that these universes cannot be compatible.

234 if self.namespace != other.namespace:

235 return False

236 if self.version != other.version:

237 _LOG.info(

238 "Universes share a namespace %r but have differing versions (%d != %d). "

239 " This could be okay but may be responsible for dimension errors later.",

240 self.namespace,

241 self.version,

242 other.version,

243 )

244

245 # For now assume compatibility if versions differ.

246 return True

247

248 def __repr__(self) -> str:

249 return f"DimensionUniverse({self._version}, {self._namespace})"

250

251 def __getitem__(self, name: str) -> DimensionElement:

252 return self._elements[name]

253

254 def __contains__(self, name: Any) -> bool:

255 return name in self._elements

256

257 def get(self, name: str, default: DimensionElement | None = None) -> DimensionElement | None:

258 """Return the `DimensionElement` with the given name or a default.

259

260 Parameters

261 ----------

262 name : `str`

263 Name of the element.

264 default : `DimensionElement`, optional

265 Element to return if the named one does not exist. Defaults to

266 `None`.

267

268 Returns

269 -------

270 element : `DimensionElement`

271 The named element.

272 """

273 return self._elements.get(name, default)

274

275 def getStaticElements(self) -> NamedValueAbstractSet[DimensionElement]:

276 """Return a set of all static elements in this universe.

277

278 Non-static elements that are created as needed may also exist, but

279 these are guaranteed to have no direct relationships to other elements

280 (though they may have spatial or temporal relationships).

281

282 Returns

283 -------

284 elements : `NamedValueAbstractSet` [ `DimensionElement` ]

285 A frozen set of `DimensionElement` instances.

286 """

287 return self._elements

288

289 def getStaticDimensions(self) -> NamedValueAbstractSet[Dimension]:

290 """Return a set of all static dimensions in this universe.

291

292 Non-static dimensions that are created as needed may also exist, but

293 these are guaranteed to have no direct relationships to other elements

294 (though they may have spatial or temporal relationships).

295

296 Returns

297 -------

298 dimensions : `NamedValueAbstractSet` [ `Dimension` ]

299 A frozen set of `Dimension` instances.

300 """

301 return self._dimensions

302

303 def getGovernorDimensions(self) -> NamedValueAbstractSet[GovernorDimension]:

304 """Return a set of all `GovernorDimension` instances in this universe.

305

306 Returns

307 -------

308 governors : `NamedValueAbstractSet` [ `GovernorDimension` ]

309 A frozen set of `GovernorDimension` instances.

310 """

311 return self.governor_dimensions

312

313 def getDatabaseElements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]:

314 """Return set of all `DatabaseDimensionElement` instances in universe.

315

316 This does not include `GovernorDimension` instances, which are backed

317 by the database but do not inherit from `DatabaseDimensionElement`.

318

319 Returns

320 -------

321 elements : `NamedValueAbstractSet` [ `DatabaseDimensionElement` ]

322 A frozen set of `DatabaseDimensionElement` instances.

323 """

324 return self.database_elements

325

326 @property

327 def elements(self) -> NamedValueAbstractSet[DimensionElement]:

328 """All dimension elements defined in this universe."""

329 return self._elements

330

331 @property

332 def dimensions(self) -> NamedValueAbstractSet[Dimension]:

333 """All dimensions defined in this universe."""

334 return self._dimensions

335

336 @property

337 @cached_getter

338 def governor_dimensions(self) -> NamedValueAbstractSet[GovernorDimension]:

339 """All governor dimensions defined in this universe.

340

341 Governor dimensions serve as special required dependencies of other

342 dimensions, with special handling in dimension query expressions and

343 collection summaries. Governor dimension records are stored in the

344 database but the set of such values is expected to be small enough

345 for all values to be cached by all clients.

346 """

347 return NamedValueSet(d for d in self._dimensions if isinstance(d, GovernorDimension)).freeze()

348

349 @property

350 @cached_getter

351 def skypix_dimensions(self) -> NamedValueAbstractSet[SkyPixDimension]:

352 """All skypix dimensions defined in this universe.

353

354 Skypix dimension records are always generated on-the-fly rather than

355 stored in the database, and they always represent a tiling of the sky

356 with no overlaps.

357 """

358 result = NamedValueSet[SkyPixDimension]()

359 for system in self.skypix:

360 result.update(system)

361 return result.freeze()

362

363 @property

364 @cached_getter

365 def database_elements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]:

366 """All dimension elements whose records are stored in the database,

367 except governor dimensions.

368 """

369 return NamedValueSet(d for d in self._elements if isinstance(d, DatabaseDimensionElement)).freeze()

370

371 @property

372 @cached_getter

373 def skypix(self) -> NamedValueAbstractSet[SkyPixSystem]:

374 """All skypix systems known to this universe.

375

376 (`NamedValueAbstractSet` [ `SkyPixSystem` ]).

377 """

378 return NamedValueSet(

379 [

380 family

381 for family in self._topology[TopologicalSpace.SPATIAL]

382 if isinstance(family, SkyPixSystem)

383 ]

384 ).freeze()

385

386 def getElementIndex(self, name: str) -> int:

387 """Return the position of the named dimension element.

388

389 The position is in this universe's sorting of all elements.

390

391 Parameters

392 ----------

393 name : `str`

394 Name of the element.

395

396 Returns

397 -------

398 index : `int`

399 Sorting index for this element.

400 """

401 return self._elementIndices[name]

402

403 def getDimensionIndex(self, name: str) -> int:

404 """Return the position of the named dimension.

405

406 This position is in this universe's sorting of all dimensions.

407

408 Parameters

409 ----------

410 name : `str`

411 Name of the dimension.

412

413 Returns

414 -------

415 index : `int`

416 Sorting index for this dimension.

417

418 Notes

419 -----

420 The dimension sort order for a universe is consistent with the element

421 order (all dimensions are elements), and either can be used to sort

422 dimensions if used consistently. But there are also some contexts in

423 which contiguous dimension-only indices are necessary or at least

424 desirable.

425 """

426 return self._dimensionIndices[name]

427

428 def conform(

429 self,

430 dimensions: Iterable[str] | str | DimensionGroup,

431 /,

432 ) -> DimensionGroup:

433 """Construct a dimension group from an iterable of dimension names.

434

435 Parameters

436 ----------

437 dimensions : `~collections.abc.Iterable` [ `str` ], `str`, or \

438 `DimensionGroup`

439 Dimensions that must be included in the returned group; their

440 dependencies will be as well.

441

442 Returns

443 -------

444 group : `DimensionGroup`

445 A `DimensionGroup` instance containing all given dimensions.

446 """

447 match dimensions:

448 case DimensionGroup():

449 return dimensions

450 case str() as name:

451 return self[name].minimal_group

452 case iterable:

453 return DimensionGroup(self, set(iterable))

454

455 @overload

456 def sorted(self, elements: Iterable[Dimension], *, reverse: bool = False) -> Sequence[Dimension]: ... 456 ↛ exitline 456 didn't return from function 'sorted' because

457

458 @overload

459 def sorted( 459 ↛ exitline 459 didn't return from function 'sorted' because

460 self, elements: Iterable[DimensionElement | str], *, reverse: bool = False

461 ) -> Sequence[DimensionElement]: ...

462

463 def sorted(self, elements: Iterable[Any], *, reverse: bool = False) -> list[Any]:

464 """Return a sorted version of the given iterable of dimension elements.

465

466 The universe's sort order is topological (an element's dependencies

467 precede it), with an unspecified (but deterministic) approach to

468 breaking ties.

469

470 Parameters

471 ----------

472 elements : `~collections.abc.Iterable` of `DimensionElement`

473 Elements to be sorted.

474 reverse : `bool`, optional

475 If `True`, sort in the opposite order.

476

477 Returns

478 -------

479 sorted : `~collections.abc.Sequence` [ `Dimension` or \

480 `DimensionElement` ]

481 A sorted sequence containing the same elements that were given.

482 """

483 s = set(elements)

484 result = [element for element in self._elements if element in s or element.name in s]

485 if reverse:

486 result.reverse()

487 return result

488

489 def get_elements_populated_by(self, dimension: Dimension) -> NamedValueAbstractSet[DimensionElement]:

490 """Return the set of `DimensionElement` objects whose

491 `~DimensionElement.populated_by` attribute is the given dimension.

492

493 Parameters

494 ----------

495 dimension : `Dimension`

496 The dimension of interest.

497

498 Returns

499 -------

500 populated_by : `NamedValueAbstractSet` [ `DimensionElement` ]

501 The set of elements who say they are populated by the given

502 dimension.

503 """

504 return self._populates[dimension.name]

505

506 @property

507 def empty(self) -> DimensionGroup:

508 """The `DimensionGroup` that contains no dimensions."""

509 return self._empty

510

511 @classmethod

512 def _unpickle(cls, version: int, namespace: str | None = None) -> DimensionUniverse:

513 """Return an unpickled dimension universe.

514

515 Callable used for unpickling.

516

517 For internal use only.

518 """

519 if namespace is None:

520 # Old pickled universe.

521 namespace = _DEFAULT_NAMESPACE

522 instance = cls._instances.get((version, namespace))

523 if instance is None:

524 raise pickle.UnpicklingError(

525 f"DimensionUniverse with version '{version}' and namespace {namespace!r} "

526 "not found. Note that DimensionUniverse objects are not "

527 "truly serialized; when using pickle to transfer them "

528 "between processes, an equivalent instance with the same "

529 "version must already exist in the receiving process."

530 )

531 return instance

532

533 def __reduce__(self) -> tuple:

534 return (self._unpickle, (self._version, self._namespace))

535

536 def __deepcopy__(self, memo: dict) -> DimensionUniverse:

537 # DimensionUniverse is recursively immutable; see note in @immutable

538 # decorator.

539 return self

540

541 # Class attributes below are shadowed by instance attributes, and are

542 # present just to hold the docstrings for those instance attributes.

543

544 commonSkyPix: SkyPixDimension

545 """The special skypix dimension that is used to relate all other spatial

546 dimensions in the `Registry` database (`SkyPixDimension`).

547 """

548

549 dimensionConfig: DimensionConfig

550 """The configuration used to create this Universe (`DimensionConfig`)."""

551

552 _cached_groups: ThreadSafeCache[frozenset[str], DimensionGroup]

553

554 _dimensions: NamedValueAbstractSet[Dimension]

555

556 _elements: NamedValueAbstractSet[DimensionElement]

557

558 _empty: DimensionGroup

559

560 _topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]]

561

562 _dimensionIndices: dict[str, int]

563

564 _elementIndices: dict[str, int]

565

566 _populates: defaultdict[str, NamedValueSet[DimensionElement]]

567

568 _version: int

569

570 _namespace: str

Coverage for python / lsst / daf / butler / dimensions / _universe.py: 40%

175 statements