Coverage for python/lsst/daf/butler/dimensions/_config.py: 88%
228 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-07-22 17:01 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30__all__ = ("DimensionConfig", "SerializedDimensionConfig")
32import textwrap
33from collections.abc import Mapping, Sequence, Set
34from typing import Any, ClassVar, Literal, Union, final
36import pydantic
38from lsst.resources import ResourcePath, ResourcePathExpression
39from lsst.sphgeom import PixelizationABC
40from lsst.utils.doImport import doImportType
42from .._config import Config, ConfigSubset
43from .._named import NamedValueSet
44from .._topology import TopologicalSpace
45from ._database import DatabaseTopologicalFamilyConstructionVisitor
46from ._elements import Dimension, KeyColumnSpec, MetadataColumnSpec
47from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor
49# The default namespace to use on older dimension config files that only
50# have a version.
51_DEFAULT_NAMESPACE = "daf_butler"
54class DimensionConfig(ConfigSubset):
55 """Configuration that defines a `DimensionUniverse`.
57 The configuration tree for dimensions is a (nested) dictionary
58 with five top-level entries:
60 - version: an integer version number, used as keys in a singleton registry
61 of all `DimensionUniverse` instances;
63 - namespace: a string to be associated with the version in the singleton
64 registry of all `DimensionUnivers` instances;
66 - skypix: a dictionary whose entries each define a `SkyPixSystem`,
67 along with a special "common" key whose value is the name of a skypix
68 dimension that is used to relate all other spatial dimensions in the
69 `Registry` database;
71 - elements: a nested dictionary whose entries each define
72 `StandardDimension` or `StandardDimensionCombination`.
74 - topology: a nested dictionary with ``spatial`` and ``temporal`` keys,
75 with dictionary values that each define a `StandardTopologicalFamily`.
77 - packers: ignored.
79 See the documentation for the linked classes above for more information
80 on the configuration syntax.
82 Parameters
83 ----------
84 other : `Config` or `str` or `dict`, optional
85 Argument specifying the configuration information as understood
86 by `Config`. If `None` is passed then defaults are loaded from
87 "dimensions.yaml", otherwise defaults are not loaded.
88 validate : `bool`, optional
89 If `True` required keys will be checked to ensure configuration
90 consistency.
91 searchPaths : `list` or `tuple`, optional
92 Explicit additional paths to search for defaults. They should
93 be supplied in priority order. These paths have higher priority
94 than those read from the environment in
95 `ConfigSubset.defaultSearchPaths()`. Paths can be `str` referring to
96 the local file system or URIs, `lsst.resources.ResourcePath`.
97 """
99 requiredKeys = ("version", "elements", "skypix")
100 defaultConfigFile = "dimensions.yaml"
102 def __init__(
103 self,
104 other: Config | ResourcePathExpression | Mapping[str, Any] | None = None,
105 validate: bool = True,
106 searchPaths: Sequence[ResourcePathExpression] | None = None,
107 ):
108 # if argument is not None then do not load/merge defaults
109 mergeDefaults = other is None
110 super().__init__(other=other, validate=validate, mergeDefaults=mergeDefaults, searchPaths=searchPaths)
112 def _updateWithConfigsFromPath(
113 self, searchPaths: Sequence[str | ResourcePath], configFile: ResourcePath | str
114 ) -> None:
115 """Search the supplied paths reading config from first found.
117 Raises
118 ------
119 FileNotFoundError
120 Raised if config file is not found in any of given locations.
122 Notes
123 -----
124 This method overrides base class method with different behavior.
125 Instead of merging all found files into a single configuration it
126 finds first matching file and reads it.
127 """
128 uri = ResourcePath(configFile)
129 if uri.isabs() and uri.exists(): 129 ↛ 131line 129 didn't jump to line 131 because the condition on line 129 was never true
130 # Assume this resource exists
131 self._updateWithOtherConfigFile(configFile)
132 self.filesRead.append(configFile)
133 else:
134 for pathDir in searchPaths: 134 ↛ 145line 134 didn't jump to line 145 because the loop on line 134 didn't complete
135 if isinstance(pathDir, str | ResourcePath): 135 ↛ 143line 135 didn't jump to line 143 because the condition on line 135 was always true
136 pathDir = ResourcePath(pathDir, forceDirectory=True)
137 file = pathDir.join(configFile)
138 if file.exists(): 138 ↛ 134line 138 didn't jump to line 134 because the condition on line 138 was always true
139 self.filesRead.append(file)
140 self._updateWithOtherConfigFile(file)
141 break
142 else:
143 raise TypeError(f"Unexpected search path type encountered: {pathDir!r}")
144 else:
145 raise FileNotFoundError(f"Could not find {configFile} in search path {searchPaths}")
147 def _updateWithOtherConfigFile(self, file: Config | str | ResourcePath | Mapping[str, Any]) -> None:
148 """Override for base class method.
150 Parameters
151 ----------
152 file : `Config`, `str`, `lsst.resources.ResourcePath`, or `dict`
153 Entity that can be converted to a `ConfigSubset`.
154 """
155 # Use this class to read the defaults so that subsetting can happen
156 # correctly.
157 externalConfig = type(self)(file, validate=False)
158 self.update(externalConfig)
160 def to_simple(self) -> SerializedDimensionConfig:
161 """Convert this configuration to a serializable Pydantic model.
163 Returns
164 -------
165 model : `SerializedDimensionConfig`
166 Serializable Pydantic version of this configuration.
167 """
168 return SerializedDimensionConfig.model_validate(self.toDict())
170 @staticmethod
171 def from_simple(simple: SerializedDimensionConfig) -> DimensionConfig:
172 """Load the configuration from a serialized version.
174 Parameters
175 ----------
176 simple : `SerializedDimensionConfig`
177 Serialized configuration to be loaded.
179 Returns
180 -------
181 config : `DimensionConfig`
182 Dimension configuration.
183 """
184 return DimensionConfig(
185 simple.model_dump(
186 # Force sets back to lists for storage in the config.
187 # The config does not do this sanitation itself and so
188 # without this the config can not be serialized to JSON
189 # form using the dump() method.
190 mode="json",
191 # Some of the fields in Pydantic model config have aliases
192 # (e.g. remapping 'class_' to 'class'). Pydantic ignores these
193 # in model_dump() by default, so we have to add by_alias to
194 # make sure that we end up with the right names in the dict.
195 by_alias=True,
196 )
197 )
199 def makeBuilder(self) -> DimensionConstructionBuilder:
200 """Construct a `DimensionConstructionBuilder`.
202 The builder will reflect this configuration.
204 Returns
205 -------
206 builder : `DimensionConstructionBuilder`
207 A builder object populated with all visitors from this
208 configuration. The `~DimensionConstructionBuilder.finish` method
209 will not have been called.
210 """
211 validated = self.to_simple()
212 builder = DimensionConstructionBuilder(
213 validated.version,
214 validated.skypix.common,
215 self,
216 namespace=validated.namespace,
217 )
218 for system_name, system_config in sorted(validated.skypix.systems.items()):
219 builder.add(system_name, system_config)
220 for element_name, element_config in validated.elements.items():
221 builder.add(element_name, element_config)
222 for family_name, members in validated.topology.spatial.items():
223 builder.add(
224 family_name,
225 DatabaseTopologicalFamilyConstructionVisitor(space=TopologicalSpace.SPATIAL, members=members),
226 )
227 for family_name, members in validated.topology.temporal.items():
228 builder.add(
229 family_name,
230 DatabaseTopologicalFamilyConstructionVisitor(
231 space=TopologicalSpace.TEMPORAL, members=members
232 ),
233 )
234 return builder
237@final
238class _SkyPixSystemConfig(pydantic.BaseModel, DimensionConstructionVisitor):
239 """Description of a hierarchical sky pixelization system in dimension
240 universe configuration.
241 """
243 class_: str = pydantic.Field(
244 alias="class",
245 description="Fully-qualified name of an `lsst.sphgeom.PixelizationABC implementation.",
246 )
248 min_level: int = 1
249 """Minimum level for this pixelization."""
251 max_level: int | None
252 """Maximum level for this pixelization."""
254 def has_dependencies_in(self, others: Set[str]) -> bool:
255 # Docstring inherited from DimensionConstructionVisitor.
256 return False
258 def visit(self, name: str, builder: DimensionConstructionBuilder) -> None:
259 # Docstring inherited from DimensionConstructionVisitor.
260 PixelizationClass = doImportType(self.class_)
261 assert issubclass(PixelizationClass, PixelizationABC)
262 if self.max_level is None: 262 ↛ 263line 262 didn't jump to line 263 because the condition on line 262 was never true
263 max_level: int | None = getattr(PixelizationClass, "MAX_LEVEL", None)
264 if max_level is None:
265 raise TypeError(
266 f"Skypix pixelization class {self.class_} does"
267 " not have MAX_LEVEL but no max level has been set explicitly."
268 )
269 self.max_level = max_level
271 from ._skypix import SkyPixSystem
273 system = SkyPixSystem(
274 name,
275 maxLevel=self.max_level,
276 PixelizationClass=PixelizationClass,
277 )
278 builder.topology[TopologicalSpace.SPATIAL].add(system)
279 for level in range(self.min_level, self.max_level + 1):
280 dimension = system[level]
281 builder.dimensions.add(dimension)
282 builder.elements.add(dimension)
285@final
286class _SkyPixSectionConfig(pydantic.BaseModel):
287 """Section of the dimension universe configuration that describes sky
288 pixelizations.
289 """
291 common: str = pydantic.Field(
292 description="Name of the dimension used to relate all other spatial dimensions."
293 )
295 systems: dict[str, _SkyPixSystemConfig] = pydantic.Field(
296 default_factory=dict, description="Descriptions of the supported sky pixelization systems."
297 )
299 model_config = pydantic.ConfigDict(extra="allow")
301 @pydantic.model_validator(mode="after")
302 def _move_extra_to_systems(self) -> _SkyPixSectionConfig:
303 """Reinterpret extra fields in this model as members of the `systems`
304 dictionary.
305 """
306 if self.__pydantic_extra__ is None: 306 ↛ 307line 306 didn't jump to line 307 because the condition on line 306 was never true
307 self.__pydantic_extra__ = {}
308 for name, data in self.__pydantic_extra__.items():
309 self.systems[name] = _SkyPixSystemConfig.model_validate(data)
310 self.__pydantic_extra__.clear()
311 return self
314@final
315class _TopologySectionConfig(pydantic.BaseModel):
316 """Section of the dimension universe configuration that describes spatial
317 and temporal relationships.
318 """
320 spatial: dict[str, list[str]] = pydantic.Field(
321 default_factory=dict,
322 description=textwrap.dedent(
323 """\
324 Dictionary of spatial dimension elements, grouped by the "family"
325 they belong to.
327 Elements in a family are ordered from fine-grained to coarse-grained.
328 """
329 ),
330 )
332 temporal: dict[str, list[str]] = pydantic.Field(
333 default_factory=dict,
334 description=textwrap.dedent(
335 """\
336 Dictionary of temporal dimension elements, grouped by the "family"
337 they belong to.
339 Elements in a family are ordered from fine-grained to coarse-grained.
340 """
341 ),
342 )
345@final
346class _LegacyGovernorDimensionStorage(pydantic.BaseModel):
347 """Legacy storage configuration for governor dimensions."""
349 cls: Literal["lsst.daf.butler.registry.dimensions.governor.BasicGovernorDimensionRecordStorage"] = (
350 "lsst.daf.butler.registry.dimensions.governor.BasicGovernorDimensionRecordStorage"
351 )
353 has_own_table: ClassVar[Literal[True]] = True
354 """Whether this dimension needs a database table to be defined."""
356 is_cached: ClassVar[Literal[True]] = True
357 """Whether this dimension's records should be cached in clients."""
359 implied_union_target: ClassVar[Literal[None]] = None
360 """Name of another dimension that implies this one, whose values for this
361 dimension define the set of allowed values for this dimension.
362 """
365@final
366class _LegacyTableDimensionStorage(pydantic.BaseModel):
367 """Legacy storage configuration for regular dimension tables stored in the
368 database.
369 """
371 cls: Literal["lsst.daf.butler.registry.dimensions.table.TableDimensionRecordStorage"] = (
372 "lsst.daf.butler.registry.dimensions.table.TableDimensionRecordStorage"
373 )
375 has_own_table: ClassVar[Literal[True]] = True
376 """Whether this dimension element needs a database table to be defined."""
378 is_cached: ClassVar[Literal[False]] = False
379 """Whether this dimension element's records should be cached in clients."""
381 implied_union_target: ClassVar[Literal[None]] = None
382 """Name of another dimension that implies this one, whose values for this
383 dimension define the set of allowed values for this dimension.
384 """
387@final
388class _LegacyImpliedUnionDimensionStorage(pydantic.BaseModel):
389 """Legacy storage configuration for dimensions whose allowable values are
390 computed from the union of the values in another dimension that implies
391 this one.
392 """
394 cls: Literal["lsst.daf.butler.registry.dimensions.query.QueryDimensionRecordStorage"] = (
395 "lsst.daf.butler.registry.dimensions.query.QueryDimensionRecordStorage"
396 )
398 view_of: str
399 """The dimension that implies this one and defines its values."""
401 has_own_table: ClassVar[Literal[False]] = False
402 """Whether this dimension needs a database table to be defined."""
404 is_cached: ClassVar[Literal[False]] = False
405 """Whether this dimension element's records should be cached in clients."""
407 @property
408 def implied_union_target(self) -> str:
409 """Name of another dimension that implies this one, whose values for
410 this dimension define the set of allowed values for this dimension.
411 """
412 return self.view_of
415@final
416class _LegacyCachingDimensionStorage(pydantic.BaseModel):
417 """Legacy storage configuration that wraps another to indicate that its
418 records should be cached.
419 """
421 cls: Literal["lsst.daf.butler.registry.dimensions.caching.CachingDimensionRecordStorage"] = (
422 "lsst.daf.butler.registry.dimensions.caching.CachingDimensionRecordStorage"
423 )
425 nested: _LegacyTableDimensionStorage | _LegacyImpliedUnionDimensionStorage
426 """Dimension storage configuration wrapped by this one."""
428 @property
429 def has_own_table(self) -> bool:
430 """Whether this dimension needs a database table to be defined."""
431 return self.nested.has_own_table
433 is_cached: ClassVar[Literal[True]] = True
434 """Whether this dimension element's records should be cached in clients."""
436 @property
437 def implied_union_target(self) -> str | None:
438 """Name of another dimension that implies this one, whose values for
439 this dimension define the set of allowed values for this dimension.
440 """
441 return self.nested.implied_union_target
444@final
445class _ElementConfig(pydantic.BaseModel, DimensionConstructionVisitor):
446 """Description of a single dimension or dimension join relation in
447 dimension universe configuration.
448 """
450 doc: str = pydantic.Field(default="", description="Documentation for the dimension or relationship.")
452 keys: list[KeyColumnSpec] = pydantic.Field(
453 default_factory=list,
454 description=textwrap.dedent(
455 """\
456 Key columns that (along with required dependency values) uniquely
457 identify the dimension's records.
459 The first columns in this list is the primary key and is used in
460 data coordinate values. Other columns are alternative keys and
461 are defined with SQL ``UNIQUE`` constraints.
462 """
463 ),
464 )
466 requires: set[str] = pydantic.Field(
467 default_factory=set,
468 description=(
469 "Other dimensions whose primary keys are part of this dimension element's (compound) primary key."
470 ),
471 )
473 implies: set[str] = pydantic.Field(
474 default_factory=set,
475 description="Other dimensions whose primary keys appear as foreign keys in this dimension element.",
476 )
478 metadata: list[MetadataColumnSpec] = pydantic.Field(
479 default_factory=list,
480 description="Non-key columns that provide extra information about a dimension element.",
481 )
483 is_cached: bool = pydantic.Field(
484 default=False,
485 description="Whether this element's records should be cached in the client.",
486 )
488 implied_union_target: str | None = pydantic.Field(
489 default=None,
490 description=textwrap.dedent(
491 """\
492 Another dimension whose stored values for this dimension form the
493 set of all allowed values.
495 The target dimension must have this dimension in its "implies"
496 list. This means the current dimension will have no table of its
497 own in the database.
498 """
499 ),
500 )
502 governor: bool = pydantic.Field(
503 default=False,
504 description=textwrap.dedent(
505 """\
506 Whether this is a governor dimension.
508 Governor dimensions are expected to have a tiny number of rows and
509 must be explicitly provided in any dimension expression in which
510 dependent dimensions appear.
512 Implies is_cached=True.
513 """
514 ),
515 )
517 always_join: bool = pydantic.Field(
518 default=False,
519 description=textwrap.dedent(
520 """\
521 Whether this dimension join relation should always be included in
522 any query where its required dependencies appear.
523 """
524 ),
525 )
527 populated_by: str | None = pydantic.Field(
528 default=None,
529 description=textwrap.dedent(
530 """\
531 The name of a required dimension that this dimension join
532 relation's rows should transferred alongside.
533 """
534 ),
535 )
537 storage: Union[
538 _LegacyGovernorDimensionStorage,
539 _LegacyTableDimensionStorage,
540 _LegacyImpliedUnionDimensionStorage,
541 _LegacyCachingDimensionStorage,
542 None,
543 ] = pydantic.Field(
544 description="How this dimension element's rows should be stored in the database and client.",
545 discriminator="cls",
546 default=None,
547 )
549 def has_dependencies_in(self, others: Set[str]) -> bool:
550 # Docstring inherited from DimensionConstructionVisitor.
551 return not (self.requires.isdisjoint(others) and self.implies.isdisjoint(others))
553 def visit(self, name: str, builder: DimensionConstructionBuilder) -> None:
554 # Docstring inherited from DimensionConstructionVisitor.
555 if self.governor:
556 from ._governor import GovernorDimension
558 governor = GovernorDimension(
559 name,
560 metadata_columns=NamedValueSet(self.metadata).freeze(),
561 unique_keys=NamedValueSet(self.keys).freeze(),
562 doc=self.doc,
563 )
564 builder.dimensions.add(governor)
565 builder.elements.add(governor)
566 return
567 # Expand required dependencies.
568 for dependency_name in tuple(self.requires): # iterate over copy
569 self.requires.update(builder.dimensions[dependency_name].required.names)
570 # Transform required and implied Dimension names into instances,
571 # and reorder to match builder's order.
572 required: NamedValueSet[Dimension] = NamedValueSet()
573 implied: NamedValueSet[Dimension] = NamedValueSet()
574 for dimension in builder.dimensions:
575 if dimension.name in self.requires:
576 required.add(dimension)
577 if dimension.name in self.implies:
578 implied.add(dimension)
579 # Elements with keys are Dimensions; the rest are
580 # DimensionCombinations.
581 if self.keys:
582 from ._database import DatabaseDimension
584 dimension = DatabaseDimension(
585 name,
586 required=required,
587 implied=implied.freeze(),
588 metadata_columns=NamedValueSet(self.metadata).freeze(),
589 unique_keys=NamedValueSet(self.keys).freeze(),
590 is_cached=self.is_cached,
591 implied_union_target=self.implied_union_target,
592 doc=self.doc,
593 )
594 builder.dimensions.add(dimension)
595 builder.elements.add(dimension)
596 else:
597 from ._database import DatabaseDimensionCombination
599 combination = DatabaseDimensionCombination(
600 name,
601 required=required,
602 implied=implied.freeze(),
603 doc=self.doc,
604 metadata_columns=NamedValueSet(self.metadata).freeze(),
605 is_cached=self.is_cached,
606 always_join=self.always_join,
607 populated_by=(
608 builder.dimensions[self.populated_by] if self.populated_by is not None else None
609 ),
610 )
611 builder.elements.add(combination)
613 @pydantic.model_validator(mode="after")
614 def _primary_key_types(self) -> _ElementConfig:
615 if self.keys and self.keys[0].type not in ("int", "string"): 615 ↛ 616line 615 didn't jump to line 616 because the condition on line 615 was never true
616 raise ValueError(
617 "The dimension primary key type (the first entry in the keys list) must be 'int' "
618 f"or 'string'; got '{self.keys[0].type}'."
619 )
620 return self
622 @pydantic.model_validator(mode="after")
623 def _not_nullable_keys(self) -> _ElementConfig:
624 for key in self.keys:
625 key.nullable = False
626 return self
628 @pydantic.model_validator(mode="after")
629 def _invalid_dimension_fields(self) -> _ElementConfig:
630 if self.keys:
631 if self.always_join: 631 ↛ 632line 631 didn't jump to line 632 because the condition on line 631 was never true
632 raise ValueError("Dimensions (elements with key columns) may not have always_join=True.")
633 if self.populated_by: 633 ↛ 634line 633 didn't jump to line 634 because the condition on line 633 was never true
634 raise ValueError("Dimensions (elements with key columns) may not have populated_by.")
635 return self
637 @pydantic.model_validator(mode="after")
638 def _storage(self) -> _ElementConfig:
639 if self.storage is not None: 639 ↛ 645line 639 didn't jump to line 645 because the condition on line 639 was always true
640 # 'storage' is legacy; pull its implications into the regular
641 # attributes and set it to None for consistency.
642 self.is_cached = self.storage.is_cached
643 self.implied_union_target = self.storage.implied_union_target
644 self.storage = None
645 if self.governor:
646 self.is_cached = True
647 if self.implied_union_target is not None:
648 if self.requires: 648 ↛ 649line 648 didn't jump to line 649 because the condition on line 648 was never true
649 raise ValueError("Implied-union dimension may not have required dependencies.")
650 if self.implies: 650 ↛ 651line 650 didn't jump to line 651 because the condition on line 650 was never true
651 raise ValueError("Implied-union dimension may not have implied dependencies.")
652 if len(self.keys) > 1: 652 ↛ 653line 652 didn't jump to line 653 because the condition on line 652 was never true
653 raise ValueError("Implied-union dimension may not have alternate keys.")
654 if self.metadata: 654 ↛ 655line 654 didn't jump to line 655 because the condition on line 654 was never true
655 raise ValueError("Implied-union dimension may not have metadata columns.")
656 return self
658 @pydantic.model_validator(mode="after")
659 def _relationship_dependencies(self) -> _ElementConfig:
660 if not self.keys and not self.requires: 660 ↛ 661line 660 didn't jump to line 661 because the condition on line 660 was never true
661 raise ValueError(
662 "Dimension relationships (elements with no key columns) must have at least one "
663 "required dependency."
664 )
665 return self
668@final
669class SerializedDimensionConfig(pydantic.BaseModel):
670 """Configuration that describes a complete dimension data model."""
672 version: int = pydantic.Field(
673 default=0,
674 description=textwrap.dedent(
675 """\
676 Integer version number for this universe.
678 This and 'namespace' are expected to uniquely identify a
679 dimension universe.
680 """
681 ),
682 )
684 namespace: str = pydantic.Field(
685 default=_DEFAULT_NAMESPACE,
686 description=textwrap.dedent(
687 """\
688 String namespace for this universe.
690 This and 'version' are expected to uniquely identify a
691 dimension universe.
692 """
693 ),
694 )
696 skypix: _SkyPixSectionConfig = pydantic.Field(
697 description="Hierarchical sky pixelization systems recognized by this dimension universe."
698 )
700 elements: dict[str, _ElementConfig] = pydantic.Field(
701 default_factory=dict, description="Non-skypix dimensions and dimension join relations."
702 )
704 topology: _TopologySectionConfig = pydantic.Field(
705 description="Spatial and temporal relationships between dimensions.",
706 default_factory=_TopologySectionConfig,
707 )