Coverage for python / lsst / daf / butler / dimensions / _config.py: 40%
228 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-04-24 08:17 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30__all__ = ("DimensionConfig", "SerializedDimensionConfig")
32import textwrap
33from collections.abc import Mapping, Sequence, Set
34from typing import Any, ClassVar, Literal, Union, final
36import pydantic
38from lsst.resources import ResourcePath, ResourcePathExpression
39from lsst.sphgeom import PixelizationABC
40from lsst.utils.doImport import doImportType
42from .._config import Config, ConfigSubset
43from .._named import NamedValueSet
44from .._topology import TopologicalSpace
45from ._database import DatabaseTopologicalFamilyConstructionVisitor
46from ._elements import Dimension, KeyColumnSpec, MetadataColumnSpec
47from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor
49# The default namespace to use on older dimension config files that only
50# have a version.
51_DEFAULT_NAMESPACE = "daf_butler"
54class DimensionConfig(ConfigSubset):
55 """Configuration that defines a `DimensionUniverse`.
57 The configuration tree for dimensions is a (nested) dictionary
58 with five top-level entries:
60 - version: an integer version number, used as keys in a singleton registry
61 of all `DimensionUniverse` instances;
63 - namespace: a string to be associated with the version in the singleton
64 registry of all `DimensionUnivers` instances;
66 - skypix: a dictionary whose entries each define a `SkyPixSystem`,
67 along with a special "common" key whose value is the name of a skypix
68 dimension that is used to relate all other spatial dimensions in the
69 `Registry` database;
71 - elements: a nested dictionary whose entries each define
72 `StandardDimension` or `StandardDimensionCombination`.
74 - topology: a nested dictionary with ``spatial`` and ``temporal`` keys,
75 with dictionary values that each define a `StandardTopologicalFamily`.
77 - packers: ignored.
79 See the documentation for the linked classes above for more information
80 on the configuration syntax.
82 Parameters
83 ----------
84 other : `Config` or `str` or `dict`, optional
85 Argument specifying the configuration information as understood
86 by `Config`. If `None` is passed then defaults are loaded from
87 "dimensions.yaml", otherwise defaults are not loaded.
88 validate : `bool`, optional
89 If `True` required keys will be checked to ensure configuration
90 consistency.
91 searchPaths : `list` or `tuple`, optional
92 Explicit additional paths to search for defaults. They should
93 be supplied in priority order. These paths have higher priority
94 than those read from the environment in
95 `ConfigSubset.defaultSearchPaths()`. Paths can be `str` referring to
96 the local file system or URIs, `lsst.resources.ResourcePath`.
97 """
99 requiredKeys = ("version", "elements", "skypix")
100 defaultConfigFile = "dimensions.yaml"
102 def __init__(
103 self,
104 other: Config | ResourcePathExpression | Mapping[str, Any] | None = None,
105 validate: bool = True,
106 searchPaths: Sequence[ResourcePathExpression] | None = None,
107 ):
108 # if argument is not None then do not load/merge defaults
109 mergeDefaults = other is None
110 super().__init__(other=other, validate=validate, mergeDefaults=mergeDefaults, searchPaths=searchPaths)
112 def _updateWithConfigsFromPath(
113 self, searchPaths: Sequence[str | ResourcePath], configFile: ResourcePath | str
114 ) -> None:
115 """Search the supplied paths reading config from first found.
117 Raises
118 ------
119 FileNotFoundError
120 Raised if config file is not found in any of given locations.
122 Notes
123 -----
124 This method overrides base class method with different behavior.
125 Instead of merging all found files into a single configuration it
126 finds first matching file and reads it.
127 """
128 uri = ResourcePath(configFile)
129 if uri.isabs() and uri.exists():
130 # Assume this resource exists
131 self._updateWithOtherConfigFile(configFile)
132 self.filesRead.append(configFile)
133 else:
134 for pathDir in searchPaths:
135 if isinstance(pathDir, str | ResourcePath):
136 pathDir = ResourcePath(pathDir, forceDirectory=True)
137 file = pathDir.join(configFile)
138 if file.exists():
139 self.filesRead.append(file)
140 self._updateWithOtherConfigFile(file)
141 break
142 else:
143 raise TypeError(f"Unexpected search path type encountered: {pathDir!r}")
144 else:
145 raise FileNotFoundError(f"Could not find {configFile} in search path {searchPaths}")
147 def _updateWithOtherConfigFile(self, file: Config | str | ResourcePath | Mapping[str, Any]) -> None:
148 """Override for base class method.
150 Parameters
151 ----------
152 file : `Config`, `str`, `lsst.resources.ResourcePath`, or `dict`
153 Entity that can be converted to a `ConfigSubset`.
154 """
155 # Use this class to read the defaults so that subsetting can happen
156 # correctly.
157 externalConfig = type(self)(file, validate=False)
158 self.update(externalConfig)
160 def to_simple(self) -> SerializedDimensionConfig:
161 """Convert this configuration to a serializable Pydantic model.
163 Returns
164 -------
165 model : `SerializedDimensionConfig`
166 Serializable Pydantic version of this configuration.
167 """
168 return SerializedDimensionConfig.model_validate(self.toDict())
170 @staticmethod
171 def from_simple(simple: SerializedDimensionConfig) -> DimensionConfig:
172 """Load the configuration from a serialized version.
174 Parameters
175 ----------
176 simple : `SerializedDimensionConfig`
177 Serialized configuration to be loaded.
179 Returns
180 -------
181 config : `DimensionConfig`
182 Dimension configuration.
183 """
184 return DimensionConfig(
185 simple.model_dump(
186 # Some of the fields in Pydantic model config have aliases
187 # (e.g. remapping 'class_' to 'class'). Pydantic ignores these
188 # in model_dump() by default, so we have to add by_alias to
189 # make sure that we end up with the right names in the dict.
190 by_alias=True
191 )
192 )
194 def makeBuilder(self) -> DimensionConstructionBuilder:
195 """Construct a `DimensionConstructionBuilder`.
197 The builder will reflect this configuration.
199 Returns
200 -------
201 builder : `DimensionConstructionBuilder`
202 A builder object populated with all visitors from this
203 configuration. The `~DimensionConstructionBuilder.finish` method
204 will not have been called.
205 """
206 validated = self.to_simple()
207 builder = DimensionConstructionBuilder(
208 validated.version,
209 validated.skypix.common,
210 self,
211 namespace=validated.namespace,
212 )
213 for system_name, system_config in sorted(validated.skypix.systems.items()):
214 builder.add(system_name, system_config)
215 for element_name, element_config in validated.elements.items():
216 builder.add(element_name, element_config)
217 for family_name, members in validated.topology.spatial.items():
218 builder.add(
219 family_name,
220 DatabaseTopologicalFamilyConstructionVisitor(space=TopologicalSpace.SPATIAL, members=members),
221 )
222 for family_name, members in validated.topology.temporal.items():
223 builder.add(
224 family_name,
225 DatabaseTopologicalFamilyConstructionVisitor(
226 space=TopologicalSpace.TEMPORAL, members=members
227 ),
228 )
229 return builder
232@final
233class _SkyPixSystemConfig(pydantic.BaseModel, DimensionConstructionVisitor):
234 """Description of a hierarchical sky pixelization system in dimension
235 universe configuration.
236 """
238 class_: str = pydantic.Field(
239 alias="class",
240 description="Fully-qualified name of an `lsst.sphgeom.PixelizationABC implementation.",
241 )
243 min_level: int = 1
244 """Minimum level for this pixelization."""
246 max_level: int | None
247 """Maximum level for this pixelization."""
249 def has_dependencies_in(self, others: Set[str]) -> bool:
250 # Docstring inherited from DimensionConstructionVisitor.
251 return False
253 def visit(self, name: str, builder: DimensionConstructionBuilder) -> None:
254 # Docstring inherited from DimensionConstructionVisitor.
255 PixelizationClass = doImportType(self.class_)
256 assert issubclass(PixelizationClass, PixelizationABC)
257 if self.max_level is None:
258 max_level: int | None = getattr(PixelizationClass, "MAX_LEVEL", None)
259 if max_level is None:
260 raise TypeError(
261 f"Skypix pixelization class {self.class_} does"
262 " not have MAX_LEVEL but no max level has been set explicitly."
263 )
264 self.max_level = max_level
266 from ._skypix import SkyPixSystem
268 system = SkyPixSystem(
269 name,
270 maxLevel=self.max_level,
271 PixelizationClass=PixelizationClass,
272 )
273 builder.topology[TopologicalSpace.SPATIAL].add(system)
274 for level in range(self.min_level, self.max_level + 1):
275 dimension = system[level]
276 builder.dimensions.add(dimension)
277 builder.elements.add(dimension)
280@final
281class _SkyPixSectionConfig(pydantic.BaseModel):
282 """Section of the dimension universe configuration that describes sky
283 pixelizations.
284 """
286 common: str = pydantic.Field(
287 description="Name of the dimension used to relate all other spatial dimensions."
288 )
290 systems: dict[str, _SkyPixSystemConfig] = pydantic.Field(
291 default_factory=dict, description="Descriptions of the supported sky pixelization systems."
292 )
294 model_config = pydantic.ConfigDict(extra="allow")
296 @pydantic.model_validator(mode="after")
297 def _move_extra_to_systems(self) -> _SkyPixSectionConfig:
298 """Reinterpret extra fields in this model as members of the `systems`
299 dictionary.
300 """
301 if self.__pydantic_extra__ is None:
302 self.__pydantic_extra__ = {}
303 for name, data in self.__pydantic_extra__.items():
304 self.systems[name] = _SkyPixSystemConfig.model_validate(data)
305 self.__pydantic_extra__.clear()
306 return self
309@final
310class _TopologySectionConfig(pydantic.BaseModel):
311 """Section of the dimension universe configuration that describes spatial
312 and temporal relationships.
313 """
315 spatial: dict[str, list[str]] = pydantic.Field(
316 default_factory=dict,
317 description=textwrap.dedent(
318 """\
319 Dictionary of spatial dimension elements, grouped by the "family"
320 they belong to.
322 Elements in a family are ordered from fine-grained to coarse-grained.
323 """
324 ),
325 )
327 temporal: dict[str, list[str]] = pydantic.Field(
328 default_factory=dict,
329 description=textwrap.dedent(
330 """\
331 Dictionary of temporal dimension elements, grouped by the "family"
332 they belong to.
334 Elements in a family are ordered from fine-grained to coarse-grained.
335 """
336 ),
337 )
340@final
341class _LegacyGovernorDimensionStorage(pydantic.BaseModel):
342 """Legacy storage configuration for governor dimensions."""
344 cls: Literal["lsst.daf.butler.registry.dimensions.governor.BasicGovernorDimensionRecordStorage"] = (
345 "lsst.daf.butler.registry.dimensions.governor.BasicGovernorDimensionRecordStorage"
346 )
348 has_own_table: ClassVar[Literal[True]] = True
349 """Whether this dimension needs a database table to be defined."""
351 is_cached: ClassVar[Literal[True]] = True
352 """Whether this dimension's records should be cached in clients."""
354 implied_union_target: ClassVar[Literal[None]] = None
355 """Name of another dimension that implies this one, whose values for this
356 dimension define the set of allowed values for this dimension.
357 """
360@final
361class _LegacyTableDimensionStorage(pydantic.BaseModel):
362 """Legacy storage configuration for regular dimension tables stored in the
363 database.
364 """
366 cls: Literal["lsst.daf.butler.registry.dimensions.table.TableDimensionRecordStorage"] = (
367 "lsst.daf.butler.registry.dimensions.table.TableDimensionRecordStorage"
368 )
370 has_own_table: ClassVar[Literal[True]] = True
371 """Whether this dimension element needs a database table to be defined."""
373 is_cached: ClassVar[Literal[False]] = False
374 """Whether this dimension element's records should be cached in clients."""
376 implied_union_target: ClassVar[Literal[None]] = None
377 """Name of another dimension that implies this one, whose values for this
378 dimension define the set of allowed values for this dimension.
379 """
382@final
383class _LegacyImpliedUnionDimensionStorage(pydantic.BaseModel):
384 """Legacy storage configuration for dimensions whose allowable values are
385 computed from the union of the values in another dimension that implies
386 this one.
387 """
389 cls: Literal["lsst.daf.butler.registry.dimensions.query.QueryDimensionRecordStorage"] = (
390 "lsst.daf.butler.registry.dimensions.query.QueryDimensionRecordStorage"
391 )
393 view_of: str
394 """The dimension that implies this one and defines its values."""
396 has_own_table: ClassVar[Literal[False]] = False
397 """Whether this dimension needs a database table to be defined."""
399 is_cached: ClassVar[Literal[False]] = False
400 """Whether this dimension element's records should be cached in clients."""
402 @property
403 def implied_union_target(self) -> str:
404 """Name of another dimension that implies this one, whose values for
405 this dimension define the set of allowed values for this dimension.
406 """
407 return self.view_of
410@final
411class _LegacyCachingDimensionStorage(pydantic.BaseModel):
412 """Legacy storage configuration that wraps another to indicate that its
413 records should be cached.
414 """
416 cls: Literal["lsst.daf.butler.registry.dimensions.caching.CachingDimensionRecordStorage"] = (
417 "lsst.daf.butler.registry.dimensions.caching.CachingDimensionRecordStorage"
418 )
420 nested: _LegacyTableDimensionStorage | _LegacyImpliedUnionDimensionStorage
421 """Dimension storage configuration wrapped by this one."""
423 @property
424 def has_own_table(self) -> bool:
425 """Whether this dimension needs a database table to be defined."""
426 return self.nested.has_own_table
428 is_cached: ClassVar[Literal[True]] = True
429 """Whether this dimension element's records should be cached in clients."""
431 @property
432 def implied_union_target(self) -> str | None:
433 """Name of another dimension that implies this one, whose values for
434 this dimension define the set of allowed values for this dimension.
435 """
436 return self.nested.implied_union_target
439@final
440class _ElementConfig(pydantic.BaseModel, DimensionConstructionVisitor):
441 """Description of a single dimension or dimension join relation in
442 dimension universe configuration.
443 """
445 doc: str = pydantic.Field(default="", description="Documentation for the dimension or relationship.")
447 keys: list[KeyColumnSpec] = pydantic.Field(
448 default_factory=list,
449 description=textwrap.dedent(
450 """\
451 Key columns that (along with required dependency values) uniquely
452 identify the dimension's records.
454 The first columns in this list is the primary key and is used in
455 data coordinate values. Other columns are alternative keys and
456 are defined with SQL ``UNIQUE`` constraints.
457 """
458 ),
459 )
461 requires: set[str] = pydantic.Field(
462 default_factory=set,
463 description=(
464 "Other dimensions whose primary keys are part of this dimension element's (compound) primary key."
465 ),
466 )
468 implies: set[str] = pydantic.Field(
469 default_factory=set,
470 description="Other dimensions whose primary keys appear as foreign keys in this dimension element.",
471 )
473 metadata: list[MetadataColumnSpec] = pydantic.Field(
474 default_factory=list,
475 description="Non-key columns that provide extra information about a dimension element.",
476 )
478 is_cached: bool = pydantic.Field(
479 default=False,
480 description="Whether this element's records should be cached in the client.",
481 )
483 implied_union_target: str | None = pydantic.Field(
484 default=None,
485 description=textwrap.dedent(
486 """\
487 Another dimension whose stored values for this dimension form the
488 set of all allowed values.
490 The target dimension must have this dimension in its "implies"
491 list. This means the current dimension will have no table of its
492 own in the database.
493 """
494 ),
495 )
497 governor: bool = pydantic.Field(
498 default=False,
499 description=textwrap.dedent(
500 """\
501 Whether this is a governor dimension.
503 Governor dimensions are expected to have a tiny number of rows and
504 must be explicitly provided in any dimension expression in which
505 dependent dimensions appear.
507 Implies is_cached=True.
508 """
509 ),
510 )
512 always_join: bool = pydantic.Field(
513 default=False,
514 description=textwrap.dedent(
515 """\
516 Whether this dimension join relation should always be included in
517 any query where its required dependencies appear.
518 """
519 ),
520 )
522 populated_by: str | None = pydantic.Field(
523 default=None,
524 description=textwrap.dedent(
525 """\
526 The name of a required dimension that this dimension join
527 relation's rows should transferred alongside.
528 """
529 ),
530 )
532 storage: Union[
533 _LegacyGovernorDimensionStorage,
534 _LegacyTableDimensionStorage,
535 _LegacyImpliedUnionDimensionStorage,
536 _LegacyCachingDimensionStorage,
537 None,
538 ] = pydantic.Field(
539 description="How this dimension element's rows should be stored in the database and client.",
540 discriminator="cls",
541 default=None,
542 )
544 def has_dependencies_in(self, others: Set[str]) -> bool:
545 # Docstring inherited from DimensionConstructionVisitor.
546 return not (self.requires.isdisjoint(others) and self.implies.isdisjoint(others))
548 def visit(self, name: str, builder: DimensionConstructionBuilder) -> None:
549 # Docstring inherited from DimensionConstructionVisitor.
550 if self.governor:
551 from ._governor import GovernorDimension
553 governor = GovernorDimension(
554 name,
555 metadata_columns=NamedValueSet(self.metadata).freeze(),
556 unique_keys=NamedValueSet(self.keys).freeze(),
557 doc=self.doc,
558 )
559 builder.dimensions.add(governor)
560 builder.elements.add(governor)
561 return
562 # Expand required dependencies.
563 for dependency_name in tuple(self.requires): # iterate over copy
564 self.requires.update(builder.dimensions[dependency_name].required.names)
565 # Transform required and implied Dimension names into instances,
566 # and reorder to match builder's order.
567 required: NamedValueSet[Dimension] = NamedValueSet()
568 implied: NamedValueSet[Dimension] = NamedValueSet()
569 for dimension in builder.dimensions:
570 if dimension.name in self.requires:
571 required.add(dimension)
572 if dimension.name in self.implies:
573 implied.add(dimension)
574 # Elements with keys are Dimensions; the rest are
575 # DimensionCombinations.
576 if self.keys:
577 from ._database import DatabaseDimension
579 dimension = DatabaseDimension(
580 name,
581 required=required,
582 implied=implied.freeze(),
583 metadata_columns=NamedValueSet(self.metadata).freeze(),
584 unique_keys=NamedValueSet(self.keys).freeze(),
585 is_cached=self.is_cached,
586 implied_union_target=self.implied_union_target,
587 doc=self.doc,
588 )
589 builder.dimensions.add(dimension)
590 builder.elements.add(dimension)
591 else:
592 from ._database import DatabaseDimensionCombination
594 combination = DatabaseDimensionCombination(
595 name,
596 required=required,
597 implied=implied.freeze(),
598 doc=self.doc,
599 metadata_columns=NamedValueSet(self.metadata).freeze(),
600 is_cached=self.is_cached,
601 always_join=self.always_join,
602 populated_by=(
603 builder.dimensions[self.populated_by] if self.populated_by is not None else None
604 ),
605 )
606 builder.elements.add(combination)
608 @pydantic.model_validator(mode="after")
609 def _primary_key_types(self) -> _ElementConfig:
610 if self.keys and self.keys[0].type not in ("int", "string"):
611 raise ValueError(
612 "The dimension primary key type (the first entry in the keys list) must be 'int' "
613 f"or 'string'; got '{self.keys[0].type}'."
614 )
615 return self
617 @pydantic.model_validator(mode="after")
618 def _not_nullable_keys(self) -> _ElementConfig:
619 for key in self.keys:
620 key.nullable = False
621 return self
623 @pydantic.model_validator(mode="after")
624 def _invalid_dimension_fields(self) -> _ElementConfig:
625 if self.keys:
626 if self.always_join:
627 raise ValueError("Dimensions (elements with key columns) may not have always_join=True.")
628 if self.populated_by:
629 raise ValueError("Dimensions (elements with key columns) may not have populated_by.")
630 return self
632 @pydantic.model_validator(mode="after")
633 def _storage(self) -> _ElementConfig:
634 if self.storage is not None:
635 # 'storage' is legacy; pull its implications into the regular
636 # attributes and set it to None for consistency.
637 self.is_cached = self.storage.is_cached
638 self.implied_union_target = self.storage.implied_union_target
639 self.storage = None
640 if self.governor:
641 self.is_cached = True
642 if self.implied_union_target is not None:
643 if self.requires:
644 raise ValueError("Implied-union dimension may not have required dependencies.")
645 if self.implies:
646 raise ValueError("Implied-union dimension may not have implied dependencies.")
647 if len(self.keys) > 1:
648 raise ValueError("Implied-union dimension may not have alternate keys.")
649 if self.metadata:
650 raise ValueError("Implied-union dimension may not have metadata columns.")
651 return self
653 @pydantic.model_validator(mode="after")
654 def _relationship_dependencies(self) -> _ElementConfig:
655 if not self.keys and not self.requires:
656 raise ValueError(
657 "Dimension relationships (elements with no key columns) must have at least one "
658 "required dependency."
659 )
660 return self
663@final
664class SerializedDimensionConfig(pydantic.BaseModel):
665 """Configuration that describes a complete dimension data model."""
667 version: int = pydantic.Field(
668 default=0,
669 description=textwrap.dedent(
670 """\
671 Integer version number for this universe.
673 This and 'namespace' are expected to uniquely identify a
674 dimension universe.
675 """
676 ),
677 )
679 namespace: str = pydantic.Field(
680 default=_DEFAULT_NAMESPACE,
681 description=textwrap.dedent(
682 """\
683 String namespace for this universe.
685 This and 'version' are expected to uniquely identify a
686 dimension universe.
687 """
688 ),
689 )
691 skypix: _SkyPixSectionConfig = pydantic.Field(
692 description="Hierarchical sky pixelization systems recognized by this dimension universe."
693 )
695 elements: dict[str, _ElementConfig] = pydantic.Field(
696 default_factory=dict, description="Non-skypix dimensions and dimension join relations."
697 )
699 topology: _TopologySectionConfig = pydantic.Field(
700 description="Spatial and temporal relationships between dimensions.",
701 default_factory=_TopologySectionConfig,
702 )