Coverage for python/lsst/daf/butler/dimensions/_config.py: 47%
223 statements
« prev ^ index » next coverage.py v7.5.0, created at 2024-05-02 03:16 -0700
« prev ^ index » next coverage.py v7.5.0, created at 2024-05-02 03:16 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30__all__ = ("DimensionConfig",)
32import textwrap
33from collections.abc import Mapping, Sequence, Set
34from typing import Any, ClassVar, Literal, Union, final
36import pydantic
37from lsst.resources import ResourcePath, ResourcePathExpression
38from lsst.sphgeom import PixelizationABC
39from lsst.utils.doImport import doImportType
41from .._config import Config, ConfigSubset
42from .._named import NamedValueSet
43from .._topology import TopologicalSpace
44from ._database import DatabaseTopologicalFamilyConstructionVisitor
45from ._elements import Dimension, KeyColumnSpec, MetadataColumnSpec
46from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor
48# The default namespace to use on older dimension config files that only
49# have a version.
50_DEFAULT_NAMESPACE = "daf_butler"
53class DimensionConfig(ConfigSubset):
54 """Configuration that defines a `DimensionUniverse`.
56 The configuration tree for dimensions is a (nested) dictionary
57 with five top-level entries:
59 - version: an integer version number, used as keys in a singleton registry
60 of all `DimensionUniverse` instances;
62 - namespace: a string to be associated with the version in the singleton
63 registry of all `DimensionUnivers` instances;
65 - skypix: a dictionary whose entries each define a `SkyPixSystem`,
66 along with a special "common" key whose value is the name of a skypix
67 dimension that is used to relate all other spatial dimensions in the
68 `Registry` database;
70 - elements: a nested dictionary whose entries each define
71 `StandardDimension` or `StandardDimensionCombination`.
73 - topology: a nested dictionary with ``spatial`` and ``temporal`` keys,
74 with dictionary values that each define a `StandardTopologicalFamily`.
76 - packers: ignored.
78 See the documentation for the linked classes above for more information
79 on the configuration syntax.
81 Parameters
82 ----------
83 other : `Config` or `str` or `dict`, optional
84 Argument specifying the configuration information as understood
85 by `Config`. If `None` is passed then defaults are loaded from
86 "dimensions.yaml", otherwise defaults are not loaded.
87 validate : `bool`, optional
88 If `True` required keys will be checked to ensure configuration
89 consistency.
90 searchPaths : `list` or `tuple`, optional
91 Explicit additional paths to search for defaults. They should
92 be supplied in priority order. These paths have higher priority
93 than those read from the environment in
94 `ConfigSubset.defaultSearchPaths()`. Paths can be `str` referring to
95 the local file system or URIs, `lsst.resources.ResourcePath`.
96 """
98 requiredKeys = ("version", "elements", "skypix")
99 defaultConfigFile = "dimensions.yaml"
101 def __init__(
102 self,
103 other: Config | ResourcePathExpression | Mapping[str, Any] | None = None,
104 validate: bool = True,
105 searchPaths: Sequence[ResourcePathExpression] | None = None,
106 ):
107 # if argument is not None then do not load/merge defaults
108 mergeDefaults = other is None
109 super().__init__(other=other, validate=validate, mergeDefaults=mergeDefaults, searchPaths=searchPaths)
111 def _updateWithConfigsFromPath(
112 self, searchPaths: Sequence[str | ResourcePath], configFile: ResourcePath | str
113 ) -> None:
114 """Search the supplied paths reading config from first found.
116 Raises
117 ------
118 FileNotFoundError
119 Raised if config file is not found in any of given locations.
121 Notes
122 -----
123 This method overrides base class method with different behavior.
124 Instead of merging all found files into a single configuration it
125 finds first matching file and reads it.
126 """
127 uri = ResourcePath(configFile)
128 if uri.isabs() and uri.exists():
129 # Assume this resource exists
130 self._updateWithOtherConfigFile(configFile)
131 self.filesRead.append(configFile)
132 else:
133 for pathDir in searchPaths:
134 if isinstance(pathDir, str | ResourcePath):
135 pathDir = ResourcePath(pathDir, forceDirectory=True)
136 file = pathDir.join(configFile)
137 if file.exists():
138 self.filesRead.append(file)
139 self._updateWithOtherConfigFile(file)
140 break
141 else:
142 raise TypeError(f"Unexpected search path type encountered: {pathDir!r}")
143 else:
144 raise FileNotFoundError(f"Could not find {configFile} in search path {searchPaths}")
146 def _updateWithOtherConfigFile(self, file: Config | str | ResourcePath | Mapping[str, Any]) -> None:
147 """Override for base class method.
149 Parameters
150 ----------
151 file : `Config`, `str`, `lsst.resources.ResourcePath`, or `dict`
152 Entity that can be converted to a `ConfigSubset`.
153 """
154 # Use this class to read the defaults so that subsetting can happen
155 # correctly.
156 externalConfig = type(self)(file, validate=False)
157 self.update(externalConfig)
159 def makeBuilder(self) -> DimensionConstructionBuilder:
160 """Construct a `DimensionConstructionBuilder`.
162 The builder will reflect this configuration.
164 Returns
165 -------
166 builder : `DimensionConstructionBuilder`
167 A builder object populated with all visitors from this
168 configuration. The `~DimensionConstructionBuilder.finish` method
169 will not have been called.
170 """
171 validated = _UniverseConfig.model_validate(self.toDict())
172 builder = DimensionConstructionBuilder(
173 validated.version,
174 validated.skypix.common,
175 self,
176 namespace=validated.namespace,
177 )
178 for system_name, system_config in sorted(validated.skypix.systems.items()):
179 builder.add(system_name, system_config)
180 for element_name, element_config in validated.elements.items():
181 builder.add(element_name, element_config)
182 for family_name, members in validated.topology.spatial.items():
183 builder.add(
184 family_name,
185 DatabaseTopologicalFamilyConstructionVisitor(space=TopologicalSpace.SPATIAL, members=members),
186 )
187 for family_name, members in validated.topology.temporal.items():
188 builder.add(
189 family_name,
190 DatabaseTopologicalFamilyConstructionVisitor(
191 space=TopologicalSpace.TEMPORAL, members=members
192 ),
193 )
194 return builder
197@final
198class _SkyPixSystemConfig(pydantic.BaseModel, DimensionConstructionVisitor):
199 """Description of a hierarchical sky pixelization system in dimension
200 universe configuration.
201 """
203 class_: str = pydantic.Field(
204 alias="class",
205 description="Fully-qualified name of an `lsst.sphgeom.PixelizationABC implementation.",
206 )
208 min_level: int = 1
209 """Minimum level for this pixelization."""
211 max_level: int | None
212 """Maximum level for this pixelization."""
214 def has_dependencies_in(self, others: Set[str]) -> bool:
215 # Docstring inherited from DimensionConstructionVisitor.
216 return False
218 def visit(self, name: str, builder: DimensionConstructionBuilder) -> None:
219 # Docstring inherited from DimensionConstructionVisitor.
220 PixelizationClass = doImportType(self.class_)
221 assert issubclass(PixelizationClass, PixelizationABC)
222 if self.max_level is None:
223 max_level: int | None = getattr(PixelizationClass, "MAX_LEVEL", None)
224 if max_level is None:
225 raise TypeError(
226 f"Skypix pixelization class {self.class_} does"
227 " not have MAX_LEVEL but no max level has been set explicitly."
228 )
229 self.max_level = max_level
231 from ._skypix import SkyPixSystem
233 system = SkyPixSystem(
234 name,
235 maxLevel=self.max_level,
236 PixelizationClass=PixelizationClass,
237 )
238 builder.topology[TopologicalSpace.SPATIAL].add(system)
239 for level in range(self.min_level, self.max_level + 1):
240 dimension = system[level]
241 builder.dimensions.add(dimension)
242 builder.elements.add(dimension)
245@final
246class _SkyPixSectionConfig(pydantic.BaseModel):
247 """Section of the dimension universe configuration that describes sky
248 pixelizations.
249 """
251 common: str = pydantic.Field(
252 description="Name of the dimension used to relate all other spatial dimensions."
253 )
255 systems: dict[str, _SkyPixSystemConfig] = pydantic.Field(
256 default_factory=dict, description="Descriptions of the supported sky pixelization systems."
257 )
259 model_config = pydantic.ConfigDict(extra="allow")
261 @pydantic.model_validator(mode="after")
262 def _move_extra_to_systems(self) -> _SkyPixSectionConfig:
263 """Reinterpret extra fields in this model as members of the `systems`
264 dictionary.
265 """
266 if self.__pydantic_extra__ is None:
267 self.__pydantic_extra__ = {}
268 for name, data in self.__pydantic_extra__.items():
269 self.systems[name] = _SkyPixSystemConfig.model_validate(data)
270 self.__pydantic_extra__.clear()
271 return self
274@final
275class _TopologySectionConfig(pydantic.BaseModel):
276 """Section of the dimension universe configuration that describes spatial
277 and temporal relationships.
278 """
280 spatial: dict[str, list[str]] = pydantic.Field(
281 default_factory=dict,
282 description=textwrap.dedent(
283 """\
284 Dictionary of spatial dimension elements, grouped by the "family"
285 they belong to.
287 Elements in a family are ordered from fine-grained to coarse-grained.
288 """
289 ),
290 )
292 temporal: dict[str, list[str]] = pydantic.Field(
293 default_factory=dict,
294 description=textwrap.dedent(
295 """\
296 Dictionary of temporal dimension elements, grouped by the "family"
297 they belong to.
299 Elements in a family are ordered from fine-grained to coarse-grained.
300 """
301 ),
302 )
305@final
306class _LegacyGovernorDimensionStorage(pydantic.BaseModel):
307 """Legacy storage configuration for governor dimensions."""
309 cls: Literal["lsst.daf.butler.registry.dimensions.governor.BasicGovernorDimensionRecordStorage"] = (
310 "lsst.daf.butler.registry.dimensions.governor.BasicGovernorDimensionRecordStorage"
311 )
313 has_own_table: ClassVar[Literal[True]] = True
314 """Whether this dimension needs a database table to be defined."""
316 is_cached: ClassVar[Literal[True]] = True
317 """Whether this dimension's records should be cached in clients."""
319 implied_union_target: ClassVar[Literal[None]] = None
320 """Name of another dimension that implies this one, whose values for this
321 dimension define the set of allowed values for this dimension.
322 """
325@final
326class _LegacyTableDimensionStorage(pydantic.BaseModel):
327 """Legacy storage configuration for regular dimension tables stored in the
328 database.
329 """
331 cls: Literal["lsst.daf.butler.registry.dimensions.table.TableDimensionRecordStorage"] = (
332 "lsst.daf.butler.registry.dimensions.table.TableDimensionRecordStorage"
333 )
335 has_own_table: ClassVar[Literal[True]] = True
336 """Whether this dimension element needs a database table to be defined."""
338 is_cached: ClassVar[Literal[False]] = False
339 """Whether this dimension element's records should be cached in clients."""
341 implied_union_target: ClassVar[Literal[None]] = None
342 """Name of another dimension that implies this one, whose values for this
343 dimension define the set of allowed values for this dimension.
344 """
347@final
348class _LegacyImpliedUnionDimensionStorage(pydantic.BaseModel):
349 """Legacy storage configuration for dimensions whose allowable values are
350 computed from the union of the values in another dimension that implies
351 this one.
352 """
354 cls: Literal["lsst.daf.butler.registry.dimensions.query.QueryDimensionRecordStorage"] = (
355 "lsst.daf.butler.registry.dimensions.query.QueryDimensionRecordStorage"
356 )
358 view_of: str
359 """The dimension that implies this one and defines its values."""
361 has_own_table: ClassVar[Literal[False]] = False
362 """Whether this dimension needs a database table to be defined."""
364 is_cached: ClassVar[Literal[False]] = False
365 """Whether this dimension element's records should be cached in clients."""
367 @property
368 def implied_union_target(self) -> str:
369 """Name of another dimension that implies this one, whose values for
370 this dimension define the set of allowed values for this dimension.
371 """
372 return self.view_of
375@final
376class _LegacyCachingDimensionStorage(pydantic.BaseModel):
377 """Legacy storage configuration that wraps another to indicate that its
378 records should be cached.
379 """
381 cls: Literal["lsst.daf.butler.registry.dimensions.caching.CachingDimensionRecordStorage"] = (
382 "lsst.daf.butler.registry.dimensions.caching.CachingDimensionRecordStorage"
383 )
385 nested: _LegacyTableDimensionStorage | _LegacyImpliedUnionDimensionStorage
386 """Dimension storage configuration wrapped by this one."""
388 @property
389 def has_own_table(self) -> bool:
390 """Whether this dimension needs a database table to be defined."""
391 return self.nested.has_own_table
393 is_cached: ClassVar[Literal[True]] = True
394 """Whether this dimension element's records should be cached in clients."""
396 @property
397 def implied_union_target(self) -> str | None:
398 """Name of another dimension that implies this one, whose values for
399 this dimension define the set of allowed values for this dimension.
400 """
401 return self.nested.implied_union_target
404@final
405class _ElementConfig(pydantic.BaseModel, DimensionConstructionVisitor):
406 """Description of a single dimension or dimension join relation in
407 dimension universe configuration.
408 """
410 doc: str = pydantic.Field(default="", description="Documentation for the dimension or relationship.")
412 keys: list[KeyColumnSpec] = pydantic.Field(
413 default_factory=list,
414 description=textwrap.dedent(
415 """\
416 Key columns that (along with required dependency values) uniquely
417 identify the dimension's records.
419 The first columns in this list is the primary key and is used in
420 data coordinate values. Other columns are alternative keys and
421 are defined with SQL ``UNIQUE`` constraints.
422 """
423 ),
424 )
426 requires: set[str] = pydantic.Field(
427 default_factory=set,
428 description=(
429 "Other dimensions whose primary keys are part of this dimension element's (compound) primary key."
430 ),
431 )
433 implies: set[str] = pydantic.Field(
434 default_factory=set,
435 description="Other dimensions whose primary keys appear as foreign keys in this dimension element.",
436 )
438 metadata: list[MetadataColumnSpec] = pydantic.Field(
439 default_factory=list,
440 description="Non-key columns that provide extra information about a dimension element.",
441 )
443 is_cached: bool = pydantic.Field(
444 default=False,
445 description="Whether this element's records should be cached in the client.",
446 )
448 implied_union_target: str | None = pydantic.Field(
449 default=None,
450 description=textwrap.dedent(
451 """\
452 Another dimension whose stored values for this dimension form the
453 set of all allowed values.
455 The target dimension must have this dimension in its "implies"
456 list. This means the current dimension will have no table of its
457 own in the database.
458 """
459 ),
460 )
462 governor: bool = pydantic.Field(
463 default=False,
464 description=textwrap.dedent(
465 """\
466 Whether this is a governor dimension.
468 Governor dimensions are expected to have a tiny number of rows and
469 must be explicitly provided in any dimension expression in which
470 dependent dimensions appear.
472 Implies is_cached=True.
473 """
474 ),
475 )
477 always_join: bool = pydantic.Field(
478 default=False,
479 description=textwrap.dedent(
480 """\
481 Whether this dimension join relation should always be included in
482 any query where its required dependencies appear.
483 """
484 ),
485 )
487 populated_by: str | None = pydantic.Field(
488 default=None,
489 description=textwrap.dedent(
490 """\
491 The name of a required dimension that this dimension join
492 relation's rows should transferred alongside.
493 """
494 ),
495 )
497 storage: Union[
498 _LegacyGovernorDimensionStorage,
499 _LegacyTableDimensionStorage,
500 _LegacyImpliedUnionDimensionStorage,
501 _LegacyCachingDimensionStorage,
502 None,
503 ] = pydantic.Field(
504 description="How this dimension element's rows should be stored in the database and client.",
505 discriminator="cls",
506 default=None,
507 )
509 def has_dependencies_in(self, others: Set[str]) -> bool:
510 # Docstring inherited from DimensionConstructionVisitor.
511 return not (self.requires.isdisjoint(others) and self.implies.isdisjoint(others))
513 def visit(self, name: str, builder: DimensionConstructionBuilder) -> None:
514 # Docstring inherited from DimensionConstructionVisitor.
515 if self.governor:
516 from ._governor import GovernorDimension
518 governor = GovernorDimension(
519 name,
520 metadata_columns=NamedValueSet(self.metadata).freeze(),
521 unique_keys=NamedValueSet(self.keys).freeze(),
522 doc=self.doc,
523 )
524 builder.dimensions.add(governor)
525 builder.elements.add(governor)
526 return
527 # Expand required dependencies.
528 for dependency_name in tuple(self.requires): # iterate over copy
529 self.requires.update(builder.dimensions[dependency_name].required.names)
530 # Transform required and implied Dimension names into instances,
531 # and reorder to match builder's order.
532 required: NamedValueSet[Dimension] = NamedValueSet()
533 implied: NamedValueSet[Dimension] = NamedValueSet()
534 for dimension in builder.dimensions:
535 if dimension.name in self.requires:
536 required.add(dimension)
537 if dimension.name in self.implies:
538 implied.add(dimension)
539 # Elements with keys are Dimensions; the rest are
540 # DimensionCombinations.
541 if self.keys:
542 from ._database import DatabaseDimension
544 dimension = DatabaseDimension(
545 name,
546 required=required,
547 implied=implied.freeze(),
548 metadata_columns=NamedValueSet(self.metadata).freeze(),
549 unique_keys=NamedValueSet(self.keys).freeze(),
550 is_cached=self.is_cached,
551 implied_union_target=self.implied_union_target,
552 doc=self.doc,
553 )
554 builder.dimensions.add(dimension)
555 builder.elements.add(dimension)
556 else:
557 from ._database import DatabaseDimensionCombination
559 combination = DatabaseDimensionCombination(
560 name,
561 required=required,
562 implied=implied.freeze(),
563 doc=self.doc,
564 metadata_columns=NamedValueSet(self.metadata).freeze(),
565 is_cached=self.is_cached,
566 always_join=self.always_join,
567 populated_by=(
568 builder.dimensions[self.populated_by] if self.populated_by is not None else None
569 ),
570 )
571 builder.elements.add(combination)
573 @pydantic.model_validator(mode="after")
574 def _primary_key_types(self) -> _ElementConfig:
575 if self.keys and self.keys[0].type not in ("int", "string"):
576 raise ValueError(
577 "The dimension primary key type (the first entry in the keys list) must be 'int' "
578 f"or 'string'; got '{self.keys[0].type}'."
579 )
580 return self
582 @pydantic.model_validator(mode="after")
583 def _not_nullable_keys(self) -> _ElementConfig:
584 for key in self.keys:
585 key.nullable = False
586 return self
588 @pydantic.model_validator(mode="after")
589 def _invalid_dimension_fields(self) -> _ElementConfig:
590 if self.keys:
591 if self.always_join:
592 raise ValueError("Dimensions (elements with key columns) may not have always_join=True.")
593 if self.populated_by:
594 raise ValueError("Dimensions (elements with key columns) may not have populated_by.")
595 return self
597 @pydantic.model_validator(mode="after")
598 def _storage(self) -> _ElementConfig:
599 if self.storage is not None:
600 # 'storage' is legacy; pull its implications into the regular
601 # attributes and set it to None for consistency.
602 self.is_cached = self.storage.is_cached
603 self.implied_union_target = self.storage.implied_union_target
604 self.storage = None
605 if self.governor:
606 self.is_cached = True
607 if self.implied_union_target is not None:
608 if self.requires:
609 raise ValueError("Implied-union dimension may not have required dependencies.")
610 if self.implies:
611 raise ValueError("Implied-union dimension may not have implied dependencies.")
612 if len(self.keys) > 1:
613 raise ValueError("Implied-union dimension may not have alternate keys.")
614 if self.metadata:
615 raise ValueError("Implied-union dimension may not have metadata columns.")
616 return self
618 @pydantic.model_validator(mode="after")
619 def _relationship_dependencies(self) -> _ElementConfig:
620 if not self.keys and not self.requires:
621 raise ValueError(
622 "Dimension relationships (elements with no key columns) must have at least one "
623 "required dependency."
624 )
625 return self
628@final
629class _UniverseConfig(pydantic.BaseModel):
630 """Configuration that describes a complete dimension data model."""
632 version: int = pydantic.Field(
633 default=0,
634 description=textwrap.dedent(
635 """\
636 Integer version number for this universe.
638 This and 'namespace' are expected to uniquely identify a
639 dimension universe.
640 """
641 ),
642 )
644 namespace: str = pydantic.Field(
645 default=_DEFAULT_NAMESPACE,
646 description=textwrap.dedent(
647 """\
648 String namespace for this universe.
650 This and 'version' are expected to uniquely identify a
651 dimension universe.
652 """
653 ),
654 )
656 skypix: _SkyPixSectionConfig = pydantic.Field(
657 description="Hierarchical sky pixelization systems recognized by this dimension universe."
658 )
660 elements: dict[str, _ElementConfig] = pydantic.Field(
661 default_factory=dict, description="Non-skypix dimensions and dimension join relations."
662 )
664 topology: _TopologySectionConfig = pydantic.Field(
665 description="Spatial and temporal relationships between dimensions.",
666 default_factory=_TopologySectionConfig,
667 )