Coverage for python/lsst/daf/butler/dimensions/_config.py: 24%
74 statements
« prev ^ index » next coverage.py v7.4.0, created at 2024-01-16 10:44 +0000
« prev ^ index » next coverage.py v7.4.0, created at 2024-01-16 10:44 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30__all__ = ("DimensionConfig",)
32from collections.abc import Iterator, Mapping, Sequence
33from typing import Any
35import pydantic
36from lsst.resources import ResourcePath, ResourcePathExpression
38from .._config import Config, ConfigSubset
39from .._topology import TopologicalSpace
40from ._database import (
41 DatabaseDimensionElementConstructionVisitor,
42 DatabaseTopologicalFamilyConstructionVisitor,
43)
44from ._elements import KeyColumnSpec, MetadataColumnSpec
45from ._governor import GovernorDimensionConstructionVisitor
46from ._skypix import SkyPixConstructionVisitor
47from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor
49# The default namespace to use on older dimension config files that only
50# have a version.
51_DEFAULT_NAMESPACE = "daf_butler"
54class DimensionConfig(ConfigSubset):
55 """Configuration that defines a `DimensionUniverse`.
57 The configuration tree for dimensions is a (nested) dictionary
58 with five top-level entries:
60 - version: an integer version number, used as keys in a singleton registry
61 of all `DimensionUniverse` instances;
63 - namespace: a string to be associated with the version in the singleton
64 registry of all `DimensionUnivers` instances;
66 - skypix: a dictionary whose entries each define a `SkyPixSystem`,
67 along with a special "common" key whose value is the name of a skypix
68 dimension that is used to relate all other spatial dimensions in the
69 `Registry` database;
71 - elements: a nested dictionary whose entries each define
72 `StandardDimension` or `StandardDimensionCombination`.
74 - topology: a nested dictionary with ``spatial`` and ``temporal`` keys,
75 with dictionary values that each define a `StandardTopologicalFamily`.
77 - packers: ignored.
79 See the documentation for the linked classes above for more information
80 on the configuration syntax.
82 Parameters
83 ----------
84 other : `Config` or `str` or `dict`, optional
85 Argument specifying the configuration information as understood
86 by `Config`. If `None` is passed then defaults are loaded from
87 "dimensions.yaml", otherwise defaults are not loaded.
88 validate : `bool`, optional
89 If `True` required keys will be checked to ensure configuration
90 consistency.
91 searchPaths : `list` or `tuple`, optional
92 Explicit additional paths to search for defaults. They should
93 be supplied in priority order. These paths have higher priority
94 than those read from the environment in
95 `ConfigSubset.defaultSearchPaths()`. Paths can be `str` referring to
96 the local file system or URIs, `lsst.resources.ResourcePath`.
97 """
99 requiredKeys = ("version", "elements", "skypix")
100 defaultConfigFile = "dimensions.yaml"
102 def __init__(
103 self,
104 other: Config | ResourcePathExpression | Mapping[str, Any] | None = None,
105 validate: bool = True,
106 searchPaths: Sequence[ResourcePathExpression] | None = None,
107 ):
108 # if argument is not None then do not load/merge defaults
109 mergeDefaults = other is None
110 super().__init__(other=other, validate=validate, mergeDefaults=mergeDefaults, searchPaths=searchPaths)
112 def _updateWithConfigsFromPath(
113 self, searchPaths: Sequence[str | ResourcePath], configFile: ResourcePath | str
114 ) -> None:
115 """Search the supplied paths reading config from first found.
117 Raises
118 ------
119 FileNotFoundError
120 Raised if config file is not found in any of given locations.
122 Notes
123 -----
124 This method overrides base class method with different behavior.
125 Instead of merging all found files into a single configuration it
126 finds first matching file and reads it.
127 """
128 uri = ResourcePath(configFile)
129 if uri.isabs() and uri.exists():
130 # Assume this resource exists
131 self._updateWithOtherConfigFile(configFile)
132 self.filesRead.append(configFile)
133 else:
134 for pathDir in searchPaths:
135 if isinstance(pathDir, str | ResourcePath):
136 pathDir = ResourcePath(pathDir, forceDirectory=True)
137 file = pathDir.join(configFile)
138 if file.exists():
139 self.filesRead.append(file)
140 self._updateWithOtherConfigFile(file)
141 break
142 else:
143 raise TypeError(f"Unexpected search path type encountered: {pathDir!r}")
144 else:
145 raise FileNotFoundError(f"Could not find {configFile} in search path {searchPaths}")
147 def _updateWithOtherConfigFile(self, file: Config | str | ResourcePath | Mapping[str, Any]) -> None:
148 """Override for base class method.
150 Parameters
151 ----------
152 file : `Config`, `str`, `lsst.resources.ResourcePath`, or `dict`
153 Entity that can be converted to a `ConfigSubset`.
154 """
155 # Use this class to read the defaults so that subsetting can happen
156 # correctly.
157 externalConfig = type(self)(file, validate=False)
158 self.update(externalConfig)
160 def _extractSkyPixVisitors(self) -> Iterator[DimensionConstructionVisitor]:
161 """Process the 'skypix' section of the configuration.
163 Yields a construction visitor for each `SkyPixSystem`.
165 Yields
166 ------
167 visitor : `DimensionConstructionVisitor`
168 Object that adds a skypix system and its dimensions to an
169 under-construction `DimensionUniverse`.
170 """
171 config = self["skypix"]
172 systemNames = set(config.keys())
173 systemNames.remove("common")
174 for systemName in sorted(systemNames):
175 subconfig = config[systemName]
176 pixelizationClassName = subconfig["class"]
177 maxLevel = subconfig.get("max_level", 24)
178 yield SkyPixConstructionVisitor(systemName, pixelizationClassName, maxLevel)
180 def _extractElementVisitors(self) -> Iterator[DimensionConstructionVisitor]:
181 """Process the 'elements' section of the configuration.
183 Yields a construction visitor for each `StandardDimension` or
184 `StandardDimensionCombination`.
186 Yields
187 ------
188 visitor : `DimensionConstructionVisitor`
189 Object that adds a `StandardDimension` or
190 `StandardDimensionCombination` to an under-construction
191 `DimensionUniverse`.
192 """
193 # MyPy is confused by the typing.Annotated usage and/or how
194 # Pydantic annotated TypeAdapter.
195 key_adapter: pydantic.TypeAdapter[KeyColumnSpec] = pydantic.TypeAdapter( # type: ignore
196 KeyColumnSpec # type: ignore
197 )
198 metadata_adapter: pydantic.TypeAdapter[MetadataColumnSpec] = pydantic.TypeAdapter( # type: ignore
199 MetadataColumnSpec # type: ignore
200 )
201 for name, subconfig in self["elements"].items():
202 metadata_columns = [metadata_adapter.validate_python(c) for c in subconfig.get("metadata", ())]
203 unique_keys = [key_adapter.validate_python(c) for c in subconfig.get("keys", ())]
204 for unique_key in unique_keys:
205 unique_key.nullable = False
206 if subconfig.get("governor", False):
207 unsupported = {"required", "implied", "viewOf", "alwaysJoin"}
208 if not unsupported.isdisjoint(subconfig.keys()):
209 raise RuntimeError(
210 f"Unsupported config key(s) for governor {name}: {unsupported & subconfig.keys()}."
211 )
212 if not subconfig.get("cached", True):
213 raise RuntimeError(f"Governor dimension {name} is always cached.")
214 yield GovernorDimensionConstructionVisitor(
215 name=name,
216 storage=subconfig["storage"],
217 metadata_columns=metadata_columns,
218 unique_keys=unique_keys,
219 doc=subconfig.get("doc", ""),
220 )
221 else:
222 yield DatabaseDimensionElementConstructionVisitor(
223 name=name,
224 storage=subconfig["storage"],
225 required=set(subconfig.get("requires", ())),
226 implied=set(subconfig.get("implies", ())),
227 metadata_columns=metadata_columns,
228 alwaysJoin=subconfig.get("always_join", False),
229 unique_keys=unique_keys,
230 populated_by=subconfig.get("populated_by", None),
231 doc=subconfig.get("doc", ""),
232 )
234 def _extractTopologyVisitors(self) -> Iterator[DimensionConstructionVisitor]:
235 """Process the 'topology' section of the configuration.
237 Yields a construction visitor for each `StandardTopologicalFamily`.
239 Yields
240 ------
241 visitor : `DimensionConstructionVisitor`
242 Object that adds a `StandardTopologicalFamily` to an
243 under-construction `DimensionUniverse` and updates its member
244 `DimensionElement` instances.
245 """
246 for spaceName, subconfig in self.get("topology", {}).items():
247 space = TopologicalSpace.__members__[spaceName.upper()]
248 for name, members in subconfig.items():
249 yield DatabaseTopologicalFamilyConstructionVisitor(
250 name=name,
251 space=space,
252 members=members,
253 )
255 def makeBuilder(self) -> DimensionConstructionBuilder:
256 """Construct a `DinmensionConstructionBuilder`.
258 The builder will reflect this configuration.
260 Returns
261 -------
262 builder : `DimensionConstructionBuilder`
263 A builder object populated with all visitors from this
264 configuration. The `~DimensionConstructionBuilder.finish` method
265 will not have been called.
266 """
267 builder = DimensionConstructionBuilder(
268 self["version"],
269 self["skypix", "common"],
270 self,
271 namespace=self.get("namespace", _DEFAULT_NAMESPACE),
272 )
273 builder.update(self._extractSkyPixVisitors())
274 builder.update(self._extractElementVisitors())
275 builder.update(self._extractTopologyVisitors())
276 return builder