Coverage for python/lsst/daf/butler/core/dimensions/_config.py: 23%
79 statements
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-17 09:33 +0000
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-17 09:33 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ("DimensionConfig",)
26import warnings
27from collections.abc import Iterator, Mapping, Sequence
28from typing import Any
30from lsst.resources import ResourcePath, ResourcePathExpression
32from .. import ddl
33from .._topology import TopologicalSpace
34from ..config import Config, ConfigSubset
35from ._database import (
36 DatabaseDimensionElementConstructionVisitor,
37 DatabaseTopologicalFamilyConstructionVisitor,
38)
39from ._governor import GovernorDimensionConstructionVisitor
40from ._packer import DimensionPackerConstructionVisitor
41from ._skypix import SkyPixConstructionVisitor
42from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor
44# The default namespace to use on older dimension config files that only
45# have a version.
46_DEFAULT_NAMESPACE = "daf_butler"
49class DimensionConfig(ConfigSubset):
50 """Configuration that defines a `DimensionUniverse`.
52 The configuration tree for dimensions is a (nested) dictionary
53 with five top-level entries:
55 - version: an integer version number, used as keys in a singleton registry
56 of all `DimensionUniverse` instances;
58 - namespace: a string to be associated with the version in the singleton
59 registry of all `DimensionUnivers` instances;
61 - skypix: a dictionary whose entries each define a `SkyPixSystem`,
62 along with a special "common" key whose value is the name of a skypix
63 dimension that is used to relate all other spatial dimensions in the
64 `Registry` database;
66 - elements: a nested dictionary whose entries each define
67 `StandardDimension` or `StandardDimensionCombination`.
69 - topology: a nested dictionary with ``spatial`` and ``temporal`` keys,
70 with dictionary values that each define a `StandardTopologicalFamily`.
72 - packers: a nested dictionary whose entries define factories for a
73 `DimensionPacker` instance.
75 See the documentation for the linked classes above for more information
76 on the configuration syntax.
78 Parameters
79 ----------
80 other : `Config` or `str` or `dict`, optional
81 Argument specifying the configuration information as understood
82 by `Config`. If `None` is passed then defaults are loaded from
83 "dimensions.yaml", otherwise defaults are not loaded.
84 validate : `bool`, optional
85 If `True` required keys will be checked to ensure configuration
86 consistency.
87 searchPaths : `list` or `tuple`, optional
88 Explicit additional paths to search for defaults. They should
89 be supplied in priority order. These paths have higher priority
90 than those read from the environment in
91 `ConfigSubset.defaultSearchPaths()`. Paths can be `str` referring to
92 the local file system or URIs, `lsst.resources.ResourcePath`.
93 """
95 requiredKeys = ("version", "elements", "skypix")
96 defaultConfigFile = "dimensions.yaml"
98 def __init__(
99 self,
100 other: Config | ResourcePathExpression | Mapping[str, Any] | None = None,
101 validate: bool = True,
102 searchPaths: Sequence[ResourcePathExpression] | None = None,
103 ):
104 # if argument is not None then do not load/merge defaults
105 mergeDefaults = other is None
106 super().__init__(other=other, validate=validate, mergeDefaults=mergeDefaults, searchPaths=searchPaths)
108 def _updateWithConfigsFromPath(
109 self, searchPaths: Sequence[str | ResourcePath], configFile: ResourcePath | str
110 ) -> None:
111 """Search the supplied paths reading config from first found.
113 Raises
114 ------
115 FileNotFoundError
116 Raised if config file is not found in any of given locations.
118 Notes
119 -----
120 This method overrides base class method with different behavior.
121 Instead of merging all found files into a single configuration it
122 finds first matching file and reads it.
123 """
124 uri = ResourcePath(configFile)
125 if uri.isabs() and uri.exists():
126 # Assume this resource exists
127 self._updateWithOtherConfigFile(configFile)
128 self.filesRead.append(configFile)
129 else:
130 for pathDir in searchPaths:
131 if isinstance(pathDir, (str, ResourcePath)):
132 pathDir = ResourcePath(pathDir, forceDirectory=True)
133 file = pathDir.join(configFile)
134 if file.exists():
135 self.filesRead.append(file)
136 self._updateWithOtherConfigFile(file)
137 break
138 else:
139 raise TypeError(f"Unexpected search path type encountered: {pathDir!r}")
140 else:
141 raise FileNotFoundError(f"Could not find {configFile} in search path {searchPaths}")
143 def _updateWithOtherConfigFile(self, file: Config | str | ResourcePath | Mapping[str, Any]) -> None:
144 """Override for base class method.
146 Parameters
147 ----------
148 file : `Config`, `str`, `lsst.resources.ResourcePath`, or `dict`
149 Entity that can be converted to a `ConfigSubset`.
150 """
151 # Use this class to read the defaults so that subsetting can happen
152 # correctly.
153 externalConfig = type(self)(file, validate=False)
154 self.update(externalConfig)
156 def _extractSkyPixVisitors(self) -> Iterator[DimensionConstructionVisitor]:
157 """Process the 'skypix' section of the configuration.
159 Yields a construction visitor for each `SkyPixSystem`.
161 Yields
162 ------
163 visitor : `DimensionConstructionVisitor`
164 Object that adds a skypix system and its dimensions to an
165 under-construction `DimensionUniverse`.
166 """
167 config = self["skypix"]
168 systemNames = set(config.keys())
169 systemNames.remove("common")
170 for systemName in sorted(systemNames):
171 subconfig = config[systemName]
172 pixelizationClassName = subconfig["class"]
173 maxLevel = subconfig.get("max_level", 24)
174 yield SkyPixConstructionVisitor(systemName, pixelizationClassName, maxLevel)
176 def _extractElementVisitors(self) -> Iterator[DimensionConstructionVisitor]:
177 """Process the 'elements' section of the configuration.
179 Yields a construction visitor for each `StandardDimension` or
180 `StandardDimensionCombination`.
182 Yields
183 ------
184 visitor : `DimensionConstructionVisitor`
185 Object that adds a `StandardDimension` or
186 `StandardDimensionCombination` to an under-construction
187 `DimensionUniverse`.
188 """
189 for name, subconfig in self["elements"].items():
190 metadata = [ddl.FieldSpec.fromConfig(c) for c in subconfig.get("metadata", ())]
191 uniqueKeys = [ddl.FieldSpec.fromConfig(c, nullable=False) for c in subconfig.get("keys", ())]
192 if uniqueKeys:
193 uniqueKeys[0].primaryKey = True
194 if subconfig.get("governor", False):
195 unsupported = {"required", "implied", "viewOf", "alwaysJoin"}
196 if not unsupported.isdisjoint(subconfig.keys()):
197 raise RuntimeError(
198 f"Unsupported config key(s) for governor {name}: {unsupported & subconfig.keys()}."
199 )
200 if not subconfig.get("cached", True):
201 raise RuntimeError(f"Governor dimension {name} is always cached.")
202 yield GovernorDimensionConstructionVisitor(
203 name=name,
204 storage=subconfig["storage"],
205 metadata=metadata,
206 uniqueKeys=uniqueKeys,
207 )
208 else:
209 yield DatabaseDimensionElementConstructionVisitor(
210 name=name,
211 storage=subconfig["storage"],
212 required=set(subconfig.get("requires", ())),
213 implied=set(subconfig.get("implies", ())),
214 metadata=metadata,
215 alwaysJoin=subconfig.get("always_join", False),
216 uniqueKeys=uniqueKeys,
217 )
219 def _extractTopologyVisitors(self) -> Iterator[DimensionConstructionVisitor]:
220 """Process the 'topology' section of the configuration.
222 Yields a construction visitor for each `StandardTopologicalFamily`.
224 Yields
225 ------
226 visitor : `DimensionConstructionVisitor`
227 Object that adds a `StandardTopologicalFamily` to an
228 under-construction `DimensionUniverse` and updates its member
229 `DimensionElement` instances.
230 """
231 for spaceName, subconfig in self.get("topology", {}).items():
232 space = TopologicalSpace.__members__[spaceName.upper()]
233 for name, members in subconfig.items():
234 yield DatabaseTopologicalFamilyConstructionVisitor(
235 name=name,
236 space=space,
237 members=members,
238 )
240 # TODO: remove this method and callers on DM-38687.
241 # Note that the corresponding entries in the dimensions config should
242 # not be removed at that time, because that's formally a schema migration.
243 def _extractPackerVisitors(self) -> Iterator[DimensionConstructionVisitor]:
244 """Process the 'packers' section of the configuration.
246 Yields construction visitors for each `DimensionPackerFactory`.
248 Yields
249 ------
250 visitor : `DimensionConstructionVisitor`
251 Object that adds a `DinmensionPackerFactory` to an
252 under-construction `DimensionUniverse`.
253 """
254 with warnings.catch_warnings():
255 # Don't warn when deprecated code calls other deprecated code.
256 warnings.simplefilter("ignore", FutureWarning)
257 for name, subconfig in self["packers"].items():
258 yield DimensionPackerConstructionVisitor(
259 name=name,
260 clsName=subconfig["cls"],
261 fixed=subconfig["fixed"],
262 dimensions=subconfig["dimensions"],
263 )
265 def makeBuilder(self) -> DimensionConstructionBuilder:
266 """Construct a `DinmensionConstructionBuilder`.
268 The builder will reflect this configuration.
270 Returns
271 -------
272 builder : `DimensionConstructionBuilder`
273 A builder object populated with all visitors from this
274 configuration. The `~DimensionConstructionBuilder.finish` method
275 will not have been called.
276 """
277 builder = DimensionConstructionBuilder(
278 self["version"],
279 self["skypix", "common"],
280 self,
281 namespace=self.get("namespace", _DEFAULT_NAMESPACE),
282 )
283 builder.update(self._extractSkyPixVisitors())
284 builder.update(self._extractElementVisitors())
285 builder.update(self._extractTopologyVisitors())
286 builder.update(self._extractPackerVisitors())
287 return builder