Coverage for python/lsst/daf/butler/core/dimensions/_config.py: 24%
76 statements
« prev ^ index » next coverage.py v7.2.3, created at 2023-04-22 02:18 -0700
« prev ^ index » next coverage.py v7.2.3, created at 2023-04-22 02:18 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ("DimensionConfig",)
26from collections.abc import Iterator, Mapping, Sequence
27from typing import Any
29from lsst.resources import ResourcePath, ResourcePathExpression
31from .. import ddl
32from .._topology import TopologicalSpace
33from ..config import Config, ConfigSubset
34from ._database import (
35 DatabaseDimensionElementConstructionVisitor,
36 DatabaseTopologicalFamilyConstructionVisitor,
37)
38from ._governor import GovernorDimensionConstructionVisitor
39from ._packer import DimensionPackerConstructionVisitor
40from ._skypix import SkyPixConstructionVisitor
41from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor
43# The default namespace to use on older dimension config files that only
44# have a version.
45_DEFAULT_NAMESPACE = "daf_butler"
48class DimensionConfig(ConfigSubset):
49 """Configuration that defines a `DimensionUniverse`.
51 The configuration tree for dimensions is a (nested) dictionary
52 with five top-level entries:
54 - version: an integer version number, used as keys in a singleton registry
55 of all `DimensionUniverse` instances;
57 - namespace: a string to be associated with the version in the singleton
58 registry of all `DimensionUnivers` instances;
60 - skypix: a dictionary whose entries each define a `SkyPixSystem`,
61 along with a special "common" key whose value is the name of a skypix
62 dimension that is used to relate all other spatial dimensions in the
63 `Registry` database;
65 - elements: a nested dictionary whose entries each define
66 `StandardDimension` or `StandardDimensionCombination`.
68 - topology: a nested dictionary with ``spatial`` and ``temporal`` keys,
69 with dictionary values that each define a `StandardTopologicalFamily`.
71 - packers: a nested dictionary whose entries define factories for a
72 `DimensionPacker` instance.
74 See the documentation for the linked classes above for more information
75 on the configuration syntax.
77 Parameters
78 ----------
79 other : `Config` or `str` or `dict`, optional
80 Argument specifying the configuration information as understood
81 by `Config`. If `None` is passed then defaults are loaded from
82 "dimensions.yaml", otherwise defaults are not loaded.
83 validate : `bool`, optional
84 If `True` required keys will be checked to ensure configuration
85 consistency.
86 searchPaths : `list` or `tuple`, optional
87 Explicit additional paths to search for defaults. They should
88 be supplied in priority order. These paths have higher priority
89 than those read from the environment in
90 `ConfigSubset.defaultSearchPaths()`. Paths can be `str` referring to
91 the local file system or URIs, `lsst.resources.ResourcePath`.
92 """
94 requiredKeys = ("version", "elements", "skypix")
95 defaultConfigFile = "dimensions.yaml"
97 def __init__(
98 self,
99 other: Config | ResourcePathExpression | Mapping[str, Any] | None = None,
100 validate: bool = True,
101 searchPaths: Sequence[ResourcePathExpression] | None = None,
102 ):
103 # if argument is not None then do not load/merge defaults
104 mergeDefaults = other is None
105 super().__init__(other=other, validate=validate, mergeDefaults=mergeDefaults, searchPaths=searchPaths)
107 def _updateWithConfigsFromPath(
108 self, searchPaths: Sequence[str | ResourcePath], configFile: ResourcePath | str
109 ) -> None:
110 """Search the supplied paths reading config from first found.
112 Raises
113 ------
114 FileNotFoundError
115 Raised if config file is not found in any of given locations.
117 Notes
118 -----
119 This method overrides base class method with different behavior.
120 Instead of merging all found files into a single configuration it
121 finds first matching file and reads it.
122 """
123 uri = ResourcePath(configFile)
124 if uri.isabs() and uri.exists():
125 # Assume this resource exists
126 self._updateWithOtherConfigFile(configFile)
127 self.filesRead.append(configFile)
128 else:
129 for pathDir in searchPaths:
130 if isinstance(pathDir, (str, ResourcePath)):
131 pathDir = ResourcePath(pathDir, forceDirectory=True)
132 file = pathDir.join(configFile)
133 if file.exists():
134 self.filesRead.append(file)
135 self._updateWithOtherConfigFile(file)
136 break
137 else:
138 raise TypeError(f"Unexpected search path type encountered: {pathDir!r}")
139 else:
140 raise FileNotFoundError(f"Could not find {configFile} in search path {searchPaths}")
142 def _updateWithOtherConfigFile(self, file: Config | str | ResourcePath | Mapping[str, Any]) -> None:
143 """Override for base class method.
145 Parameters
146 ----------
147 file : `Config`, `str`, `lsst.resources.ResourcePath`, or `dict`
148 Entity that can be converted to a `ConfigSubset`.
149 """
150 # Use this class to read the defaults so that subsetting can happen
151 # correctly.
152 externalConfig = type(self)(file, validate=False)
153 self.update(externalConfig)
155 def _extractSkyPixVisitors(self) -> Iterator[DimensionConstructionVisitor]:
156 """Process the 'skypix' section of the configuration.
158 Yields a construction visitor for each `SkyPixSystem`.
160 Yields
161 ------
162 visitor : `DimensionConstructionVisitor`
163 Object that adds a skypix system and its dimensions to an
164 under-construction `DimensionUniverse`.
165 """
166 config = self["skypix"]
167 systemNames = set(config.keys())
168 systemNames.remove("common")
169 for systemName in sorted(systemNames):
170 subconfig = config[systemName]
171 pixelizationClassName = subconfig["class"]
172 maxLevel = subconfig.get("max_level", 24)
173 yield SkyPixConstructionVisitor(systemName, pixelizationClassName, maxLevel)
175 def _extractElementVisitors(self) -> Iterator[DimensionConstructionVisitor]:
176 """Process the 'elements' section of the configuration.
178 Yields a construction visitor for each `StandardDimension` or
179 `StandardDimensionCombination`.
181 Yields
182 ------
183 visitor : `DimensionConstructionVisitor`
184 Object that adds a `StandardDimension` or
185 `StandardDimensionCombination` to an under-construction
186 `DimensionUniverse`.
187 """
188 for name, subconfig in self["elements"].items():
189 metadata = [ddl.FieldSpec.fromConfig(c) for c in subconfig.get("metadata", ())]
190 uniqueKeys = [ddl.FieldSpec.fromConfig(c, nullable=False) for c in subconfig.get("keys", ())]
191 if uniqueKeys:
192 uniqueKeys[0].primaryKey = True
193 if subconfig.get("governor", False):
194 unsupported = {"required", "implied", "viewOf", "alwaysJoin"}
195 if not unsupported.isdisjoint(subconfig.keys()):
196 raise RuntimeError(
197 f"Unsupported config key(s) for governor {name}: {unsupported & subconfig.keys()}."
198 )
199 if not subconfig.get("cached", True):
200 raise RuntimeError(f"Governor dimension {name} is always cached.")
201 yield GovernorDimensionConstructionVisitor(
202 name=name,
203 storage=subconfig["storage"],
204 metadata=metadata,
205 uniqueKeys=uniqueKeys,
206 )
207 else:
208 yield DatabaseDimensionElementConstructionVisitor(
209 name=name,
210 storage=subconfig["storage"],
211 required=set(subconfig.get("requires", ())),
212 implied=set(subconfig.get("implies", ())),
213 metadata=metadata,
214 alwaysJoin=subconfig.get("always_join", False),
215 uniqueKeys=uniqueKeys,
216 )
218 def _extractTopologyVisitors(self) -> Iterator[DimensionConstructionVisitor]:
219 """Process the 'topology' section of the configuration.
221 Yields a construction visitor for each `StandardTopologicalFamily`.
223 Yields
224 ------
225 visitor : `DimensionConstructionVisitor`
226 Object that adds a `StandardTopologicalFamily` to an
227 under-construction `DimensionUniverse` and updates its member
228 `DimensionElement` instances.
229 """
230 for spaceName, subconfig in self.get("topology", {}).items():
231 space = TopologicalSpace.__members__[spaceName.upper()]
232 for name, members in subconfig.items():
233 yield DatabaseTopologicalFamilyConstructionVisitor(
234 name=name,
235 space=space,
236 members=members,
237 )
239 def _extractPackerVisitors(self) -> Iterator[DimensionConstructionVisitor]:
240 """Process the 'packers' section of the configuration.
242 Yields construction visitors for each `DimensionPackerFactory`.
244 Yields
245 ------
246 visitor : `DimensionConstructionVisitor`
247 Object that adds a `DinmensionPackerFactory` to an
248 under-construction `DimensionUniverse`.
249 """
250 for name, subconfig in self["packers"].items():
251 yield DimensionPackerConstructionVisitor(
252 name=name,
253 clsName=subconfig["cls"],
254 fixed=subconfig["fixed"],
255 dimensions=subconfig["dimensions"],
256 )
258 def makeBuilder(self) -> DimensionConstructionBuilder:
259 """Construct a `DinmensionConstructionBuilder`.
261 The builder will reflect this configuration.
263 Returns
264 -------
265 builder : `DimensionConstructionBuilder`
266 A builder object populated with all visitors from this
267 configuration. The `~DimensionConstructionBuilder.finish` method
268 will not have been called.
269 """
270 builder = DimensionConstructionBuilder(
271 self["version"],
272 self["skypix", "common"],
273 self,
274 namespace=self.get("namespace", _DEFAULT_NAMESPACE),
275 )
276 builder.update(self._extractSkyPixVisitors())
277 builder.update(self._extractElementVisitors())
278 builder.update(self._extractTopologyVisitors())
279 builder.update(self._extractPackerVisitors())
280 return builder