Coverage for python/lsst/daf/butler/core/dimensions/_config.py: 23%
75 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-15 01:59 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-15 01:59 -0800
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ("DimensionConfig",)
26from typing import Iterable, Iterator, Optional, Union
28from lsst.resources import ResourcePath, ResourcePathExpression
30from .. import ddl
31from .._topology import TopologicalSpace
32from ..config import Config, ConfigSubset
33from ._database import (
34 DatabaseDimensionElementConstructionVisitor,
35 DatabaseTopologicalFamilyConstructionVisitor,
36)
37from ._governor import GovernorDimensionConstructionVisitor
38from ._packer import DimensionPackerConstructionVisitor
39from ._skypix import SkyPixConstructionVisitor
40from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor
42# The default namespace to use on older dimension config files that only
43# have a version.
44_DEFAULT_NAMESPACE = "daf_butler"
47class DimensionConfig(ConfigSubset):
48 """Configuration that defines a `DimensionUniverse`.
50 The configuration tree for dimensions is a (nested) dictionary
51 with five top-level entries:
53 - version: an integer version number, used as keys in a singleton registry
54 of all `DimensionUniverse` instances;
56 - namespace: a string to be associated with the version in the singleton
57 registry of all `DimensionUnivers` instances;
59 - skypix: a dictionary whose entries each define a `SkyPixSystem`,
60 along with a special "common" key whose value is the name of a skypix
61 dimension that is used to relate all other spatial dimensions in the
62 `Registry` database;
64 - elements: a nested dictionary whose entries each define
65 `StandardDimension` or `StandardDimensionCombination`.
67 - topology: a nested dictionary with ``spatial`` and ``temporal`` keys,
68 with dictionary values that each define a `StandardTopologicalFamily`.
70 - packers: a nested dictionary whose entries define factories for a
71 `DimensionPacker` instance.
73 See the documentation for the linked classes above for more information
74 on the configuration syntax.
76 Parameters
77 ----------
78 other : `Config` or `str` or `dict`, optional
79 Argument specifying the configuration information as understood
80 by `Config`. If `None` is passed then defaults are loaded from
81 "dimensions.yaml", otherwise defaults are not loaded.
82 validate : `bool`, optional
83 If `True` required keys will be checked to ensure configuration
84 consistency.
85 searchPaths : `list` or `tuple`, optional
86 Explicit additional paths to search for defaults. They should
87 be supplied in priority order. These paths have higher priority
88 than those read from the environment in
89 `ConfigSubset.defaultSearchPaths()`. Paths can be `str` referring to
90 the local file system or URIs, `lsst.resources.ResourcePath`.
91 """
93 requiredKeys = ("version", "elements", "skypix")
94 defaultConfigFile = "dimensions.yaml"
96 def __init__(
97 self,
98 other: Union[Config, ResourcePathExpression, None] = None,
99 validate: bool = True,
100 searchPaths: Optional[Iterable[ResourcePathExpression]] = None,
101 ):
102 # if argument is not None then do not load/merge defaults
103 mergeDefaults = other is None
104 super().__init__(other=other, validate=validate, mergeDefaults=mergeDefaults, searchPaths=searchPaths)
106 def _updateWithConfigsFromPath(
107 self, searchPaths: Iterable[ResourcePathExpression], configFile: str
108 ) -> None:
109 """Search the supplied paths reading config from first found.
111 Raises
112 ------
113 FileNotFoundError
114 Raised if config file is not found in any of given locations.
116 Notes
117 -----
118 This method overrides base class method with different behavior.
119 Instead of merging all found files into a single configuration it
120 finds first matching file and reads it.
121 """
122 uri = ResourcePath(configFile)
123 if uri.isabs() and uri.exists():
124 # Assume this resource exists
125 self._updateWithOtherConfigFile(configFile)
126 self.filesRead.append(configFile)
127 else:
128 for pathDir in searchPaths:
129 if isinstance(pathDir, (str, ResourcePath)):
130 pathDir = ResourcePath(pathDir, forceDirectory=True)
131 file = pathDir.join(configFile)
132 if file.exists():
133 self.filesRead.append(file)
134 self._updateWithOtherConfigFile(file)
135 break
136 else:
137 raise TypeError(f"Unexpected search path type encountered: {pathDir!r}")
138 else:
139 raise FileNotFoundError(f"Could not find {configFile} in search path {searchPaths}")
141 def _updateWithOtherConfigFile(self, file: Union[ResourcePath, str]) -> None:
142 """Override for base class method.
144 Parameters
145 ----------
146 file : `Config`, `str`, `lsst.resources.ResourcePath`, or `dict`
147 Entity that can be converted to a `ConfigSubset`.
148 """
149 # Use this class to read the defaults so that subsetting can happen
150 # correctly.
151 externalConfig = type(self)(file, validate=False)
152 self.update(externalConfig)
154 def _extractSkyPixVisitors(self) -> Iterator[DimensionConstructionVisitor]:
155 """Process the 'skypix' section of the configuration.
157 Yields a construction visitor for each `SkyPixSystem`.
159 Yields
160 ------
161 visitor : `DimensionConstructionVisitor`
162 Object that adds a skypix system and its dimensions to an
163 under-construction `DimensionUniverse`.
164 """
165 config = self["skypix"]
166 systemNames = set(config.keys())
167 systemNames.remove("common")
168 for systemName in sorted(systemNames):
169 subconfig = config[systemName]
170 pixelizationClassName = subconfig["class"]
171 maxLevel = subconfig.get("max_level", 24)
172 yield SkyPixConstructionVisitor(systemName, pixelizationClassName, maxLevel)
174 def _extractElementVisitors(self) -> Iterator[DimensionConstructionVisitor]:
175 """Process the 'elements' section of the configuration.
177 Yields a construction visitor for each `StandardDimension` or
178 `StandardDimensionCombination`.
180 Yields
181 ------
182 visitor : `DimensionConstructionVisitor`
183 Object that adds a `StandardDimension` or
184 `StandardDimensionCombination` to an under-construction
185 `DimensionUniverse`.
186 """
187 for name, subconfig in self["elements"].items():
188 metadata = [ddl.FieldSpec.fromConfig(c) for c in subconfig.get("metadata", ())]
189 uniqueKeys = [ddl.FieldSpec.fromConfig(c, nullable=False) for c in subconfig.get("keys", ())]
190 if uniqueKeys:
191 uniqueKeys[0].primaryKey = True
192 if subconfig.get("governor", False):
193 unsupported = {"required", "implied", "viewOf", "alwaysJoin"}
194 if not unsupported.isdisjoint(subconfig.keys()):
195 raise RuntimeError(
196 f"Unsupported config key(s) for governor {name}: {unsupported & subconfig.keys()}."
197 )
198 if not subconfig.get("cached", True):
199 raise RuntimeError(f"Governor dimension {name} is always cached.")
200 yield GovernorDimensionConstructionVisitor(
201 name=name,
202 storage=subconfig["storage"],
203 metadata=metadata,
204 uniqueKeys=uniqueKeys,
205 )
206 else:
207 yield DatabaseDimensionElementConstructionVisitor(
208 name=name,
209 storage=subconfig["storage"],
210 required=set(subconfig.get("requires", ())),
211 implied=set(subconfig.get("implies", ())),
212 metadata=metadata,
213 alwaysJoin=subconfig.get("always_join", False),
214 uniqueKeys=uniqueKeys,
215 )
217 def _extractTopologyVisitors(self) -> Iterator[DimensionConstructionVisitor]:
218 """Process the 'topology' section of the configuration.
220 Yields a construction visitor for each `StandardTopologicalFamily`.
222 Yields
223 ------
224 visitor : `DimensionConstructionVisitor`
225 Object that adds a `StandardTopologicalFamily` to an
226 under-construction `DimensionUniverse` and updates its member
227 `DimensionElement` instances.
228 """
229 for spaceName, subconfig in self.get("topology", {}).items():
230 space = TopologicalSpace.__members__[spaceName.upper()]
231 for name, members in subconfig.items():
232 yield DatabaseTopologicalFamilyConstructionVisitor(
233 name=name,
234 space=space,
235 members=members,
236 )
238 def _extractPackerVisitors(self) -> Iterator[DimensionConstructionVisitor]:
239 """Process the 'packers' section of the configuration.
241 Yields construction visitors for each `DimensionPackerFactory`.
243 Yields
244 ------
245 visitor : `DimensionConstructionVisitor`
246 Object that adds a `DinmensionPackerFactory` to an
247 under-construction `DimensionUniverse`.
248 """
249 for name, subconfig in self["packers"].items():
250 yield DimensionPackerConstructionVisitor(
251 name=name,
252 clsName=subconfig["cls"],
253 fixed=subconfig["fixed"],
254 dimensions=subconfig["dimensions"],
255 )
257 def makeBuilder(self) -> DimensionConstructionBuilder:
258 """Construct a `DinmensionConstructionBuilder`.
260 The builder will reflect this configuration.
262 Returns
263 -------
264 builder : `DimensionConstructionBuilder`
265 A builder object populated with all visitors from this
266 configuration. The `~DimensionConstructionBuilder.finish` method
267 will not have been called.
268 """
269 builder = DimensionConstructionBuilder(
270 self["version"],
271 self["skypix", "common"],
272 self,
273 namespace=self.get("namespace", _DEFAULT_NAMESPACE),
274 )
275 builder.update(self._extractSkyPixVisitors())
276 builder.update(self._extractElementVisitors())
277 builder.update(self._extractTopologyVisitors())
278 builder.update(self._extractPackerVisitors())
279 return builder