Coverage for python/lsst/daf/butler/core/dimensions/_config.py : 22%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ("DimensionConfig",)
26from typing import Iterator, Iterable, Optional, Union
28from ..config import Config, ConfigSubset
29from .. import ddl
30from .._butlerUri import ButlerURI
31from .._topology import TopologicalSpace
32from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor
33from ._governor import GovernorDimensionConstructionVisitor
34from ._packer import DimensionPackerConstructionVisitor
35from ._skypix import SkyPixConstructionVisitor
36from ._database import (
37 DatabaseDimensionElementConstructionVisitor,
38 DatabaseTopologicalFamilyConstructionVisitor,
39)
42class DimensionConfig(ConfigSubset):
43 """Configuration that defines a `DimensionUniverse`.
45 The configuration tree for dimensions is a (nested) dictionary
46 with five top-level entries:
48 - version: an integer version number, used as keys in a singleton registry
49 of all `DimensionUniverse` instances;
51 - skypix: a dictionary whose entries each define a `SkyPixSystem`,
52 along with a special "common" key whose value is the name of a skypix
53 dimension that is used to relate all other spatial dimensions in the
54 `Registry` database;
56 - elements: a nested dictionary whose entries each define
57 `StandardDimension` or `StandardDimensionCombination`.
59 - topology: a nested dictionary with ``spatial`` and ``temporal`` keys,
60 with dictionary values that each define a `StandardTopologicalFamily`.
62 - packers: a nested dictionary whose entries define factories for a
63 `DimensionPacker` instance.
65 See the documentation for the linked classes above for more information
66 on the configuration syntax.
68 Parameters
69 ----------
70 other : `Config` or `str` or `dict`, optional
71 Argument specifying the configuration information as understood
72 by `Config`. If `None` is passed then defaults are loaded from
73 "dimensions.yaml", otherwise defaults are not loaded.
74 validate : `bool`, optional
75 If `True` required keys will be checked to ensure configuration
76 consistency.
77 searchPaths : `list` or `tuple`, optional
78 Explicit additional paths to search for defaults. They should
79 be supplied in priority order. These paths have higher priority
80 than those read from the environment in
81 `ConfigSubset.defaultSearchPaths()`. Paths can be `str` referring to
82 the local file system or URIs, `ButlerURI`.
83 """
85 requiredKeys = ("version", "elements", "skypix")
86 defaultConfigFile = "dimensions.yaml"
88 def __init__(self, other: Union[Config, ButlerURI, str, None] = None, validate: bool = True,
89 searchPaths: Optional[Iterable[Union[str, ButlerURI]]] = None):
90 # if argument is not None then do not load/merge defaults
91 mergeDefaults = other is None
92 super().__init__(other=other, validate=validate, mergeDefaults=mergeDefaults,
93 searchPaths=searchPaths)
95 def _updateWithConfigsFromPath(self, searchPaths: Iterable[Union[str, ButlerURI]],
96 configFile: str) -> None:
97 """Search the supplied paths reading config from first found.
99 Raises
100 ------
101 FileNotFoundError
102 Raised if config file is not found in any of given locations.
104 Notes
105 -----
106 This method overrides base class method with different behavior.
107 Instead of merging all found files into a single configuration it
108 finds first matching file and reads it.
109 """
110 uri = ButlerURI(configFile)
111 if uri.isabs() and uri.exists():
112 # Assume this resource exists
113 self._updateWithOtherConfigFile(configFile)
114 self.filesRead.append(configFile)
115 else:
116 for pathDir in searchPaths:
117 if isinstance(pathDir, (str, ButlerURI)):
118 pathDir = ButlerURI(pathDir, forceDirectory=True)
119 file = pathDir.join(configFile)
120 if file.exists():
121 self.filesRead.append(file)
122 self._updateWithOtherConfigFile(file)
123 break
124 else:
125 raise TypeError(f"Unexpected search path type encountered: {pathDir!r}")
126 else:
127 raise FileNotFoundError(f"Could not find {configFile} in search path {searchPaths}")
129 def _updateWithOtherConfigFile(self, file: Union[ButlerURI, str]) -> None:
130 """Override for base class method.
132 Parameters
133 ----------
134 file : `Config`, `str`, `ButlerURI`, or `dict`
135 Entity that can be converted to a `ConfigSubset`.
136 """
137 # Use this class to read the defaults so that subsetting can happen
138 # correctly.
139 externalConfig = type(self)(file, validate=False)
140 self.update(externalConfig)
142 def _extractSkyPixVisitors(self) -> Iterator[DimensionConstructionVisitor]:
143 """Process the 'skypix' section of the configuration, yielding a
144 construction visitor for each `SkyPixSystem`.
146 Yields
147 ------
148 visitor : `DimensionConstructionVisitor`
149 Object that adds a skypix system and its dimensions to an
150 under-construction `DimensionUniverse`.
151 """
152 config = self["skypix"]
153 systemNames = set(config.keys())
154 systemNames.remove("common")
155 for systemName in sorted(systemNames):
156 subconfig = config[systemName]
157 pixelizationClassName = subconfig["class"]
158 maxLevel = subconfig.get("max_level", 24)
159 yield SkyPixConstructionVisitor(systemName, pixelizationClassName, maxLevel)
161 def _extractElementVisitors(self) -> Iterator[DimensionConstructionVisitor]:
162 """Process the 'elements' section of the configuration, yielding a
163 construction visitor for each `StandardDimension` or
164 `StandardDimensionCombination`.
166 Yields
167 ------
168 visitor : `DimensionConstructionVisitor`
169 Object that adds a `StandardDimension` or
170 `StandardDimensionCombination` to an under-construction
171 `DimensionUniverse`.
172 """
173 for name, subconfig in self["elements"].items():
174 metadata = [ddl.FieldSpec.fromConfig(c) for c in subconfig.get("metadata", ())]
175 uniqueKeys = [ddl.FieldSpec.fromConfig(c, nullable=False) for c in subconfig.get("keys", ())]
176 if uniqueKeys:
177 uniqueKeys[0].primaryKey = True
178 if subconfig.get("governor", False):
179 unsupported = {"required", "implied", "viewOf", "alwaysJoin"}
180 if not unsupported.isdisjoint(subconfig.keys()):
181 raise RuntimeError(
182 f"Unsupported config key(s) for governor {name}: {unsupported & subconfig.keys()}."
183 )
184 if not subconfig.get("cached", True):
185 raise RuntimeError(
186 f"Governor dimension {name} is always cached."
187 )
188 yield GovernorDimensionConstructionVisitor(
189 name=name,
190 storage=subconfig["storage"],
191 metadata=metadata,
192 uniqueKeys=uniqueKeys,
193 )
194 else:
195 yield DatabaseDimensionElementConstructionVisitor(
196 name=name,
197 storage=subconfig["storage"],
198 required=set(subconfig.get("requires", ())),
199 implied=set(subconfig.get("implies", ())),
200 metadata=metadata,
201 alwaysJoin=subconfig.get("always_join", False),
202 uniqueKeys=uniqueKeys,
203 )
205 def _extractTopologyVisitors(self) -> Iterator[DimensionConstructionVisitor]:
206 """Process the 'topology' section of the configuration, yielding a
207 construction visitor for each `StandardTopologicalFamily`.
209 Yields
210 ------
211 visitor : `DimensionConstructionVisitor`
212 Object that adds a `StandardTopologicalFamily` to an
213 under-construction `DimensionUniverse` and updates its member
214 `DimensionElement` instances.
215 """
216 for spaceName, subconfig in self.get("topology", {}).items():
217 space = TopologicalSpace.__members__[spaceName.upper()]
218 for name, members in subconfig.items():
219 yield DatabaseTopologicalFamilyConstructionVisitor(
220 name=name,
221 space=space,
222 members=members,
223 )
225 def _extractPackerVisitors(self) -> Iterator[DimensionConstructionVisitor]:
226 """Process the 'packers' section of the configuration, yielding
227 construction visitors for each `DimensionPackerFactory`.
229 Yields
230 ------
231 visitor : `DimensionConstructionVisitor`
232 Object that adds a `DinmensionPackerFactory` to an
233 under-construction `DimensionUniverse`.
234 """
235 for name, subconfig in self["packers"].items():
236 yield DimensionPackerConstructionVisitor(
237 name=name,
238 clsName=subconfig["cls"],
239 fixed=subconfig["fixed"],
240 dimensions=subconfig["dimensions"],
241 )
243 def makeBuilder(self) -> DimensionConstructionBuilder:
244 """Construct a `DinmensionConstructionBuilder` that reflects this
245 configuration.
247 Returns
248 -------
249 builder : `DimensionConstructionBuilder`
250 A builder object populated with all visitors from this
251 configuration. The `~DimensionConstructionBuilder.finish` method
252 will not have been called.
253 """
254 builder = DimensionConstructionBuilder(self["version"], self["skypix", "common"])
255 builder.update(self._extractSkyPixVisitors())
256 builder.update(self._extractElementVisitors())
257 builder.update(self._extractTopologyVisitors())
258 builder.update(self._extractPackerVisitors())
259 return builder