Coverage for python/lsst/daf/butler/core/dimensions/_config.py: 23%
79 statements
« prev ^ index » next coverage.py v7.3.1, created at 2023-10-02 08:00 +0000
« prev ^ index » next coverage.py v7.3.1, created at 2023-10-02 08:00 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30__all__ = ("DimensionConfig",)
32import warnings
33from collections.abc import Iterator, Mapping, Sequence
34from typing import Any
36from lsst.resources import ResourcePath, ResourcePathExpression
38from .. import ddl
39from .._topology import TopologicalSpace
40from ..config import Config, ConfigSubset
41from ._database import (
42 DatabaseDimensionElementConstructionVisitor,
43 DatabaseTopologicalFamilyConstructionVisitor,
44)
45from ._governor import GovernorDimensionConstructionVisitor
46from ._packer import DimensionPackerConstructionVisitor
47from ._skypix import SkyPixConstructionVisitor
48from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor
50# The default namespace to use on older dimension config files that only
51# have a version.
52_DEFAULT_NAMESPACE = "daf_butler"
55class DimensionConfig(ConfigSubset):
56 """Configuration that defines a `DimensionUniverse`.
58 The configuration tree for dimensions is a (nested) dictionary
59 with five top-level entries:
61 - version: an integer version number, used as keys in a singleton registry
62 of all `DimensionUniverse` instances;
64 - namespace: a string to be associated with the version in the singleton
65 registry of all `DimensionUnivers` instances;
67 - skypix: a dictionary whose entries each define a `SkyPixSystem`,
68 along with a special "common" key whose value is the name of a skypix
69 dimension that is used to relate all other spatial dimensions in the
70 `Registry` database;
72 - elements: a nested dictionary whose entries each define
73 `StandardDimension` or `StandardDimensionCombination`.
75 - topology: a nested dictionary with ``spatial`` and ``temporal`` keys,
76 with dictionary values that each define a `StandardTopologicalFamily`.
78 - packers: a nested dictionary whose entries define factories for a
79 `DimensionPacker` instance.
81 See the documentation for the linked classes above for more information
82 on the configuration syntax.
84 Parameters
85 ----------
86 other : `Config` or `str` or `dict`, optional
87 Argument specifying the configuration information as understood
88 by `Config`. If `None` is passed then defaults are loaded from
89 "dimensions.yaml", otherwise defaults are not loaded.
90 validate : `bool`, optional
91 If `True` required keys will be checked to ensure configuration
92 consistency.
93 searchPaths : `list` or `tuple`, optional
94 Explicit additional paths to search for defaults. They should
95 be supplied in priority order. These paths have higher priority
96 than those read from the environment in
97 `ConfigSubset.defaultSearchPaths()`. Paths can be `str` referring to
98 the local file system or URIs, `lsst.resources.ResourcePath`.
99 """
101 requiredKeys = ("version", "elements", "skypix")
102 defaultConfigFile = "dimensions.yaml"
104 def __init__(
105 self,
106 other: Config | ResourcePathExpression | Mapping[str, Any] | None = None,
107 validate: bool = True,
108 searchPaths: Sequence[ResourcePathExpression] | None = None,
109 ):
110 # if argument is not None then do not load/merge defaults
111 mergeDefaults = other is None
112 super().__init__(other=other, validate=validate, mergeDefaults=mergeDefaults, searchPaths=searchPaths)
114 def _updateWithConfigsFromPath(
115 self, searchPaths: Sequence[str | ResourcePath], configFile: ResourcePath | str
116 ) -> None:
117 """Search the supplied paths reading config from first found.
119 Raises
120 ------
121 FileNotFoundError
122 Raised if config file is not found in any of given locations.
124 Notes
125 -----
126 This method overrides base class method with different behavior.
127 Instead of merging all found files into a single configuration it
128 finds first matching file and reads it.
129 """
130 uri = ResourcePath(configFile)
131 if uri.isabs() and uri.exists():
132 # Assume this resource exists
133 self._updateWithOtherConfigFile(configFile)
134 self.filesRead.append(configFile)
135 else:
136 for pathDir in searchPaths:
137 if isinstance(pathDir, str | ResourcePath):
138 pathDir = ResourcePath(pathDir, forceDirectory=True)
139 file = pathDir.join(configFile)
140 if file.exists():
141 self.filesRead.append(file)
142 self._updateWithOtherConfigFile(file)
143 break
144 else:
145 raise TypeError(f"Unexpected search path type encountered: {pathDir!r}")
146 else:
147 raise FileNotFoundError(f"Could not find {configFile} in search path {searchPaths}")
149 def _updateWithOtherConfigFile(self, file: Config | str | ResourcePath | Mapping[str, Any]) -> None:
150 """Override for base class method.
152 Parameters
153 ----------
154 file : `Config`, `str`, `lsst.resources.ResourcePath`, or `dict`
155 Entity that can be converted to a `ConfigSubset`.
156 """
157 # Use this class to read the defaults so that subsetting can happen
158 # correctly.
159 externalConfig = type(self)(file, validate=False)
160 self.update(externalConfig)
162 def _extractSkyPixVisitors(self) -> Iterator[DimensionConstructionVisitor]:
163 """Process the 'skypix' section of the configuration.
165 Yields a construction visitor for each `SkyPixSystem`.
167 Yields
168 ------
169 visitor : `DimensionConstructionVisitor`
170 Object that adds a skypix system and its dimensions to an
171 under-construction `DimensionUniverse`.
172 """
173 config = self["skypix"]
174 systemNames = set(config.keys())
175 systemNames.remove("common")
176 for systemName in sorted(systemNames):
177 subconfig = config[systemName]
178 pixelizationClassName = subconfig["class"]
179 maxLevel = subconfig.get("max_level", 24)
180 yield SkyPixConstructionVisitor(systemName, pixelizationClassName, maxLevel)
182 def _extractElementVisitors(self) -> Iterator[DimensionConstructionVisitor]:
183 """Process the 'elements' section of the configuration.
185 Yields a construction visitor for each `StandardDimension` or
186 `StandardDimensionCombination`.
188 Yields
189 ------
190 visitor : `DimensionConstructionVisitor`
191 Object that adds a `StandardDimension` or
192 `StandardDimensionCombination` to an under-construction
193 `DimensionUniverse`.
194 """
195 for name, subconfig in self["elements"].items():
196 metadata = [ddl.FieldSpec.fromConfig(c) for c in subconfig.get("metadata", ())]
197 uniqueKeys = [ddl.FieldSpec.fromConfig(c, nullable=False) for c in subconfig.get("keys", ())]
198 if uniqueKeys:
199 uniqueKeys[0].primaryKey = True
200 if subconfig.get("governor", False):
201 unsupported = {"required", "implied", "viewOf", "alwaysJoin"}
202 if not unsupported.isdisjoint(subconfig.keys()):
203 raise RuntimeError(
204 f"Unsupported config key(s) for governor {name}: {unsupported & subconfig.keys()}."
205 )
206 if not subconfig.get("cached", True):
207 raise RuntimeError(f"Governor dimension {name} is always cached.")
208 yield GovernorDimensionConstructionVisitor(
209 name=name,
210 storage=subconfig["storage"],
211 metadata=metadata,
212 uniqueKeys=uniqueKeys,
213 )
214 else:
215 yield DatabaseDimensionElementConstructionVisitor(
216 name=name,
217 storage=subconfig["storage"],
218 required=set(subconfig.get("requires", ())),
219 implied=set(subconfig.get("implies", ())),
220 metadata=metadata,
221 alwaysJoin=subconfig.get("always_join", False),
222 uniqueKeys=uniqueKeys,
223 populated_by=subconfig.get("populated_by", None),
224 )
226 def _extractTopologyVisitors(self) -> Iterator[DimensionConstructionVisitor]:
227 """Process the 'topology' section of the configuration.
229 Yields a construction visitor for each `StandardTopologicalFamily`.
231 Yields
232 ------
233 visitor : `DimensionConstructionVisitor`
234 Object that adds a `StandardTopologicalFamily` to an
235 under-construction `DimensionUniverse` and updates its member
236 `DimensionElement` instances.
237 """
238 for spaceName, subconfig in self.get("topology", {}).items():
239 space = TopologicalSpace.__members__[spaceName.upper()]
240 for name, members in subconfig.items():
241 yield DatabaseTopologicalFamilyConstructionVisitor(
242 name=name,
243 space=space,
244 members=members,
245 )
247 # TODO: remove this method and callers on DM-38687.
248 # Note that the corresponding entries in the dimensions config should
249 # not be removed at that time, because that's formally a schema migration.
250 def _extractPackerVisitors(self) -> Iterator[DimensionConstructionVisitor]:
251 """Process the 'packers' section of the configuration.
253 Yields construction visitors for each `DimensionPackerFactory`.
255 Yields
256 ------
257 visitor : `DimensionConstructionVisitor`
258 Object that adds a `DinmensionPackerFactory` to an
259 under-construction `DimensionUniverse`.
260 """
261 with warnings.catch_warnings():
262 # Don't warn when deprecated code calls other deprecated code.
263 warnings.simplefilter("ignore", FutureWarning)
264 for name, subconfig in self["packers"].items():
265 yield DimensionPackerConstructionVisitor(
266 name=name,
267 clsName=subconfig["cls"],
268 fixed=subconfig["fixed"],
269 dimensions=subconfig["dimensions"],
270 )
272 def makeBuilder(self) -> DimensionConstructionBuilder:
273 """Construct a `DinmensionConstructionBuilder`.
275 The builder will reflect this configuration.
277 Returns
278 -------
279 builder : `DimensionConstructionBuilder`
280 A builder object populated with all visitors from this
281 configuration. The `~DimensionConstructionBuilder.finish` method
282 will not have been called.
283 """
284 builder = DimensionConstructionBuilder(
285 self["version"],
286 self["skypix", "common"],
287 self,
288 namespace=self.get("namespace", _DEFAULT_NAMESPACE),
289 )
290 builder.update(self._extractSkyPixVisitors())
291 builder.update(self._extractElementVisitors())
292 builder.update(self._extractTopologyVisitors())
293 builder.update(self._extractPackerVisitors())
294 return builder