Coverage for python/lsst/daf/butler/_butlerConfig.py: 15%
68 statements
« prev ^ index » next coverage.py v7.3.1, created at 2023-10-02 08:00 +0000
« prev ^ index » next coverage.py v7.3.1, created at 2023-10-02 08:00 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28"""Configuration classes specific to the Butler.
29"""
30from __future__ import annotations
32__all__ = ("ButlerConfig",)
34import contextlib
35import copy
36import os
37from collections.abc import Sequence
39from lsst.resources import ResourcePath, ResourcePathExpression
41from ._butlerRepoIndex import ButlerRepoIndex
42from .core import Config, DatastoreConfig, StorageClassConfig
43from .registry import RegistryConfig
44from .transfers import RepoTransferFormatConfig
46CONFIG_COMPONENT_CLASSES = (RegistryConfig, StorageClassConfig, DatastoreConfig, RepoTransferFormatConfig)
49class ButlerConfig(Config):
50 """Contains the configuration for a `Butler`.
52 The configuration is read and merged with default configurations for
53 the particular classes. The defaults are read according to the rules
54 outlined in `ConfigSubset`. Each component of the configuration associated
55 with a configuration class reads its own defaults.
57 Parameters
58 ----------
59 other : `str`, `Config`, `ButlerConfig`, optional
60 Path to butler configuration YAML file or a directory containing a
61 "butler.yaml" file. If `None` the butler will
62 be configured based entirely on defaults read from the environment
63 or from ``searchPaths``.
64 No defaults will be read if a `ButlerConfig` is supplied directly.
65 searchPaths : `list` or `tuple`, optional
66 Explicit additional paths to search for defaults. They should
67 be supplied in priority order. These paths have higher priority
68 than those read from the environment in
69 `ConfigSubset.defaultSearchPaths()`. They are only read if ``other``
70 refers to a configuration file or directory.
71 without_datastore : `bool`, optional
72 If `True` remove the datastore configuration.
73 """
75 def __init__(
76 self,
77 other: ResourcePathExpression | Config | None = None,
78 searchPaths: Sequence[ResourcePathExpression] | None = None,
79 without_datastore: bool = False,
80 ):
81 self.configDir: ResourcePath | None = None
83 # If this is already a ButlerConfig we assume that defaults
84 # have already been loaded.
85 if other is not None and isinstance(other, ButlerConfig):
86 super().__init__(other)
87 # Ensure that the configuration directory propagates
88 self.configDir = copy.copy(other.configDir)
89 return
91 # If a string is given it *could* be an alias that should be
92 # expanded by the repository index system.
93 original_other = other
94 resolved_alias = False
95 if isinstance(other, str):
96 with contextlib.suppress(Exception):
97 # Force back to a string because the resolved URI
98 # might not refer explicitly to a directory and we have
99 # check below to guess that.
100 other = str(ButlerRepoIndex.get_repo_uri(other, True))
101 if other != original_other:
102 resolved_alias = True
104 # Include ResourcePath here in case it refers to a directory.
105 # Creating a ResourcePath from a ResourcePath is a no-op.
106 if isinstance(other, str | os.PathLike | ResourcePath):
107 # This will only allow supported schemes
108 uri = ResourcePath(other)
110 # We allow the butler configuration file to be left off the
111 # URI supplied by the user. If a directory-like URI is given
112 # we add the default configuration name.
114 # It's easy to miss a trailing / for remote URIs so try to guess
115 # we have been given a directory-like URI if there is no
116 # file extension. Local URIs do not need any guess work.
117 if not uri.isLocal and not uri.getExtension():
118 uri = ResourcePath(other, forceDirectory=True)
120 if uri.isdir():
121 # Could also be butler.json (for example in the butler
122 # server) but checking for existence will slow things
123 # down given that this might involve two checks and then
124 # the config read below would still do the read.
125 other = uri.join("butler.yaml")
127 # Create an empty config for us to populate
128 super().__init__()
130 # Read the supplied config so that we can work out which other
131 # defaults to use.
132 try:
133 butlerConfig = Config(other)
134 except FileNotFoundError as e:
135 # No reason to talk about aliases unless we were given a
136 # string and the alias was not resolved.
137 if isinstance(original_other, str):
138 if not resolved_alias:
139 # No alias was resolved. List known aliases if we have
140 # them or else explain a reason why aliasing might not
141 # have happened.
142 if known := ButlerRepoIndex.get_known_repos():
143 aliases = f"(given {original_other!r} and known aliases: {', '.join(known)})"
144 else:
145 failure_reason = ButlerRepoIndex.get_failure_reason()
146 if failure_reason:
147 failure_reason = f": {failure_reason}"
148 aliases = f"(given {original_other!r} and no known aliases{failure_reason})"
149 else:
150 aliases = f"(resolved from alias {original_other!r})"
151 errmsg = f"{e} {aliases}"
152 else:
153 errmsg = str(e)
154 raise FileNotFoundError(errmsg) from e
156 configFile = butlerConfig.configFile
157 if configFile is not None:
158 uri = ResourcePath(configFile)
159 self.configFile = uri
160 self.configDir = uri.dirname()
162 # A Butler config contains defaults defined by each of the component
163 # configuration classes. We ask each of them to apply defaults to
164 # the values we have been supplied by the user.
165 for configClass in CONFIG_COMPONENT_CLASSES:
166 assert configClass.component is not None, "Config class component cannot be None"
168 if without_datastore and configClass is DatastoreConfig:
169 if configClass.component in butlerConfig:
170 del butlerConfig[configClass.component]
171 continue
173 # Only send the parent config if the child
174 # config component is present (otherwise it assumes that the
175 # keys from other components are part of the child)
176 localOverrides = None
177 if configClass.component in butlerConfig:
178 localOverrides = butlerConfig
179 config = configClass(localOverrides, searchPaths=searchPaths)
180 # Re-attach it using the global namespace
181 self.update({configClass.component: config})
182 # Remove the key from the butlerConfig since we have already
183 # merged that information.
184 if configClass.component in butlerConfig:
185 del butlerConfig[configClass.component]
187 # Now that we have all the defaults we can merge the externally
188 # provided config into the defaults.
189 # Not needed if there is never information in a butler config file
190 # not present in component configurations
191 self.update(butlerConfig)