Coverage for python/lsst/daf/butler/_butler_config.py: 17%

81 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-11 03:16 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Configuration classes specific to the Butler. 

29""" 

30from __future__ import annotations 

31 

32__all__ = ("ButlerConfig",) 

33 

34import contextlib 

35import copy 

36import os 

37from collections.abc import Sequence 

38from enum import Enum 

39 

40from lsst.resources import ResourcePath, ResourcePathExpression 

41 

42from ._butler_repo_index import ButlerRepoIndex 

43from ._config import Config 

44from ._storage_class import StorageClassConfig 

45from .datastore import DatastoreConfig 

46from .registry import RegistryConfig 

47from .transfers import RepoTransferFormatConfig 

48 

49CONFIG_COMPONENT_CLASSES = (RegistryConfig, StorageClassConfig, DatastoreConfig, RepoTransferFormatConfig) 

50 

51ButlerType = Enum("ButlerType", ["DIRECT", "REMOTE"]) 

52 

53 

54class ButlerConfig(Config): 

55 """Contains the configuration for a `Butler`. 

56 

57 The configuration is read and merged with default configurations for 

58 the particular classes. The defaults are read according to the rules 

59 outlined in `ConfigSubset`. Each component of the configuration associated 

60 with a configuration class reads its own defaults. 

61 

62 Parameters 

63 ---------- 

64 other : `str`, `Config`, `ButlerConfig`, optional 

65 Path to butler configuration YAML file or a directory containing a 

66 "butler.yaml" file. If `None` the butler will 

67 be configured based entirely on defaults read from the environment 

68 or from ``searchPaths``. 

69 No defaults will be read if a `ButlerConfig` is supplied directly. 

70 searchPaths : `list` or `tuple`, optional 

71 Explicit additional paths to search for defaults. They should 

72 be supplied in priority order. These paths have higher priority 

73 than those read from the environment in 

74 `ConfigSubset.defaultSearchPaths()`. They are only read if ``other`` 

75 refers to a configuration file or directory. 

76 without_datastore : `bool`, optional 

77 If `True` remove the datastore configuration. 

78 """ 

79 

80 def __init__( 

81 self, 

82 other: ResourcePathExpression | Config | None = None, 

83 searchPaths: Sequence[ResourcePathExpression] | None = None, 

84 without_datastore: bool = False, 

85 ): 

86 self.configDir: ResourcePath | None = None 

87 

88 # If this is already a ButlerConfig we assume that defaults 

89 # have already been loaded. 

90 if other is not None and isinstance(other, ButlerConfig): 

91 super().__init__(other) 

92 # Ensure that the configuration directory propagates 

93 self.configDir = copy.copy(other.configDir) 

94 return 

95 

96 # If a string is given it *could* be an alias that should be 

97 # expanded by the repository index system. 

98 original_other = other 

99 resolved_alias = False 

100 if isinstance(other, str): 

101 with contextlib.suppress(Exception): 

102 # Force back to a string because the resolved URI 

103 # might not refer explicitly to a directory and we have 

104 # check below to guess that. 

105 other = str(ButlerRepoIndex.get_repo_uri(other, False)) 

106 if other != original_other: 

107 resolved_alias = True 

108 

109 # Include ResourcePath here in case it refers to a directory. 

110 # Creating a ResourcePath from a ResourcePath is a no-op. 

111 if isinstance(other, str | os.PathLike | ResourcePath): 

112 # This will only allow supported schemes 

113 uri = ResourcePath(other) 

114 

115 # We allow the butler configuration file to be left off the 

116 # URI supplied by the user. If a directory-like URI is given 

117 # we add the default configuration name. 

118 

119 # It's easy to miss a trailing / for remote URIs so try to guess 

120 # we have been given a directory-like URI if there is no 

121 # file extension. Local URIs do not need any guess work. 

122 if not uri.isLocal and not uri.getExtension(): 

123 uri = ResourcePath(other, forceDirectory=True) 

124 

125 if uri.isdir(): 

126 # Could also be butler.json (for example in the butler 

127 # server) but checking for existence will slow things 

128 # down given that this might involve two checks and then 

129 # the config read below would still do the read. 

130 other = uri.join("butler.yaml", forceDirectory=False) 

131 

132 # Create an empty config for us to populate 

133 super().__init__() 

134 

135 # Read the supplied config so that we can work out which other 

136 # defaults to use. 

137 try: 

138 butlerConfig = Config(other) 

139 except FileNotFoundError as e: 

140 # No reason to talk about aliases unless we were given a 

141 # string and the alias was not resolved. 

142 if isinstance(original_other, str): 

143 if not resolved_alias: 

144 # No alias was resolved. List known aliases if we have 

145 # them or else explain a reason why aliasing might not 

146 # have happened. 

147 if known := ButlerRepoIndex.get_known_repos(): 

148 aliases = f"(given {original_other!r} and known aliases: {', '.join(known)})" 

149 else: 

150 failure_reason = ButlerRepoIndex.get_failure_reason() 

151 if failure_reason: 

152 failure_reason = f": {failure_reason}" 

153 aliases = f"(given {original_other!r} and no known aliases{failure_reason})" 

154 else: 

155 aliases = f"(resolved from alias {original_other!r})" 

156 errmsg = f"{e} {aliases}" 

157 else: 

158 errmsg = str(e) 

159 raise FileNotFoundError(errmsg) from e 

160 

161 configFile = butlerConfig.configFile 

162 if configFile is not None: 

163 uri = ResourcePath(configFile) 

164 self.configFile = uri 

165 self.configDir = uri.dirname() 

166 

167 # A Butler config contains defaults defined by each of the component 

168 # configuration classes. We ask each of them to apply defaults to 

169 # the values we have been supplied by the user. 

170 for configClass in CONFIG_COMPONENT_CLASSES: 

171 assert configClass.component is not None, "Config class component cannot be None" 

172 

173 if without_datastore and configClass is DatastoreConfig: 

174 if configClass.component in butlerConfig: 

175 del butlerConfig[configClass.component] 

176 continue 

177 

178 # Only send the parent config if the child 

179 # config component is present (otherwise it assumes that the 

180 # keys from other components are part of the child) 

181 localOverrides = None 

182 if configClass.component in butlerConfig: 

183 localOverrides = butlerConfig 

184 config = configClass(localOverrides, searchPaths=searchPaths) 

185 # Re-attach it using the global namespace 

186 self.update({configClass.component: config}) 

187 # Remove the key from the butlerConfig since we have already 

188 # merged that information. 

189 if configClass.component in butlerConfig: 

190 del butlerConfig[configClass.component] 

191 

192 # Now that we have all the defaults we can merge the externally 

193 # provided config into the defaults. 

194 # Not needed if there is never information in a butler config file 

195 # not present in component configurations 

196 self.update(butlerConfig) 

197 

198 def get_butler_type(self) -> ButlerType: 

199 # Configuration optionally includes a class name specifying which 

200 # implementation to use, DirectButler or RemoteButler. 

201 butler_class_name = self.get("cls") 

202 if butler_class_name is None: 

203 # There are many existing DirectButler configurations that are 

204 # missing the ``cls`` property. 

205 return ButlerType.DIRECT 

206 elif butler_class_name == "lsst.daf.butler.direct_butler.DirectButler": 

207 return ButlerType.DIRECT 

208 elif butler_class_name == "lsst.daf.butler.remote_butler.RemoteButler": 

209 return ButlerType.REMOTE 

210 else: 

211 raise ValueError( 

212 f"Butler configuration requests to load unknown Butler class {butler_class_name}" 

213 )