Coverage for python/lsst/daf/butler/_butler_config.py: 16%

70 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-10-27 09:44 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Configuration classes specific to the Butler. 

29""" 

30from __future__ import annotations 

31 

32__all__ = ("ButlerConfig",) 

33 

34import contextlib 

35import copy 

36import os 

37from collections.abc import Sequence 

38 

39from lsst.resources import ResourcePath, ResourcePathExpression 

40 

41from ._butler_repo_index import ButlerRepoIndex 

42from ._config import Config 

43from ._storage_class import StorageClassConfig 

44from .datastore import DatastoreConfig 

45from .registry import RegistryConfig 

46from .transfers import RepoTransferFormatConfig 

47 

48CONFIG_COMPONENT_CLASSES = (RegistryConfig, StorageClassConfig, DatastoreConfig, RepoTransferFormatConfig) 

49 

50 

51class ButlerConfig(Config): 

52 """Contains the configuration for a `Butler`. 

53 

54 The configuration is read and merged with default configurations for 

55 the particular classes. The defaults are read according to the rules 

56 outlined in `ConfigSubset`. Each component of the configuration associated 

57 with a configuration class reads its own defaults. 

58 

59 Parameters 

60 ---------- 

61 other : `str`, `Config`, `ButlerConfig`, optional 

62 Path to butler configuration YAML file or a directory containing a 

63 "butler.yaml" file. If `None` the butler will 

64 be configured based entirely on defaults read from the environment 

65 or from ``searchPaths``. 

66 No defaults will be read if a `ButlerConfig` is supplied directly. 

67 searchPaths : `list` or `tuple`, optional 

68 Explicit additional paths to search for defaults. They should 

69 be supplied in priority order. These paths have higher priority 

70 than those read from the environment in 

71 `ConfigSubset.defaultSearchPaths()`. They are only read if ``other`` 

72 refers to a configuration file or directory. 

73 without_datastore : `bool`, optional 

74 If `True` remove the datastore configuration. 

75 """ 

76 

77 def __init__( 

78 self, 

79 other: ResourcePathExpression | Config | None = None, 

80 searchPaths: Sequence[ResourcePathExpression] | None = None, 

81 without_datastore: bool = False, 

82 ): 

83 self.configDir: ResourcePath | None = None 

84 

85 # If this is already a ButlerConfig we assume that defaults 

86 # have already been loaded. 

87 if other is not None and isinstance(other, ButlerConfig): 

88 super().__init__(other) 

89 # Ensure that the configuration directory propagates 

90 self.configDir = copy.copy(other.configDir) 

91 return 

92 

93 # If a string is given it *could* be an alias that should be 

94 # expanded by the repository index system. 

95 original_other = other 

96 resolved_alias = False 

97 if isinstance(other, str): 

98 with contextlib.suppress(Exception): 

99 # Force back to a string because the resolved URI 

100 # might not refer explicitly to a directory and we have 

101 # check below to guess that. 

102 other = str(ButlerRepoIndex.get_repo_uri(other, True)) 

103 if other != original_other: 

104 resolved_alias = True 

105 

106 # Include ResourcePath here in case it refers to a directory. 

107 # Creating a ResourcePath from a ResourcePath is a no-op. 

108 if isinstance(other, str | os.PathLike | ResourcePath): 

109 # This will only allow supported schemes 

110 uri = ResourcePath(other) 

111 

112 # We allow the butler configuration file to be left off the 

113 # URI supplied by the user. If a directory-like URI is given 

114 # we add the default configuration name. 

115 

116 # It's easy to miss a trailing / for remote URIs so try to guess 

117 # we have been given a directory-like URI if there is no 

118 # file extension. Local URIs do not need any guess work. 

119 if not uri.isLocal and not uri.getExtension(): 

120 uri = ResourcePath(other, forceDirectory=True) 

121 

122 if uri.isdir(): 

123 # Could also be butler.json (for example in the butler 

124 # server) but checking for existence will slow things 

125 # down given that this might involve two checks and then 

126 # the config read below would still do the read. 

127 other = uri.join("butler.yaml") 

128 

129 # Create an empty config for us to populate 

130 super().__init__() 

131 

132 # Read the supplied config so that we can work out which other 

133 # defaults to use. 

134 try: 

135 butlerConfig = Config(other) 

136 except FileNotFoundError as e: 

137 # No reason to talk about aliases unless we were given a 

138 # string and the alias was not resolved. 

139 if isinstance(original_other, str): 

140 if not resolved_alias: 

141 # No alias was resolved. List known aliases if we have 

142 # them or else explain a reason why aliasing might not 

143 # have happened. 

144 if known := ButlerRepoIndex.get_known_repos(): 

145 aliases = f"(given {original_other!r} and known aliases: {', '.join(known)})" 

146 else: 

147 failure_reason = ButlerRepoIndex.get_failure_reason() 

148 if failure_reason: 

149 failure_reason = f": {failure_reason}" 

150 aliases = f"(given {original_other!r} and no known aliases{failure_reason})" 

151 else: 

152 aliases = f"(resolved from alias {original_other!r})" 

153 errmsg = f"{e} {aliases}" 

154 else: 

155 errmsg = str(e) 

156 raise FileNotFoundError(errmsg) from e 

157 

158 configFile = butlerConfig.configFile 

159 if configFile is not None: 

160 uri = ResourcePath(configFile) 

161 self.configFile = uri 

162 self.configDir = uri.dirname() 

163 

164 # A Butler config contains defaults defined by each of the component 

165 # configuration classes. We ask each of them to apply defaults to 

166 # the values we have been supplied by the user. 

167 for configClass in CONFIG_COMPONENT_CLASSES: 

168 assert configClass.component is not None, "Config class component cannot be None" 

169 

170 if without_datastore and configClass is DatastoreConfig: 

171 if configClass.component in butlerConfig: 

172 del butlerConfig[configClass.component] 

173 continue 

174 

175 # Only send the parent config if the child 

176 # config component is present (otherwise it assumes that the 

177 # keys from other components are part of the child) 

178 localOverrides = None 

179 if configClass.component in butlerConfig: 

180 localOverrides = butlerConfig 

181 config = configClass(localOverrides, searchPaths=searchPaths) 

182 # Re-attach it using the global namespace 

183 self.update({configClass.component: config}) 

184 # Remove the key from the butlerConfig since we have already 

185 # merged that information. 

186 if configClass.component in butlerConfig: 

187 del butlerConfig[configClass.component] 

188 

189 # Now that we have all the defaults we can merge the externally 

190 # provided config into the defaults. 

191 # Not needed if there is never information in a butler config file 

192 # not present in component configurations 

193 self.update(butlerConfig)