Coverage for python/lsst/daf/butler/_butlerConfig.py: 15%

68 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-08-12 09:20 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Configuration classes specific to the Butler. 

23""" 

24from __future__ import annotations 

25 

26__all__ = ("ButlerConfig",) 

27 

28import contextlib 

29import copy 

30import os 

31from collections.abc import Sequence 

32 

33from lsst.resources import ResourcePath, ResourcePathExpression 

34 

35from ._butlerRepoIndex import ButlerRepoIndex 

36from .core import Config, DatastoreConfig, StorageClassConfig 

37from .registry import RegistryConfig 

38from .transfers import RepoTransferFormatConfig 

39 

40CONFIG_COMPONENT_CLASSES = (RegistryConfig, StorageClassConfig, DatastoreConfig, RepoTransferFormatConfig) 

41 

42 

43class ButlerConfig(Config): 

44 """Contains the configuration for a `Butler`. 

45 

46 The configuration is read and merged with default configurations for 

47 the particular classes. The defaults are read according to the rules 

48 outlined in `ConfigSubset`. Each component of the configuration associated 

49 with a configuration class reads its own defaults. 

50 

51 Parameters 

52 ---------- 

53 other : `str`, `Config`, `ButlerConfig`, optional 

54 Path to butler configuration YAML file or a directory containing a 

55 "butler.yaml" file. If `None` the butler will 

56 be configured based entirely on defaults read from the environment 

57 or from ``searchPaths``. 

58 No defaults will be read if a `ButlerConfig` is supplied directly. 

59 searchPaths : `list` or `tuple`, optional 

60 Explicit additional paths to search for defaults. They should 

61 be supplied in priority order. These paths have higher priority 

62 than those read from the environment in 

63 `ConfigSubset.defaultSearchPaths()`. They are only read if ``other`` 

64 refers to a configuration file or directory. 

65 without_datastore : `bool`, optional 

66 If `True` remove the datastore configuration. 

67 """ 

68 

69 def __init__( 

70 self, 

71 other: ResourcePathExpression | Config | None = None, 

72 searchPaths: Sequence[ResourcePathExpression] | None = None, 

73 without_datastore: bool = False, 

74 ): 

75 self.configDir: ResourcePath | None = None 

76 

77 # If this is already a ButlerConfig we assume that defaults 

78 # have already been loaded. 

79 if other is not None and isinstance(other, ButlerConfig): 

80 super().__init__(other) 

81 # Ensure that the configuration directory propagates 

82 self.configDir = copy.copy(other.configDir) 

83 return 

84 

85 # If a string is given it *could* be an alias that should be 

86 # expanded by the repository index system. 

87 original_other = other 

88 resolved_alias = False 

89 if isinstance(other, str): 

90 with contextlib.suppress(Exception): 

91 # Force back to a string because the resolved URI 

92 # might not refer explicitly to a directory and we have 

93 # check below to guess that. 

94 other = str(ButlerRepoIndex.get_repo_uri(other, True)) 

95 if other != original_other: 

96 resolved_alias = True 

97 

98 # Include ResourcePath here in case it refers to a directory. 

99 # Creating a ResourcePath from a ResourcePath is a no-op. 

100 if isinstance(other, str | os.PathLike | ResourcePath): 

101 # This will only allow supported schemes 

102 uri = ResourcePath(other) 

103 

104 # We allow the butler configuration file to be left off the 

105 # URI supplied by the user. If a directory-like URI is given 

106 # we add the default configuration name. 

107 

108 # It's easy to miss a trailing / for remote URIs so try to guess 

109 # we have been given a directory-like URI if there is no 

110 # file extension. Local URIs do not need any guess work. 

111 if not uri.isLocal and not uri.getExtension(): 

112 uri = ResourcePath(other, forceDirectory=True) 

113 

114 if uri.isdir(): 

115 # Could also be butler.json (for example in the butler 

116 # server) but checking for existence will slow things 

117 # down given that this might involve two checks and then 

118 # the config read below would still do the read. 

119 other = uri.join("butler.yaml") 

120 

121 # Create an empty config for us to populate 

122 super().__init__() 

123 

124 # Read the supplied config so that we can work out which other 

125 # defaults to use. 

126 try: 

127 butlerConfig = Config(other) 

128 except FileNotFoundError as e: 

129 # No reason to talk about aliases unless we were given a 

130 # string and the alias was not resolved. 

131 if isinstance(original_other, str): 

132 if not resolved_alias: 

133 # No alias was resolved. List known aliases if we have 

134 # them or else explain a reason why aliasing might not 

135 # have happened. 

136 if known := ButlerRepoIndex.get_known_repos(): 

137 aliases = f"(given {original_other!r} and known aliases: {', '.join(known)})" 

138 else: 

139 failure_reason = ButlerRepoIndex.get_failure_reason() 

140 if failure_reason: 

141 failure_reason = f": {failure_reason}" 

142 aliases = f"(given {original_other!r} and no known aliases{failure_reason})" 

143 else: 

144 aliases = f"(resolved from alias {original_other!r})" 

145 errmsg = f"{e} {aliases}" 

146 else: 

147 errmsg = str(e) 

148 raise FileNotFoundError(errmsg) from e 

149 

150 configFile = butlerConfig.configFile 

151 if configFile is not None: 

152 uri = ResourcePath(configFile) 

153 self.configFile = uri 

154 self.configDir = uri.dirname() 

155 

156 # A Butler config contains defaults defined by each of the component 

157 # configuration classes. We ask each of them to apply defaults to 

158 # the values we have been supplied by the user. 

159 for configClass in CONFIG_COMPONENT_CLASSES: 

160 assert configClass.component is not None, "Config class component cannot be None" 

161 

162 if without_datastore and configClass is DatastoreConfig: 

163 if configClass.component in butlerConfig: 

164 del butlerConfig[configClass.component] 

165 continue 

166 

167 # Only send the parent config if the child 

168 # config component is present (otherwise it assumes that the 

169 # keys from other components are part of the child) 

170 localOverrides = None 

171 if configClass.component in butlerConfig: 

172 localOverrides = butlerConfig 

173 config = configClass(localOverrides, searchPaths=searchPaths) 

174 # Re-attach it using the global namespace 

175 self.update({configClass.component: config}) 

176 # Remove the key from the butlerConfig since we have already 

177 # merged that information. 

178 if configClass.component in butlerConfig: 

179 del butlerConfig[configClass.component] 

180 

181 # Now that we have all the defaults we can merge the externally 

182 # provided config into the defaults. 

183 # Not needed if there is never information in a butler config file 

184 # not present in component configurations 

185 self.update(butlerConfig)