Coverage for python/lsst/daf/butler/_butlerConfig.py: 15%

65 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-14 19:21 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Configuration classes specific to the Butler. 

23""" 

24from __future__ import annotations 

25 

26__all__ = ("ButlerConfig",) 

27 

28import copy 

29import os 

30from collections.abc import Sequence 

31 

32from lsst.resources import ResourcePath, ResourcePathExpression 

33 

34from ._butlerRepoIndex import ButlerRepoIndex 

35from .core import Config, DatastoreConfig, StorageClassConfig 

36from .registry import RegistryConfig 

37from .transfers import RepoTransferFormatConfig 

38 

39CONFIG_COMPONENT_CLASSES = (RegistryConfig, StorageClassConfig, DatastoreConfig, RepoTransferFormatConfig) 

40 

41 

42class ButlerConfig(Config): 

43 """Contains the configuration for a `Butler`. 

44 

45 The configuration is read and merged with default configurations for 

46 the particular classes. The defaults are read according to the rules 

47 outlined in `ConfigSubset`. Each component of the configuration associated 

48 with a configuration class reads its own defaults. 

49 

50 Parameters 

51 ---------- 

52 other : `str`, `Config`, `ButlerConfig`, optional 

53 Path to butler configuration YAML file or a directory containing a 

54 "butler.yaml" file. If `None` the butler will 

55 be configured based entirely on defaults read from the environment 

56 or from ``searchPaths``. 

57 No defaults will be read if a `ButlerConfig` is supplied directly. 

58 searchPaths : `list` or `tuple`, optional 

59 Explicit additional paths to search for defaults. They should 

60 be supplied in priority order. These paths have higher priority 

61 than those read from the environment in 

62 `ConfigSubset.defaultSearchPaths()`. They are only read if ``other`` 

63 refers to a configuration file or directory. 

64 """ 

65 

66 def __init__( 

67 self, 

68 other: ResourcePathExpression | Config | None = None, 

69 searchPaths: Sequence[ResourcePathExpression] | None = None, 

70 ): 

71 self.configDir: ResourcePath | None = None 

72 

73 # If this is already a ButlerConfig we assume that defaults 

74 # have already been loaded. 

75 if other is not None and isinstance(other, ButlerConfig): 

76 super().__init__(other) 

77 # Ensure that the configuration directory propagates 

78 self.configDir = copy.copy(other.configDir) 

79 return 

80 

81 # If a string is given it *could* be an alias that should be 

82 # expanded by the repository index system. 

83 original_other = other 

84 resolved_alias = False 

85 if isinstance(other, str): 

86 try: 

87 # Force back to a string because the resolved URI 

88 # might not refer explicitly to a directory and we have 

89 # check below to guess that. 

90 other = str(ButlerRepoIndex.get_repo_uri(other, True)) 

91 except Exception: 

92 pass 

93 if other != original_other: 

94 resolved_alias = True 

95 

96 # Include ResourcePath here in case it refers to a directory. 

97 # Creating a ResourcePath from a ResourcePath is a no-op. 

98 if isinstance(other, (str, os.PathLike, ResourcePath)): 

99 # This will only allow supported schemes 

100 uri = ResourcePath(other) 

101 

102 # We allow the butler configuration file to be left off the 

103 # URI supplied by the user. If a directory-like URI is given 

104 # we add the default configuration name. 

105 

106 # It's easy to miss a trailing / for remote URIs so try to guess 

107 # we have been given a directory-like URI if there is no 

108 # file extension. Local URIs do not need any guess work. 

109 if not uri.isLocal and not uri.getExtension(): 

110 uri = ResourcePath(other, forceDirectory=True) 

111 

112 if uri.isdir(): 

113 # Could also be butler.json (for example in the butler 

114 # server) but checking for existence will slow things 

115 # down given that this might involve two checks and then 

116 # the config read below would still do the read. 

117 other = uri.join("butler.yaml") 

118 

119 # Create an empty config for us to populate 

120 super().__init__() 

121 

122 # Read the supplied config so that we can work out which other 

123 # defaults to use. 

124 try: 

125 butlerConfig = Config(other) 

126 except FileNotFoundError as e: 

127 # No reason to talk about aliases unless we were given a 

128 # string and the alias was not resolved. 

129 if isinstance(original_other, str): 

130 if not resolved_alias: 

131 # No alias was resolved. List known aliases if we have 

132 # them or else explain a reason why aliasing might not 

133 # have happened. 

134 if known := ButlerRepoIndex.get_known_repos(): 

135 aliases = f"(given {original_other!r} and known aliases: {', '.join(known)})" 

136 else: 

137 failure_reason = ButlerRepoIndex.get_failure_reason() 

138 if failure_reason: 

139 failure_reason = f": {failure_reason}" 

140 aliases = f"(given {original_other!r} and no known aliases{failure_reason})" 

141 else: 

142 aliases = f"(resolved from alias {original_other!r})" 

143 errmsg = f"{e} {aliases}" 

144 else: 

145 errmsg = str(e) 

146 raise FileNotFoundError(errmsg) from e 

147 

148 configFile = butlerConfig.configFile 

149 if configFile is not None: 

150 uri = ResourcePath(configFile) 

151 self.configFile = uri 

152 self.configDir = uri.dirname() 

153 

154 # A Butler config contains defaults defined by each of the component 

155 # configuration classes. We ask each of them to apply defaults to 

156 # the values we have been supplied by the user. 

157 for configClass in CONFIG_COMPONENT_CLASSES: 

158 # Only send the parent config if the child 

159 # config component is present (otherwise it assumes that the 

160 # keys from other components are part of the child) 

161 localOverrides = None 

162 if configClass.component in butlerConfig: 

163 localOverrides = butlerConfig 

164 config = configClass(localOverrides, searchPaths=searchPaths) 

165 # Re-attach it using the global namespace 

166 assert configClass.component is not None, "Config class component cannot be None" 

167 self.update({configClass.component: config}) 

168 # Remove the key from the butlerConfig since we have already 

169 # merged that information. 

170 if configClass.component in butlerConfig: 

171 del butlerConfig[configClass.component] 

172 

173 # Now that we have all the defaults we can merge the externally 

174 # provided config into the defaults. 

175 # Not needed if there is never information in a butler config file 

176 # not present in component configurations 

177 self.update(butlerConfig)