Coverage for python/lsst/daf/butler/_labeled_butler_factory.py: 22%

62 statements  

« prev     ^ index     » next       coverage.py v7.5.0, created at 2024-05-02 03:16 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28__all__ = ("LabeledButlerFactory",) 

29 

30from collections.abc import Callable, Mapping 

31 

32from lsst.resources import ResourcePathExpression 

33 

34from ._butler import Butler 

35from ._butler_config import ButlerConfig, ButlerType 

36from ._butler_repo_index import ButlerRepoIndex 

37from ._utilities.named_locks import NamedLocks 

38from ._utilities.thread_safe_cache import ThreadSafeCache 

39 

40_FactoryFunction = Callable[[str | None], Butler] 

41"""Function that takes an access token string or `None`, and returns a Butler 

42instance.""" 

43 

44 

45class LabeledButlerFactory: 

46 """Factory for efficiently instantiating Butler instances from the 

47 repository index file. This is intended for use from long-lived services 

48 that want to instantiate a separate Butler instance for each end user 

49 request. 

50 

51 Parameters 

52 ---------- 

53 repositories : `~collections.abc.Mapping` [`str`, `str`], optional 

54 Keys are arbitrary labels, and values are URIs to Butler configuration 

55 files. If not provided, defaults to the global repository index 

56 configured by the ``DAF_BUTLER_REPOSITORY_INDEX`` environment variable 

57 -- see `ButlerRepoIndex`. 

58 

59 Notes 

60 ----- 

61 This interface is currently considered experimental and is subject to 

62 change. 

63 

64 For each label in the repository index, caches shared state to allow fast 

65 instantiation of new instances. 

66 

67 Instance methods on this class are threadsafe -- a single instance of 

68 `LabeledButlerFactory` can be used concurrently by multiple threads. It is 

69 NOT safe for a single `Butler` instance returned by this factory to be used 

70 concurrently by multiple threads. However, separate `Butler` instances can 

71 safely be used by separate threads. 

72 """ 

73 

74 def __init__(self, repositories: Mapping[str, str] | None = None) -> None: 

75 if repositories is None: 

76 self._repositories = None 

77 else: 

78 self._repositories = dict(repositories) 

79 

80 self._factories = ThreadSafeCache[str, _FactoryFunction]() 

81 self._initialization_locks = NamedLocks() 

82 

83 # This may be overridden by unit tests. 

84 self._preload_direct_butler_cache = True 

85 

86 def create_butler(self, *, label: str, access_token: str | None) -> Butler: 

87 """Create a Butler instance. 

88 

89 Parameters 

90 ---------- 

91 label : `str` 

92 Label of the repository to instantiate, from the ``repositories`` 

93 parameter to the `LabeledButlerFactory` constructor or the global 

94 repository index file. 

95 access_token : `str` | `None` 

96 Gafaelfawr access token used to authenticate to a Butler server. 

97 This is required for any repositories configured to use 

98 `RemoteButler`. If you only use `DirectButler`, this may be 

99 `None`. 

100 

101 Raises 

102 ------ 

103 KeyError 

104 Raised if the label is not found in the index. 

105 

106 Notes 

107 ----- 

108 For a service making requests on behalf of end users, the access token 

109 should normally be a "delegated" token so that access permissions are 

110 based on the end user instead of the service. See 

111 https://gafaelfawr.lsst.io/user-guide/gafaelfawringress.html#requesting-delegated-tokens 

112 """ 

113 factory = self._get_or_create_butler_factory_function(label) 

114 return factory(access_token) 

115 

116 def _get_or_create_butler_factory_function(self, label: str) -> _FactoryFunction: 

117 # We maintain a separate lock per label. We only want to instantiate 

118 # one factory function per label, because creating the factory sets up 

119 # shared state that should only exist once per repository. However, we 

120 # don't want other repositories' instance creation to block on one 

121 # repository that is slow to initialize. 

122 with self._initialization_locks.lock(label): 

123 if (factory := self._factories.get(label)) is not None: 

124 return factory 

125 

126 factory = self._create_butler_factory_function(label) 

127 return self._factories.set_or_get(label, factory) 

128 

129 def _create_butler_factory_function(self, label: str) -> _FactoryFunction: 

130 config_uri = self._get_config_uri(label) 

131 config = ButlerConfig(config_uri) 

132 butler_type = config.get_butler_type() 

133 

134 match butler_type: 

135 case ButlerType.DIRECT: 

136 return _create_direct_butler_factory(config, self._preload_direct_butler_cache) 

137 case ButlerType.REMOTE: 

138 return _create_remote_butler_factory(config) 

139 case _: 

140 raise TypeError(f"Unknown butler type '{butler_type}' for label '{label}'") 

141 

142 def _get_config_uri(self, label: str) -> ResourcePathExpression: 

143 if self._repositories is None: 

144 return ButlerRepoIndex.get_repo_uri(label) 

145 else: 

146 config_uri = self._repositories.get(label) 

147 if config_uri is None: 

148 raise KeyError(f"Unknown repository label '{label}'") 

149 return config_uri 

150 

151 

152def _create_direct_butler_factory(config: ButlerConfig, preload_cache: bool) -> _FactoryFunction: 

153 import lsst.daf.butler.direct_butler 

154 

155 # Create a 'template' Butler that will be cloned when callers request an 

156 # instance. 

157 butler = Butler.from_config(config) 

158 assert isinstance(butler, lsst.daf.butler.direct_butler.DirectButler) 

159 

160 # Load caches so that data is available in cloned instances without 

161 # needing to refetch it from the database for every instance. 

162 if preload_cache: 

163 butler._preload_cache() 

164 

165 def create_butler(access_token: str | None) -> Butler: 

166 # Access token is ignored because DirectButler does not use Gafaelfawr 

167 # authentication. 

168 return butler._clone() 

169 

170 return create_butler 

171 

172 

173def _create_remote_butler_factory(config: ButlerConfig) -> _FactoryFunction: 

174 import lsst.daf.butler.remote_butler 

175 

176 factory = lsst.daf.butler.remote_butler.RemoteButlerFactory.create_factory_from_config(config) 

177 

178 def create_butler(access_token: str | None) -> Butler: 

179 if access_token is None: 

180 raise ValueError("Access token is required to connect to a Butler server") 

181 return factory.create_butler_for_access_token(access_token) 

182 

183 return create_butler