Coverage for python/lsst/daf/butler/_labeled_butler_factory.py: 24%

60 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-04-05 02:53 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28__all__ = ("LabeledButlerFactory",) 

29 

30from collections.abc import Callable, Mapping 

31 

32from lsst.resources import ResourcePathExpression 

33 

34from ._butler import Butler 

35from ._butler_config import ButlerConfig, ButlerType 

36from ._butler_repo_index import ButlerRepoIndex 

37from ._utilities.named_locks import NamedLocks 

38from ._utilities.thread_safe_cache import ThreadSafeCache 

39 

40_FactoryFunction = Callable[[str | None], Butler] 

41"""Function that takes an access token string or `None`, and returns a Butler 

42instance.""" 

43 

44 

45class LabeledButlerFactory: 

46 """Factory for efficiently instantiating Butler instances from the 

47 repository index file. This is intended for use from long-lived services 

48 that want to instantiate a separate Butler instance for each end user 

49 request. 

50 

51 Parameters 

52 ---------- 

53 repositories : `~collections.abc.Mapping` [`str`, `str`], optional 

54 Keys are arbitrary labels, and values are URIs to Butler configuration 

55 files. If not provided, defaults to the global repository index 

56 configured by the ``DAF_BUTLER_REPOSITORY_INDEX`` environment variable 

57 -- see `ButlerRepoIndex`. 

58 

59 Notes 

60 ----- 

61 This interface is currently considered experimental and is subject to 

62 change. 

63 

64 For each label in the repository index, caches shared state to allow fast 

65 instantiation of new instances. 

66 

67 Instance methods on this class are threadsafe -- a single instance of 

68 `LabeledButlerFactory` can be used concurrently by multiple threads. It is 

69 NOT safe for a single `Butler` instance returned by this factory to be used 

70 concurrently by multiple threads. However, separate `Butler` instances can 

71 safely be used by separate threads. 

72 """ 

73 

74 def __init__(self, repositories: Mapping[str, str] | None = None) -> None: 

75 if repositories is None: 

76 self._repositories = None 

77 else: 

78 self._repositories = dict(repositories) 

79 

80 self._factories = ThreadSafeCache[str, _FactoryFunction]() 

81 self._initialization_locks = NamedLocks() 

82 

83 def create_butler(self, *, label: str, access_token: str | None) -> Butler: 

84 """Create a Butler instance. 

85 

86 Parameters 

87 ---------- 

88 label : `str` 

89 Label of the repository to instantiate, from the ``repositories`` 

90 parameter to the `LabeledButlerFactory` constructor or the global 

91 repository index file. 

92 access_token : `str` | `None` 

93 Gafaelfawr access token used to authenticate to a Butler server. 

94 This is required for any repositories configured to use 

95 `RemoteButler`. If you only use `DirectButler`, this may be 

96 `None`. 

97 

98 Raises 

99 ------ 

100 KeyError 

101 Raised if the label is not found in the index. 

102 

103 Notes 

104 ----- 

105 For a service making requests on behalf of end users, the access token 

106 should normally be a "delegated" token so that access permissions are 

107 based on the end user instead of the service. See 

108 https://gafaelfawr.lsst.io/user-guide/gafaelfawringress.html#requesting-delegated-tokens 

109 """ 

110 factory = self._get_or_create_butler_factory_function(label) 

111 return factory(access_token) 

112 

113 def _get_or_create_butler_factory_function(self, label: str) -> _FactoryFunction: 

114 # We maintain a separate lock per label. We only want to instantiate 

115 # one factory function per label, because creating the factory sets up 

116 # shared state that should only exist once per repository. However, we 

117 # don't want other repositories' instance creation to block on one 

118 # repository that is slow to initialize. 

119 with self._initialization_locks.lock(label): 

120 if (factory := self._factories.get(label)) is not None: 

121 return factory 

122 

123 factory = self._create_butler_factory_function(label) 

124 return self._factories.set_or_get(label, factory) 

125 

126 def _create_butler_factory_function(self, label: str) -> _FactoryFunction: 

127 config_uri = self._get_config_uri(label) 

128 config = ButlerConfig(config_uri) 

129 butler_type = config.get_butler_type() 

130 

131 match butler_type: 

132 case ButlerType.DIRECT: 

133 return _create_direct_butler_factory(config) 

134 case ButlerType.REMOTE: 

135 return _create_remote_butler_factory(config) 

136 case _: 

137 raise TypeError(f"Unknown butler type '{butler_type}' for label '{label}'") 

138 

139 def _get_config_uri(self, label: str) -> ResourcePathExpression: 

140 if self._repositories is None: 

141 return ButlerRepoIndex.get_repo_uri(label) 

142 else: 

143 config_uri = self._repositories.get(label) 

144 if config_uri is None: 

145 raise KeyError(f"Unknown repository label '{label}'") 

146 return config_uri 

147 

148 

149def _create_direct_butler_factory(config: ButlerConfig) -> _FactoryFunction: 

150 import lsst.daf.butler.direct_butler 

151 

152 # Create a 'template' Butler that will be cloned when callers request an 

153 # instance. 

154 butler = Butler.from_config(config) 

155 assert isinstance(butler, lsst.daf.butler.direct_butler.DirectButler) 

156 

157 # Load caches so that data is available in cloned instances without 

158 # needing to refetch it from the database for every instance. 

159 butler._preload_cache() 

160 

161 def create_butler(access_token: str | None) -> Butler: 

162 # Access token is ignored because DirectButler does not use Gafaelfawr 

163 # authentication. 

164 return butler._clone() 

165 

166 return create_butler 

167 

168 

169def _create_remote_butler_factory(config: ButlerConfig) -> _FactoryFunction: 

170 import lsst.daf.butler.remote_butler 

171 

172 factory = lsst.daf.butler.remote_butler.RemoteButlerFactory.create_factory_from_config(config) 

173 

174 def create_butler(access_token: str | None) -> Butler: 

175 if access_token is None: 

176 raise ValueError("Access token is required to connect to a Butler server") 

177 return factory.create_butler_for_access_token(access_token) 

178 

179 return create_butler