Coverage for python/lsst/daf/butler/_labeled_butler_factory.py: 25%

57 statements  

« prev     ^ index     » next       coverage.py v7.4.0, created at 2024-01-16 10:44 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28__all__ = ("LabeledButlerFactory",) 

29 

30from typing import Callable, Mapping 

31 

32from lsst.resources import ResourcePathExpression 

33 

34from ._butler import Butler 

35from ._butler_config import ButlerConfig, ButlerType 

36from ._butler_repo_index import ButlerRepoIndex 

37from ._utilities.named_locks import NamedLocks 

38from ._utilities.thread_safe_cache import ThreadSafeCache 

39 

40_FactoryFunction = Callable[[str | None], Butler] 

41"""Function that takes an access token string or `None`, and returns a Butler 

42instance.""" 

43 

44 

45class LabeledButlerFactory: 

46 """Factory for efficiently instantiating Butler instances from the 

47 repository index file. This is intended for use from long-lived services 

48 that want to instantiate a separate Butler instance for each end user 

49 request. 

50 

51 Parameters 

52 ---------- 

53 repositories : `~collections.abc.Mapping` [`str`, `str`], optional 

54 Keys are arbitrary labels, and values are URIs to Butler configuration 

55 files. If not provided, defaults to the global repository index 

56 configured by the ``DAF_BUTLER_REPOSITORY_INDEX`` environment variable 

57 -- see `ButlerRepoIndex`. 

58 

59 Notes 

60 ----- 

61 This interface is currently considered experimental and is subject to 

62 change. 

63 

64 For each label in the repository index, caches shared state to allow fast 

65 instantiation of new instances. 

66 

67 Instance methods on this class are threadsafe. A single instance of 

68 `LabeledButlerFactory` can be shared between multiple threads. Note that 

69 ``DirectButler`` itself is not currently threadsafe, so this guarantee does 

70 not buy you much. See DM-42317. 

71 """ 

72 

73 def __init__(self, repositories: Mapping[str, str] | None = None) -> None: 

74 if repositories is None: 

75 self._repositories = None 

76 else: 

77 self._repositories = dict(repositories) 

78 

79 self._factories = ThreadSafeCache[str, _FactoryFunction]() 

80 self._initialization_locks = NamedLocks() 

81 

82 def create_butler(self, *, label: str, access_token: str | None) -> Butler: 

83 """Create a Butler instance. 

84 

85 Parameters 

86 ---------- 

87 label : `str` 

88 Label of the repository to instantiate, from the ``repositories`` 

89 parameter to the `LabeledButlerFactory` constructor or the global 

90 repository index file. 

91 access_token : `str` | `None` 

92 Gafaelfawr access token used to authenticate to a Butler server. 

93 This is required for any repositories configured to use 

94 `RemoteButler`. If you only use `DirectButler`, this may be 

95 `None`. 

96 

97 Raises 

98 ------ 

99 KeyError 

100 Raised if the label is not found in the index. 

101 

102 Notes 

103 ----- 

104 For a service making requests on behalf of end users, the access token 

105 should normally be a "delegated" token so that access permissions are 

106 based on the end user instead of the service. See 

107 https://gafaelfawr.lsst.io/user-guide/gafaelfawringress.html#requesting-delegated-tokens 

108 """ 

109 factory = self._get_or_create_butler_factory_function(label) 

110 return factory(access_token) 

111 

112 def _get_or_create_butler_factory_function(self, label: str) -> _FactoryFunction: 

113 # We maintain a separate lock per label. We only want to instantiate 

114 # one factory function per label, because creating the factory sets up 

115 # shared state that should only exist once per repository. However, we 

116 # don't want other repositories' instance creation to block on one 

117 # repository that is slow to initialize. 

118 with self._initialization_locks.lock(label): 

119 if (factory := self._factories.get(label)) is not None: 

120 return factory 

121 

122 factory = self._create_butler_factory_function(label) 

123 return self._factories.set_or_get(label, factory) 

124 

125 def _create_butler_factory_function(self, label: str) -> _FactoryFunction: 

126 config_uri = self._get_config_uri(label) 

127 config = ButlerConfig(config_uri) 

128 butler_type = config.get_butler_type() 

129 

130 match butler_type: 

131 case ButlerType.DIRECT: 

132 return _create_direct_butler_factory(config) 

133 case ButlerType.REMOTE: 

134 return _create_remote_butler_factory(config) 

135 case _: 

136 raise TypeError(f"Unknown butler type '{butler_type}' for label '{label}'") 

137 

138 def _get_config_uri(self, label: str) -> ResourcePathExpression: 

139 if self._repositories is None: 

140 return ButlerRepoIndex.get_repo_uri(label) 

141 else: 

142 config_uri = self._repositories.get(label) 

143 if config_uri is None: 

144 raise KeyError(f"Unknown repository label '{label}'") 

145 return config_uri 

146 

147 

148def _create_direct_butler_factory(config: ButlerConfig) -> _FactoryFunction: 

149 butler = Butler.from_config(config) 

150 

151 def create_butler(access_token: str | None) -> Butler: 

152 # Access token is ignored because DirectButler does not use Gafaelfawr 

153 # authentication. 

154 

155 # TODO DM-42317: This is not actually safe in its current form, because 

156 # clone returns an object that has non-thread-safe mutable state shared 

157 # between the original and cloned instance. 

158 # However, current services are already sharing a single global 

159 # non-cloned Butler instance, so this isn't making things worse than 

160 # they already are. 

161 return butler._clone() 

162 

163 return create_butler 

164 

165 

166def _create_remote_butler_factory(config: ButlerConfig) -> _FactoryFunction: 

167 import lsst.daf.butler.remote_butler 

168 

169 factory = lsst.daf.butler.remote_butler.RemoteButlerFactory.create_factory_from_config(config) 

170 

171 def create_butler(access_token: str | None) -> Butler: 

172 if access_token is None: 

173 raise ValueError("Access token is required to connect to a Butler server") 

174 return factory.create_butler_for_access_token(access_token) 

175 

176 return create_butler