Coverage for python/lsst/daf/butler/_labeled_butler_factory.py: 27%
68 statements
« prev ^ index » next coverage.py v7.11.0, created at 2025-11-04 01:36 -0800
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28__all__ = ("LabeledButlerFactory", "LabeledButlerFactoryProtocol")
30from collections.abc import Callable, Mapping
31from typing import Protocol
33from lsst.resources import ResourcePathExpression
35from ._butler import Butler
36from ._butler_config import ButlerConfig, ButlerType
37from ._butler_repo_index import ButlerRepoIndex
38from ._utilities.named_locks import NamedLocks
39from ._utilities.thread_safe_cache import ThreadSafeCache
41_FactoryFunction = Callable[[str | None], Butler]
42"""Function that takes an access token string or `None`, and returns a Butler
43instance."""
46class LabeledButlerFactoryProtocol(Protocol):
47 """Callable to retrieve a butler from a label."""
49 def __call__(self, label: str) -> Butler: ... 49 ↛ exitline 49 didn't return from function '__call__' because
52class LabeledButlerFactory:
53 """Factory for efficiently instantiating Butler instances from the
54 repository index file. This is intended for use from long-lived services
55 that want to instantiate a separate Butler instance for each end user
56 request.
58 Parameters
59 ----------
60 repositories : `~collections.abc.Mapping` [`str`, `str`], optional
61 Keys are arbitrary labels, and values are URIs to Butler configuration
62 files. If not provided, defaults to the global repository index
63 configured by the ``DAF_BUTLER_REPOSITORY_INDEX`` environment variable
64 -- see `ButlerRepoIndex`.
66 Notes
67 -----
68 This interface is currently considered experimental and is subject to
69 change.
71 For each label in the repository index, caches shared state to allow fast
72 instantiation of new instances.
74 Instance methods on this class are threadsafe -- a single instance of
75 `LabeledButlerFactory` can be used concurrently by multiple threads. It is
76 NOT safe for a single `Butler` instance returned by this factory to be used
77 concurrently by multiple threads. However, separate `Butler` instances can
78 safely be used by separate threads.
79 """
81 def __init__(self, repositories: Mapping[str, str] | None = None) -> None:
82 if repositories is None:
83 self._repositories = None
84 else:
85 self._repositories = dict(repositories)
87 self._factories = ThreadSafeCache[str, _FactoryFunction]()
88 self._initialization_locks = NamedLocks()
90 # This may be overridden by unit tests.
91 self._preload_unsafe_direct_butler_caches = True
93 def bind(self, access_token: str | None) -> LabeledButlerFactoryProtocol:
94 """Create a callable factory function for generating Butler instances
95 with out needing to specify access tokans again.
97 Parameters
98 ----------
99 access_token : `str` or `None`
100 An optional access token to use for authentication with the Butler.
102 Returns
103 -------
104 bound : `LabeledButlerFactoryProtocol`
105 A callable that takes a label as input and returns a Butler
106 instance.
107 """
109 def create(label: str) -> Butler:
110 return self.create_butler(label=label, access_token=access_token)
112 return create
114 def create_butler(self, *, label: str, access_token: str | None) -> Butler:
115 """Create a Butler instance.
117 Parameters
118 ----------
119 label : `str`
120 Label of the repository to instantiate, from the ``repositories``
121 parameter to the `LabeledButlerFactory` constructor or the global
122 repository index file.
123 access_token : `str` | `None`
124 Gafaelfawr access token used to authenticate to a Butler server.
125 This is required for any repositories configured to use
126 `RemoteButler`. If you only use `DirectButler`, this may be
127 `None`.
129 Raises
130 ------
131 KeyError
132 Raised if the label is not found in the index.
134 Notes
135 -----
136 For a service making requests on behalf of end users, the access token
137 should normally be a "delegated" token so that access permissions are
138 based on the end user instead of the service. See
139 https://gafaelfawr.lsst.io/user-guide/gafaelfawringress.html#requesting-delegated-tokens
140 """
141 factory = self._get_or_create_butler_factory_function(label)
142 return factory(access_token)
144 def _get_or_create_butler_factory_function(self, label: str) -> _FactoryFunction:
145 # We maintain a separate lock per label. We only want to instantiate
146 # one factory function per label, because creating the factory sets up
147 # shared state that should only exist once per repository. However, we
148 # don't want other repositories' instance creation to block on one
149 # repository that is slow to initialize.
150 with self._initialization_locks.lock(label):
151 if (factory := self._factories.get(label)) is not None:
152 return factory
154 factory = self._create_butler_factory_function(label)
155 return self._factories.set_or_get(label, factory)
157 def _create_butler_factory_function(self, label: str) -> _FactoryFunction:
158 config_uri = self._get_config_uri(label)
159 config = ButlerConfig(config_uri)
160 butler_type = config.get_butler_type()
162 match butler_type:
163 case ButlerType.DIRECT:
164 return _create_direct_butler_factory(config, self._preload_unsafe_direct_butler_caches)
165 case ButlerType.REMOTE:
166 return _create_remote_butler_factory(config)
167 case _:
168 raise TypeError(f"Unknown butler type '{butler_type}' for label '{label}'")
170 def _get_config_uri(self, label: str) -> ResourcePathExpression:
171 if self._repositories is None:
172 return ButlerRepoIndex.get_repo_uri(label)
173 else:
174 config_uri = self._repositories.get(label)
175 if config_uri is None:
176 raise KeyError(f"Unknown repository label '{label}'")
177 return config_uri
180def _create_direct_butler_factory(config: ButlerConfig, preload_unsafe_caches: bool) -> _FactoryFunction:
181 import lsst.daf.butler.direct_butler
183 # Create a 'template' Butler that will be cloned when callers request an
184 # instance.
185 butler = Butler.from_config(config)
186 assert isinstance(butler, lsst.daf.butler.direct_butler.DirectButler)
188 # Load caches so that data is available in cloned instances without
189 # needing to refetch it from the database for every instance.
190 butler._preload_cache(load_dimension_record_cache=preload_unsafe_caches)
192 def create_butler(access_token: str | None) -> Butler:
193 # Access token is ignored because DirectButler does not use Gafaelfawr
194 # authentication.
195 return butler.clone()
197 return create_butler
200def _create_remote_butler_factory(config: ButlerConfig) -> _FactoryFunction:
201 import lsst.daf.butler.remote_butler._factory
203 factory = lsst.daf.butler.remote_butler._factory.RemoteButlerFactory.create_factory_from_config(config)
205 def create_butler(access_token: str | None) -> Butler:
206 if access_token is None:
207 raise ValueError("Access token is required to connect to a Butler server")
208 return factory.create_butler_for_access_token(access_token)
210 return create_butler