Coverage for python/lsst/daf/butler/_limited_butler.py: 67%
70 statements
« prev ^ index » next coverage.py v7.10.6, created at 2025-10-09 03:23 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30__all__ = ("LimitedButler",)
32import logging
33from abc import ABC, abstractmethod
34from collections.abc import Iterable, Iterator
35from contextlib import contextmanager
36from typing import Any, ClassVar
38from lsst.resources import ResourcePath
40from ._butler_metrics import ButlerMetrics
41from ._dataset_provenance import DatasetProvenance
42from ._dataset_ref import DatasetRef
43from ._deferredDatasetHandle import DeferredDatasetHandle
44from ._storage_class import StorageClass, StorageClassFactory
45from .datastore import DatasetRefURIs, Datastore, FileTransferSource
46from .dimensions import DimensionUniverse
48log = logging.getLogger(__name__)
51class LimitedButler(ABC):
52 """A minimal butler interface that is sufficient to back
53 `~lsst.pipe.base.PipelineTask` execution.
54 """
56 GENERATION: ClassVar[int] = 3
57 """This is a Generation 3 Butler.
59 This attribute may be removed in the future, once the Generation 2 Butler
60 interface has been fully retired; it should only be used in transitional
61 code.
62 """
64 @abstractmethod
65 def isWriteable(self) -> bool:
66 """Return `True` if this `Butler` supports write operations."""
67 raise NotImplementedError()
69 @abstractmethod
70 def put(self, obj: Any, ref: DatasetRef, /, *, provenance: DatasetProvenance | None = None) -> DatasetRef:
71 """Store a dataset that already has a UUID and ``RUN`` collection.
73 Parameters
74 ----------
75 obj : `object`
76 The dataset.
77 ref : `DatasetRef`
78 Resolved reference for a not-yet-stored dataset.
79 provenance : `DatasetProvenance` or `None`, optional
80 Any provenance that should be attached to the serialized dataset.
81 Not supported by all serialization mechanisms.
83 Returns
84 -------
85 ref : `DatasetRef`
86 The same as the given, for convenience and symmetry with
87 `Butler.put`.
89 Raises
90 ------
91 TypeError
92 Raised if the butler is read-only.
94 Notes
95 -----
96 Whether this method inserts the given dataset into a ``Registry`` is
97 implementation defined (some `LimitedButler` subclasses do not have a
98 `Registry`), but it always adds the dataset to a `Datastore`, and the
99 given ``ref.id`` and ``ref.run`` are always preserved.
100 """
101 raise NotImplementedError()
103 def get(
104 self,
105 ref: DatasetRef,
106 /,
107 *,
108 parameters: dict[str, Any] | None = None,
109 storageClass: StorageClass | str | None = None,
110 ) -> Any:
111 """Retrieve a stored dataset.
113 Parameters
114 ----------
115 ref : `DatasetRef`
116 A resolved `DatasetRef` directly associated with a dataset.
117 parameters : `dict`
118 Additional StorageClass-defined options to control reading,
119 typically used to efficiently read only a subset of the dataset.
120 storageClass : `StorageClass` or `str`, optional
121 The storage class to be used to override the Python type
122 returned by this method. By default the returned type matches
123 the dataset type definition for this dataset. Specifying a
124 read `StorageClass` can force a different type to be returned.
125 This type must be compatible with the original type.
127 Returns
128 -------
129 obj : `object`
130 The dataset.
132 Notes
133 -----
134 In a `LimitedButler` the only allowable way to specify a dataset is
135 to use a resolved `DatasetRef`. Subclasses can support more options.
136 """
137 log.debug("Butler get: %s, parameters=%s, storageClass: %s", ref, parameters, storageClass)
138 with self._metrics.instrument_get(log, msg="Retrieved dataset"):
139 return self._datastore.get(ref, parameters=parameters, storageClass=storageClass)
141 def getDeferred(
142 self,
143 ref: DatasetRef,
144 /,
145 *,
146 parameters: dict[str, Any] | None = None,
147 storageClass: str | StorageClass | None = None,
148 ) -> DeferredDatasetHandle:
149 """Create a `DeferredDatasetHandle` which can later retrieve a dataset,
150 after an immediate registry lookup.
152 Parameters
153 ----------
154 ref : `DatasetRef`
155 For the default implementation of a `LimitedButler`, the only
156 acceptable parameter is a resolved `DatasetRef`.
157 parameters : `dict`
158 Additional StorageClass-defined options to control reading,
159 typically used to efficiently read only a subset of the dataset.
160 storageClass : `StorageClass` or `str`, optional
161 The storage class to be used to override the Python type
162 returned by this method. By default the returned type matches
163 the dataset type definition for this dataset. Specifying a
164 read `StorageClass` can force a different type to be returned.
165 This type must be compatible with the original type.
167 Returns
168 -------
169 obj : `DeferredDatasetHandle`
170 A handle which can be used to retrieve a dataset at a later time.
172 Notes
173 -----
174 In a `LimitedButler` the only allowable way to specify a dataset is
175 to use a resolved `DatasetRef`. Subclasses can support more options.
176 """
177 return DeferredDatasetHandle(butler=self, ref=ref, parameters=parameters, storageClass=storageClass)
179 def get_datastore_names(self) -> tuple[str, ...]:
180 """Return the names of the datastores associated with this butler.
182 Returns
183 -------
184 names : `tuple` [`str`, ...]
185 The names of the datastores.
186 """
187 return self._datastore.names
189 def get_datastore_roots(self) -> dict[str, ResourcePath | None]:
190 """Return the defined root URIs for all registered datastores.
192 Returns
193 -------
194 roots : `dict` [`str`, `~lsst.resources.ResourcePath` | `None`]
195 A mapping from datastore name to datastore root URI. The root
196 can be `None` if the datastore does not have any concept of a root
197 URI.
198 """
199 return self._datastore.roots
201 def getURIs(
202 self,
203 ref: DatasetRef,
204 /,
205 *,
206 predict: bool = False,
207 ) -> DatasetRefURIs:
208 """Return the URIs associated with the dataset.
210 Parameters
211 ----------
212 ref : `DatasetRef`
213 A `DatasetRef` for which URIs are requested.
214 predict : `bool`
215 If `True`, allow URIs to be returned of datasets that have not
216 been written.
218 Returns
219 -------
220 uris : `DatasetRefURIs`
221 The URI to the primary artifact associated with this dataset (if
222 the dataset was disassembled within the datastore this may be
223 `None`), and the URIs to any components associated with the dataset
224 artifact (can be empty if there are no components).
225 """
226 return self._datastore.getURIs(ref, predict)
228 def getURI(
229 self,
230 ref: DatasetRef,
231 /,
232 *,
233 predict: bool = False,
234 ) -> ResourcePath:
235 """Return the URI to the Dataset.
237 Parameters
238 ----------
239 ref : `DatasetRef`
240 A `DatasetRef` for which a single URI is requested.
241 predict : `bool`
242 If `True`, allow URIs to be returned of datasets that have not
243 been written.
245 Returns
246 -------
247 uri : `lsst.resources.ResourcePath`
248 URI pointing to the Dataset within the datastore. If the
249 Dataset does not exist in the datastore, and if ``predict`` is
250 `True`, the URI will be a prediction and will include a URI
251 fragment "#predicted".
252 If the datastore does not have entities that relate well
253 to the concept of a URI the returned URI string will be
254 descriptive. The returned URI is not guaranteed to be obtainable.
256 Raises
257 ------
258 RuntimeError
259 Raised if a URI is requested for a dataset that consists of
260 multiple artifacts.
261 """
262 primary, components = self.getURIs(ref, predict=predict)
264 if primary is None or components:
265 raise RuntimeError(
266 f"Dataset ({ref}) includes distinct URIs for components. Use LimitedButler.getURIs() instead."
267 )
268 return primary
270 def get_many_uris(
271 self,
272 refs: Iterable[DatasetRef],
273 predict: bool = False,
274 allow_missing: bool = False,
275 ) -> dict[DatasetRef, DatasetRefURIs]:
276 """Return URIs associated with many datasets.
278 Parameters
279 ----------
280 refs : iterable of `DatasetIdRef`
281 References to the required datasets.
282 predict : `bool`, optional
283 If `True`, allow URIs to be returned of datasets that have not
284 been written.
285 allow_missing : `bool`
286 If `False`, and ``predict`` is `False`, will raise if a
287 `DatasetRef` does not exist.
289 Returns
290 -------
291 URIs : `dict` of [`DatasetRef`, `DatasetRefURIs`]
292 A dict of primary and component URIs, indexed by the passed-in
293 refs.
295 Raises
296 ------
297 FileNotFoundError
298 A URI has been requested for a dataset that does not exist and
299 guessing is not allowed.
301 Notes
302 -----
303 In file-based datastores, get_many_uris does not check that the file is
304 present. It assumes that if datastore is aware of the file then it
305 actually exists.
306 """
307 return self._datastore.getManyURIs(refs, predict=predict, allow_missing=allow_missing)
309 def stored(self, ref: DatasetRef) -> bool:
310 """Indicate whether the dataset's artifacts are present in the
311 Datastore.
313 Parameters
314 ----------
315 ref : `DatasetRef`
316 Resolved reference to a dataset.
318 Returns
319 -------
320 stored : `bool`
321 Whether the dataset artifact exists in the datastore and can be
322 retrieved.
323 """
324 return self.stored_many([ref])[ref]
326 def stored_many(
327 self,
328 refs: Iterable[DatasetRef],
329 ) -> dict[DatasetRef, bool]:
330 """Check the datastore for artifact existence of multiple datasets
331 at once.
333 Parameters
334 ----------
335 refs : iterable of `DatasetRef`
336 The datasets to be checked.
338 Returns
339 -------
340 existence : `dict` of [`DatasetRef`, `bool`]
341 Mapping from given dataset refs to boolean indicating artifact
342 existence.
343 """
344 return self._datastore.mexists(refs)
346 def markInputUnused(self, ref: DatasetRef) -> None:
347 """Indicate that a predicted input was not actually used when
348 processing a `Quantum`.
350 Parameters
351 ----------
352 ref : `DatasetRef`
353 Reference to the unused dataset.
355 Notes
356 -----
357 By default, a dataset is considered "actually used" if it is accessed
358 via `get` or a handle to it is obtained via `getDeferred`
359 (even if the handle is not used). This method must be called after one
360 of those in order to remove the dataset from the actual input list.
362 This method does nothing for butlers that do not store provenance
363 information (which is the default implementation provided by the base
364 class).
365 """
366 pass
368 @abstractmethod
369 def pruneDatasets(
370 self,
371 refs: Iterable[DatasetRef],
372 *,
373 disassociate: bool = True,
374 unstore: bool = False,
375 tags: Iterable[str] = (),
376 purge: bool = False,
377 ) -> None:
378 """Remove one or more datasets from a collection and/or storage.
380 Parameters
381 ----------
382 refs : `~collections.abc.Iterable` of `DatasetRef`
383 Datasets to prune. These must be "resolved" references (not just
384 a `DatasetType` and data ID).
385 disassociate : `bool`, optional
386 Disassociate pruned datasets from ``tags``, or from all collections
387 if ``purge=True``.
388 unstore : `bool`, optional
389 If `True` (`False` is default) remove these datasets from all
390 datastores known to this butler. Note that this will make it
391 impossible to retrieve these datasets even via other collections.
392 Datasets that are already not stored are ignored by this option.
393 tags : `~collections.abc.Iterable` [ `str` ], optional
394 `~CollectionType.TAGGED` collections to disassociate the datasets
395 from. Ignored if ``disassociate`` is `False` or ``purge`` is
396 `True`.
397 purge : `bool`, optional
398 If `True` (`False` is default), completely remove the dataset from
399 the `Registry`. To prevent accidental deletions, ``purge`` may
400 only be `True` if all of the following conditions are met:
402 - ``disassociate`` is `True`;
403 - ``unstore`` is `True`.
405 This mode may remove provenance information from datasets other
406 than those provided, and should be used with extreme care.
408 Raises
409 ------
410 TypeError
411 Raised if the butler is read-only, if no collection was provided,
412 or the conditions for ``purge=True`` were not met.
413 """
414 raise NotImplementedError()
416 @contextmanager
417 def record_metrics(self, metrics: ButlerMetrics | None = None) -> Iterator[ButlerMetrics]:
418 """Enable new metrics recording context.
420 Parameters
421 ----------
422 metrics : `lsst.daf.butler.ButlerMetrics`
423 Optional override metrics object. If given, this will be the
424 same object returned by the context manager.
426 Yields
427 ------
428 metrics : `lsst.daf.butler.ButlerMetrics`
429 Metrics recorded within this context. This temporarily replaces
430 any existing metrics object associated with this butler.
431 """
432 old_metrics = self._metrics
433 new_metrics = metrics if metrics is not None else ButlerMetrics()
434 try:
435 self._metrics = new_metrics
436 yield new_metrics
437 finally:
438 self._metrics = old_metrics
440 @property
441 @abstractmethod
442 def dimensions(self) -> DimensionUniverse:
443 """Structure managing all dimensions recognized by this data
444 repository (`DimensionUniverse`).
445 """
446 raise NotImplementedError()
448 @property
449 def _file_transfer_source(self) -> FileTransferSource:
450 """Object that manages the transfer of files between Butler
451 repositories.
452 """
453 return self._datastore
455 _datastore: Datastore
456 """The object that manages actual dataset storage (`Datastore`)."""
458 storageClasses: StorageClassFactory
459 """An object that maps known storage class names to objects that fully
460 describe them (`StorageClassFactory`).
461 """
463 _metrics: ButlerMetrics
464 """An object for recording metrics associated with this butler.
465 (`ButlerMetrics`)
466 """