Coverage for python/lsst/daf/butler/_butler.py: 30%
316 statements
« prev ^ index » next coverage.py v7.11.0, created at 2025-11-21 01:20 -0800
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30__all__ = ["Butler"]
32import dataclasses
33import urllib.parse
34import uuid
35from abc import abstractmethod
36from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence
37from contextlib import AbstractContextManager
38from types import EllipsisType
39from typing import TYPE_CHECKING, Any, Literal, Self, TextIO
41from lsst.resources import ResourcePath, ResourcePathExpression
42from lsst.utils import doImportType
43from lsst.utils.iteration import ensure_iterable
44from lsst.utils.logging import getLogger
46from ._butler_collections import ButlerCollections
47from ._butler_config import ButlerConfig, ButlerType
48from ._butler_instance_options import ButlerInstanceOptions
49from ._butler_metrics import ButlerMetrics
50from ._butler_repo_index import ButlerRepoIndex
51from ._config import Config, ConfigSubset
52from ._exceptions import EmptyQueryResultError, InvalidQueryError
53from ._limited_butler import LimitedButler
54from ._query_all_datasets import QueryAllDatasetsParameters
55from .datastore import Datastore
56from .dimensions import DataCoordinate, DimensionConfig
57from .registry import RegistryConfig, _RegistryFactory
58from .repo_relocation import BUTLER_ROOT_TAG
59from .utils import has_globs
61if TYPE_CHECKING:
62 from ._dataset_existence import DatasetExistence
63 from ._dataset_provenance import DatasetProvenance
64 from ._dataset_ref import DatasetId, DatasetRef
65 from ._dataset_type import DatasetType
66 from ._deferredDatasetHandle import DeferredDatasetHandle
67 from ._file_dataset import FileDataset
68 from ._labeled_butler_factory import LabeledButlerFactoryProtocol
69 from ._storage_class import StorageClass
70 from ._timespan import Timespan
71 from .datastore import DatasetRefURIs
72 from .dimensions import DataId, DimensionGroup, DimensionRecord
73 from .queries import Query
74 from .registry import CollectionArgType, Registry
75 from .transfers import RepoExportContext
77_LOG = getLogger(__name__)
80@dataclasses.dataclass
81class ParsedButlerDatasetURI:
82 label: str
83 dataset_id: uuid.UUID
84 uri: str
87@dataclasses.dataclass
88class SpecificButlerDataset:
89 butler: Butler
90 dataset: DatasetRef | None
93class _DeprecatedDefault:
94 """Default value for a deprecated parameter."""
97class Butler(LimitedButler, AbstractContextManager): # numpydoc ignore=PR02
98 """Interface for data butler and factory for Butler instances.
100 Parameters
101 ----------
102 config : `ButlerConfig`, `Config` or `str`, optional
103 Configuration. Anything acceptable to the `ButlerConfig` constructor.
104 If a directory path is given the configuration will be read from a
105 ``butler.yaml`` file in that location. If `None` is given default
106 values will be used. If ``config`` contains "cls" key then its value is
107 used as a name of butler class and it must be a sub-class of this
108 class, otherwise `DirectButler` is instantiated.
109 collections : `str` or `~collections.abc.Iterable` [ `str` ], optional
110 An expression specifying the collections to be searched (in order) when
111 reading datasets.
112 This may be a `str` collection name or an iterable thereof.
113 See :ref:`daf_butler_collection_expressions` for more information.
114 These collections are not registered automatically and must be
115 manually registered before they are used by any method, but they may be
116 manually registered after the `Butler` is initialized.
117 run : `str`, optional
118 Name of the `~CollectionType.RUN` collection new datasets should be
119 inserted into. If ``collections`` is `None` and ``run`` is not `None`,
120 ``collections`` will be set to ``[run]``. If not `None`, this
121 collection will automatically be registered. If this is not set (and
122 ``writeable`` is not set either), a read-only butler will be created.
123 searchPaths : `list` of `str`, optional
124 Directory paths to search when calculating the full Butler
125 configuration. Not used if the supplied config is already a
126 `ButlerConfig`.
127 writeable : `bool`, optional
128 Explicitly sets whether the butler supports write operations. If not
129 provided, a read-write butler is created if any of ``run``, ``tags``,
130 or ``chains`` is non-empty.
131 inferDefaults : `bool`, optional
132 If `True` (default) infer default data ID values from the values
133 present in the datasets in ``collections``: if all collections have the
134 same value (or no value) for a governor dimension, that value will be
135 the default for that dimension. Nonexistent collections are ignored.
136 If a default value is provided explicitly for a governor dimension via
137 ``**kwargs``, no default will be inferred for that dimension.
138 without_datastore : `bool`, optional
139 If `True` do not attach a datastore to this butler. Any attempts
140 to use a datastore will fail.
141 **kwargs : `Any`
142 Additional keyword arguments passed to a constructor of actual butler
143 class.
145 Notes
146 -----
147 The preferred way to instantiate Butler is via the `from_config` method.
148 The call to ``Butler(...)`` is equivalent to ``Butler.from_config(...)``,
149 but ``mypy`` will complain about the former.
150 """
152 def __new__(
153 cls,
154 config: Config | ResourcePathExpression | None = None,
155 *,
156 collections: Any = None,
157 run: str | None = None,
158 searchPaths: Sequence[ResourcePathExpression] | None = None,
159 writeable: bool | None = None,
160 inferDefaults: bool = True,
161 without_datastore: bool = False,
162 metrics: ButlerMetrics | None = None,
163 **kwargs: Any,
164 ) -> Butler:
165 if cls is Butler:
166 return Butler.from_config(
167 config=config,
168 collections=collections,
169 run=run,
170 searchPaths=searchPaths,
171 writeable=writeable,
172 inferDefaults=inferDefaults,
173 without_datastore=without_datastore,
174 metrics=metrics,
175 **kwargs,
176 )
178 # Note: we do not pass any parameters to __new__, Python will pass them
179 # to __init__ after __new__ returns sub-class instance.
180 return super().__new__(cls)
182 @classmethod
183 def from_config(
184 cls,
185 config: Config | ResourcePathExpression | None = None,
186 *,
187 collections: Any = None,
188 run: str | None = None,
189 searchPaths: Sequence[ResourcePathExpression] | None = None,
190 writeable: bool | None = None,
191 inferDefaults: bool = True,
192 without_datastore: bool = False,
193 metrics: ButlerMetrics | None = None,
194 **kwargs: Any,
195 ) -> Butler:
196 """Create butler instance from configuration.
198 Parameters
199 ----------
200 config : `ButlerConfig`, `Config` or `str`, optional
201 Configuration. Anything acceptable to the `ButlerConfig`
202 constructor. If a directory path is given the configuration will be
203 read from a ``butler.yaml`` file in that location. If `None` is
204 given default values will be used. If ``config`` contains "cls" key
205 then its value is used as a name of butler class and it must be a
206 sub-class of this class, otherwise `DirectButler` is instantiated.
207 collections : `str` or `~collections.abc.Iterable` [ `str` ], optional
208 An expression specifying the collections to be searched (in order)
209 when reading datasets.
210 This may be a `str` collection name or an iterable thereof.
211 See :ref:`daf_butler_collection_expressions` for more information.
212 These collections are not registered automatically and must be
213 manually registered before they are used by any method, but they
214 may be manually registered after the `Butler` is initialized.
215 run : `str`, optional
216 Name of the `~CollectionType.RUN` collection new datasets should be
217 inserted into. If ``collections`` is `None` and ``run`` is not
218 `None`, ``collections`` will be set to ``[run]``. If not `None`,
219 this collection will automatically be registered. If this is not
220 set (and ``writeable`` is not set either), a read-only butler will
221 be created.
222 searchPaths : `list` of `str`, optional
223 Directory paths to search when calculating the full Butler
224 configuration. Not used if the supplied config is already a
225 `ButlerConfig`.
226 writeable : `bool`, optional
227 Explicitly sets whether the butler supports write operations. If
228 not provided, a read-write butler is created if any of ``run``,
229 ``tags``, or ``chains`` is non-empty.
230 inferDefaults : `bool`, optional
231 If `True` (default) infer default data ID values from the values
232 present in the datasets in ``collections``: if all collections have
233 the same value (or no value) for a governor dimension, that value
234 will be the default for that dimension. Nonexistent collections
235 are ignored. If a default value is provided explicitly for a
236 governor dimension via ``**kwargs``, no default will be inferred
237 for that dimension.
238 without_datastore : `bool`, optional
239 If `True` do not attach a datastore to this butler. Any attempts
240 to use a datastore will fail.
241 metrics : `ButlerMetrics` or `None`, optional
242 Metrics object to record butler usage statistics.
243 **kwargs : `Any`
244 Default data ID key-value pairs. These may only identify
245 "governor" dimensions like ``instrument`` and ``skymap``.
247 Returns
248 -------
249 butler : `Butler`
250 A `Butler` constructed from the given configuration.
252 Notes
253 -----
254 Calling this factory method is identical to calling
255 ``Butler(config, ...)``. Its only raison d'être is that ``mypy``
256 complains about ``Butler()`` call.
258 Examples
259 --------
260 While there are many ways to control exactly how a `Butler` interacts
261 with the collections in its `Registry`, the most common cases are still
262 simple.
264 For a read-only `Butler` that searches one collection, do::
266 butler = Butler.from_config(
267 "/path/to/repo", collections=["u/alice/DM-50000"]
268 )
270 For a read-write `Butler` that writes to and reads from a
271 `~CollectionType.RUN` collection::
273 butler = Butler.from_config(
274 "/path/to/repo", run="u/alice/DM-50000/a"
275 )
277 The `Butler` passed to a ``PipelineTask`` is often much more complex,
278 because we want to write to one `~CollectionType.RUN` collection but
279 read from several others (as well)::
281 butler = Butler.from_config(
282 "/path/to/repo",
283 run="u/alice/DM-50000/a",
284 collections=[
285 "u/alice/DM-50000/a",
286 "u/bob/DM-49998",
287 "HSC/defaults",
288 ],
289 )
291 This butler will `put` new datasets to the run ``u/alice/DM-50000/a``.
292 Datasets will be read first from that run (since it appears first in
293 the chain), and then from ``u/bob/DM-49998`` and finally
294 ``HSC/defaults``.
296 Finally, one can always create a `Butler` with no collections::
298 butler = Butler.from_config("/path/to/repo", writeable=True)
300 This can be extremely useful when you just want to use
301 ``butler.registry``, e.g. for inserting dimension data or managing
302 collections, or when the collections you want to use with the butler
303 are not consistent. Passing ``writeable`` explicitly here is only
304 necessary if you want to be able to make changes to the repo - usually
305 the value for ``writeable`` can be guessed from the collection
306 arguments provided, but it defaults to `False` when there are not
307 collection arguments.
308 """
309 # DirectButler used to have a way to specify a "copy constructor" by
310 # passing the "butler" parameter to its constructor. This has
311 # been moved out of the constructor into Butler.clone().
312 butler = kwargs.pop("butler", None)
313 metrics = metrics if metrics is not None else ButlerMetrics()
314 if butler is not None:
315 if not isinstance(butler, Butler):
316 raise TypeError("'butler' parameter must be a Butler instance")
317 if config is not None or searchPaths is not None or writeable is not None:
318 raise TypeError(
319 "Cannot pass 'config', 'searchPaths', or 'writeable' arguments with 'butler' argument."
320 )
321 return butler.clone(
322 collections=collections, run=run, inferDefaults=inferDefaults, metrics=metrics, dataId=kwargs
323 )
325 options = ButlerInstanceOptions(
326 collections=collections,
327 run=run,
328 writeable=writeable,
329 inferDefaults=inferDefaults,
330 metrics=metrics,
331 kwargs=kwargs,
332 )
334 # Load the Butler configuration. This may involve searching the
335 # environment to locate a configuration file.
336 butler_config = ButlerConfig(config, searchPaths=searchPaths, without_datastore=without_datastore)
337 butler_type = butler_config.get_butler_type()
339 # Make DirectButler if class is not specified.
340 match butler_type:
341 case ButlerType.DIRECT:
342 from .direct_butler import DirectButler
344 return DirectButler.create_from_config(
345 butler_config,
346 options=options,
347 without_datastore=without_datastore,
348 )
349 case ButlerType.REMOTE:
350 from .remote_butler._factory import RemoteButlerFactory
352 # Assume this is being created by a client who would like
353 # default caching of remote datasets.
354 factory = RemoteButlerFactory.create_factory_from_config(butler_config)
355 return factory.create_butler_with_credentials_from_environment(
356 butler_options=options, enable_datastore_cache=True
357 )
358 case _:
359 raise TypeError(f"Unknown Butler type '{butler_type}'")
361 def __enter__(self) -> Self:
362 return self
364 def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> Literal[False]:
365 try:
366 self.close()
367 except Exception:
368 _LOG.exception("An exception occured during Butler.close()")
369 return False
371 @staticmethod
372 def makeRepo(
373 root: ResourcePathExpression,
374 config: Config | str | None = None,
375 dimensionConfig: Config | str | None = None,
376 standalone: bool = False,
377 searchPaths: list[str] | None = None,
378 forceConfigRoot: bool = True,
379 outfile: ResourcePathExpression | None = None,
380 overwrite: bool = False,
381 ) -> Config:
382 """Create an empty data repository by adding a butler.yaml config
383 to a repository root directory.
385 Parameters
386 ----------
387 root : `lsst.resources.ResourcePathExpression`
388 Path or URI to the root location of the new repository. Will be
389 created if it does not exist.
390 config : `Config` or `str`, optional
391 Configuration to write to the repository, after setting any
392 root-dependent Registry or Datastore config options. Can not
393 be a `ButlerConfig` or a `ConfigSubset`. If `None`, default
394 configuration will be used. Root-dependent config options
395 specified in this config are overwritten if ``forceConfigRoot``
396 is `True`.
397 dimensionConfig : `Config` or `str`, optional
398 Configuration for dimensions, will be used to initialize registry
399 database.
400 standalone : `bool`
401 If True, write all expanded defaults, not just customized or
402 repository-specific settings.
403 This (mostly) decouples the repository from the default
404 configuration, insulating it from changes to the defaults (which
405 may be good or bad, depending on the nature of the changes).
406 Future *additions* to the defaults will still be picked up when
407 initializing `Butlers` to repos created with ``standalone=True``.
408 searchPaths : `list` of `str`, optional
409 Directory paths to search when calculating the full butler
410 configuration.
411 forceConfigRoot : `bool`, optional
412 If `False`, any values present in the supplied ``config`` that
413 would normally be reset are not overridden and will appear
414 directly in the output config. This allows non-standard overrides
415 of the root directory for a datastore or registry to be given.
416 If this parameter is `True` the values for ``root`` will be
417 forced into the resulting config if appropriate.
418 outfile : `lss.resources.ResourcePathExpression`, optional
419 If not-`None`, the output configuration will be written to this
420 location rather than into the repository itself. Can be a URI
421 string. Can refer to a directory that will be used to write
422 ``butler.yaml``.
423 overwrite : `bool`, optional
424 Create a new configuration file even if one already exists
425 in the specified output location. Default is to raise
426 an exception.
428 Returns
429 -------
430 config : `Config`
431 The updated `Config` instance written to the repo.
433 Raises
434 ------
435 ValueError
436 Raised if a ButlerConfig or ConfigSubset is passed instead of a
437 regular Config (as these subclasses would make it impossible to
438 support ``standalone=False``).
439 FileExistsError
440 Raised if the output config file already exists.
441 os.error
442 Raised if the directory does not exist, exists but is not a
443 directory, or cannot be created.
445 Notes
446 -----
447 Note that when ``standalone=False`` (the default), the configuration
448 search path (see `ConfigSubset.defaultSearchPaths`) that was used to
449 construct the repository should also be used to construct any Butlers
450 to avoid configuration inconsistencies.
451 """
452 if isinstance(config, ButlerConfig | ConfigSubset):
453 raise ValueError("makeRepo must be passed a regular Config without defaults applied.")
455 # Ensure that the root of the repository exists or can be made
456 root_uri = ResourcePath(root, forceDirectory=True)
457 root_uri.mkdir()
459 config = Config(config)
461 # If we are creating a new repo from scratch with relative roots,
462 # do not propagate an explicit root from the config file
463 if "root" in config:
464 del config["root"]
466 full = ButlerConfig(config, searchPaths=searchPaths) # this applies defaults
467 imported_class = doImportType(full["datastore", "cls"])
468 if not issubclass(imported_class, Datastore):
469 raise TypeError(f"Imported datastore class {full['datastore', 'cls']} is not a Datastore")
470 datastoreClass: type[Datastore] = imported_class
471 datastoreClass.setConfigRoot(BUTLER_ROOT_TAG, config, full, overwrite=forceConfigRoot)
473 # if key exists in given config, parse it, otherwise parse the defaults
474 # in the expanded config
475 if config.get(("registry", "db")):
476 registryConfig = RegistryConfig(config)
477 else:
478 registryConfig = RegistryConfig(full)
479 defaultDatabaseUri = registryConfig.makeDefaultDatabaseUri(BUTLER_ROOT_TAG)
480 if defaultDatabaseUri is not None:
481 Config.updateParameters(
482 RegistryConfig, config, full, toUpdate={"db": defaultDatabaseUri}, overwrite=forceConfigRoot
483 )
484 else:
485 Config.updateParameters(RegistryConfig, config, full, toCopy=("db",), overwrite=forceConfigRoot)
487 if standalone:
488 config.merge(full)
489 else:
490 # Always expand the registry.managers section into the per-repo
491 # config, because after the database schema is created, it's not
492 # allowed to change anymore. Note that in the standalone=True
493 # branch, _everything_ in the config is expanded, so there's no
494 # need to special case this.
495 Config.updateParameters(RegistryConfig, config, full, toMerge=("managers",), overwrite=False)
496 configURI: ResourcePathExpression
497 if outfile is not None:
498 # When writing to a separate location we must include
499 # the root of the butler repo in the config else it won't know
500 # where to look.
501 config["root"] = root_uri.geturl()
502 configURI = outfile
503 else:
504 configURI = root_uri
505 # Strip obscore configuration, if it is present, before writing config
506 # to a file, obscore config will be stored in registry.
507 if (obscore_config_key := ("registry", "managers", "obscore", "config")) in config:
508 config_to_write = config.copy()
509 del config_to_write[obscore_config_key]
510 config_to_write.dumpToUri(configURI, overwrite=overwrite)
511 # configFile attribute is updated, need to copy it to original.
512 config.configFile = config_to_write.configFile
513 else:
514 config.dumpToUri(configURI, overwrite=overwrite)
516 # Create Registry and populate tables
517 registryConfig = RegistryConfig(config.get("registry"))
518 dimensionConfig = DimensionConfig(dimensionConfig)
519 registry = _RegistryFactory(registryConfig).create_from_config(
520 dimensionConfig=dimensionConfig, butlerRoot=root_uri
521 )
522 registry.close()
524 _LOG.verbose("Wrote new Butler configuration file to %s", configURI)
526 return config
528 @classmethod
529 def get_repo_uri(cls, label: str, return_label: bool = False) -> ResourcePath:
530 """Look up the label in a butler repository index.
532 Parameters
533 ----------
534 label : `str`
535 Label of the Butler repository to look up.
536 return_label : `bool`, optional
537 If ``label`` cannot be found in the repository index (either
538 because index is not defined or ``label`` is not in the index) and
539 ``return_label`` is `True` then return ``ResourcePath(label)``.
540 If ``return_label`` is `False` (default) then an exception will be
541 raised instead.
543 Returns
544 -------
545 uri : `lsst.resources.ResourcePath`
546 URI to the Butler repository associated with the given label or
547 default value if it is provided.
549 Raises
550 ------
551 KeyError
552 Raised if the label is not found in the index, or if an index
553 is not defined, and ``return_label`` is `False`.
555 Notes
556 -----
557 See `~lsst.daf.butler.ButlerRepoIndex` for details on how the
558 information is discovered.
559 """
560 return ButlerRepoIndex.get_repo_uri(label, return_label)
562 @classmethod
563 def get_known_repos(cls) -> set[str]:
564 """Retrieve the list of known repository labels.
566 Returns
567 -------
568 repos : `set` of `str`
569 All the known labels. Can be empty if no index can be found.
571 Notes
572 -----
573 See `~lsst.daf.butler.ButlerRepoIndex` for details on how the
574 information is discovered.
575 """
576 return ButlerRepoIndex.get_known_repos()
578 @classmethod
579 def parse_dataset_uri(cls, uri: str) -> ParsedButlerDatasetURI:
580 """Extract the butler label and dataset ID from a dataset URI.
582 Parameters
583 ----------
584 uri : `str`
585 The dataset URI to parse.
587 Returns
588 -------
589 parsed : `ParsedButlerDatasetURI`
590 The label associated with the butler repository from which this
591 dataset originates and the ID of the dataset.
593 Notes
594 -----
595 Supports dataset URIs of the forms
596 ``ivo://org.rubinobs/usdac/dr1?repo=butler_label&id=UUID`` (see
597 DMTN-302) and ``butler://butler_label/UUID``. The ``butler`` URI is
598 deprecated and can not include ``/`` in the label string. ``ivo`` URIs
599 can include anything supported by the `Butler` constructor, including
600 paths to repositories and alias labels.
602 ivo://org.rubinobs/dr1?repo=/repo/main&id=UUID
604 will return a label of ``/repo/main``.
606 This method does not attempt to check that the dataset exists in the
607 labeled butler.
609 Since the IVOID can be issued by any publisher to represent a Butler
610 dataset there is no validation of the path or netloc component of the
611 URI. The only requirement is that there are ``id`` and ``repo`` keys
612 in the ``ivo`` URI query component.
613 """
614 parsed = urllib.parse.urlparse(uri)
615 parsed_scheme = parsed.scheme.lower()
616 if parsed_scheme == "ivo":
617 # Do not validate the netloc or the path values.
618 qs = urllib.parse.parse_qs(parsed.query)
619 if "repo" not in qs or "id" not in qs:
620 raise ValueError(f"Missing 'repo' and/or 'id' query parameters in IVOID {uri}.")
621 if len(qs["repo"]) != 1 or len(qs["id"]) != 1:
622 raise ValueError(f"Butler IVOID only supports a single value of repo and id, got {uri}")
623 label = qs["repo"][0]
624 id_ = qs["id"][0]
625 elif parsed_scheme == "butler":
626 label = parsed.netloc # Butler label is case sensitive.
627 # Need to strip the leading /.
628 id_ = parsed.path[1:]
629 else:
630 raise ValueError(f"Unrecognized URI scheme: {uri!r}")
631 # Strip trailing/leading whitespace from label.
632 label = label.strip()
633 if not label:
634 raise ValueError(f"No butler repository label found in uri {uri!r}")
635 try:
636 dataset_id = uuid.UUID(hex=id_)
637 except Exception as e:
638 e.add_note(f"Error extracting dataset ID from uri {uri!r} with dataset ID string {id_!r}")
639 raise
641 return ParsedButlerDatasetURI(label=label, dataset_id=dataset_id, uri=uri)
643 @classmethod
644 def get_dataset_from_uri(
645 cls, uri: str, factory: LabeledButlerFactoryProtocol | None = None
646 ) -> SpecificButlerDataset:
647 """Get the dataset associated with the given dataset URI.
649 Parameters
650 ----------
651 uri : `str`
652 The URI associated with a dataset.
653 factory : `LabeledButlerFactoryProtocol` or `None`, optional
654 Bound factory function that will be given the butler label
655 and receive a `Butler`. If this is not provided the label
656 will be tried directly.
658 Returns
659 -------
660 result : `SpecificButlerDataset`
661 The butler associated with this URI and the dataset itself.
662 The dataset can be `None` if the UUID is valid but the dataset
663 is not known to this butler.
664 """
665 parsed = cls.parse_dataset_uri(uri)
666 butler: Butler | None = None
667 if factory is not None:
668 # If the label is not recognized, it might be a path.
669 try:
670 butler = factory(parsed.label)
671 except KeyError:
672 pass
673 if butler is None:
674 butler = cls.from_config(parsed.label)
675 return SpecificButlerDataset(butler=butler, dataset=butler.get_dataset(parsed.dataset_id))
677 @abstractmethod
678 def _caching_context(self) -> AbstractContextManager[None]:
679 """Context manager that enables caching."""
680 raise NotImplementedError()
682 @abstractmethod
683 def transaction(self) -> AbstractContextManager[None]:
684 """Context manager supporting `Butler` transactions.
686 Transactions can be nested.
687 """
688 raise NotImplementedError()
690 @abstractmethod
691 def put(
692 self,
693 obj: Any,
694 datasetRefOrType: DatasetRef | DatasetType | str,
695 /,
696 dataId: DataId | None = None,
697 *,
698 run: str | None = None,
699 provenance: DatasetProvenance | None = None,
700 **kwargs: Any,
701 ) -> DatasetRef:
702 """Store and register a dataset.
704 Parameters
705 ----------
706 obj : `object`
707 The dataset.
708 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
709 When `DatasetRef` is provided, ``dataId`` should be `None`.
710 Otherwise the `DatasetType` or name thereof. If a fully resolved
711 `DatasetRef` is given the run and ID are used directly.
712 dataId : `dict` or `DataCoordinate`
713 A `dict` of `Dimension` link name, value pairs that label the
714 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
715 should be provided as the second argument.
716 run : `str`, optional
717 The name of the run the dataset should be added to, overriding
718 ``self.run``. Not used if a resolved `DatasetRef` is provided.
719 provenance : `DatasetProvenance` or `None`, optional
720 Any provenance that should be attached to the serialized dataset.
721 Not supported by all serialization mechanisms.
722 **kwargs
723 Additional keyword arguments used to augment or construct a
724 `DataCoordinate`. See `DataCoordinate.standardize`
725 parameters. Not used if a resolve `DatasetRef` is provided.
727 Returns
728 -------
729 ref : `DatasetRef`
730 A reference to the stored dataset, updated with the correct id if
731 given.
733 Raises
734 ------
735 TypeError
736 Raised if the butler is read-only or if no run has been provided.
737 """
738 raise NotImplementedError()
740 @abstractmethod
741 def getDeferred(
742 self,
743 datasetRefOrType: DatasetRef | DatasetType | str,
744 /,
745 dataId: DataId | None = None,
746 *,
747 parameters: dict | None = None,
748 collections: Any = None,
749 storageClass: str | StorageClass | None = None,
750 timespan: Timespan | None = None,
751 **kwargs: Any,
752 ) -> DeferredDatasetHandle:
753 """Create a `DeferredDatasetHandle` which can later retrieve a dataset,
754 after an immediate registry lookup.
756 Parameters
757 ----------
758 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
759 When `DatasetRef` the `dataId` should be `None`.
760 Otherwise the `DatasetType` or name thereof.
761 dataId : `dict` or `DataCoordinate`, optional
762 A `dict` of `Dimension` link name, value pairs that label the
763 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
764 should be provided as the first argument.
765 parameters : `dict`
766 Additional StorageClass-defined options to control reading,
767 typically used to efficiently read only a subset of the dataset.
768 collections : Any, optional
769 Collections to be searched, overriding ``self.collections``.
770 Can be any of the types supported by the ``collections`` argument
771 to butler construction.
772 storageClass : `StorageClass` or `str`, optional
773 The storage class to be used to override the Python type
774 returned by this method. By default the returned type matches
775 the dataset type definition for this dataset. Specifying a
776 read `StorageClass` can force a different type to be returned.
777 This type must be compatible with the original type.
778 timespan : `Timespan` or `None`, optional
779 A timespan that the validity range of the dataset must overlap.
780 If not provided and this is a calibration dataset type, an attempt
781 will be made to find the timespan from any temporal coordinate
782 in the data ID.
783 **kwargs
784 Additional keyword arguments used to augment or construct a
785 `DataId`. See `DataId` parameters.
787 Returns
788 -------
789 obj : `DeferredDatasetHandle`
790 A handle which can be used to retrieve a dataset at a later time.
792 Raises
793 ------
794 LookupError
795 Raised if no matching dataset exists in the `Registry` or
796 datastore.
797 ValueError
798 Raised if a resolved `DatasetRef` was passed as an input, but it
799 differs from the one found in the registry.
800 TypeError
801 Raised if no collections were provided.
802 """
803 raise NotImplementedError()
805 @abstractmethod
806 def get(
807 self,
808 datasetRefOrType: DatasetRef | DatasetType | str,
809 /,
810 dataId: DataId | None = None,
811 *,
812 parameters: dict[str, Any] | None = None,
813 collections: Any = None,
814 storageClass: StorageClass | str | None = None,
815 timespan: Timespan | None = None,
816 **kwargs: Any,
817 ) -> Any:
818 """Retrieve a stored dataset.
820 Parameters
821 ----------
822 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
823 When `DatasetRef` the `dataId` should be `None`.
824 Otherwise the `DatasetType` or name thereof.
825 If a resolved `DatasetRef`, the associated dataset
826 is returned directly without additional querying.
827 dataId : `dict` or `DataCoordinate`
828 A `dict` of `Dimension` link name, value pairs that label the
829 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
830 should be provided as the first argument.
831 parameters : `dict`
832 Additional StorageClass-defined options to control reading,
833 typically used to efficiently read only a subset of the dataset.
834 collections : Any, optional
835 Collections to be searched, overriding ``self.collections``.
836 Can be any of the types supported by the ``collections`` argument
837 to butler construction.
838 storageClass : `StorageClass` or `str`, optional
839 The storage class to be used to override the Python type
840 returned by this method. By default the returned type matches
841 the dataset type definition for this dataset. Specifying a
842 read `StorageClass` can force a different type to be returned.
843 This type must be compatible with the original type.
844 timespan : `Timespan` or `None`, optional
845 A timespan that the validity range of the dataset must overlap.
846 If not provided and this is a calibration dataset type, an attempt
847 will be made to find the timespan from any temporal coordinate
848 in the data ID.
849 **kwargs
850 Additional keyword arguments used to augment or construct a
851 `DataCoordinate`. See `DataCoordinate.standardize`
852 parameters.
854 Returns
855 -------
856 obj : `object`
857 The dataset.
859 Raises
860 ------
861 LookupError
862 Raised if no matching dataset exists in the `Registry`.
863 TypeError
864 Raised if no collections were provided.
866 Notes
867 -----
868 When looking up datasets in a `~CollectionType.CALIBRATION` collection,
869 this method requires that the given data ID include temporal dimensions
870 beyond the dimensions of the dataset type itself, in order to find the
871 dataset with the appropriate validity range. For example, a "bias"
872 dataset with native dimensions ``{instrument, detector}`` could be
873 fetched with a ``{instrument, detector, exposure}`` data ID, because
874 ``exposure`` is a temporal dimension.
875 """
876 raise NotImplementedError()
878 @abstractmethod
879 def getURIs(
880 self,
881 datasetRefOrType: DatasetRef | DatasetType | str,
882 /,
883 dataId: DataId | None = None,
884 *,
885 predict: bool = False,
886 collections: Any = None,
887 run: str | None = None,
888 **kwargs: Any,
889 ) -> DatasetRefURIs:
890 """Return the URIs associated with the dataset.
892 Parameters
893 ----------
894 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
895 When `DatasetRef` the `dataId` should be `None`.
896 Otherwise the `DatasetType` or name thereof.
897 dataId : `dict` or `DataCoordinate`
898 A `dict` of `Dimension` link name, value pairs that label the
899 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
900 should be provided as the first argument.
901 predict : `bool`
902 If `True`, allow URIs to be returned of datasets that have not
903 been written.
904 collections : Any, optional
905 Collections to be searched, overriding ``self.collections``.
906 Can be any of the types supported by the ``collections`` argument
907 to butler construction.
908 run : `str`, optional
909 Run to use for predictions, overriding ``self.run``.
910 **kwargs
911 Additional keyword arguments used to augment or construct a
912 `DataCoordinate`. See `DataCoordinate.standardize`
913 parameters.
915 Returns
916 -------
917 uris : `DatasetRefURIs`
918 The URI to the primary artifact associated with this dataset (if
919 the dataset was disassembled within the datastore this may be
920 `None`), and the URIs to any components associated with the dataset
921 artifact. (can be empty if there are no components).
922 """
923 raise NotImplementedError()
925 def getURI(
926 self,
927 datasetRefOrType: DatasetRef | DatasetType | str,
928 /,
929 dataId: DataId | None = None,
930 *,
931 predict: bool = False,
932 collections: Any = None,
933 run: str | None = None,
934 **kwargs: Any,
935 ) -> ResourcePath:
936 """Return the URI to the Dataset.
938 Parameters
939 ----------
940 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
941 When `DatasetRef` the `dataId` should be `None`.
942 Otherwise the `DatasetType` or name thereof.
943 dataId : `dict` or `DataCoordinate`
944 A `dict` of `Dimension` link name, value pairs that label the
945 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
946 should be provided as the first argument.
947 predict : `bool`
948 If `True`, allow URIs to be returned of datasets that have not
949 been written.
950 collections : Any, optional
951 Collections to be searched, overriding ``self.collections``.
952 Can be any of the types supported by the ``collections`` argument
953 to butler construction.
954 run : `str`, optional
955 Run to use for predictions, overriding ``self.run``.
956 **kwargs
957 Additional keyword arguments used to augment or construct a
958 `DataCoordinate`. See `DataCoordinate.standardize`
959 parameters.
961 Returns
962 -------
963 uri : `lsst.resources.ResourcePath`
964 URI pointing to the Dataset within the datastore. If the
965 Dataset does not exist in the datastore, and if ``predict`` is
966 `True`, the URI will be a prediction and will include a URI
967 fragment "#predicted".
968 If the datastore does not have entities that relate well
969 to the concept of a URI the returned URI string will be
970 descriptive. The returned URI is not guaranteed to be obtainable.
972 Raises
973 ------
974 LookupError
975 A URI has been requested for a dataset that does not exist and
976 guessing is not allowed.
977 ValueError
978 Raised if a resolved `DatasetRef` was passed as an input, but it
979 differs from the one found in the registry.
980 TypeError
981 Raised if no collections were provided.
982 RuntimeError
983 Raised if a URI is requested for a dataset that consists of
984 multiple artifacts.
985 """
986 primary, components = self.getURIs(
987 datasetRefOrType, dataId=dataId, predict=predict, collections=collections, run=run, **kwargs
988 )
990 if primary is None or components:
991 raise RuntimeError(
992 f"Dataset ({datasetRefOrType}) includes distinct URIs for components. "
993 "Use Butler.getURIs() instead."
994 )
995 return primary
997 @abstractmethod
998 def get_dataset_type(self, name: str) -> DatasetType:
999 """Get the `DatasetType`.
1001 Parameters
1002 ----------
1003 name : `str`
1004 Name of the type.
1006 Returns
1007 -------
1008 type : `DatasetType`
1009 The `DatasetType` associated with the given name.
1011 Raises
1012 ------
1013 lsst.daf.butler.MissingDatasetTypeError
1014 Raised if the requested dataset type has not been registered.
1016 Notes
1017 -----
1018 This method handles component dataset types automatically, though most
1019 other operations do not.
1020 """
1021 raise NotImplementedError()
1023 @abstractmethod
1024 def get_dataset(
1025 self,
1026 id: DatasetId | str,
1027 *,
1028 storage_class: str | StorageClass | None = None,
1029 dimension_records: bool = False,
1030 datastore_records: bool = False,
1031 ) -> DatasetRef | None:
1032 """Retrieve a Dataset entry.
1034 Parameters
1035 ----------
1036 id : `DatasetId`
1037 The unique identifier for the dataset, as an instance of
1038 `uuid.UUID` or a string containing a hexadecimal number.
1039 storage_class : `str` or `StorageClass` or `None`
1040 A storage class to use when creating the returned entry. If given
1041 it must be compatible with the default storage class.
1042 dimension_records : `bool`, optional
1043 If `True` the ref will be expanded and contain dimension records.
1044 datastore_records : `bool`, optional
1045 If `True` the ref will contain associated datastore records.
1047 Returns
1048 -------
1049 ref : `DatasetRef` or `None`
1050 A ref to the Dataset, or `None` if no matching Dataset
1051 was found.
1052 """
1053 raise NotImplementedError()
1055 @abstractmethod
1056 def get_many_datasets(self, ids: Iterable[DatasetId | str]) -> list[DatasetRef]:
1057 """Retrieve a list of dataset entries.
1059 Parameters
1060 ----------
1061 ids : `~collections.abc.Iterable` [ `DatasetId` or `str` ]
1062 The unique identifiers for the datasets, as instances of
1063 `uuid.UUID` or strings containing a hexadecimal number.
1065 Returns
1066 -------
1067 refs : `list` [ `DatasetRef` ]
1068 A list containing a `DatasetRef` for each of the given dataset IDs.
1069 If a dataset was not found, no error is thrown -- it is just not
1070 included in the list. The returned datasets are in no particular
1071 order.
1072 """
1073 raise NotImplementedError()
1075 @abstractmethod
1076 def find_dataset(
1077 self,
1078 dataset_type: DatasetType | str,
1079 data_id: DataId | None = None,
1080 *,
1081 collections: str | Sequence[str] | None = None,
1082 timespan: Timespan | None = None,
1083 storage_class: str | StorageClass | None = None,
1084 dimension_records: bool = False,
1085 datastore_records: bool = False,
1086 **kwargs: Any,
1087 ) -> DatasetRef | None:
1088 """Find a dataset given its `DatasetType` and data ID.
1090 This can be used to obtain a `DatasetRef` that permits the dataset to
1091 be read from a `Datastore`. If the dataset is a component and can not
1092 be found using the provided dataset type, a dataset ref for the parent
1093 will be returned instead but with the correct dataset type.
1095 Parameters
1096 ----------
1097 dataset_type : `DatasetType` or `str`
1098 A `DatasetType` or the name of one. If this is a `DatasetType`
1099 instance, its storage class will be respected and propagated to
1100 the output, even if it differs from the dataset type definition
1101 in the registry, as long as the storage classes are convertible.
1102 data_id : `dict` or `DataCoordinate`, optional
1103 A `dict`-like object containing the `Dimension` links that identify
1104 the dataset within a collection. If it is a `dict` the dataId
1105 can include dimension record values such as ``day_obs`` and
1106 ``seq_num`` or ``full_name`` that can be used to derive the
1107 primary dimension.
1108 collections : `str` or `list` [`str`], optional
1109 A an ordered list of collections to search for the dataset.
1110 Defaults to ``self.defaults.collections``.
1111 timespan : `Timespan`, optional
1112 A timespan that the validity range of the dataset must overlap.
1113 If not provided, any `~CollectionType.CALIBRATION` collections
1114 matched by the ``collections`` argument will not be searched.
1115 storage_class : `str` or `StorageClass` or `None`
1116 A storage class to use when creating the returned entry. If given
1117 it must be compatible with the default storage class.
1118 dimension_records : `bool`, optional
1119 If `True` the ref will be expanded and contain dimension records.
1120 datastore_records : `bool`, optional
1121 If `True` the ref will contain associated datastore records.
1122 **kwargs
1123 Additional keyword arguments passed to
1124 `DataCoordinate.standardize` to convert ``dataId`` to a true
1125 `DataCoordinate` or augment an existing one. This can also include
1126 dimension record metadata that can be used to derive a primary
1127 dimension value.
1129 Returns
1130 -------
1131 ref : `DatasetRef`
1132 A reference to the dataset, or `None` if no matching Dataset
1133 was found.
1135 Raises
1136 ------
1137 lsst.daf.butler.NoDefaultCollectionError
1138 Raised if ``collections`` is `None` and
1139 ``self.collections`` is `None`.
1140 LookupError
1141 Raised if one or more data ID keys are missing.
1142 lsst.daf.butler.MissingDatasetTypeError
1143 Raised if the dataset type does not exist.
1144 lsst.daf.butler.MissingCollectionError
1145 Raised if any of ``collections`` does not exist in the registry.
1147 Notes
1148 -----
1149 This method simply returns `None` and does not raise an exception even
1150 when the set of collections searched is intrinsically incompatible with
1151 the dataset type, e.g. if ``datasetType.isCalibration() is False``, but
1152 only `~CollectionType.CALIBRATION` collections are being searched.
1153 This may make it harder to debug some lookup failures, but the behavior
1154 is intentional; we consider it more important that failed searches are
1155 reported consistently, regardless of the reason, and that adding
1156 additional collections that do not contain a match to the search path
1157 never changes the behavior.
1159 This method handles component dataset types automatically, though most
1160 other query operations do not.
1161 """
1162 raise NotImplementedError()
1164 @abstractmethod
1165 def retrieve_artifacts_zip(
1166 self,
1167 refs: Iterable[DatasetRef],
1168 destination: ResourcePathExpression,
1169 overwrite: bool = True,
1170 ) -> ResourcePath:
1171 """Retrieve artifacts from a Butler and place in ZIP file.
1173 Parameters
1174 ----------
1175 refs : `~collections.abc.Iterable` [ `DatasetRef` ]
1176 The datasets to be included in the zip file.
1177 destination : `lsst.resources.ResourcePathExpression`
1178 Directory to write the new ZIP file. This directory will
1179 also be used as a staging area for the datasets being downloaded
1180 from the datastore.
1181 overwrite : `bool`, optional
1182 If `False` the output Zip will not be written if a file of the
1183 same name is already present in ``destination``.
1185 Returns
1186 -------
1187 zip_file : `lsst.resources.ResourcePath`
1188 The path to the new ZIP file.
1190 Raises
1191 ------
1192 ValueError
1193 Raised if there are no refs to retrieve.
1194 """
1195 raise NotImplementedError()
1197 @abstractmethod
1198 def retrieveArtifacts(
1199 self,
1200 refs: Iterable[DatasetRef],
1201 destination: ResourcePathExpression,
1202 transfer: str = "auto",
1203 preserve_path: bool = True,
1204 overwrite: bool = False,
1205 ) -> list[ResourcePath]:
1206 """Retrieve the artifacts associated with the supplied refs.
1208 Parameters
1209 ----------
1210 refs : iterable of `DatasetRef`
1211 The datasets for which artifacts are to be retrieved.
1212 A single ref can result in multiple artifacts. The refs must
1213 be resolved.
1214 destination : `lsst.resources.ResourcePath` or `str`
1215 Location to write the artifacts.
1216 transfer : `str`, optional
1217 Method to use to transfer the artifacts. Must be one of the options
1218 supported by `~lsst.resources.ResourcePath.transfer_from()`.
1219 "move" is not allowed.
1220 preserve_path : `bool`, optional
1221 If `True` the full path of the artifact within the datastore
1222 is preserved. If `False` the final file component of the path
1223 is used.
1224 overwrite : `bool`, optional
1225 If `True` allow transfers to overwrite existing files at the
1226 destination.
1228 Returns
1229 -------
1230 targets : `list` of `lsst.resources.ResourcePath`
1231 URIs of file artifacts in destination location. Order is not
1232 preserved.
1234 Notes
1235 -----
1236 For non-file datastores the artifacts written to the destination
1237 may not match the representation inside the datastore. For example
1238 a hierarchical data structure in a NoSQL database may well be stored
1239 as a JSON file.
1240 """
1241 raise NotImplementedError()
1243 @abstractmethod
1244 def exists(
1245 self,
1246 dataset_ref_or_type: DatasetRef | DatasetType | str,
1247 /,
1248 data_id: DataId | None = None,
1249 *,
1250 full_check: bool = True,
1251 collections: Any = None,
1252 **kwargs: Any,
1253 ) -> DatasetExistence:
1254 """Indicate whether a dataset is known to Butler registry and
1255 datastore.
1257 Parameters
1258 ----------
1259 dataset_ref_or_type : `DatasetRef`, `DatasetType`, or `str`
1260 When `DatasetRef` the `dataId` should be `None`.
1261 Otherwise the `DatasetType` or name thereof.
1262 data_id : `dict` or `DataCoordinate`
1263 A `dict` of `Dimension` link name, value pairs that label the
1264 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
1265 should be provided as the first argument.
1266 full_check : `bool`, optional
1267 If `True`, a check will be made for the actual existence of a
1268 dataset artifact. This will involve additional overhead due to
1269 the need to query an external system. If `False`, this check will
1270 be omitted, and the registry and datastore will solely be asked
1271 if they know about the dataset but no direct check for the
1272 artifact will be performed.
1273 collections : Any, optional
1274 Collections to be searched, overriding ``self.collections``.
1275 Can be any of the types supported by the ``collections`` argument
1276 to butler construction.
1277 **kwargs
1278 Additional keyword arguments used to augment or construct a
1279 `DataCoordinate`. See `DataCoordinate.standardize`
1280 parameters.
1282 Returns
1283 -------
1284 existence : `DatasetExistence`
1285 Object indicating whether the dataset is known to registry and
1286 datastore. Evaluates to `True` if the dataset is present and known
1287 to both.
1288 """
1289 raise NotImplementedError()
1291 @abstractmethod
1292 def _exists_many(
1293 self,
1294 refs: Iterable[DatasetRef],
1295 /,
1296 *,
1297 full_check: bool = True,
1298 ) -> dict[DatasetRef, DatasetExistence]:
1299 """Indicate whether multiple datasets are known to Butler registry and
1300 datastore.
1302 This is an experimental API that may change at any moment.
1304 Parameters
1305 ----------
1306 refs : iterable of `DatasetRef`
1307 The datasets to be checked.
1308 full_check : `bool`, optional
1309 If `True`, a check will be made for the actual existence of each
1310 dataset artifact. This will involve additional overhead due to
1311 the need to query an external system. If `False`, this check will
1312 be omitted, and the registry and datastore will solely be asked
1313 if they know about the dataset(s) but no direct check for the
1314 artifact(s) will be performed.
1316 Returns
1317 -------
1318 existence : dict of [`DatasetRef`, `DatasetExistence`]
1319 Mapping from the given dataset refs to an enum indicating the
1320 status of the dataset in registry and datastore.
1321 Each value evaluates to `True` if the dataset is present and known
1322 to both.
1323 """
1324 raise NotImplementedError()
1326 @abstractmethod
1327 def removeRuns(
1328 self,
1329 names: Iterable[str],
1330 unstore: bool | type[_DeprecatedDefault] = _DeprecatedDefault,
1331 *,
1332 unlink_from_chains: bool = False,
1333 ) -> None:
1334 """Remove one or more `~CollectionType.RUN` collections and the
1335 datasets within them.
1337 Parameters
1338 ----------
1339 names : `~collections.abc.Iterable` [ `str` ]
1340 The names of the collections to remove.
1341 unstore : `bool`, optional
1342 If `True` (default), delete datasets from all datastores in which
1343 they are present, and attempt to rollback the registry deletions if
1344 datastore deletions fail (which may not always be possible). If
1345 `False`, datastore records for these datasets are still removed,
1346 but any artifacts (e.g. files) will not be. This parameter is now
1347 deprecated and no longer has any effect. Files are always deleted
1348 from datastores unless they were ingested using full URIs.
1349 unlink_from_chains : `bool`, optional
1350 If `True` remove the RUN collection from any chains prior to
1351 removing the RUN. If `False` the removal will fail if any chains
1352 still refer to the RUN.
1354 Raises
1355 ------
1356 TypeError
1357 Raised if one or more collections are not of type
1358 `~CollectionType.RUN`.
1359 """
1360 raise NotImplementedError()
1362 @abstractmethod
1363 def ingest(
1364 self,
1365 *datasets: FileDataset,
1366 transfer: str | None = "auto",
1367 record_validation_info: bool = True,
1368 skip_existing: bool = False,
1369 ) -> None:
1370 """Store and register one or more datasets that already exist on disk.
1372 Parameters
1373 ----------
1374 *datasets : `FileDataset`
1375 Each positional argument is a struct containing information about
1376 a file to be ingested, including its URI (either absolute or
1377 relative to the datastore root, if applicable), a resolved
1378 `DatasetRef`, and optionally a formatter class or its
1379 fully-qualified string name. If a formatter is not provided, the
1380 formatter that would be used for `put` is assumed. On successful
1381 ingest all `FileDataset.formatter` attributes will be set to the
1382 formatter class used. `FileDataset.path` attributes may be modified
1383 to put paths in whatever the datastore considers a standardized
1384 form.
1385 transfer : `str`, optional
1386 If not `None`, must be one of 'auto', 'move', 'copy', 'direct',
1387 'split', 'hardlink', 'relsymlink' or 'symlink', indicating how to
1388 transfer the file.
1389 record_validation_info : `bool`, optional
1390 If `True`, the default, the datastore can record validation
1391 information associated with the file. If `False` the datastore
1392 will not attempt to track any information such as checksums
1393 or file sizes. This can be useful if such information is tracked
1394 in an external system or if the file is to be compressed in place.
1395 It is up to the datastore whether this parameter is relevant.
1396 skip_existing : `bool`, optional
1397 If `True`, a dataset will not be ingested if a dataset with the
1398 same dataset ID already exists in the datastore.
1399 If `False` (the default), a `ConflictingDefinitionError` will be
1400 raised if any datasets with the same dataset ID already exist
1401 in the datastore.
1403 Raises
1404 ------
1405 TypeError
1406 Raised if the butler is read-only or if no run was provided.
1407 NotImplementedError
1408 Raised if the `Datastore` does not support the given transfer mode.
1409 DatasetTypeNotSupportedError
1410 Raised if one or more files to be ingested have a dataset type that
1411 is not supported by the `Datastore`..
1412 FileNotFoundError
1413 Raised if one of the given files does not exist.
1414 FileExistsError
1415 Raised if transfer is not `None` but the (internal) location the
1416 file would be moved to is already occupied.
1417 ConflictingDefinitionError
1418 Raised if a dataset already exists in the repository and
1419 ``skip_existing`` is `False`.
1421 Notes
1422 -----
1423 This operation is not fully exception safe: if a database operation
1424 fails, the given `FileDataset` instances may be only partially updated.
1426 It is atomic in terms of database operations (they will either all
1427 succeed or all fail) providing the database engine implements
1428 transactions correctly. It will attempt to be atomic in terms of
1429 filesystem operations as well, but this cannot be implemented
1430 rigorously for most datastores.
1431 """
1432 raise NotImplementedError()
1434 @abstractmethod
1435 def ingest_zip(
1436 self,
1437 zip_file: ResourcePathExpression,
1438 transfer: str = "auto",
1439 *,
1440 transfer_dimensions: bool = False,
1441 dry_run: bool = False,
1442 ) -> None:
1443 """Ingest a Zip file into this butler.
1445 The Zip file must have been created by `retrieve_artifacts_zip`.
1447 Parameters
1448 ----------
1449 zip_file : `lsst.resources.ResourcePathExpression`
1450 Path to the Zip file.
1451 transfer : `str`, optional
1452 Method to use to transfer the Zip into the datastore.
1453 transfer_dimensions : `bool`, optional
1454 If `True`, dimension record data associated with the new datasets
1455 will be transferred from the Zip file, if present.
1456 dry_run : `bool`, optional
1457 If `True` the ingest will be processed without any modifications
1458 made to the target butler and as if the target butler did not
1459 have any of the datasets.
1461 Notes
1462 -----
1463 Run collections and dataset types are created as needed.
1464 """
1465 raise NotImplementedError()
1467 @abstractmethod
1468 def export(
1469 self,
1470 *,
1471 directory: str | None = None,
1472 filename: str | None = None,
1473 format: str | None = None,
1474 transfer: str | None = None,
1475 ) -> AbstractContextManager[RepoExportContext]:
1476 """Export datasets from the repository represented by this `Butler`.
1478 This method is a context manager that returns a helper object
1479 (`RepoExportContext`) that is used to indicate what information from
1480 the repository should be exported.
1482 Parameters
1483 ----------
1484 directory : `str`, optional
1485 Directory dataset files should be written to if ``transfer`` is not
1486 `None`.
1487 filename : `str`, optional
1488 Name for the file that will include database information associated
1489 with the exported datasets. If this is not an absolute path and
1490 ``directory`` is not `None`, it will be written to ``directory``
1491 instead of the current working directory. Defaults to
1492 "export.{format}".
1493 format : `str`, optional
1494 File format for the database information file. If `None`, the
1495 extension of ``filename`` will be used.
1496 transfer : `str`, optional
1497 Transfer mode passed to `Datastore.export`.
1499 Raises
1500 ------
1501 TypeError
1502 Raised if the set of arguments passed is inconsistent.
1504 Examples
1505 --------
1506 Typically the `Registry.queryDataIds` and `Registry.queryDatasets`
1507 methods are used to provide the iterables over data IDs and/or datasets
1508 to be exported::
1510 with butler.export("exports.yaml") as export:
1511 # Export all flats, but none of the dimension element rows
1512 # (i.e. data ID information) associated with them.
1513 export.saveDatasets(
1514 butler.registry.queryDatasets("flat"), elements=()
1515 )
1516 # Export all datasets that start with "deepCoadd_" and all of
1517 # their associated data ID information.
1518 export.saveDatasets(butler.registry.queryDatasets("deepCoadd_*"))
1519 """
1520 raise NotImplementedError()
1522 @abstractmethod
1523 def import_(
1524 self,
1525 *,
1526 directory: ResourcePathExpression | None = None,
1527 filename: ResourcePathExpression | TextIO | None = None,
1528 format: str | None = None,
1529 transfer: str | None = None,
1530 skip_dimensions: set | None = None,
1531 record_validation_info: bool = True,
1532 without_datastore: bool = False,
1533 ) -> None:
1534 """Import datasets into this repository that were exported from a
1535 different butler repository via `~lsst.daf.butler.Butler.export`.
1537 Parameters
1538 ----------
1539 directory : `~lsst.resources.ResourcePathExpression`, optional
1540 Directory containing dataset files to import from. If `None`,
1541 ``filename`` and all dataset file paths specified therein must
1542 be absolute.
1543 filename : `~lsst.resources.ResourcePathExpression` or `TextIO`
1544 A stream or name of file that contains database information
1545 associated with the exported datasets, typically generated by
1546 `~lsst.daf.butler.Butler.export`. If this a string (name) or
1547 `~lsst.resources.ResourcePath` and is not an absolute path,
1548 it will first be looked for relative to ``directory`` and if not
1549 found there it will be looked for in the current working
1550 directory. Defaults to "export.{format}".
1551 format : `str`, optional
1552 File format for ``filename``. If `None`, the extension of
1553 ``filename`` will be used.
1554 transfer : `str`, optional
1555 Transfer mode passed to `~lsst.daf.butler.Datastore.ingest`.
1556 skip_dimensions : `set`, optional
1557 Names of dimensions that should be skipped and not imported.
1558 record_validation_info : `bool`, optional
1559 If `True`, the default, the datastore can record validation
1560 information associated with the file. If `False` the datastore
1561 will not attempt to track any information such as checksums
1562 or file sizes. This can be useful if such information is tracked
1563 in an external system or if the file is to be compressed in place.
1564 It is up to the datastore whether this parameter is relevant.
1565 without_datastore : `bool`, optional
1566 If `True` only registry records will be imported and the datastore
1567 will be ignored.
1569 Raises
1570 ------
1571 TypeError
1572 Raised if the set of arguments passed is inconsistent, or if the
1573 butler is read-only.
1574 """
1575 raise NotImplementedError()
1577 @abstractmethod
1578 def transfer_dimension_records_from(
1579 self, source_butler: LimitedButler | Butler, source_refs: Iterable[DatasetRef]
1580 ) -> None:
1581 """Transfer dimension records to this Butler from another Butler.
1583 Parameters
1584 ----------
1585 source_butler : `LimitedButler` or `Butler`
1586 Butler from which the records are to be transferred. If data IDs
1587 in ``source_refs`` are not expanded then this has to be a full
1588 `Butler` whose registry will be used to expand data IDs. If the
1589 source refs contain coordinates that are used to populate other
1590 records then this will also need to be a full `Butler`.
1591 source_refs : iterable of `DatasetRef`
1592 Datasets defined in the source butler whose dimension records
1593 should be transferred to this butler. In most circumstances.
1594 transfer is faster if the dataset refs are expanded.
1595 """
1596 raise NotImplementedError()
1598 @abstractmethod
1599 def transfer_from(
1600 self,
1601 source_butler: LimitedButler,
1602 source_refs: Iterable[DatasetRef],
1603 transfer: str = "auto",
1604 skip_missing: bool = True,
1605 register_dataset_types: bool = False,
1606 transfer_dimensions: bool = False,
1607 dry_run: bool = False,
1608 ) -> Collection[DatasetRef]:
1609 """Transfer datasets to this Butler from a run in another Butler.
1611 Parameters
1612 ----------
1613 source_butler : `LimitedButler`
1614 Butler from which the datasets are to be transferred. If data IDs
1615 in ``source_refs`` are not expanded then this has to be a full
1616 `Butler` whose registry will be used to expand data IDs.
1617 source_refs : iterable of `DatasetRef`
1618 Datasets defined in the source butler that should be transferred to
1619 this butler. In most circumstances, ``transfer_from`` is faster if
1620 the dataset refs are expanded.
1621 transfer : `str`, optional
1622 Transfer mode passed to `~lsst.daf.butler.Datastore.transfer_from`.
1623 skip_missing : `bool`
1624 If `True`, datasets with no datastore artifact associated with
1625 them are not transferred. If `False` a registry entry will be
1626 created even if no datastore record is created (and so will
1627 look equivalent to the dataset being unstored).
1628 register_dataset_types : `bool`
1629 If `True` any missing dataset types are registered. Otherwise
1630 an exception is raised.
1631 transfer_dimensions : `bool`, optional
1632 If `True`, dimension record data associated with the new datasets
1633 will be transferred.
1634 dry_run : `bool`, optional
1635 If `True` the transfer will be processed without any modifications
1636 made to the target butler and as if the target butler did not
1637 have any of the datasets.
1639 Returns
1640 -------
1641 refs : `list` of `DatasetRef`
1642 The refs added to this Butler.
1644 Notes
1645 -----
1646 The datastore artifact has to exist for a transfer
1647 to be made but non-existence is not an error.
1649 Datasets that already exist in this run will be skipped.
1651 The datasets are imported as part of a transaction, although
1652 dataset types are registered before the transaction is started.
1653 This means that it is possible for a dataset type to be registered
1654 even though transfer has failed.
1655 """
1656 raise NotImplementedError()
1658 @abstractmethod
1659 def validateConfiguration(
1660 self,
1661 logFailures: bool = False,
1662 datasetTypeNames: Iterable[str] | None = None,
1663 ignore: Iterable[str] | None = None,
1664 ) -> None:
1665 """Validate butler configuration.
1667 Checks that each `DatasetType` can be stored in the `Datastore`.
1669 Parameters
1670 ----------
1671 logFailures : `bool`, optional
1672 If `True`, output a log message for every validation error
1673 detected.
1674 datasetTypeNames : iterable of `str`, optional
1675 The `DatasetType` names that should be checked. This allows
1676 only a subset to be selected.
1677 ignore : iterable of `str`, optional
1678 Names of DatasetTypes to skip over. This can be used to skip
1679 known problems. If a named `DatasetType` corresponds to a
1680 composite, all components of that `DatasetType` will also be
1681 ignored.
1683 Raises
1684 ------
1685 ButlerValidationError
1686 Raised if there is some inconsistency with how this Butler
1687 is configured.
1688 """
1689 raise NotImplementedError()
1691 @property
1692 @abstractmethod
1693 def collection_chains(self) -> ButlerCollections:
1694 """Object with methods for modifying collection chains
1695 (`~lsst.daf.butler.ButlerCollections`).
1697 Deprecated. Replaced with ``collections`` property.
1698 """
1699 raise NotImplementedError()
1701 @property
1702 @abstractmethod
1703 def collections(self) -> ButlerCollections:
1704 """Object with methods for modifying and querying collections
1705 (`~lsst.daf.butler.ButlerCollections`).
1707 Use of this object is preferred over `registry` wherever possible.
1708 """
1709 raise NotImplementedError()
1711 @property
1712 @abstractmethod
1713 def run(self) -> str | None:
1714 """Name of the run this butler writes outputs to by default (`str` or
1715 `None`).
1716 """
1717 raise NotImplementedError()
1719 @property
1720 @abstractmethod
1721 def registry(self) -> Registry:
1722 """The object that manages dataset metadata and relationships
1723 (`Registry`).
1725 Many operations that don't involve reading or writing butler datasets
1726 are accessible only via `Registry` methods. Eventually these methods
1727 will be replaced by equivalent `Butler` methods.
1728 """
1729 raise NotImplementedError()
1731 @abstractmethod
1732 def query(self) -> AbstractContextManager[Query]:
1733 """Context manager returning a `Query` object used for construction
1734 and execution of complex queries.
1735 """
1736 raise NotImplementedError()
1738 def query_data_ids(
1739 self,
1740 dimensions: DimensionGroup | Iterable[str] | str,
1741 *,
1742 data_id: DataId | None = None,
1743 where: str = "",
1744 bind: Mapping[str, Any] | None = None,
1745 with_dimension_records: bool = False,
1746 order_by: Iterable[str] | str | None = None,
1747 limit: int | None = -20_000,
1748 explain: bool = True,
1749 **kwargs: Any,
1750 ) -> list[DataCoordinate]:
1751 """Query for data IDs matching user-provided criteria.
1753 Parameters
1754 ----------
1755 dimensions : `DimensionGroup`, `str`, or \
1756 `~collections.abc.Iterable` [`str`]
1757 The dimensions of the data IDs to yield, as either `DimensionGroup`
1758 instances or `str`. Will be automatically expanded to a complete
1759 `DimensionGroup`.
1760 data_id : `dict` or `DataCoordinate`, optional
1761 A data ID whose key-value pairs are used as equality constraints
1762 in the query.
1763 where : `str`, optional
1764 A string expression similar to a SQL WHERE clause. May involve
1765 any column of a dimension table or (as a shortcut for the primary
1766 key column of a dimension table) dimension name. See
1767 :ref:`daf_butler_dimension_expressions` for more information.
1768 bind : `~collections.abc.Mapping`, optional
1769 Mapping containing literal values that should be injected into the
1770 ``where`` expression, keyed by the identifiers they replace.
1771 Values of collection type can be expanded in some cases; see
1772 :ref:`daf_butler_dimension_expressions_identifiers` for more
1773 information.
1774 with_dimension_records : `bool`, optional
1775 If `True` (default is `False`) then returned data IDs will have
1776 dimension records.
1777 order_by : `~collections.abc.Iterable` [`str`] or `str`, optional
1778 Names of the columns/dimensions to use for ordering returned data
1779 IDs. Column name can be prefixed with minus (``-``) to use
1780 descending ordering.
1781 limit : `int` or `None`, optional
1782 Upper limit on the number of returned records. `None` can be used
1783 if no limit is wanted. A limit of ``0`` means that the query will
1784 be executed and validated but no results will be returned. In this
1785 case there will be no exception even if ``explain`` is `True`.
1786 If a negative value is given a warning will be issued if the number
1787 of results is capped by that limit.
1788 explain : `bool`, optional
1789 If `True` (default) then `EmptyQueryResultError` exception is
1790 raised when resulting list is empty. The exception contains
1791 non-empty list of strings explaining possible causes for empty
1792 result.
1793 **kwargs
1794 Additional keyword arguments are forwarded to
1795 `DataCoordinate.standardize` when processing the ``data_id``
1796 argument (and may be used to provide a constraining data ID even
1797 when the ``data_id`` argument is `None`).
1799 Returns
1800 -------
1801 dataIds : `list` [`DataCoordinate`]
1802 Data IDs matching the given query parameters. These are always
1803 guaranteed to identify all dimensions (`DataCoordinate.hasFull`
1804 returns `True`).
1806 Raises
1807 ------
1808 lsst.daf.butler.registry.DataIdError
1809 Raised when ``data_id`` or keyword arguments specify unknown
1810 dimensions or values, or when they contain inconsistent values.
1811 lsst.daf.butler.registry.UserExpressionError
1812 Raised when ``where`` expression is invalid.
1813 lsst.daf.butler.EmptyQueryResultError
1814 Raised when query generates empty result and ``explain`` is set to
1815 `True`.
1816 TypeError
1817 Raised when the arguments are incompatible.
1818 """
1819 if data_id is None:
1820 data_id = DataCoordinate.make_empty(self.dimensions)
1821 if order_by is None:
1822 order_by = []
1823 query_limit = limit
1824 warn_limit = False
1825 if limit is not None and limit < 0:
1826 query_limit = abs(limit) + 1
1827 warn_limit = True
1828 with self.query() as query:
1829 result = (
1830 query.data_ids(dimensions)
1831 .where(data_id, where, bind=bind, **kwargs)
1832 .order_by(*ensure_iterable(order_by))
1833 .limit(query_limit)
1834 )
1835 if with_dimension_records:
1836 result = result.with_dimension_records()
1837 data_ids = list(result)
1838 if warn_limit and len(data_ids) == query_limit:
1839 # We asked for one too many so must remove that from the list.
1840 data_ids.pop(-1)
1841 assert limit is not None # For mypy.
1842 _LOG.warning("More data IDs are available than the requested limit of %d.", abs(limit))
1843 if explain and (limit is None or limit != 0) and not data_ids:
1844 raise EmptyQueryResultError(list(result.explain_no_results()))
1845 return data_ids
1847 def query_datasets(
1848 self,
1849 dataset_type: str | DatasetType,
1850 collections: str | Iterable[str] | None = None,
1851 *,
1852 find_first: bool = True,
1853 data_id: DataId | None = None,
1854 where: str = "",
1855 bind: Mapping[str, Any] | None = None,
1856 with_dimension_records: bool = False,
1857 order_by: Iterable[str] | str | None = None,
1858 limit: int | None = -20_000,
1859 explain: bool = True,
1860 **kwargs: Any,
1861 ) -> list[DatasetRef]:
1862 """Query for dataset references matching user-provided criteria.
1864 Parameters
1865 ----------
1866 dataset_type : `str` or `DatasetType`
1867 Dataset type object or name to search for.
1868 collections : collection expression, optional
1869 A collection name or iterable of collection names to search. If not
1870 provided, the default collections are used. Can be a wildcard if
1871 ``find_first`` is `False` (if find first is requested the order
1872 of collections matters and wildcards make the order indeterminate).
1873 See :ref:`daf_butler_collection_expressions` for more information.
1874 find_first : `bool`, optional
1875 If `True` (default), for each result data ID, only yield one
1876 `DatasetRef` of each `DatasetType`, from the first collection in
1877 which a dataset of that dataset type appears (according to the
1878 order of ``collections`` passed in). If `True`, ``collections``
1879 must not contain wildcards.
1880 data_id : `dict` or `DataCoordinate`, optional
1881 A data ID whose key-value pairs are used as equality constraints in
1882 the query.
1883 where : `str`, optional
1884 A string expression similar to a SQL WHERE clause. May involve any
1885 column of a dimension table or (as a shortcut for the primary key
1886 column of a dimension table) dimension name. See
1887 :ref:`daf_butler_dimension_expressions` for more information.
1888 bind : `~collections.abc.Mapping`, optional
1889 Mapping containing literal values that should be injected into the
1890 ``where`` expression, keyed by the identifiers they replace. Values
1891 of collection type can be expanded in some cases; see
1892 :ref:`daf_butler_dimension_expressions_identifiers` for more
1893 information.
1894 with_dimension_records : `bool`, optional
1895 If `True` (default is `False`) then returned data IDs will have
1896 dimension records.
1897 order_by : `~collections.abc.Iterable` [`str`] or `str`, optional
1898 Names of the columns/dimensions to use for ordering returned data
1899 IDs. Column name can be prefixed with minus (``-``) to use
1900 descending ordering.
1901 limit : `int` or `None`, optional
1902 Upper limit on the number of returned records. `None` can be used
1903 if no limit is wanted. A limit of ``0`` means that the query will
1904 be executed and validated but no results will be returned. In this
1905 case there will be no exception even if ``explain`` is `True`.
1906 If a negative value is given a warning will be issued if the number
1907 of results is capped by that limit.
1908 explain : `bool`, optional
1909 If `True` (default) then `EmptyQueryResultError` exception is
1910 raised when resulting list is empty. The exception contains
1911 non-empty list of strings explaining possible causes for empty
1912 result.
1913 **kwargs
1914 Additional keyword arguments are forwarded to
1915 `DataCoordinate.standardize` when processing the ``data_id``
1916 argument (and may be used to provide a constraining data ID even
1917 when the ``data_id`` argument is `None`).
1919 Returns
1920 -------
1921 refs : `.queries.DatasetRefQueryResults`
1922 Dataset references matching the given query criteria. Nested data
1923 IDs are guaranteed to include values for all implied dimensions
1924 (i.e. `DataCoordinate.hasFull` will return `True`).
1926 Raises
1927 ------
1928 lsst.daf.butler.registry.DatasetTypeExpressionError
1929 Raised when ``dataset_type`` expression is invalid.
1930 lsst.daf.butler.registry.DataIdError
1931 Raised when ``data_id`` or keyword arguments specify unknown
1932 dimensions or values, or when they contain inconsistent values.
1933 lsst.daf.butler.registry.UserExpressionError
1934 Raised when ``where`` expression is invalid.
1935 lsst.daf.butler.EmptyQueryResultError
1936 Raised when query generates empty result and ``explain`` is set to
1937 `True`.
1938 TypeError
1939 Raised when the arguments are incompatible, such as when a
1940 collection wildcard is passed when ``find_first`` is `True`, or
1941 when ``collections`` is `None` and default butler collections are
1942 not defined.
1943 """
1944 if data_id is None:
1945 data_id = DataCoordinate.make_empty(self.dimensions)
1946 if order_by is None:
1947 order_by = []
1948 if collections and has_globs(collections):
1949 # Wild cards need to be expanded but can only be allowed if
1950 # find_first=False because expanding wildcards does not return
1951 # a guaranteed ordering. Querying collection registry to expand
1952 # collections when we do not have wildcards is expensive so only
1953 # do it if we need it.
1954 if find_first:
1955 raise InvalidQueryError(
1956 f"Can not use wildcards in collections when find_first=True (given {collections})"
1957 )
1958 collections = self.collections.query(collections)
1959 query_limit = limit
1960 warn_limit = False
1961 if limit is not None and limit < 0:
1962 query_limit = abs(limit) + 1
1963 warn_limit = True
1964 with self.query() as query:
1965 result = (
1966 query.datasets(dataset_type, collections=collections, find_first=find_first)
1967 .where(data_id, where, bind=bind, **kwargs)
1968 .order_by(*ensure_iterable(order_by))
1969 .limit(query_limit)
1970 )
1971 if with_dimension_records:
1972 result = result.with_dimension_records()
1973 refs = list(result)
1974 if warn_limit and len(refs) == query_limit:
1975 # We asked for one too many so must remove that from the list.
1976 refs.pop(-1)
1977 assert limit is not None # For mypy.
1978 _LOG.warning("More datasets are available than the requested limit of %d.", abs(limit))
1979 if explain and (limit is None or limit != 0) and not refs:
1980 raise EmptyQueryResultError(list(result.explain_no_results()))
1981 return refs
1983 def query_dimension_records(
1984 self,
1985 element: str,
1986 *,
1987 data_id: DataId | None = None,
1988 where: str = "",
1989 bind: Mapping[str, Any] | None = None,
1990 order_by: Iterable[str] | str | None = None,
1991 limit: int | None = -20_000,
1992 explain: bool = True,
1993 **kwargs: Any,
1994 ) -> list[DimensionRecord]:
1995 """Query for dimension information matching user-provided criteria.
1997 Parameters
1998 ----------
1999 element : `str`
2000 The name of a dimension element to obtain records for.
2001 data_id : `dict` or `DataCoordinate`, optional
2002 A data ID whose key-value pairs are used as equality constraints
2003 in the query.
2004 where : `str`, optional
2005 A string expression similar to a SQL WHERE clause. See
2006 `queryDataIds` and :ref:`daf_butler_dimension_expressions` for more
2007 information.
2008 bind : `~collections.abc.Mapping`, optional
2009 Mapping containing literal values that should be injected into the
2010 ``where`` expression, keyed by the identifiers they replace.
2011 Values of collection type can be expanded in some cases; see
2012 :ref:`daf_butler_dimension_expressions_identifiers` for more
2013 information.
2014 order_by : `~collections.abc.Iterable` [`str`] or `str`, optional
2015 Names of the columns/dimensions to use for ordering returned data
2016 IDs. Column name can be prefixed with minus (``-``) to use
2017 descending ordering.
2018 limit : `int` or `None`, optional
2019 Upper limit on the number of returned records. `None` can be used
2020 if no limit is wanted. A limit of ``0`` means that the query will
2021 be executed and validated but no results will be returned. In this
2022 case there will be no exception even if ``explain`` is `True`.
2023 If a negative value is given a warning will be issued if the number
2024 of results is capped by that limit.
2025 explain : `bool`, optional
2026 If `True` (default) then `EmptyQueryResultError` exception is
2027 raised when resulting list is empty. The exception contains
2028 non-empty list of strings explaining possible causes for empty
2029 result.
2030 **kwargs
2031 Additional keyword arguments are forwarded to
2032 `DataCoordinate.standardize` when processing the ``data_id``
2033 argument (and may be used to provide a constraining data ID even
2034 when the ``data_id`` argument is `None`).
2036 Returns
2037 -------
2038 records : `list`[`DimensionRecord`]
2039 Dimension records matching the given query parameters.
2041 Raises
2042 ------
2043 lsst.daf.butler.registry.DataIdError
2044 Raised when ``data_id`` or keyword arguments specify unknown
2045 dimensions or values, or when they contain inconsistent values.
2046 lsst.daf.butler.registry.UserExpressionError
2047 Raised when ``where`` expression is invalid.
2048 lsst.daf.butler.EmptyQueryResultError
2049 Raised when query generates empty result and ``explain`` is set to
2050 `True`.
2051 TypeError
2052 Raised when the arguments are incompatible, such as when a
2053 collection wildcard is passed when ``find_first`` is `True`, or
2054 when ``collections`` is `None` and default butler collections are
2055 not defined.
2056 """
2057 if data_id is None:
2058 data_id = DataCoordinate.make_empty(self.dimensions)
2059 if order_by is None:
2060 order_by = []
2061 query_limit = limit
2062 warn_limit = False
2063 if limit is not None and limit < 0:
2064 query_limit = abs(limit) + 1
2065 warn_limit = True
2066 with self.query() as query:
2067 result = (
2068 query.dimension_records(element)
2069 .where(data_id, where, bind=bind, **kwargs)
2070 .order_by(*ensure_iterable(order_by))
2071 .limit(query_limit)
2072 )
2073 dimension_records = list(result)
2074 if warn_limit and len(dimension_records) == query_limit:
2075 # We asked for one too many so must remove that from the list.
2076 dimension_records.pop(-1)
2077 assert limit is not None # For mypy.
2078 _LOG.warning(
2079 "More dimension records are available than the requested limit of %d.", abs(limit)
2080 )
2081 if explain and (limit is None or limit != 0) and not dimension_records:
2082 raise EmptyQueryResultError(list(result.explain_no_results()))
2083 return dimension_records
2085 def query_all_datasets(
2086 self,
2087 collections: str | Iterable[str] | None = None,
2088 *,
2089 name: str | Iterable[str] = "*",
2090 find_first: bool = True,
2091 data_id: DataId | None = None,
2092 where: str = "",
2093 bind: Mapping[str, Any] | None = None,
2094 limit: int | None = -20_000,
2095 **kwargs: Any,
2096 ) -> list[DatasetRef]:
2097 """Query for datasets of potentially multiple types.
2099 Parameters
2100 ----------
2101 collections : `str` or `~collections.abc.Iterable` [ `str` ], optional
2102 The collection or collections to search, in order. If not provided
2103 or `None`, the default collection search path for this butler is
2104 used.
2105 name : `str` or `~collections.abc.Iterable` [ `str` ], optional
2106 Names or name patterns (glob-style) that returned dataset type
2107 names must match. If an iterable, items are OR'd together. The
2108 default is to include all dataset types in the given collections.
2109 find_first : `bool`, optional
2110 If `True` (default), for each result data ID, only yield one
2111 `DatasetRef` of each `DatasetType`, from the first collection in
2112 which a dataset of that dataset type appears (according to the
2113 order of ``collections`` passed in).
2114 data_id : `dict` or `DataCoordinate`, optional
2115 A data ID whose key-value pairs are used as equality constraints in
2116 the query.
2117 where : `str`, optional
2118 A string expression similar to a SQL WHERE clause. May involve any
2119 column of a dimension table or (as a shortcut for the primary key
2120 column of a dimension table) dimension name. See
2121 :ref:`daf_butler_dimension_expressions` for more information.
2122 bind : `~collections.abc.Mapping`, optional
2123 Mapping containing literal values that should be injected into the
2124 ``where`` expression, keyed by the identifiers they replace. Values
2125 of collection type can be expanded in some cases; see
2126 :ref:`daf_butler_dimension_expressions_identifiers` for more
2127 information.
2128 limit : `int` or `None`, optional
2129 Upper limit on the number of returned records. `None` can be used
2130 if no limit is wanted. A limit of ``0`` means that the query will
2131 be executed and validated but no results will be returned.
2132 If a negative value is given a warning will be issued if the number
2133 of results is capped by that limit. If no limit is provided, by
2134 default a maximum of 20,000 records will be returned.
2135 **kwargs
2136 Additional keyword arguments are forwarded to
2137 `DataCoordinate.standardize` when processing the ``data_id``
2138 argument (and may be used to provide a constraining data ID even
2139 when the ``data_id`` argument is `None`).
2141 Raises
2142 ------
2143 MissingDatasetTypeError
2144 When no dataset types match ``name``, or an explicit (non-glob)
2145 dataset type in ``name`` does not exist.
2146 InvalidQueryError
2147 If the parameters to the query are inconsistent or malformed.
2148 MissingCollectionError
2149 If a given collection is not found.
2151 Returns
2152 -------
2153 refs : `list` [ `DatasetRef` ]
2154 Dataset references matching the given query criteria. Nested data
2155 IDs are guaranteed to include values for all implied dimensions
2156 (i.e. `DataCoordinate.hasFull` will return `True`), but will not
2157 include dimension records (`DataCoordinate.hasRecords` will be
2158 `False`).
2159 """
2160 if collections is None:
2161 collections = list(self.collections.defaults)
2162 else:
2163 collections = list(ensure_iterable(collections))
2165 if bind is None:
2166 bind = {}
2167 if data_id is None:
2168 data_id = {}
2170 warn_limit = False
2171 if limit is not None and limit < 0:
2172 # Add one to the limit so we can detect if we have exceeded it.
2173 limit = abs(limit) + 1
2174 warn_limit = True
2176 args = QueryAllDatasetsParameters(
2177 collections=collections,
2178 name=list(ensure_iterable(name)),
2179 find_first=find_first,
2180 data_id=data_id,
2181 where=where,
2182 limit=limit,
2183 bind=bind,
2184 kwargs=kwargs,
2185 with_dimension_records=False,
2186 )
2187 with self._query_all_datasets_by_page(args) as pages:
2188 result = []
2189 for page in pages:
2190 result.extend(page)
2192 if warn_limit and limit is not None and len(result) >= limit:
2193 # Remove the extra dataset we added for the limit check.
2194 result.pop()
2195 _LOG.warning("More datasets are available than the requested limit of %d.", limit - 1)
2197 return result
2199 @abstractmethod
2200 def _query_all_datasets_by_page(
2201 self, args: QueryAllDatasetsParameters
2202 ) -> AbstractContextManager[Iterator[list[DatasetRef]]]:
2203 raise NotImplementedError()
2205 def clone(
2206 self,
2207 *,
2208 collections: CollectionArgType | None | EllipsisType = ...,
2209 run: str | None | EllipsisType = ...,
2210 inferDefaults: bool | EllipsisType = ...,
2211 dataId: dict[str, str] | EllipsisType = ...,
2212 metrics: ButlerMetrics | None = None,
2213 ) -> Butler:
2214 """Return a new Butler instance connected to the same repository
2215 as this one, optionally overriding ``collections``, ``run``,
2216 ``inferDefaults``, and default data ID.
2218 Parameters
2219 ----------
2220 collections : `~lsst.daf.butler.registry.CollectionArgType` or `None`,\
2221 optional
2222 Same as constructor. If omitted, uses value from original object.
2223 run : `str` or `None`, optional
2224 Same as constructor. If `None`, no default run is used. If
2225 omitted, copies value from original object.
2226 inferDefaults : `bool`, optional
2227 Same as constructor. If omitted, copies value from original
2228 object.
2229 dataId : `str`
2230 Same as ``kwargs`` passed to the constructor. If omitted, copies
2231 values from original object.
2232 metrics : `ButlerMetrics` or `None`, optional
2233 Metrics object to record butler statistics.
2234 """
2235 raise NotImplementedError()
2237 @abstractmethod
2238 def close(self) -> None:
2239 """Release all resources associated with this Butler instance. The
2240 instance may no longer be used after this is called.
2242 Notes
2243 -----
2244 Instead of calling ``close()``directly, you can use the Butler object
2245 as a context manager. For example::
2246 with Butler(...) as butler:
2247 butler.get(...)
2248 # butler is closed after exiting the block.
2249 """
2250 raise NotImplementedError()