Coverage for python/lsst/daf/butler/_butler.py: 30%
305 statements
« prev ^ index » next coverage.py v7.11.0, created at 2025-11-19 01:32 -0800
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30__all__ = ["Butler"]
32import dataclasses
33import urllib.parse
34import uuid
35from abc import abstractmethod
36from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence
37from contextlib import AbstractContextManager
38from types import EllipsisType
39from typing import TYPE_CHECKING, Any, TextIO
41from lsst.resources import ResourcePath, ResourcePathExpression
42from lsst.utils import doImportType
43from lsst.utils.iteration import ensure_iterable
44from lsst.utils.logging import getLogger
46from ._butler_collections import ButlerCollections
47from ._butler_config import ButlerConfig, ButlerType
48from ._butler_instance_options import ButlerInstanceOptions
49from ._butler_metrics import ButlerMetrics
50from ._butler_repo_index import ButlerRepoIndex
51from ._config import Config, ConfigSubset
52from ._exceptions import EmptyQueryResultError, InvalidQueryError
53from ._limited_butler import LimitedButler
54from ._query_all_datasets import QueryAllDatasetsParameters
55from .datastore import Datastore
56from .dimensions import DataCoordinate, DimensionConfig
57from .registry import RegistryConfig, _RegistryFactory
58from .repo_relocation import BUTLER_ROOT_TAG
59from .utils import has_globs
61if TYPE_CHECKING:
62 from ._dataset_existence import DatasetExistence
63 from ._dataset_provenance import DatasetProvenance
64 from ._dataset_ref import DatasetId, DatasetRef
65 from ._dataset_type import DatasetType
66 from ._deferredDatasetHandle import DeferredDatasetHandle
67 from ._file_dataset import FileDataset
68 from ._labeled_butler_factory import LabeledButlerFactoryProtocol
69 from ._storage_class import StorageClass
70 from ._timespan import Timespan
71 from .datastore import DatasetRefURIs
72 from .dimensions import DataId, DimensionGroup, DimensionRecord
73 from .queries import Query
74 from .registry import CollectionArgType, Registry
75 from .transfers import RepoExportContext
77_LOG = getLogger(__name__)
80@dataclasses.dataclass
81class ParsedButlerDatasetURI:
82 label: str
83 dataset_id: uuid.UUID
84 uri: str
87@dataclasses.dataclass
88class SpecificButlerDataset:
89 butler: Butler
90 dataset: DatasetRef | None
93class _DeprecatedDefault:
94 """Default value for a deprecated parameter."""
97class Butler(LimitedButler): # numpydoc ignore=PR02
98 """Interface for data butler and factory for Butler instances.
100 Parameters
101 ----------
102 config : `ButlerConfig`, `Config` or `str`, optional
103 Configuration. Anything acceptable to the `ButlerConfig` constructor.
104 If a directory path is given the configuration will be read from a
105 ``butler.yaml`` file in that location. If `None` is given default
106 values will be used. If ``config`` contains "cls" key then its value is
107 used as a name of butler class and it must be a sub-class of this
108 class, otherwise `DirectButler` is instantiated.
109 collections : `str` or `~collections.abc.Iterable` [ `str` ], optional
110 An expression specifying the collections to be searched (in order) when
111 reading datasets.
112 This may be a `str` collection name or an iterable thereof.
113 See :ref:`daf_butler_collection_expressions` for more information.
114 These collections are not registered automatically and must be
115 manually registered before they are used by any method, but they may be
116 manually registered after the `Butler` is initialized.
117 run : `str`, optional
118 Name of the `~CollectionType.RUN` collection new datasets should be
119 inserted into. If ``collections`` is `None` and ``run`` is not `None`,
120 ``collections`` will be set to ``[run]``. If not `None`, this
121 collection will automatically be registered. If this is not set (and
122 ``writeable`` is not set either), a read-only butler will be created.
123 searchPaths : `list` of `str`, optional
124 Directory paths to search when calculating the full Butler
125 configuration. Not used if the supplied config is already a
126 `ButlerConfig`.
127 writeable : `bool`, optional
128 Explicitly sets whether the butler supports write operations. If not
129 provided, a read-write butler is created if any of ``run``, ``tags``,
130 or ``chains`` is non-empty.
131 inferDefaults : `bool`, optional
132 If `True` (default) infer default data ID values from the values
133 present in the datasets in ``collections``: if all collections have the
134 same value (or no value) for a governor dimension, that value will be
135 the default for that dimension. Nonexistent collections are ignored.
136 If a default value is provided explicitly for a governor dimension via
137 ``**kwargs``, no default will be inferred for that dimension.
138 without_datastore : `bool`, optional
139 If `True` do not attach a datastore to this butler. Any attempts
140 to use a datastore will fail.
141 **kwargs : `Any`
142 Additional keyword arguments passed to a constructor of actual butler
143 class.
145 Notes
146 -----
147 The preferred way to instantiate Butler is via the `from_config` method.
148 The call to ``Butler(...)`` is equivalent to ``Butler.from_config(...)``,
149 but ``mypy`` will complain about the former.
150 """
152 def __new__(
153 cls,
154 config: Config | ResourcePathExpression | None = None,
155 *,
156 collections: Any = None,
157 run: str | None = None,
158 searchPaths: Sequence[ResourcePathExpression] | None = None,
159 writeable: bool | None = None,
160 inferDefaults: bool = True,
161 without_datastore: bool = False,
162 metrics: ButlerMetrics | None = None,
163 **kwargs: Any,
164 ) -> Butler:
165 if cls is Butler:
166 return Butler.from_config(
167 config=config,
168 collections=collections,
169 run=run,
170 searchPaths=searchPaths,
171 writeable=writeable,
172 inferDefaults=inferDefaults,
173 without_datastore=without_datastore,
174 metrics=metrics,
175 **kwargs,
176 )
178 # Note: we do not pass any parameters to __new__, Python will pass them
179 # to __init__ after __new__ returns sub-class instance.
180 return super().__new__(cls)
182 @classmethod
183 def from_config(
184 cls,
185 config: Config | ResourcePathExpression | None = None,
186 *,
187 collections: Any = None,
188 run: str | None = None,
189 searchPaths: Sequence[ResourcePathExpression] | None = None,
190 writeable: bool | None = None,
191 inferDefaults: bool = True,
192 without_datastore: bool = False,
193 metrics: ButlerMetrics | None = None,
194 **kwargs: Any,
195 ) -> Butler:
196 """Create butler instance from configuration.
198 Parameters
199 ----------
200 config : `ButlerConfig`, `Config` or `str`, optional
201 Configuration. Anything acceptable to the `ButlerConfig`
202 constructor. If a directory path is given the configuration will be
203 read from a ``butler.yaml`` file in that location. If `None` is
204 given default values will be used. If ``config`` contains "cls" key
205 then its value is used as a name of butler class and it must be a
206 sub-class of this class, otherwise `DirectButler` is instantiated.
207 collections : `str` or `~collections.abc.Iterable` [ `str` ], optional
208 An expression specifying the collections to be searched (in order)
209 when reading datasets.
210 This may be a `str` collection name or an iterable thereof.
211 See :ref:`daf_butler_collection_expressions` for more information.
212 These collections are not registered automatically and must be
213 manually registered before they are used by any method, but they
214 may be manually registered after the `Butler` is initialized.
215 run : `str`, optional
216 Name of the `~CollectionType.RUN` collection new datasets should be
217 inserted into. If ``collections`` is `None` and ``run`` is not
218 `None`, ``collections`` will be set to ``[run]``. If not `None`,
219 this collection will automatically be registered. If this is not
220 set (and ``writeable`` is not set either), a read-only butler will
221 be created.
222 searchPaths : `list` of `str`, optional
223 Directory paths to search when calculating the full Butler
224 configuration. Not used if the supplied config is already a
225 `ButlerConfig`.
226 writeable : `bool`, optional
227 Explicitly sets whether the butler supports write operations. If
228 not provided, a read-write butler is created if any of ``run``,
229 ``tags``, or ``chains`` is non-empty.
230 inferDefaults : `bool`, optional
231 If `True` (default) infer default data ID values from the values
232 present in the datasets in ``collections``: if all collections have
233 the same value (or no value) for a governor dimension, that value
234 will be the default for that dimension. Nonexistent collections
235 are ignored. If a default value is provided explicitly for a
236 governor dimension via ``**kwargs``, no default will be inferred
237 for that dimension.
238 without_datastore : `bool`, optional
239 If `True` do not attach a datastore to this butler. Any attempts
240 to use a datastore will fail.
241 metrics : `ButlerMetrics` or `None`, optional
242 Metrics object to record butler usage statistics.
243 **kwargs : `Any`
244 Default data ID key-value pairs. These may only identify
245 "governor" dimensions like ``instrument`` and ``skymap``.
247 Returns
248 -------
249 butler : `Butler`
250 A `Butler` constructed from the given configuration.
252 Notes
253 -----
254 Calling this factory method is identical to calling
255 ``Butler(config, ...)``. Its only raison d'être is that ``mypy``
256 complains about ``Butler()`` call.
258 Examples
259 --------
260 While there are many ways to control exactly how a `Butler` interacts
261 with the collections in its `Registry`, the most common cases are still
262 simple.
264 For a read-only `Butler` that searches one collection, do::
266 butler = Butler.from_config(
267 "/path/to/repo", collections=["u/alice/DM-50000"]
268 )
270 For a read-write `Butler` that writes to and reads from a
271 `~CollectionType.RUN` collection::
273 butler = Butler.from_config(
274 "/path/to/repo", run="u/alice/DM-50000/a"
275 )
277 The `Butler` passed to a ``PipelineTask`` is often much more complex,
278 because we want to write to one `~CollectionType.RUN` collection but
279 read from several others (as well)::
281 butler = Butler.from_config(
282 "/path/to/repo",
283 run="u/alice/DM-50000/a",
284 collections=[
285 "u/alice/DM-50000/a",
286 "u/bob/DM-49998",
287 "HSC/defaults",
288 ],
289 )
291 This butler will `put` new datasets to the run ``u/alice/DM-50000/a``.
292 Datasets will be read first from that run (since it appears first in
293 the chain), and then from ``u/bob/DM-49998`` and finally
294 ``HSC/defaults``.
296 Finally, one can always create a `Butler` with no collections::
298 butler = Butler.from_config("/path/to/repo", writeable=True)
300 This can be extremely useful when you just want to use
301 ``butler.registry``, e.g. for inserting dimension data or managing
302 collections, or when the collections you want to use with the butler
303 are not consistent. Passing ``writeable`` explicitly here is only
304 necessary if you want to be able to make changes to the repo - usually
305 the value for ``writeable`` can be guessed from the collection
306 arguments provided, but it defaults to `False` when there are not
307 collection arguments.
308 """
309 # DirectButler used to have a way to specify a "copy constructor" by
310 # passing the "butler" parameter to its constructor. This has
311 # been moved out of the constructor into Butler.clone().
312 butler = kwargs.pop("butler", None)
313 metrics = metrics if metrics is not None else ButlerMetrics()
314 if butler is not None:
315 if not isinstance(butler, Butler):
316 raise TypeError("'butler' parameter must be a Butler instance")
317 if config is not None or searchPaths is not None or writeable is not None:
318 raise TypeError(
319 "Cannot pass 'config', 'searchPaths', or 'writeable' arguments with 'butler' argument."
320 )
321 return butler.clone(
322 collections=collections, run=run, inferDefaults=inferDefaults, metrics=metrics, dataId=kwargs
323 )
325 options = ButlerInstanceOptions(
326 collections=collections,
327 run=run,
328 writeable=writeable,
329 inferDefaults=inferDefaults,
330 metrics=metrics,
331 kwargs=kwargs,
332 )
334 # Load the Butler configuration. This may involve searching the
335 # environment to locate a configuration file.
336 butler_config = ButlerConfig(config, searchPaths=searchPaths, without_datastore=without_datastore)
337 butler_type = butler_config.get_butler_type()
339 # Make DirectButler if class is not specified.
340 match butler_type:
341 case ButlerType.DIRECT:
342 from .direct_butler import DirectButler
344 return DirectButler.create_from_config(
345 butler_config,
346 options=options,
347 without_datastore=without_datastore,
348 )
349 case ButlerType.REMOTE:
350 from .remote_butler._factory import RemoteButlerFactory
352 # Assume this is being created by a client who would like
353 # default caching of remote datasets.
354 factory = RemoteButlerFactory.create_factory_from_config(butler_config)
355 return factory.create_butler_with_credentials_from_environment(
356 butler_options=options, enable_datastore_cache=True
357 )
358 case _:
359 raise TypeError(f"Unknown Butler type '{butler_type}'")
361 @staticmethod
362 def makeRepo(
363 root: ResourcePathExpression,
364 config: Config | str | None = None,
365 dimensionConfig: Config | str | None = None,
366 standalone: bool = False,
367 searchPaths: list[str] | None = None,
368 forceConfigRoot: bool = True,
369 outfile: ResourcePathExpression | None = None,
370 overwrite: bool = False,
371 ) -> Config:
372 """Create an empty data repository by adding a butler.yaml config
373 to a repository root directory.
375 Parameters
376 ----------
377 root : `lsst.resources.ResourcePathExpression`
378 Path or URI to the root location of the new repository. Will be
379 created if it does not exist.
380 config : `Config` or `str`, optional
381 Configuration to write to the repository, after setting any
382 root-dependent Registry or Datastore config options. Can not
383 be a `ButlerConfig` or a `ConfigSubset`. If `None`, default
384 configuration will be used. Root-dependent config options
385 specified in this config are overwritten if ``forceConfigRoot``
386 is `True`.
387 dimensionConfig : `Config` or `str`, optional
388 Configuration for dimensions, will be used to initialize registry
389 database.
390 standalone : `bool`
391 If True, write all expanded defaults, not just customized or
392 repository-specific settings.
393 This (mostly) decouples the repository from the default
394 configuration, insulating it from changes to the defaults (which
395 may be good or bad, depending on the nature of the changes).
396 Future *additions* to the defaults will still be picked up when
397 initializing `Butlers` to repos created with ``standalone=True``.
398 searchPaths : `list` of `str`, optional
399 Directory paths to search when calculating the full butler
400 configuration.
401 forceConfigRoot : `bool`, optional
402 If `False`, any values present in the supplied ``config`` that
403 would normally be reset are not overridden and will appear
404 directly in the output config. This allows non-standard overrides
405 of the root directory for a datastore or registry to be given.
406 If this parameter is `True` the values for ``root`` will be
407 forced into the resulting config if appropriate.
408 outfile : `lss.resources.ResourcePathExpression`, optional
409 If not-`None`, the output configuration will be written to this
410 location rather than into the repository itself. Can be a URI
411 string. Can refer to a directory that will be used to write
412 ``butler.yaml``.
413 overwrite : `bool`, optional
414 Create a new configuration file even if one already exists
415 in the specified output location. Default is to raise
416 an exception.
418 Returns
419 -------
420 config : `Config`
421 The updated `Config` instance written to the repo.
423 Raises
424 ------
425 ValueError
426 Raised if a ButlerConfig or ConfigSubset is passed instead of a
427 regular Config (as these subclasses would make it impossible to
428 support ``standalone=False``).
429 FileExistsError
430 Raised if the output config file already exists.
431 os.error
432 Raised if the directory does not exist, exists but is not a
433 directory, or cannot be created.
435 Notes
436 -----
437 Note that when ``standalone=False`` (the default), the configuration
438 search path (see `ConfigSubset.defaultSearchPaths`) that was used to
439 construct the repository should also be used to construct any Butlers
440 to avoid configuration inconsistencies.
441 """
442 if isinstance(config, ButlerConfig | ConfigSubset):
443 raise ValueError("makeRepo must be passed a regular Config without defaults applied.")
445 # Ensure that the root of the repository exists or can be made
446 root_uri = ResourcePath(root, forceDirectory=True)
447 root_uri.mkdir()
449 config = Config(config)
451 # If we are creating a new repo from scratch with relative roots,
452 # do not propagate an explicit root from the config file
453 if "root" in config:
454 del config["root"]
456 full = ButlerConfig(config, searchPaths=searchPaths) # this applies defaults
457 imported_class = doImportType(full["datastore", "cls"])
458 if not issubclass(imported_class, Datastore):
459 raise TypeError(f"Imported datastore class {full['datastore', 'cls']} is not a Datastore")
460 datastoreClass: type[Datastore] = imported_class
461 datastoreClass.setConfigRoot(BUTLER_ROOT_TAG, config, full, overwrite=forceConfigRoot)
463 # if key exists in given config, parse it, otherwise parse the defaults
464 # in the expanded config
465 if config.get(("registry", "db")):
466 registryConfig = RegistryConfig(config)
467 else:
468 registryConfig = RegistryConfig(full)
469 defaultDatabaseUri = registryConfig.makeDefaultDatabaseUri(BUTLER_ROOT_TAG)
470 if defaultDatabaseUri is not None:
471 Config.updateParameters(
472 RegistryConfig, config, full, toUpdate={"db": defaultDatabaseUri}, overwrite=forceConfigRoot
473 )
474 else:
475 Config.updateParameters(RegistryConfig, config, full, toCopy=("db",), overwrite=forceConfigRoot)
477 if standalone:
478 config.merge(full)
479 else:
480 # Always expand the registry.managers section into the per-repo
481 # config, because after the database schema is created, it's not
482 # allowed to change anymore. Note that in the standalone=True
483 # branch, _everything_ in the config is expanded, so there's no
484 # need to special case this.
485 Config.updateParameters(RegistryConfig, config, full, toMerge=("managers",), overwrite=False)
486 configURI: ResourcePathExpression
487 if outfile is not None:
488 # When writing to a separate location we must include
489 # the root of the butler repo in the config else it won't know
490 # where to look.
491 config["root"] = root_uri.geturl()
492 configURI = outfile
493 else:
494 configURI = root_uri
495 # Strip obscore configuration, if it is present, before writing config
496 # to a file, obscore config will be stored in registry.
497 if (obscore_config_key := ("registry", "managers", "obscore", "config")) in config:
498 config_to_write = config.copy()
499 del config_to_write[obscore_config_key]
500 config_to_write.dumpToUri(configURI, overwrite=overwrite)
501 # configFile attribute is updated, need to copy it to original.
502 config.configFile = config_to_write.configFile
503 else:
504 config.dumpToUri(configURI, overwrite=overwrite)
506 # Create Registry and populate tables
507 registryConfig = RegistryConfig(config.get("registry"))
508 dimensionConfig = DimensionConfig(dimensionConfig)
509 _RegistryFactory(registryConfig).create_from_config(
510 dimensionConfig=dimensionConfig, butlerRoot=root_uri
511 )
513 _LOG.verbose("Wrote new Butler configuration file to %s", configURI)
515 return config
517 @classmethod
518 def get_repo_uri(cls, label: str, return_label: bool = False) -> ResourcePath:
519 """Look up the label in a butler repository index.
521 Parameters
522 ----------
523 label : `str`
524 Label of the Butler repository to look up.
525 return_label : `bool`, optional
526 If ``label`` cannot be found in the repository index (either
527 because index is not defined or ``label`` is not in the index) and
528 ``return_label`` is `True` then return ``ResourcePath(label)``.
529 If ``return_label`` is `False` (default) then an exception will be
530 raised instead.
532 Returns
533 -------
534 uri : `lsst.resources.ResourcePath`
535 URI to the Butler repository associated with the given label or
536 default value if it is provided.
538 Raises
539 ------
540 KeyError
541 Raised if the label is not found in the index, or if an index
542 is not defined, and ``return_label`` is `False`.
544 Notes
545 -----
546 See `~lsst.daf.butler.ButlerRepoIndex` for details on how the
547 information is discovered.
548 """
549 return ButlerRepoIndex.get_repo_uri(label, return_label)
551 @classmethod
552 def get_known_repos(cls) -> set[str]:
553 """Retrieve the list of known repository labels.
555 Returns
556 -------
557 repos : `set` of `str`
558 All the known labels. Can be empty if no index can be found.
560 Notes
561 -----
562 See `~lsst.daf.butler.ButlerRepoIndex` for details on how the
563 information is discovered.
564 """
565 return ButlerRepoIndex.get_known_repos()
567 @classmethod
568 def parse_dataset_uri(cls, uri: str) -> ParsedButlerDatasetURI:
569 """Extract the butler label and dataset ID from a dataset URI.
571 Parameters
572 ----------
573 uri : `str`
574 The dataset URI to parse.
576 Returns
577 -------
578 parsed : `ParsedButlerDatasetURI`
579 The label associated with the butler repository from which this
580 dataset originates and the ID of the dataset.
582 Notes
583 -----
584 Supports dataset URIs of the forms
585 ``ivo://org.rubinobs/usdac/dr1?repo=butler_label&id=UUID`` (see
586 DMTN-302) and ``butler://butler_label/UUID``. The ``butler`` URI is
587 deprecated and can not include ``/`` in the label string. ``ivo`` URIs
588 can include anything supported by the `Butler` constructor, including
589 paths to repositories and alias labels.
591 ivo://org.rubinobs/dr1?repo=/repo/main&id=UUID
593 will return a label of ``/repo/main``.
595 This method does not attempt to check that the dataset exists in the
596 labeled butler.
598 Since the IVOID can be issued by any publisher to represent a Butler
599 dataset there is no validation of the path or netloc component of the
600 URI. The only requirement is that there are ``id`` and ``repo`` keys
601 in the ``ivo`` URI query component.
602 """
603 parsed = urllib.parse.urlparse(uri)
604 parsed_scheme = parsed.scheme.lower()
605 if parsed_scheme == "ivo":
606 # Do not validate the netloc or the path values.
607 qs = urllib.parse.parse_qs(parsed.query)
608 if "repo" not in qs or "id" not in qs:
609 raise ValueError(f"Missing 'repo' and/or 'id' query parameters in IVOID {uri}.")
610 if len(qs["repo"]) != 1 or len(qs["id"]) != 1:
611 raise ValueError(f"Butler IVOID only supports a single value of repo and id, got {uri}")
612 label = qs["repo"][0]
613 id_ = qs["id"][0]
614 elif parsed_scheme == "butler":
615 label = parsed.netloc # Butler label is case sensitive.
616 # Need to strip the leading /.
617 id_ = parsed.path[1:]
618 else:
619 raise ValueError(f"Unrecognized URI scheme: {uri!r}")
620 # Strip trailing/leading whitespace from label.
621 label = label.strip()
622 if not label:
623 raise ValueError(f"No butler repository label found in uri {uri!r}")
624 try:
625 dataset_id = uuid.UUID(hex=id_)
626 except Exception as e:
627 e.add_note(f"Error extracting dataset ID from uri {uri!r} with dataset ID string {id_!r}")
628 raise
630 return ParsedButlerDatasetURI(label=label, dataset_id=dataset_id, uri=uri)
632 @classmethod
633 def get_dataset_from_uri(
634 cls, uri: str, factory: LabeledButlerFactoryProtocol | None = None
635 ) -> SpecificButlerDataset:
636 """Get the dataset associated with the given dataset URI.
638 Parameters
639 ----------
640 uri : `str`
641 The URI associated with a dataset.
642 factory : `LabeledButlerFactoryProtocol` or `None`, optional
643 Bound factory function that will be given the butler label
644 and receive a `Butler`. If this is not provided the label
645 will be tried directly.
647 Returns
648 -------
649 result : `SpecificButlerDataset`
650 The butler associated with this URI and the dataset itself.
651 The dataset can be `None` if the UUID is valid but the dataset
652 is not known to this butler.
653 """
654 parsed = cls.parse_dataset_uri(uri)
655 butler: Butler | None = None
656 if factory is not None:
657 # If the label is not recognized, it might be a path.
658 try:
659 butler = factory(parsed.label)
660 except KeyError:
661 pass
662 if butler is None:
663 butler = cls.from_config(parsed.label)
664 return SpecificButlerDataset(butler=butler, dataset=butler.get_dataset(parsed.dataset_id))
666 @abstractmethod
667 def _caching_context(self) -> AbstractContextManager[None]:
668 """Context manager that enables caching."""
669 raise NotImplementedError()
671 @abstractmethod
672 def transaction(self) -> AbstractContextManager[None]:
673 """Context manager supporting `Butler` transactions.
675 Transactions can be nested.
676 """
677 raise NotImplementedError()
679 @abstractmethod
680 def put(
681 self,
682 obj: Any,
683 datasetRefOrType: DatasetRef | DatasetType | str,
684 /,
685 dataId: DataId | None = None,
686 *,
687 run: str | None = None,
688 provenance: DatasetProvenance | None = None,
689 **kwargs: Any,
690 ) -> DatasetRef:
691 """Store and register a dataset.
693 Parameters
694 ----------
695 obj : `object`
696 The dataset.
697 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
698 When `DatasetRef` is provided, ``dataId`` should be `None`.
699 Otherwise the `DatasetType` or name thereof. If a fully resolved
700 `DatasetRef` is given the run and ID are used directly.
701 dataId : `dict` or `DataCoordinate`
702 A `dict` of `Dimension` link name, value pairs that label the
703 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
704 should be provided as the second argument.
705 run : `str`, optional
706 The name of the run the dataset should be added to, overriding
707 ``self.run``. Not used if a resolved `DatasetRef` is provided.
708 provenance : `DatasetProvenance` or `None`, optional
709 Any provenance that should be attached to the serialized dataset.
710 Not supported by all serialization mechanisms.
711 **kwargs
712 Additional keyword arguments used to augment or construct a
713 `DataCoordinate`. See `DataCoordinate.standardize`
714 parameters. Not used if a resolve `DatasetRef` is provided.
716 Returns
717 -------
718 ref : `DatasetRef`
719 A reference to the stored dataset, updated with the correct id if
720 given.
722 Raises
723 ------
724 TypeError
725 Raised if the butler is read-only or if no run has been provided.
726 """
727 raise NotImplementedError()
729 @abstractmethod
730 def getDeferred(
731 self,
732 datasetRefOrType: DatasetRef | DatasetType | str,
733 /,
734 dataId: DataId | None = None,
735 *,
736 parameters: dict | None = None,
737 collections: Any = None,
738 storageClass: str | StorageClass | None = None,
739 timespan: Timespan | None = None,
740 **kwargs: Any,
741 ) -> DeferredDatasetHandle:
742 """Create a `DeferredDatasetHandle` which can later retrieve a dataset,
743 after an immediate registry lookup.
745 Parameters
746 ----------
747 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
748 When `DatasetRef` the `dataId` should be `None`.
749 Otherwise the `DatasetType` or name thereof.
750 dataId : `dict` or `DataCoordinate`, optional
751 A `dict` of `Dimension` link name, value pairs that label the
752 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
753 should be provided as the first argument.
754 parameters : `dict`
755 Additional StorageClass-defined options to control reading,
756 typically used to efficiently read only a subset of the dataset.
757 collections : Any, optional
758 Collections to be searched, overriding ``self.collections``.
759 Can be any of the types supported by the ``collections`` argument
760 to butler construction.
761 storageClass : `StorageClass` or `str`, optional
762 The storage class to be used to override the Python type
763 returned by this method. By default the returned type matches
764 the dataset type definition for this dataset. Specifying a
765 read `StorageClass` can force a different type to be returned.
766 This type must be compatible with the original type.
767 timespan : `Timespan` or `None`, optional
768 A timespan that the validity range of the dataset must overlap.
769 If not provided and this is a calibration dataset type, an attempt
770 will be made to find the timespan from any temporal coordinate
771 in the data ID.
772 **kwargs
773 Additional keyword arguments used to augment or construct a
774 `DataId`. See `DataId` parameters.
776 Returns
777 -------
778 obj : `DeferredDatasetHandle`
779 A handle which can be used to retrieve a dataset at a later time.
781 Raises
782 ------
783 LookupError
784 Raised if no matching dataset exists in the `Registry` or
785 datastore.
786 ValueError
787 Raised if a resolved `DatasetRef` was passed as an input, but it
788 differs from the one found in the registry.
789 TypeError
790 Raised if no collections were provided.
791 """
792 raise NotImplementedError()
794 @abstractmethod
795 def get(
796 self,
797 datasetRefOrType: DatasetRef | DatasetType | str,
798 /,
799 dataId: DataId | None = None,
800 *,
801 parameters: dict[str, Any] | None = None,
802 collections: Any = None,
803 storageClass: StorageClass | str | None = None,
804 timespan: Timespan | None = None,
805 **kwargs: Any,
806 ) -> Any:
807 """Retrieve a stored dataset.
809 Parameters
810 ----------
811 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
812 When `DatasetRef` the `dataId` should be `None`.
813 Otherwise the `DatasetType` or name thereof.
814 If a resolved `DatasetRef`, the associated dataset
815 is returned directly without additional querying.
816 dataId : `dict` or `DataCoordinate`
817 A `dict` of `Dimension` link name, value pairs that label the
818 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
819 should be provided as the first argument.
820 parameters : `dict`
821 Additional StorageClass-defined options to control reading,
822 typically used to efficiently read only a subset of the dataset.
823 collections : Any, optional
824 Collections to be searched, overriding ``self.collections``.
825 Can be any of the types supported by the ``collections`` argument
826 to butler construction.
827 storageClass : `StorageClass` or `str`, optional
828 The storage class to be used to override the Python type
829 returned by this method. By default the returned type matches
830 the dataset type definition for this dataset. Specifying a
831 read `StorageClass` can force a different type to be returned.
832 This type must be compatible with the original type.
833 timespan : `Timespan` or `None`, optional
834 A timespan that the validity range of the dataset must overlap.
835 If not provided and this is a calibration dataset type, an attempt
836 will be made to find the timespan from any temporal coordinate
837 in the data ID.
838 **kwargs
839 Additional keyword arguments used to augment or construct a
840 `DataCoordinate`. See `DataCoordinate.standardize`
841 parameters.
843 Returns
844 -------
845 obj : `object`
846 The dataset.
848 Raises
849 ------
850 LookupError
851 Raised if no matching dataset exists in the `Registry`.
852 TypeError
853 Raised if no collections were provided.
855 Notes
856 -----
857 When looking up datasets in a `~CollectionType.CALIBRATION` collection,
858 this method requires that the given data ID include temporal dimensions
859 beyond the dimensions of the dataset type itself, in order to find the
860 dataset with the appropriate validity range. For example, a "bias"
861 dataset with native dimensions ``{instrument, detector}`` could be
862 fetched with a ``{instrument, detector, exposure}`` data ID, because
863 ``exposure`` is a temporal dimension.
864 """
865 raise NotImplementedError()
867 @abstractmethod
868 def getURIs(
869 self,
870 datasetRefOrType: DatasetRef | DatasetType | str,
871 /,
872 dataId: DataId | None = None,
873 *,
874 predict: bool = False,
875 collections: Any = None,
876 run: str | None = None,
877 **kwargs: Any,
878 ) -> DatasetRefURIs:
879 """Return the URIs associated with the dataset.
881 Parameters
882 ----------
883 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
884 When `DatasetRef` the `dataId` should be `None`.
885 Otherwise the `DatasetType` or name thereof.
886 dataId : `dict` or `DataCoordinate`
887 A `dict` of `Dimension` link name, value pairs that label the
888 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
889 should be provided as the first argument.
890 predict : `bool`
891 If `True`, allow URIs to be returned of datasets that have not
892 been written.
893 collections : Any, optional
894 Collections to be searched, overriding ``self.collections``.
895 Can be any of the types supported by the ``collections`` argument
896 to butler construction.
897 run : `str`, optional
898 Run to use for predictions, overriding ``self.run``.
899 **kwargs
900 Additional keyword arguments used to augment or construct a
901 `DataCoordinate`. See `DataCoordinate.standardize`
902 parameters.
904 Returns
905 -------
906 uris : `DatasetRefURIs`
907 The URI to the primary artifact associated with this dataset (if
908 the dataset was disassembled within the datastore this may be
909 `None`), and the URIs to any components associated with the dataset
910 artifact. (can be empty if there are no components).
911 """
912 raise NotImplementedError()
914 def getURI(
915 self,
916 datasetRefOrType: DatasetRef | DatasetType | str,
917 /,
918 dataId: DataId | None = None,
919 *,
920 predict: bool = False,
921 collections: Any = None,
922 run: str | None = None,
923 **kwargs: Any,
924 ) -> ResourcePath:
925 """Return the URI to the Dataset.
927 Parameters
928 ----------
929 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
930 When `DatasetRef` the `dataId` should be `None`.
931 Otherwise the `DatasetType` or name thereof.
932 dataId : `dict` or `DataCoordinate`
933 A `dict` of `Dimension` link name, value pairs that label the
934 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
935 should be provided as the first argument.
936 predict : `bool`
937 If `True`, allow URIs to be returned of datasets that have not
938 been written.
939 collections : Any, optional
940 Collections to be searched, overriding ``self.collections``.
941 Can be any of the types supported by the ``collections`` argument
942 to butler construction.
943 run : `str`, optional
944 Run to use for predictions, overriding ``self.run``.
945 **kwargs
946 Additional keyword arguments used to augment or construct a
947 `DataCoordinate`. See `DataCoordinate.standardize`
948 parameters.
950 Returns
951 -------
952 uri : `lsst.resources.ResourcePath`
953 URI pointing to the Dataset within the datastore. If the
954 Dataset does not exist in the datastore, and if ``predict`` is
955 `True`, the URI will be a prediction and will include a URI
956 fragment "#predicted".
957 If the datastore does not have entities that relate well
958 to the concept of a URI the returned URI string will be
959 descriptive. The returned URI is not guaranteed to be obtainable.
961 Raises
962 ------
963 LookupError
964 A URI has been requested for a dataset that does not exist and
965 guessing is not allowed.
966 ValueError
967 Raised if a resolved `DatasetRef` was passed as an input, but it
968 differs from the one found in the registry.
969 TypeError
970 Raised if no collections were provided.
971 RuntimeError
972 Raised if a URI is requested for a dataset that consists of
973 multiple artifacts.
974 """
975 primary, components = self.getURIs(
976 datasetRefOrType, dataId=dataId, predict=predict, collections=collections, run=run, **kwargs
977 )
979 if primary is None or components:
980 raise RuntimeError(
981 f"Dataset ({datasetRefOrType}) includes distinct URIs for components. "
982 "Use Butler.getURIs() instead."
983 )
984 return primary
986 @abstractmethod
987 def get_dataset_type(self, name: str) -> DatasetType:
988 """Get the `DatasetType`.
990 Parameters
991 ----------
992 name : `str`
993 Name of the type.
995 Returns
996 -------
997 type : `DatasetType`
998 The `DatasetType` associated with the given name.
1000 Raises
1001 ------
1002 lsst.daf.butler.MissingDatasetTypeError
1003 Raised if the requested dataset type has not been registered.
1005 Notes
1006 -----
1007 This method handles component dataset types automatically, though most
1008 other operations do not.
1009 """
1010 raise NotImplementedError()
1012 @abstractmethod
1013 def get_dataset(
1014 self,
1015 id: DatasetId | str,
1016 *,
1017 storage_class: str | StorageClass | None = None,
1018 dimension_records: bool = False,
1019 datastore_records: bool = False,
1020 ) -> DatasetRef | None:
1021 """Retrieve a Dataset entry.
1023 Parameters
1024 ----------
1025 id : `DatasetId`
1026 The unique identifier for the dataset, as an instance of
1027 `uuid.UUID` or a string containing a hexadecimal number.
1028 storage_class : `str` or `StorageClass` or `None`
1029 A storage class to use when creating the returned entry. If given
1030 it must be compatible with the default storage class.
1031 dimension_records : `bool`, optional
1032 If `True` the ref will be expanded and contain dimension records.
1033 datastore_records : `bool`, optional
1034 If `True` the ref will contain associated datastore records.
1036 Returns
1037 -------
1038 ref : `DatasetRef` or `None`
1039 A ref to the Dataset, or `None` if no matching Dataset
1040 was found.
1041 """
1042 raise NotImplementedError()
1044 @abstractmethod
1045 def get_many_datasets(self, ids: Iterable[DatasetId | str]) -> list[DatasetRef]:
1046 """Retrieve a list of dataset entries.
1048 Parameters
1049 ----------
1050 ids : `~collections.abc.Iterable` [ `DatasetId` or `str` ]
1051 The unique identifiers for the datasets, as instances of
1052 `uuid.UUID` or strings containing a hexadecimal number.
1054 Returns
1055 -------
1056 refs : `list` [ `DatasetRef` ]
1057 A list containing a `DatasetRef` for each of the given dataset IDs.
1058 If a dataset was not found, no error is thrown -- it is just not
1059 included in the list. The returned datasets are in no particular
1060 order.
1061 """
1062 raise NotImplementedError()
1064 @abstractmethod
1065 def find_dataset(
1066 self,
1067 dataset_type: DatasetType | str,
1068 data_id: DataId | None = None,
1069 *,
1070 collections: str | Sequence[str] | None = None,
1071 timespan: Timespan | None = None,
1072 storage_class: str | StorageClass | None = None,
1073 dimension_records: bool = False,
1074 datastore_records: bool = False,
1075 **kwargs: Any,
1076 ) -> DatasetRef | None:
1077 """Find a dataset given its `DatasetType` and data ID.
1079 This can be used to obtain a `DatasetRef` that permits the dataset to
1080 be read from a `Datastore`. If the dataset is a component and can not
1081 be found using the provided dataset type, a dataset ref for the parent
1082 will be returned instead but with the correct dataset type.
1084 Parameters
1085 ----------
1086 dataset_type : `DatasetType` or `str`
1087 A `DatasetType` or the name of one. If this is a `DatasetType`
1088 instance, its storage class will be respected and propagated to
1089 the output, even if it differs from the dataset type definition
1090 in the registry, as long as the storage classes are convertible.
1091 data_id : `dict` or `DataCoordinate`, optional
1092 A `dict`-like object containing the `Dimension` links that identify
1093 the dataset within a collection. If it is a `dict` the dataId
1094 can include dimension record values such as ``day_obs`` and
1095 ``seq_num`` or ``full_name`` that can be used to derive the
1096 primary dimension.
1097 collections : `str` or `list` [`str`], optional
1098 A an ordered list of collections to search for the dataset.
1099 Defaults to ``self.defaults.collections``.
1100 timespan : `Timespan`, optional
1101 A timespan that the validity range of the dataset must overlap.
1102 If not provided, any `~CollectionType.CALIBRATION` collections
1103 matched by the ``collections`` argument will not be searched.
1104 storage_class : `str` or `StorageClass` or `None`
1105 A storage class to use when creating the returned entry. If given
1106 it must be compatible with the default storage class.
1107 dimension_records : `bool`, optional
1108 If `True` the ref will be expanded and contain dimension records.
1109 datastore_records : `bool`, optional
1110 If `True` the ref will contain associated datastore records.
1111 **kwargs
1112 Additional keyword arguments passed to
1113 `DataCoordinate.standardize` to convert ``dataId`` to a true
1114 `DataCoordinate` or augment an existing one. This can also include
1115 dimension record metadata that can be used to derive a primary
1116 dimension value.
1118 Returns
1119 -------
1120 ref : `DatasetRef`
1121 A reference to the dataset, or `None` if no matching Dataset
1122 was found.
1124 Raises
1125 ------
1126 lsst.daf.butler.NoDefaultCollectionError
1127 Raised if ``collections`` is `None` and
1128 ``self.collections`` is `None`.
1129 LookupError
1130 Raised if one or more data ID keys are missing.
1131 lsst.daf.butler.MissingDatasetTypeError
1132 Raised if the dataset type does not exist.
1133 lsst.daf.butler.MissingCollectionError
1134 Raised if any of ``collections`` does not exist in the registry.
1136 Notes
1137 -----
1138 This method simply returns `None` and does not raise an exception even
1139 when the set of collections searched is intrinsically incompatible with
1140 the dataset type, e.g. if ``datasetType.isCalibration() is False``, but
1141 only `~CollectionType.CALIBRATION` collections are being searched.
1142 This may make it harder to debug some lookup failures, but the behavior
1143 is intentional; we consider it more important that failed searches are
1144 reported consistently, regardless of the reason, and that adding
1145 additional collections that do not contain a match to the search path
1146 never changes the behavior.
1148 This method handles component dataset types automatically, though most
1149 other query operations do not.
1150 """
1151 raise NotImplementedError()
1153 @abstractmethod
1154 def retrieve_artifacts_zip(
1155 self,
1156 refs: Iterable[DatasetRef],
1157 destination: ResourcePathExpression,
1158 overwrite: bool = True,
1159 ) -> ResourcePath:
1160 """Retrieve artifacts from a Butler and place in ZIP file.
1162 Parameters
1163 ----------
1164 refs : `~collections.abc.Iterable` [ `DatasetRef` ]
1165 The datasets to be included in the zip file.
1166 destination : `lsst.resources.ResourcePathExpression`
1167 Directory to write the new ZIP file. This directory will
1168 also be used as a staging area for the datasets being downloaded
1169 from the datastore.
1170 overwrite : `bool`, optional
1171 If `False` the output Zip will not be written if a file of the
1172 same name is already present in ``destination``.
1174 Returns
1175 -------
1176 zip_file : `lsst.resources.ResourcePath`
1177 The path to the new ZIP file.
1179 Raises
1180 ------
1181 ValueError
1182 Raised if there are no refs to retrieve.
1183 """
1184 raise NotImplementedError()
1186 @abstractmethod
1187 def retrieveArtifacts(
1188 self,
1189 refs: Iterable[DatasetRef],
1190 destination: ResourcePathExpression,
1191 transfer: str = "auto",
1192 preserve_path: bool = True,
1193 overwrite: bool = False,
1194 ) -> list[ResourcePath]:
1195 """Retrieve the artifacts associated with the supplied refs.
1197 Parameters
1198 ----------
1199 refs : iterable of `DatasetRef`
1200 The datasets for which artifacts are to be retrieved.
1201 A single ref can result in multiple artifacts. The refs must
1202 be resolved.
1203 destination : `lsst.resources.ResourcePath` or `str`
1204 Location to write the artifacts.
1205 transfer : `str`, optional
1206 Method to use to transfer the artifacts. Must be one of the options
1207 supported by `~lsst.resources.ResourcePath.transfer_from()`.
1208 "move" is not allowed.
1209 preserve_path : `bool`, optional
1210 If `True` the full path of the artifact within the datastore
1211 is preserved. If `False` the final file component of the path
1212 is used.
1213 overwrite : `bool`, optional
1214 If `True` allow transfers to overwrite existing files at the
1215 destination.
1217 Returns
1218 -------
1219 targets : `list` of `lsst.resources.ResourcePath`
1220 URIs of file artifacts in destination location. Order is not
1221 preserved.
1223 Notes
1224 -----
1225 For non-file datastores the artifacts written to the destination
1226 may not match the representation inside the datastore. For example
1227 a hierarchical data structure in a NoSQL database may well be stored
1228 as a JSON file.
1229 """
1230 raise NotImplementedError()
1232 @abstractmethod
1233 def exists(
1234 self,
1235 dataset_ref_or_type: DatasetRef | DatasetType | str,
1236 /,
1237 data_id: DataId | None = None,
1238 *,
1239 full_check: bool = True,
1240 collections: Any = None,
1241 **kwargs: Any,
1242 ) -> DatasetExistence:
1243 """Indicate whether a dataset is known to Butler registry and
1244 datastore.
1246 Parameters
1247 ----------
1248 dataset_ref_or_type : `DatasetRef`, `DatasetType`, or `str`
1249 When `DatasetRef` the `dataId` should be `None`.
1250 Otherwise the `DatasetType` or name thereof.
1251 data_id : `dict` or `DataCoordinate`
1252 A `dict` of `Dimension` link name, value pairs that label the
1253 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
1254 should be provided as the first argument.
1255 full_check : `bool`, optional
1256 If `True`, a check will be made for the actual existence of a
1257 dataset artifact. This will involve additional overhead due to
1258 the need to query an external system. If `False`, this check will
1259 be omitted, and the registry and datastore will solely be asked
1260 if they know about the dataset but no direct check for the
1261 artifact will be performed.
1262 collections : Any, optional
1263 Collections to be searched, overriding ``self.collections``.
1264 Can be any of the types supported by the ``collections`` argument
1265 to butler construction.
1266 **kwargs
1267 Additional keyword arguments used to augment or construct a
1268 `DataCoordinate`. See `DataCoordinate.standardize`
1269 parameters.
1271 Returns
1272 -------
1273 existence : `DatasetExistence`
1274 Object indicating whether the dataset is known to registry and
1275 datastore. Evaluates to `True` if the dataset is present and known
1276 to both.
1277 """
1278 raise NotImplementedError()
1280 @abstractmethod
1281 def _exists_many(
1282 self,
1283 refs: Iterable[DatasetRef],
1284 /,
1285 *,
1286 full_check: bool = True,
1287 ) -> dict[DatasetRef, DatasetExistence]:
1288 """Indicate whether multiple datasets are known to Butler registry and
1289 datastore.
1291 This is an experimental API that may change at any moment.
1293 Parameters
1294 ----------
1295 refs : iterable of `DatasetRef`
1296 The datasets to be checked.
1297 full_check : `bool`, optional
1298 If `True`, a check will be made for the actual existence of each
1299 dataset artifact. This will involve additional overhead due to
1300 the need to query an external system. If `False`, this check will
1301 be omitted, and the registry and datastore will solely be asked
1302 if they know about the dataset(s) but no direct check for the
1303 artifact(s) will be performed.
1305 Returns
1306 -------
1307 existence : dict of [`DatasetRef`, `DatasetExistence`]
1308 Mapping from the given dataset refs to an enum indicating the
1309 status of the dataset in registry and datastore.
1310 Each value evaluates to `True` if the dataset is present and known
1311 to both.
1312 """
1313 raise NotImplementedError()
1315 @abstractmethod
1316 def removeRuns(
1317 self,
1318 names: Iterable[str],
1319 unstore: bool | type[_DeprecatedDefault] = _DeprecatedDefault,
1320 *,
1321 unlink_from_chains: bool = False,
1322 ) -> None:
1323 """Remove one or more `~CollectionType.RUN` collections and the
1324 datasets within them.
1326 Parameters
1327 ----------
1328 names : `~collections.abc.Iterable` [ `str` ]
1329 The names of the collections to remove.
1330 unstore : `bool`, optional
1331 If `True` (default), delete datasets from all datastores in which
1332 they are present, and attempt to rollback the registry deletions if
1333 datastore deletions fail (which may not always be possible). If
1334 `False`, datastore records for these datasets are still removed,
1335 but any artifacts (e.g. files) will not be. This parameter is now
1336 deprecated and no longer has any effect. Files are always deleted
1337 from datastores unless they were ingested using full URIs.
1338 unlink_from_chains : `bool`, optional
1339 If `True` remove the RUN collection from any chains prior to
1340 removing the RUN. If `False` the removal will fail if any chains
1341 still refer to the RUN.
1343 Raises
1344 ------
1345 TypeError
1346 Raised if one or more collections are not of type
1347 `~CollectionType.RUN`.
1348 """
1349 raise NotImplementedError()
1351 @abstractmethod
1352 def ingest(
1353 self,
1354 *datasets: FileDataset,
1355 transfer: str | None = "auto",
1356 record_validation_info: bool = True,
1357 skip_existing: bool = False,
1358 ) -> None:
1359 """Store and register one or more datasets that already exist on disk.
1361 Parameters
1362 ----------
1363 *datasets : `FileDataset`
1364 Each positional argument is a struct containing information about
1365 a file to be ingested, including its URI (either absolute or
1366 relative to the datastore root, if applicable), a resolved
1367 `DatasetRef`, and optionally a formatter class or its
1368 fully-qualified string name. If a formatter is not provided, the
1369 formatter that would be used for `put` is assumed. On successful
1370 ingest all `FileDataset.formatter` attributes will be set to the
1371 formatter class used. `FileDataset.path` attributes may be modified
1372 to put paths in whatever the datastore considers a standardized
1373 form.
1374 transfer : `str`, optional
1375 If not `None`, must be one of 'auto', 'move', 'copy', 'direct',
1376 'split', 'hardlink', 'relsymlink' or 'symlink', indicating how to
1377 transfer the file.
1378 record_validation_info : `bool`, optional
1379 If `True`, the default, the datastore can record validation
1380 information associated with the file. If `False` the datastore
1381 will not attempt to track any information such as checksums
1382 or file sizes. This can be useful if such information is tracked
1383 in an external system or if the file is to be compressed in place.
1384 It is up to the datastore whether this parameter is relevant.
1385 skip_existing : `bool`, optional
1386 If `True`, a dataset will not be ingested if a dataset with the
1387 same dataset ID already exists in the datastore.
1388 If `False` (the default), a `ConflictingDefinitionError` will be
1389 raised if any datasets with the same dataset ID already exist
1390 in the datastore.
1392 Raises
1393 ------
1394 TypeError
1395 Raised if the butler is read-only or if no run was provided.
1396 NotImplementedError
1397 Raised if the `Datastore` does not support the given transfer mode.
1398 DatasetTypeNotSupportedError
1399 Raised if one or more files to be ingested have a dataset type that
1400 is not supported by the `Datastore`..
1401 FileNotFoundError
1402 Raised if one of the given files does not exist.
1403 FileExistsError
1404 Raised if transfer is not `None` but the (internal) location the
1405 file would be moved to is already occupied.
1406 ConflictingDefinitionError
1407 Raised if a dataset already exists in the repository and
1408 ``skip_existing`` is `False`.
1410 Notes
1411 -----
1412 This operation is not fully exception safe: if a database operation
1413 fails, the given `FileDataset` instances may be only partially updated.
1415 It is atomic in terms of database operations (they will either all
1416 succeed or all fail) providing the database engine implements
1417 transactions correctly. It will attempt to be atomic in terms of
1418 filesystem operations as well, but this cannot be implemented
1419 rigorously for most datastores.
1420 """
1421 raise NotImplementedError()
1423 @abstractmethod
1424 def ingest_zip(
1425 self,
1426 zip_file: ResourcePathExpression,
1427 transfer: str = "auto",
1428 *,
1429 transfer_dimensions: bool = False,
1430 dry_run: bool = False,
1431 ) -> None:
1432 """Ingest a Zip file into this butler.
1434 The Zip file must have been created by `retrieve_artifacts_zip`.
1436 Parameters
1437 ----------
1438 zip_file : `lsst.resources.ResourcePathExpression`
1439 Path to the Zip file.
1440 transfer : `str`, optional
1441 Method to use to transfer the Zip into the datastore.
1442 transfer_dimensions : `bool`, optional
1443 If `True`, dimension record data associated with the new datasets
1444 will be transferred from the Zip file, if present.
1445 dry_run : `bool`, optional
1446 If `True` the ingest will be processed without any modifications
1447 made to the target butler and as if the target butler did not
1448 have any of the datasets.
1450 Notes
1451 -----
1452 Run collections and dataset types are created as needed.
1453 """
1454 raise NotImplementedError()
1456 @abstractmethod
1457 def export(
1458 self,
1459 *,
1460 directory: str | None = None,
1461 filename: str | None = None,
1462 format: str | None = None,
1463 transfer: str | None = None,
1464 ) -> AbstractContextManager[RepoExportContext]:
1465 """Export datasets from the repository represented by this `Butler`.
1467 This method is a context manager that returns a helper object
1468 (`RepoExportContext`) that is used to indicate what information from
1469 the repository should be exported.
1471 Parameters
1472 ----------
1473 directory : `str`, optional
1474 Directory dataset files should be written to if ``transfer`` is not
1475 `None`.
1476 filename : `str`, optional
1477 Name for the file that will include database information associated
1478 with the exported datasets. If this is not an absolute path and
1479 ``directory`` is not `None`, it will be written to ``directory``
1480 instead of the current working directory. Defaults to
1481 "export.{format}".
1482 format : `str`, optional
1483 File format for the database information file. If `None`, the
1484 extension of ``filename`` will be used.
1485 transfer : `str`, optional
1486 Transfer mode passed to `Datastore.export`.
1488 Raises
1489 ------
1490 TypeError
1491 Raised if the set of arguments passed is inconsistent.
1493 Examples
1494 --------
1495 Typically the `Registry.queryDataIds` and `Registry.queryDatasets`
1496 methods are used to provide the iterables over data IDs and/or datasets
1497 to be exported::
1499 with butler.export("exports.yaml") as export:
1500 # Export all flats, but none of the dimension element rows
1501 # (i.e. data ID information) associated with them.
1502 export.saveDatasets(
1503 butler.registry.queryDatasets("flat"), elements=()
1504 )
1505 # Export all datasets that start with "deepCoadd_" and all of
1506 # their associated data ID information.
1507 export.saveDatasets(butler.registry.queryDatasets("deepCoadd_*"))
1508 """
1509 raise NotImplementedError()
1511 @abstractmethod
1512 def import_(
1513 self,
1514 *,
1515 directory: ResourcePathExpression | None = None,
1516 filename: ResourcePathExpression | TextIO | None = None,
1517 format: str | None = None,
1518 transfer: str | None = None,
1519 skip_dimensions: set | None = None,
1520 record_validation_info: bool = True,
1521 without_datastore: bool = False,
1522 ) -> None:
1523 """Import datasets into this repository that were exported from a
1524 different butler repository via `~lsst.daf.butler.Butler.export`.
1526 Parameters
1527 ----------
1528 directory : `~lsst.resources.ResourcePathExpression`, optional
1529 Directory containing dataset files to import from. If `None`,
1530 ``filename`` and all dataset file paths specified therein must
1531 be absolute.
1532 filename : `~lsst.resources.ResourcePathExpression` or `TextIO`
1533 A stream or name of file that contains database information
1534 associated with the exported datasets, typically generated by
1535 `~lsst.daf.butler.Butler.export`. If this a string (name) or
1536 `~lsst.resources.ResourcePath` and is not an absolute path,
1537 it will first be looked for relative to ``directory`` and if not
1538 found there it will be looked for in the current working
1539 directory. Defaults to "export.{format}".
1540 format : `str`, optional
1541 File format for ``filename``. If `None`, the extension of
1542 ``filename`` will be used.
1543 transfer : `str`, optional
1544 Transfer mode passed to `~lsst.daf.butler.Datastore.ingest`.
1545 skip_dimensions : `set`, optional
1546 Names of dimensions that should be skipped and not imported.
1547 record_validation_info : `bool`, optional
1548 If `True`, the default, the datastore can record validation
1549 information associated with the file. If `False` the datastore
1550 will not attempt to track any information such as checksums
1551 or file sizes. This can be useful if such information is tracked
1552 in an external system or if the file is to be compressed in place.
1553 It is up to the datastore whether this parameter is relevant.
1554 without_datastore : `bool`, optional
1555 If `True` only registry records will be imported and the datastore
1556 will be ignored.
1558 Raises
1559 ------
1560 TypeError
1561 Raised if the set of arguments passed is inconsistent, or if the
1562 butler is read-only.
1563 """
1564 raise NotImplementedError()
1566 @abstractmethod
1567 def transfer_dimension_records_from(
1568 self, source_butler: LimitedButler | Butler, source_refs: Iterable[DatasetRef]
1569 ) -> None:
1570 """Transfer dimension records to this Butler from another Butler.
1572 Parameters
1573 ----------
1574 source_butler : `LimitedButler` or `Butler`
1575 Butler from which the records are to be transferred. If data IDs
1576 in ``source_refs`` are not expanded then this has to be a full
1577 `Butler` whose registry will be used to expand data IDs. If the
1578 source refs contain coordinates that are used to populate other
1579 records then this will also need to be a full `Butler`.
1580 source_refs : iterable of `DatasetRef`
1581 Datasets defined in the source butler whose dimension records
1582 should be transferred to this butler. In most circumstances.
1583 transfer is faster if the dataset refs are expanded.
1584 """
1585 raise NotImplementedError()
1587 @abstractmethod
1588 def transfer_from(
1589 self,
1590 source_butler: LimitedButler,
1591 source_refs: Iterable[DatasetRef],
1592 transfer: str = "auto",
1593 skip_missing: bool = True,
1594 register_dataset_types: bool = False,
1595 transfer_dimensions: bool = False,
1596 dry_run: bool = False,
1597 ) -> Collection[DatasetRef]:
1598 """Transfer datasets to this Butler from a run in another Butler.
1600 Parameters
1601 ----------
1602 source_butler : `LimitedButler`
1603 Butler from which the datasets are to be transferred. If data IDs
1604 in ``source_refs`` are not expanded then this has to be a full
1605 `Butler` whose registry will be used to expand data IDs.
1606 source_refs : iterable of `DatasetRef`
1607 Datasets defined in the source butler that should be transferred to
1608 this butler. In most circumstances, ``transfer_from`` is faster if
1609 the dataset refs are expanded.
1610 transfer : `str`, optional
1611 Transfer mode passed to `~lsst.daf.butler.Datastore.transfer_from`.
1612 skip_missing : `bool`
1613 If `True`, datasets with no datastore artifact associated with
1614 them are not transferred. If `False` a registry entry will be
1615 created even if no datastore record is created (and so will
1616 look equivalent to the dataset being unstored).
1617 register_dataset_types : `bool`
1618 If `True` any missing dataset types are registered. Otherwise
1619 an exception is raised.
1620 transfer_dimensions : `bool`, optional
1621 If `True`, dimension record data associated with the new datasets
1622 will be transferred.
1623 dry_run : `bool`, optional
1624 If `True` the transfer will be processed without any modifications
1625 made to the target butler and as if the target butler did not
1626 have any of the datasets.
1628 Returns
1629 -------
1630 refs : `list` of `DatasetRef`
1631 The refs added to this Butler.
1633 Notes
1634 -----
1635 The datastore artifact has to exist for a transfer
1636 to be made but non-existence is not an error.
1638 Datasets that already exist in this run will be skipped.
1640 The datasets are imported as part of a transaction, although
1641 dataset types are registered before the transaction is started.
1642 This means that it is possible for a dataset type to be registered
1643 even though transfer has failed.
1644 """
1645 raise NotImplementedError()
1647 @abstractmethod
1648 def validateConfiguration(
1649 self,
1650 logFailures: bool = False,
1651 datasetTypeNames: Iterable[str] | None = None,
1652 ignore: Iterable[str] | None = None,
1653 ) -> None:
1654 """Validate butler configuration.
1656 Checks that each `DatasetType` can be stored in the `Datastore`.
1658 Parameters
1659 ----------
1660 logFailures : `bool`, optional
1661 If `True`, output a log message for every validation error
1662 detected.
1663 datasetTypeNames : iterable of `str`, optional
1664 The `DatasetType` names that should be checked. This allows
1665 only a subset to be selected.
1666 ignore : iterable of `str`, optional
1667 Names of DatasetTypes to skip over. This can be used to skip
1668 known problems. If a named `DatasetType` corresponds to a
1669 composite, all components of that `DatasetType` will also be
1670 ignored.
1672 Raises
1673 ------
1674 ButlerValidationError
1675 Raised if there is some inconsistency with how this Butler
1676 is configured.
1677 """
1678 raise NotImplementedError()
1680 @property
1681 @abstractmethod
1682 def collection_chains(self) -> ButlerCollections:
1683 """Object with methods for modifying collection chains
1684 (`~lsst.daf.butler.ButlerCollections`).
1686 Deprecated. Replaced with ``collections`` property.
1687 """
1688 raise NotImplementedError()
1690 @property
1691 @abstractmethod
1692 def collections(self) -> ButlerCollections:
1693 """Object with methods for modifying and querying collections
1694 (`~lsst.daf.butler.ButlerCollections`).
1696 Use of this object is preferred over `registry` wherever possible.
1697 """
1698 raise NotImplementedError()
1700 @property
1701 @abstractmethod
1702 def run(self) -> str | None:
1703 """Name of the run this butler writes outputs to by default (`str` or
1704 `None`).
1705 """
1706 raise NotImplementedError()
1708 @property
1709 @abstractmethod
1710 def registry(self) -> Registry:
1711 """The object that manages dataset metadata and relationships
1712 (`Registry`).
1714 Many operations that don't involve reading or writing butler datasets
1715 are accessible only via `Registry` methods. Eventually these methods
1716 will be replaced by equivalent `Butler` methods.
1717 """
1718 raise NotImplementedError()
1720 @abstractmethod
1721 def query(self) -> AbstractContextManager[Query]:
1722 """Context manager returning a `Query` object used for construction
1723 and execution of complex queries.
1724 """
1725 raise NotImplementedError()
1727 def query_data_ids(
1728 self,
1729 dimensions: DimensionGroup | Iterable[str] | str,
1730 *,
1731 data_id: DataId | None = None,
1732 where: str = "",
1733 bind: Mapping[str, Any] | None = None,
1734 with_dimension_records: bool = False,
1735 order_by: Iterable[str] | str | None = None,
1736 limit: int | None = -20_000,
1737 explain: bool = True,
1738 **kwargs: Any,
1739 ) -> list[DataCoordinate]:
1740 """Query for data IDs matching user-provided criteria.
1742 Parameters
1743 ----------
1744 dimensions : `DimensionGroup`, `str`, or \
1745 `~collections.abc.Iterable` [`str`]
1746 The dimensions of the data IDs to yield, as either `DimensionGroup`
1747 instances or `str`. Will be automatically expanded to a complete
1748 `DimensionGroup`.
1749 data_id : `dict` or `DataCoordinate`, optional
1750 A data ID whose key-value pairs are used as equality constraints
1751 in the query.
1752 where : `str`, optional
1753 A string expression similar to a SQL WHERE clause. May involve
1754 any column of a dimension table or (as a shortcut for the primary
1755 key column of a dimension table) dimension name. See
1756 :ref:`daf_butler_dimension_expressions` for more information.
1757 bind : `~collections.abc.Mapping`, optional
1758 Mapping containing literal values that should be injected into the
1759 ``where`` expression, keyed by the identifiers they replace.
1760 Values of collection type can be expanded in some cases; see
1761 :ref:`daf_butler_dimension_expressions_identifiers` for more
1762 information.
1763 with_dimension_records : `bool`, optional
1764 If `True` (default is `False`) then returned data IDs will have
1765 dimension records.
1766 order_by : `~collections.abc.Iterable` [`str`] or `str`, optional
1767 Names of the columns/dimensions to use for ordering returned data
1768 IDs. Column name can be prefixed with minus (``-``) to use
1769 descending ordering.
1770 limit : `int` or `None`, optional
1771 Upper limit on the number of returned records. `None` can be used
1772 if no limit is wanted. A limit of ``0`` means that the query will
1773 be executed and validated but no results will be returned. In this
1774 case there will be no exception even if ``explain`` is `True`.
1775 If a negative value is given a warning will be issued if the number
1776 of results is capped by that limit.
1777 explain : `bool`, optional
1778 If `True` (default) then `EmptyQueryResultError` exception is
1779 raised when resulting list is empty. The exception contains
1780 non-empty list of strings explaining possible causes for empty
1781 result.
1782 **kwargs
1783 Additional keyword arguments are forwarded to
1784 `DataCoordinate.standardize` when processing the ``data_id``
1785 argument (and may be used to provide a constraining data ID even
1786 when the ``data_id`` argument is `None`).
1788 Returns
1789 -------
1790 dataIds : `list` [`DataCoordinate`]
1791 Data IDs matching the given query parameters. These are always
1792 guaranteed to identify all dimensions (`DataCoordinate.hasFull`
1793 returns `True`).
1795 Raises
1796 ------
1797 lsst.daf.butler.registry.DataIdError
1798 Raised when ``data_id`` or keyword arguments specify unknown
1799 dimensions or values, or when they contain inconsistent values.
1800 lsst.daf.butler.registry.UserExpressionError
1801 Raised when ``where`` expression is invalid.
1802 lsst.daf.butler.EmptyQueryResultError
1803 Raised when query generates empty result and ``explain`` is set to
1804 `True`.
1805 TypeError
1806 Raised when the arguments are incompatible.
1807 """
1808 if data_id is None:
1809 data_id = DataCoordinate.make_empty(self.dimensions)
1810 if order_by is None:
1811 order_by = []
1812 query_limit = limit
1813 warn_limit = False
1814 if limit is not None and limit < 0:
1815 query_limit = abs(limit) + 1
1816 warn_limit = True
1817 with self.query() as query:
1818 result = (
1819 query.data_ids(dimensions)
1820 .where(data_id, where, bind=bind, **kwargs)
1821 .order_by(*ensure_iterable(order_by))
1822 .limit(query_limit)
1823 )
1824 if with_dimension_records:
1825 result = result.with_dimension_records()
1826 data_ids = list(result)
1827 if warn_limit and len(data_ids) == query_limit:
1828 # We asked for one too many so must remove that from the list.
1829 data_ids.pop(-1)
1830 assert limit is not None # For mypy.
1831 _LOG.warning("More data IDs are available than the requested limit of %d.", abs(limit))
1832 if explain and (limit is None or limit != 0) and not data_ids:
1833 raise EmptyQueryResultError(list(result.explain_no_results()))
1834 return data_ids
1836 def query_datasets(
1837 self,
1838 dataset_type: str | DatasetType,
1839 collections: str | Iterable[str] | None = None,
1840 *,
1841 find_first: bool = True,
1842 data_id: DataId | None = None,
1843 where: str = "",
1844 bind: Mapping[str, Any] | None = None,
1845 with_dimension_records: bool = False,
1846 order_by: Iterable[str] | str | None = None,
1847 limit: int | None = -20_000,
1848 explain: bool = True,
1849 **kwargs: Any,
1850 ) -> list[DatasetRef]:
1851 """Query for dataset references matching user-provided criteria.
1853 Parameters
1854 ----------
1855 dataset_type : `str` or `DatasetType`
1856 Dataset type object or name to search for.
1857 collections : collection expression, optional
1858 A collection name or iterable of collection names to search. If not
1859 provided, the default collections are used. Can be a wildcard if
1860 ``find_first`` is `False` (if find first is requested the order
1861 of collections matters and wildcards make the order indeterminate).
1862 See :ref:`daf_butler_collection_expressions` for more information.
1863 find_first : `bool`, optional
1864 If `True` (default), for each result data ID, only yield one
1865 `DatasetRef` of each `DatasetType`, from the first collection in
1866 which a dataset of that dataset type appears (according to the
1867 order of ``collections`` passed in). If `True`, ``collections``
1868 must not contain wildcards.
1869 data_id : `dict` or `DataCoordinate`, optional
1870 A data ID whose key-value pairs are used as equality constraints in
1871 the query.
1872 where : `str`, optional
1873 A string expression similar to a SQL WHERE clause. May involve any
1874 column of a dimension table or (as a shortcut for the primary key
1875 column of a dimension table) dimension name. See
1876 :ref:`daf_butler_dimension_expressions` for more information.
1877 bind : `~collections.abc.Mapping`, optional
1878 Mapping containing literal values that should be injected into the
1879 ``where`` expression, keyed by the identifiers they replace. Values
1880 of collection type can be expanded in some cases; see
1881 :ref:`daf_butler_dimension_expressions_identifiers` for more
1882 information.
1883 with_dimension_records : `bool`, optional
1884 If `True` (default is `False`) then returned data IDs will have
1885 dimension records.
1886 order_by : `~collections.abc.Iterable` [`str`] or `str`, optional
1887 Names of the columns/dimensions to use for ordering returned data
1888 IDs. Column name can be prefixed with minus (``-``) to use
1889 descending ordering.
1890 limit : `int` or `None`, optional
1891 Upper limit on the number of returned records. `None` can be used
1892 if no limit is wanted. A limit of ``0`` means that the query will
1893 be executed and validated but no results will be returned. In this
1894 case there will be no exception even if ``explain`` is `True`.
1895 If a negative value is given a warning will be issued if the number
1896 of results is capped by that limit.
1897 explain : `bool`, optional
1898 If `True` (default) then `EmptyQueryResultError` exception is
1899 raised when resulting list is empty. The exception contains
1900 non-empty list of strings explaining possible causes for empty
1901 result.
1902 **kwargs
1903 Additional keyword arguments are forwarded to
1904 `DataCoordinate.standardize` when processing the ``data_id``
1905 argument (and may be used to provide a constraining data ID even
1906 when the ``data_id`` argument is `None`).
1908 Returns
1909 -------
1910 refs : `.queries.DatasetRefQueryResults`
1911 Dataset references matching the given query criteria. Nested data
1912 IDs are guaranteed to include values for all implied dimensions
1913 (i.e. `DataCoordinate.hasFull` will return `True`).
1915 Raises
1916 ------
1917 lsst.daf.butler.registry.DatasetTypeExpressionError
1918 Raised when ``dataset_type`` expression is invalid.
1919 lsst.daf.butler.registry.DataIdError
1920 Raised when ``data_id`` or keyword arguments specify unknown
1921 dimensions or values, or when they contain inconsistent values.
1922 lsst.daf.butler.registry.UserExpressionError
1923 Raised when ``where`` expression is invalid.
1924 lsst.daf.butler.EmptyQueryResultError
1925 Raised when query generates empty result and ``explain`` is set to
1926 `True`.
1927 TypeError
1928 Raised when the arguments are incompatible, such as when a
1929 collection wildcard is passed when ``find_first`` is `True`, or
1930 when ``collections`` is `None` and default butler collections are
1931 not defined.
1932 """
1933 if data_id is None:
1934 data_id = DataCoordinate.make_empty(self.dimensions)
1935 if order_by is None:
1936 order_by = []
1937 if collections and has_globs(collections):
1938 # Wild cards need to be expanded but can only be allowed if
1939 # find_first=False because expanding wildcards does not return
1940 # a guaranteed ordering. Querying collection registry to expand
1941 # collections when we do not have wildcards is expensive so only
1942 # do it if we need it.
1943 if find_first:
1944 raise InvalidQueryError(
1945 f"Can not use wildcards in collections when find_first=True (given {collections})"
1946 )
1947 collections = self.collections.query(collections)
1948 query_limit = limit
1949 warn_limit = False
1950 if limit is not None and limit < 0:
1951 query_limit = abs(limit) + 1
1952 warn_limit = True
1953 with self.query() as query:
1954 result = (
1955 query.datasets(dataset_type, collections=collections, find_first=find_first)
1956 .where(data_id, where, bind=bind, **kwargs)
1957 .order_by(*ensure_iterable(order_by))
1958 .limit(query_limit)
1959 )
1960 if with_dimension_records:
1961 result = result.with_dimension_records()
1962 refs = list(result)
1963 if warn_limit and len(refs) == query_limit:
1964 # We asked for one too many so must remove that from the list.
1965 refs.pop(-1)
1966 assert limit is not None # For mypy.
1967 _LOG.warning("More datasets are available than the requested limit of %d.", abs(limit))
1968 if explain and (limit is None or limit != 0) and not refs:
1969 raise EmptyQueryResultError(list(result.explain_no_results()))
1970 return refs
1972 def query_dimension_records(
1973 self,
1974 element: str,
1975 *,
1976 data_id: DataId | None = None,
1977 where: str = "",
1978 bind: Mapping[str, Any] | None = None,
1979 order_by: Iterable[str] | str | None = None,
1980 limit: int | None = -20_000,
1981 explain: bool = True,
1982 **kwargs: Any,
1983 ) -> list[DimensionRecord]:
1984 """Query for dimension information matching user-provided criteria.
1986 Parameters
1987 ----------
1988 element : `str`
1989 The name of a dimension element to obtain records for.
1990 data_id : `dict` or `DataCoordinate`, optional
1991 A data ID whose key-value pairs are used as equality constraints
1992 in the query.
1993 where : `str`, optional
1994 A string expression similar to a SQL WHERE clause. See
1995 `queryDataIds` and :ref:`daf_butler_dimension_expressions` for more
1996 information.
1997 bind : `~collections.abc.Mapping`, optional
1998 Mapping containing literal values that should be injected into the
1999 ``where`` expression, keyed by the identifiers they replace.
2000 Values of collection type can be expanded in some cases; see
2001 :ref:`daf_butler_dimension_expressions_identifiers` for more
2002 information.
2003 order_by : `~collections.abc.Iterable` [`str`] or `str`, optional
2004 Names of the columns/dimensions to use for ordering returned data
2005 IDs. Column name can be prefixed with minus (``-``) to use
2006 descending ordering.
2007 limit : `int` or `None`, optional
2008 Upper limit on the number of returned records. `None` can be used
2009 if no limit is wanted. A limit of ``0`` means that the query will
2010 be executed and validated but no results will be returned. In this
2011 case there will be no exception even if ``explain`` is `True`.
2012 If a negative value is given a warning will be issued if the number
2013 of results is capped by that limit.
2014 explain : `bool`, optional
2015 If `True` (default) then `EmptyQueryResultError` exception is
2016 raised when resulting list is empty. The exception contains
2017 non-empty list of strings explaining possible causes for empty
2018 result.
2019 **kwargs
2020 Additional keyword arguments are forwarded to
2021 `DataCoordinate.standardize` when processing the ``data_id``
2022 argument (and may be used to provide a constraining data ID even
2023 when the ``data_id`` argument is `None`).
2025 Returns
2026 -------
2027 records : `list`[`DimensionRecord`]
2028 Dimension records matching the given query parameters.
2030 Raises
2031 ------
2032 lsst.daf.butler.registry.DataIdError
2033 Raised when ``data_id`` or keyword arguments specify unknown
2034 dimensions or values, or when they contain inconsistent values.
2035 lsst.daf.butler.registry.UserExpressionError
2036 Raised when ``where`` expression is invalid.
2037 lsst.daf.butler.EmptyQueryResultError
2038 Raised when query generates empty result and ``explain`` is set to
2039 `True`.
2040 TypeError
2041 Raised when the arguments are incompatible, such as when a
2042 collection wildcard is passed when ``find_first`` is `True`, or
2043 when ``collections`` is `None` and default butler collections are
2044 not defined.
2045 """
2046 if data_id is None:
2047 data_id = DataCoordinate.make_empty(self.dimensions)
2048 if order_by is None:
2049 order_by = []
2050 query_limit = limit
2051 warn_limit = False
2052 if limit is not None and limit < 0:
2053 query_limit = abs(limit) + 1
2054 warn_limit = True
2055 with self.query() as query:
2056 result = (
2057 query.dimension_records(element)
2058 .where(data_id, where, bind=bind, **kwargs)
2059 .order_by(*ensure_iterable(order_by))
2060 .limit(query_limit)
2061 )
2062 dimension_records = list(result)
2063 if warn_limit and len(dimension_records) == query_limit:
2064 # We asked for one too many so must remove that from the list.
2065 dimension_records.pop(-1)
2066 assert limit is not None # For mypy.
2067 _LOG.warning(
2068 "More dimension records are available than the requested limit of %d.", abs(limit)
2069 )
2070 if explain and (limit is None or limit != 0) and not dimension_records:
2071 raise EmptyQueryResultError(list(result.explain_no_results()))
2072 return dimension_records
2074 def query_all_datasets(
2075 self,
2076 collections: str | Iterable[str] | None = None,
2077 *,
2078 name: str | Iterable[str] = "*",
2079 find_first: bool = True,
2080 data_id: DataId | None = None,
2081 where: str = "",
2082 bind: Mapping[str, Any] | None = None,
2083 limit: int | None = -20_000,
2084 **kwargs: Any,
2085 ) -> list[DatasetRef]:
2086 """Query for datasets of potentially multiple types.
2088 Parameters
2089 ----------
2090 collections : `str` or `~collections.abc.Iterable` [ `str` ], optional
2091 The collection or collections to search, in order. If not provided
2092 or `None`, the default collection search path for this butler is
2093 used.
2094 name : `str` or `~collections.abc.Iterable` [ `str` ], optional
2095 Names or name patterns (glob-style) that returned dataset type
2096 names must match. If an iterable, items are OR'd together. The
2097 default is to include all dataset types in the given collections.
2098 find_first : `bool`, optional
2099 If `True` (default), for each result data ID, only yield one
2100 `DatasetRef` of each `DatasetType`, from the first collection in
2101 which a dataset of that dataset type appears (according to the
2102 order of ``collections`` passed in).
2103 data_id : `dict` or `DataCoordinate`, optional
2104 A data ID whose key-value pairs are used as equality constraints in
2105 the query.
2106 where : `str`, optional
2107 A string expression similar to a SQL WHERE clause. May involve any
2108 column of a dimension table or (as a shortcut for the primary key
2109 column of a dimension table) dimension name. See
2110 :ref:`daf_butler_dimension_expressions` for more information.
2111 bind : `~collections.abc.Mapping`, optional
2112 Mapping containing literal values that should be injected into the
2113 ``where`` expression, keyed by the identifiers they replace. Values
2114 of collection type can be expanded in some cases; see
2115 :ref:`daf_butler_dimension_expressions_identifiers` for more
2116 information.
2117 limit : `int` or `None`, optional
2118 Upper limit on the number of returned records. `None` can be used
2119 if no limit is wanted. A limit of ``0`` means that the query will
2120 be executed and validated but no results will be returned.
2121 If a negative value is given a warning will be issued if the number
2122 of results is capped by that limit. If no limit is provided, by
2123 default a maximum of 20,000 records will be returned.
2124 **kwargs
2125 Additional keyword arguments are forwarded to
2126 `DataCoordinate.standardize` when processing the ``data_id``
2127 argument (and may be used to provide a constraining data ID even
2128 when the ``data_id`` argument is `None`).
2130 Raises
2131 ------
2132 MissingDatasetTypeError
2133 When no dataset types match ``name``, or an explicit (non-glob)
2134 dataset type in ``name`` does not exist.
2135 InvalidQueryError
2136 If the parameters to the query are inconsistent or malformed.
2137 MissingCollectionError
2138 If a given collection is not found.
2140 Returns
2141 -------
2142 refs : `list` [ `DatasetRef` ]
2143 Dataset references matching the given query criteria. Nested data
2144 IDs are guaranteed to include values for all implied dimensions
2145 (i.e. `DataCoordinate.hasFull` will return `True`), but will not
2146 include dimension records (`DataCoordinate.hasRecords` will be
2147 `False`).
2148 """
2149 if collections is None:
2150 collections = list(self.collections.defaults)
2151 else:
2152 collections = list(ensure_iterable(collections))
2154 if bind is None:
2155 bind = {}
2156 if data_id is None:
2157 data_id = {}
2159 warn_limit = False
2160 if limit is not None and limit < 0:
2161 # Add one to the limit so we can detect if we have exceeded it.
2162 limit = abs(limit) + 1
2163 warn_limit = True
2165 args = QueryAllDatasetsParameters(
2166 collections=collections,
2167 name=list(ensure_iterable(name)),
2168 find_first=find_first,
2169 data_id=data_id,
2170 where=where,
2171 limit=limit,
2172 bind=bind,
2173 kwargs=kwargs,
2174 with_dimension_records=False,
2175 )
2176 with self._query_all_datasets_by_page(args) as pages:
2177 result = []
2178 for page in pages:
2179 result.extend(page)
2181 if warn_limit and limit is not None and len(result) >= limit:
2182 # Remove the extra dataset we added for the limit check.
2183 result.pop()
2184 _LOG.warning("More datasets are available than the requested limit of %d.", limit - 1)
2186 return result
2188 @abstractmethod
2189 def _query_all_datasets_by_page(
2190 self, args: QueryAllDatasetsParameters
2191 ) -> AbstractContextManager[Iterator[list[DatasetRef]]]:
2192 raise NotImplementedError()
2194 def clone(
2195 self,
2196 *,
2197 collections: CollectionArgType | None | EllipsisType = ...,
2198 run: str | None | EllipsisType = ...,
2199 inferDefaults: bool | EllipsisType = ...,
2200 dataId: dict[str, str] | EllipsisType = ...,
2201 metrics: ButlerMetrics | None = None,
2202 ) -> Butler:
2203 """Return a new Butler instance connected to the same repository
2204 as this one, optionally overriding ``collections``, ``run``,
2205 ``inferDefaults``, and default data ID.
2207 Parameters
2208 ----------
2209 collections : `~lsst.daf.butler.registry.CollectionArgType` or `None`,\
2210 optional
2211 Same as constructor. If omitted, uses value from original object.
2212 run : `str` or `None`, optional
2213 Same as constructor. If `None`, no default run is used. If
2214 omitted, copies value from original object.
2215 inferDefaults : `bool`, optional
2216 Same as constructor. If omitted, copies value from original
2217 object.
2218 dataId : `str`
2219 Same as ``kwargs`` passed to the constructor. If omitted, copies
2220 values from original object.
2221 metrics : `ButlerMetrics` or `None`, optional
2222 Metrics object to record butler statistics.
2223 """
2224 raise NotImplementedError()